"""
MemoryUsage extension

See documentation in docs/topics/extensions.rst
"""

from __future__ import annotations

import logging
import socket
import sys
import warnings
from importlib import import_module
from pprint import pformat
from typing import TYPE_CHECKING

from scrapy import signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
from scrapy.utils.defer import _schedule_coro
from scrapy.utils.engine import get_engine_status
from scrapy.utils.misc import build_from_crawler

if TYPE_CHECKING:
    from twisted.internet.task import LoopingCall

    # typing.Self requires Python 3.11
    from typing_extensions import Self

    from scrapy.crawler import Crawler
    from scrapy.statscollectors import StatsCollector


logger = logging.getLogger(__name__)


class MemoryUsage:
    def __init__(self, crawler: Crawler):
        if not crawler.settings.getbool("MEMUSAGE_ENABLED"):
            raise NotConfigured
        try:
            # stdlib's resource module is only available on unix platforms.
            self.resource = import_module("resource")
        except ImportError as exc:
            raise NotConfigured from exc

        self.crawler: Crawler = crawler
        self._stats: StatsCollector = crawler.stats
        self.warned: bool = False
        self.notify_mails: list[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL")
        if self.notify_mails:  # pragma: no cover
            from scrapy.mail import MailSender  # noqa: PLC0415

            warnings.warn(
                "The 'MEMUSAGE_NOTIFY_MAIL' setting is deprecated and will be removed "
                "in a future release. Please use the 'memusage_warning_reached' and 'spider_closed' "
                "signals to implement custom notifications.",
                category=ScrapyDeprecationWarning,
                stacklevel=2,
            )
            self.mail = build_from_crawler(MailSender, crawler)

        self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024
        self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024
        self.check_interval: float = crawler.settings.getfloat(
            "MEMUSAGE_CHECK_INTERVAL_SECONDS"
        )
        crawler.signals.connect(self.engine_started, signal=signals.engine_started)
        crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped)

    @classmethod
    def from_crawler(cls, crawler: Crawler) -> Self:
        return cls(crawler)

    def get_virtual_size(self) -> int:
        size: int = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss
        if sys.platform != "darwin":
            # on macOS ru_maxrss is in bytes, on Linux it is in KB
            size *= 1024
        return size

    def engine_started(self) -> None:
        self._stats.set_value("memusage/startup", self.get_virtual_size())
        self.tasks: list[AsyncioLoopingCall | LoopingCall] = []
        tsk = create_looping_call(self.update)
        self.tasks.append(tsk)
        tsk.start(self.check_interval, now=True)
        if self.limit:
            tsk = create_looping_call(self._check_limit)
            self.tasks.append(tsk)
            tsk.start(self.check_interval, now=True)
        if self.warning:
            tsk = create_looping_call(self._check_warning)
            self.tasks.append(tsk)
            tsk.start(self.check_interval, now=True)

    def engine_stopped(self) -> None:
        for tsk in self.tasks:
            if tsk.running:  # pragma: no branch
                tsk.stop()

    def update(self) -> None:
        self._stats.max_value("memusage/max", self.get_virtual_size())

    def _check_limit(self) -> None:
        peak_mem_usage = self.get_virtual_size()
        if peak_mem_usage > self.limit:
            self._stats.set_value("memusage/limit_reached", 1)
            mem = self.limit / 1024 / 1024
            logger.error(
                "Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...",
                {"memusage": mem},
                extra={"crawler": self.crawler},
            )
            if self.notify_mails:  # pragma: no cover
                subj = (
                    f"{self.crawler.settings['BOT_NAME']} terminated: "
                    f"memory usage exceeded {mem}MiB at {socket.gethostname()}"
                )
                self._send_report(self.notify_mails, subj)
                self._stats.set_value("memusage/limit_notified", 1)

            if self.crawler.engine.spider is not None:
                _schedule_coro(
                    self.crawler.engine.close_spider_async(reason="memusage_exceeded")
                )
            else:
                _schedule_coro(self.crawler.stop_async())
        else:
            logger.info(
                "Peak memory usage is %(virtualsize)dMiB",
                {"virtualsize": peak_mem_usage / 1024 / 1024},
            )

    def _check_warning(self) -> None:
        if self.warned:  # warn only once
            return
        if self.get_virtual_size() > self.warning:
            self._stats.set_value("memusage/warning_reached", 1)
            self.crawler.signals.send_catch_log(signal=signals.memusage_warning_reached)
            mem = self.warning / 1024 / 1024
            logger.warning(
                "Memory usage reached %(memusage)dMiB",
                {"memusage": mem},
                extra={"crawler": self.crawler},
            )
            if self.notify_mails:  # pragma: no cover
                subj = (
                    f"{self.crawler.settings['BOT_NAME']} warning: "
                    f"memory usage reached {mem}MiB at {socket.gethostname()}"
                )
                self._send_report(self.notify_mails, subj)
                self._stats.set_value("memusage/warning_notified", 1)
            self.warned = True

    def _send_report(self, rcpts: list[str], subject: str) -> None:  # pragma: no cover
        """send notification mail with some additional useful info"""
        s = f"Memory usage at engine startup : {self._stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"
        s += f"Maximum memory usage          : {self._stats.get_value('memusage/max') / 1024 / 1024}M\r\n"
        s += f"Current memory usage          : {self.get_virtual_size() / 1024 / 1024}M\r\n"

        s += (
            "ENGINE STATUS ------------------------------------------------------- \r\n"
        )
        s += "\r\n"
        s += pformat(get_engine_status(self.crawler.engine))
        s += "\r\n"
        self.mail.send(rcpts, subject, s)
