"""Download handlers for http and https schemes"""

from __future__ import annotations

import ipaddress
import logging
import re
from contextlib import suppress
from functools import partial
from io import BytesIO
from time import monotonic
from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast
from urllib.parse import urldefrag, urlparse

from twisted.internet import ssl
from twisted.internet.defer import Deferred, succeed
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet.protocol import Factory, Protocol, connectionDone
from twisted.python.failure import Failure
from twisted.web._newclient import (
    HEADER,
    STATUS,
    HTTP11ClientProtocol,
    HTTPClientParser,
)
from twisted.web.client import (
    URI,
    Agent,
    HTTPConnectionPool,
    ResponseDone,
    ResponseFailed,
    _HTTP11ClientFactory,
)
from twisted.web.client import Response as TxResponse
from twisted.web.http import PotentialDataLoss, _DataLoss
from twisted.web.http_headers import Headers as TxHeaders
from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS, IResponse
from zope.interface import implementer

from scrapy import Request, signals
from scrapy.core.downloader._idna_patch import _install_twisted_idna_fallbacks
from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings
from scrapy.exceptions import (
    DownloadCancelledError,
    DownloadTimeoutError,
    NotConfigured,
    ResponseDataLossError,
    StopDownload,
)
from scrapy.http import Headers, Response
from scrapy.utils._download_handlers import (
    check_stop_download,
    get_dataloss_msg,
    get_maxsize_msg,
    get_warnsize_msg,
    make_response,
    normalize_bind_address,
    wrap_twisted_exceptions,
)
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.utils.ssl import _log_ssl_conn_debug_info
from scrapy.utils.url import add_http_if_no_scheme

from ._base_http import BaseHttpDownloadHandler

if TYPE_CHECKING:
    from twisted.internet.base import ReactorBase
    from twisted.internet.interfaces import IAddress, IConsumer
    from twisted.web._newclient import Request as TxRequest

    # typing.NotRequired requires Python 3.11
    from typing_extensions import NotRequired

    from scrapy.crawler import Crawler


logger = logging.getLogger(__name__)

_T = TypeVar("_T")


class _ResultT(TypedDict):
    txresponse: TxResponse
    body: NotRequired[bytes]
    flags: NotRequired[list[str] | None]
    certificate: NotRequired[ssl.Certificate | None]
    ip_address: NotRequired[ipaddress.IPv4Address | ipaddress.IPv6Address | None]
    stop_download: NotRequired[StopDownload | None]


class HTTP11DownloadHandler(BaseHttpDownloadHandler):
    def __init__(self, crawler: Crawler):
        if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"):
            raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.")
        _install_twisted_idna_fallbacks()
        super().__init__(crawler)
        self._crawler = crawler

        from twisted.internet import reactor

        self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True)
        self._pool.maxPersistentPerHost = crawler.settings.getint(
            "CONCURRENT_REQUESTS_PER_DOMAIN"
        )
        self._pool._factory = _LenientHTTP11ClientFactory

        self._contextFactory: IPolicyForHTTPS = _load_context_factory_from_settings(
            crawler
        )
        self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS")
        self._disconnect_timeout: int = 1

    async def download_request(self, request: Request) -> Response:
        if hasattr(self._crawler.spider, "download_maxsize"):  # pragma: no cover
            warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE")
        if hasattr(self._crawler.spider, "download_warnsize"):  # pragma: no cover
            warn_on_deprecated_spider_attribute(
                "download_warnsize", "DOWNLOAD_WARNSIZE"
            )

        agent = _ScrapyAgent(
            contextFactory=self._contextFactory,
            bindAddress=self._bind_address,
            pool=self._pool,
            maxsize=getattr(
                self._crawler.spider, "download_maxsize", self._default_maxsize
            ),
            warnsize=getattr(
                self._crawler.spider, "download_warnsize", self._default_warnsize
            ),
            fail_on_dataloss=self._fail_on_dataloss,
            crawler=self._crawler,
            tls_verbose_logging=self._tls_verbose_logging,
        )
        try:
            with wrap_twisted_exceptions():
                return await maybe_deferred_to_future(agent.download_request(request))
        except ResponseDataLossError:
            if not self._fail_on_dataloss_warned:
                logger.warning(get_dataloss_msg(request.url))
                self._fail_on_dataloss_warned = True
            raise

    async def close(self) -> None:
        from twisted.internet import reactor

        d: Deferred[None] = self._pool.closeCachedConnections()
        # closeCachedConnections will hang on network or server issues, so
        # we'll manually timeout the deferred.
        #
        # Twisted issue addressing this problem can be found here:
        # https://github.com/twisted/twisted/issues/7738
        #
        # closeCachedConnections doesn't handle external errbacks, so we'll
        # issue a callback after `_disconnect_timeout` seconds.
        #
        # See also https://github.com/scrapy/scrapy/issues/2653
        delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, ())

        try:
            await maybe_deferred_to_future(d)
        finally:
            if delayed_call.active():
                delayed_call.cancel()


class TunnelError(Exception):
    """An HTTP CONNECT tunnel could not be established by the proxy."""


class _TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
    """An endpoint that tunnels through proxies to allow HTTPS downloads. To
    accomplish that, this endpoint sends an HTTP CONNECT to the proxy.
    The HTTP CONNECT is always sent when using this endpoint, I think this could
    be improved as the CONNECT will be redundant if the connection associated
    with this endpoint comes from the pool and a CONNECT has already been issued
    for it.
    """

    _truncatedLength = 1000
    _responseAnswer = (
        r"HTTP/1\.. (?P<status>\d{3})(?P<reason>.{," + str(_truncatedLength) + r"})"
    )
    _responseMatcher = re.compile(_responseAnswer.encode())

    def __init__(
        self,
        reactor: ReactorBase,
        host: str,
        port: int,
        proxyConf: tuple[str, int, bytes | None],
        contextFactory: IPolicyForHTTPS,
        timeout: float = 30,
        bindAddress: tuple[str, int] | None = None,
    ):
        proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
        super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
        self._tunnelReadyDeferred: Deferred[Protocol] = Deferred()
        self._tunneledHost: str = host
        self._tunneledPort: int = port
        self._contextFactory: IPolicyForHTTPS = contextFactory
        self._connectBuffer: bytearray = bytearray()

    def requestTunnel(self, protocol: Protocol) -> Protocol:
        """Asks the proxy to open a tunnel."""
        assert protocol.transport
        tunnelReq = _tunnel_request_data(
            self._tunneledHost, self._tunneledPort, self._proxyAuthHeader
        )
        protocol.transport.write(tunnelReq)
        self._protocolDataReceived = protocol.dataReceived
        protocol.dataReceived = self.processProxyResponse  # type: ignore[method-assign]
        self._protocol = protocol
        return protocol

    def processProxyResponse(self, data: bytes) -> None:
        """Processes the response from the proxy. If the tunnel is successfully
        created, notifies the client that we are ready to send requests. If not
        raises a TunnelError.
        """
        assert self._protocol.transport
        self._connectBuffer += data
        # make sure that enough (all) bytes are consumed
        # and that we've got all HTTP headers (ending with a blank line)
        # from the proxy so that we don't send those bytes to the TLS layer
        #
        # see https://github.com/scrapy/scrapy/issues/2491
        if b"\r\n\r\n" not in self._connectBuffer:
            return
        self._protocol.dataReceived = self._protocolDataReceived  # type: ignore[method-assign]
        respm = _TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)
        if respm and int(respm.group("status")) == 200:
            # set proper Server Name Indication extension
            sslOptions = self._contextFactory.creatorForNetloc(  # type: ignore[call-arg,misc]
                self._tunneledHost,  # type: ignore[arg-type]
                self._tunneledPort,
            )
            self._protocol.transport.startTLS(sslOptions, self._protocolFactory)
            self._tunnelReadyDeferred.callback(self._protocol)
        else:
            extra: Any
            if respm:
                extra = {
                    "status": int(respm.group("status")),
                    "reason": respm.group("reason").strip(),
                }
            else:
                extra = data[: self._truncatedLength]
            self._tunnelReadyDeferred.errback(
                TunnelError(
                    "Could not open CONNECT tunnel with proxy "
                    f"{self._host}:{self._port} [{extra!r}]"
                )
            )

    def connectFailed(self, reason: Failure) -> None:
        """Propagates the errback to the appropriate deferred."""
        self._tunnelReadyDeferred.errback(reason)

    def connect(self, protocolFactory: Factory) -> Deferred[Protocol]:
        self._protocolFactory = protocolFactory
        connectDeferred = super().connect(protocolFactory)
        connectDeferred.addCallback(self.requestTunnel)
        connectDeferred.addErrback(self.connectFailed)
        return self._tunnelReadyDeferred


def _tunnel_request_data(
    host: str, port: int, proxy_auth_header: bytes | None = None
) -> bytes:
    r"""
    Return binary content of a CONNECT request.

    >>> from scrapy.utils.python import to_unicode as s
    >>> s(_tunnel_request_data("example.com", 8080))
    'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
    >>> s(_tunnel_request_data("example.com", 8080, b"123"))
    'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n'
    >>> s(_tunnel_request_data(b"example.com", "8090"))
    'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n'
    """
    host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port))
    tunnel_req = b"CONNECT " + host_value + b" HTTP/1.1\r\n"
    tunnel_req += b"Host: " + host_value + b"\r\n"
    if proxy_auth_header:
        tunnel_req += b"Proxy-Authorization: " + proxy_auth_header + b"\r\n"
    tunnel_req += b"\r\n"
    return tunnel_req


class _TunnelingAgent(Agent):
    """An agent that uses a ``_TunnelingTCP4ClientEndpoint`` to make HTTPS
    downloads. It may look strange that we have chosen to subclass Agent and not
    ProxyAgent but consider that after the tunnel is opened the proxy is
    transparent to the client; thus the agent should behave like there is no
    proxy involved.
    """

    def __init__(
        self,
        *,
        reactor: ReactorBase,
        proxyConf: tuple[str, int, bytes | None],
        contextFactory: IPolicyForHTTPS,
        connectTimeout: float | None = None,
        bindAddress: tuple[str, int] | None = None,
        pool: HTTPConnectionPool | None = None,
    ):
        super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)  # type: ignore[no-untyped-call]
        self._proxyConf: tuple[str, int, bytes | None] = proxyConf
        self._contextFactory: IPolicyForHTTPS = contextFactory

    def _getEndpoint(self, uri: URI) -> _TunnelingTCP4ClientEndpoint:
        return _TunnelingTCP4ClientEndpoint(
            reactor=self._reactor,
            host=uri.host,
            port=uri.port,
            proxyConf=self._proxyConf,
            contextFactory=self._contextFactory,
            timeout=self._endpointFactory._connectTimeout,
            bindAddress=self._endpointFactory._bindAddress,
        )

    def _requestWithEndpoint(
        self,
        key: Any,
        endpoint: TCP4ClientEndpoint,
        method: bytes,
        parsedURI: URI,
        headers: TxHeaders | None,
        bodyProducer: IBodyProducer | None,
        requestPath: bytes,
    ) -> Deferred[IResponse]:
        # proxy host and port are required for HTTP pool `key`
        # otherwise, same remote host connection request could reuse
        # a cached tunneled connection to a different proxy
        key += self._proxyConf
        return super()._requestWithEndpoint(
            key=key,
            endpoint=endpoint,
            method=method,
            parsedURI=parsedURI,
            headers=headers,
            bodyProducer=bodyProducer,
            requestPath=requestPath,
        )


class _ScrapyProxyAgent(Agent):
    def __init__(
        self,
        reactor: ReactorBase,
        proxyURI: bytes,
        contextFactory: IPolicyForHTTPS,
        connectTimeout: float | None = None,
        bindAddress: tuple[str, int] | None = None,
        pool: HTTPConnectionPool | None = None,
    ):
        super().__init__(  # type: ignore[no-untyped-call]
            reactor=reactor,
            contextFactory=contextFactory,
            connectTimeout=connectTimeout,
            bindAddress=bindAddress,
            pool=pool,
        )
        self._proxyURI: URI = URI.fromBytes(proxyURI)

    def request(
        self,
        method: bytes,
        uri: bytes,
        headers: TxHeaders | None = None,
        bodyProducer: IBodyProducer | None = None,
    ) -> Deferred[IResponse]:
        """
        Issue a new request via the configured proxy.
        """
        # Cache *all* connections under the same key, since we are only
        # connecting to a single destination, the proxy:
        return self._requestWithEndpoint(
            key=(b"http-proxy", self._proxyURI.host, self._proxyURI.port),
            endpoint=self._getEndpoint(self._proxyURI),  # type: ignore[no-untyped-call]
            method=method,
            parsedURI=URI.fromBytes(uri),
            headers=headers,
            bodyProducer=bodyProducer,
            requestPath=uri,
        )


class _ScrapyAgent:
    def __init__(
        self,
        *,
        contextFactory: IPolicyForHTTPS,
        connectTimeout: float = 10,
        bindAddress: str | tuple[str, int] | None = None,
        pool: HTTPConnectionPool | None = None,
        maxsize: int = 0,
        warnsize: int = 0,
        fail_on_dataloss: bool = True,
        crawler: Crawler,
        tls_verbose_logging: bool = False,
    ):
        self._contextFactory: IPolicyForHTTPS = contextFactory
        self._connectTimeout: float = connectTimeout
        self._bindAddress: str | tuple[str, int] | None = bindAddress
        self._pool: HTTPConnectionPool | None = pool
        self._maxsize: int = maxsize
        self._warnsize: int = warnsize
        self._fail_on_dataloss: bool = fail_on_dataloss
        self._txresponse: TxResponse | None = None
        self._crawler: Crawler = crawler
        self._tls_verbose_logging: bool = tls_verbose_logging

    def _get_agent(self, request: Request, timeout: float) -> Agent:
        from twisted.internet import reactor

        bindaddress = request.meta.get("bindaddress") or self._bindAddress
        bindaddress = normalize_bind_address(bindaddress)
        proxy = request.meta.get("proxy")
        if proxy:
            proxy = add_http_if_no_scheme(proxy)
            proxy_parsed = urlparse(proxy)
            proxy_host = proxy_parsed.hostname
            proxy_port = proxy_parsed.port
            if not proxy_port:
                proxy_port = 443 if proxy_parsed.scheme == "https" else 80
            if urlparse_cached(request).scheme == "https":
                if proxy_parsed.scheme == "https":  # pragma: no cover
                    raise NotImplementedError(
                        "HTTPS proxies for HTTPS destinations are not supported"
                    )
                assert proxy_host is not None
                proxyAuth = request.headers.get(b"Proxy-Authorization", None)
                proxyConf = (proxy_host, proxy_port, proxyAuth)
                return _TunnelingAgent(
                    reactor=reactor,
                    proxyConf=proxyConf,
                    contextFactory=self._contextFactory,
                    connectTimeout=timeout,
                    bindAddress=bindaddress,
                    pool=self._pool,
                )
            return _ScrapyProxyAgent(
                reactor=reactor,
                proxyURI=to_bytes(proxy, encoding="ascii"),
                contextFactory=self._contextFactory,
                connectTimeout=timeout,
                bindAddress=bindaddress,
                pool=self._pool,
            )

        return Agent(
            reactor=reactor,
            contextFactory=self._contextFactory,
            connectTimeout=timeout,
            bindAddress=bindaddress,
            pool=self._pool,
        )

    def download_request(self, request: Request) -> Deferred[Response]:
        from twisted.internet import reactor

        timeout = request.meta.get("download_timeout") or self._connectTimeout
        agent = self._get_agent(request, timeout)

        # request details
        url = urldefrag(request.url)[0]
        method = to_bytes(request.method)
        headers = TxHeaders(request.headers)
        if isinstance(agent, _TunnelingAgent):
            headers.removeHeader(b"Proxy-Authorization")
        bodyproducer = _RequestBodyProducer(request.body) if request.body else None
        start_time = monotonic()
        d: Deferred[IResponse] = agent.request(
            method,
            to_bytes(url, encoding="ascii"),
            headers,
            cast("IBodyProducer", bodyproducer),
        )
        # set download latency
        d.addCallback(self._cb_latency, request, start_time)
        # response body is ready to be consumed
        d2: Deferred[_ResultT] = d.addCallback(self._cb_bodyready, request)
        d3: Deferred[Response] = d2.addCallback(self._cb_bodydone, url)
        # check download timeout
        self._timeout_cl = reactor.callLater(timeout, d3.cancel)
        d3.addBoth(self._cb_timeout, request, url, timeout)
        return d3

    def _cb_timeout(self, result: _T, request: Request, url: str, timeout: float) -> _T:
        if self._timeout_cl.active():
            self._timeout_cl.cancel()
            return result
        # needed for HTTPS requests, otherwise _ResponseReader doesn't
        # receive connectionLost()
        if self._txresponse:
            self._txresponse._transport.stopProducing()

        raise DownloadTimeoutError(f"Getting {url} took longer than {timeout} seconds.")

    def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T:
        request.meta["download_latency"] = monotonic() - start_time
        return result

    @staticmethod
    def _headers_from_twisted_response(response: TxResponse) -> Headers:
        headers = Headers()
        if response.length != UNKNOWN_LENGTH:
            headers[b"Content-Length"] = str(response.length).encode()
        headers.update(response.headers.getAllRawHeaders())
        return headers

    def _cb_bodyready(
        self, txresponse: TxResponse, request: Request
    ) -> _ResultT | Deferred[_ResultT]:
        if stop_download := check_stop_download(
            signals.headers_received,
            self._crawler,
            request,
            headers=self._headers_from_twisted_response(txresponse),
            body_length=txresponse.length,
        ):
            txresponse._transport.stopProducing()
            txresponse._transport.loseConnection()
            return {
                "txresponse": txresponse,
                "stop_download": stop_download,
            }

        # deliverBody hangs for responses without body
        if cast("int", txresponse.length) == 0:
            return {
                "txresponse": txresponse,
            }

        maxsize = request.meta.get("download_maxsize", self._maxsize)
        warnsize = request.meta.get("download_warnsize", self._warnsize)
        expected_size = (
            cast("int", txresponse.length)
            if txresponse.length != UNKNOWN_LENGTH
            else -1
        )
        fail_on_dataloss = request.meta.get(
            "download_fail_on_dataloss", self._fail_on_dataloss
        )

        if maxsize and expected_size > maxsize:
            warning_msg = get_maxsize_msg(
                expected_size, maxsize, request, expected=True
            )
            logger.warning(warning_msg)
            # Abort connection immediately.
            txresponse._transport._producer.abortConnection()
            raise DownloadCancelledError(warning_msg)

        reached_warnsize = bool(warnsize and expected_size > warnsize)
        if reached_warnsize:
            logger.warning(
                get_warnsize_msg(expected_size, warnsize, request, expected=True)
            )

        d: Deferred[_ResultT] = Deferred(partial(self._cancel, txresponse=txresponse))
        txresponse.deliverBody(
            _ResponseReader(
                finished=d,
                txresponse=txresponse,
                request=request,
                maxsize=maxsize,
                warnsize=warnsize,
                reached_warnsize=reached_warnsize,
                fail_on_dataloss=fail_on_dataloss,
                crawler=self._crawler,
                tls_verbose_logging=self._tls_verbose_logging,
            )
        )

        # save response for timeouts
        self._txresponse = txresponse

        return d

    @staticmethod
    def _cancel(_: Any, txresponse: TxResponse) -> None:
        # Abort connection immediately.
        txresponse._transport._producer.abortConnection()

    def _cb_bodydone(self, result: _ResultT, url: str) -> Response:
        headers = self._headers_from_twisted_response(result["txresponse"])
        try:
            version = result["txresponse"].version
            protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}"
        except (AttributeError, TypeError, IndexError):
            protocol = None
        return make_response(
            url=url,
            status=int(result["txresponse"].code),
            headers=headers,
            body=result.get("body", b""),
            flags=result.get("flags"),
            certificate=result.get("certificate"),
            ip_address=result.get("ip_address"),
            protocol=protocol,
            stop_download=result.get("stop_download"),
        )


@implementer(IBodyProducer)
class _RequestBodyProducer:
    def __init__(self, body: bytes):
        self.body = body
        self.length = len(body)

    def startProducing(self, consumer: IConsumer) -> Deferred[None]:
        consumer.write(self.body)
        return succeed(None)

    def pauseProducing(self) -> None:
        pass

    def stopProducing(self) -> None:
        pass


class _ResponseReader(Protocol):
    def __init__(
        self,
        finished: Deferred[_ResultT],
        txresponse: TxResponse,
        request: Request,
        maxsize: int,
        warnsize: int,
        fail_on_dataloss: bool,
        crawler: Crawler,
        *,
        reached_warnsize: bool = False,
        tls_verbose_logging: bool = False,
    ):
        self._finished: Deferred[_ResultT] = finished
        self._txresponse: TxResponse = txresponse
        self._request: Request = request
        self._bodybuf: BytesIO = BytesIO()
        self._maxsize: int = maxsize
        self._warnsize: int = warnsize
        self._fail_on_dataloss: bool = fail_on_dataloss
        self._reached_warnsize: bool = reached_warnsize
        self._bytes_received: int = 0
        self._certificate: ssl.Certificate | None = None
        self._ip_address: ipaddress.IPv4Address | ipaddress.IPv6Address | None = None
        self._crawler: Crawler = crawler
        self._tls_verbose_logging: bool = tls_verbose_logging

    def _finish_response(
        self, flags: list[str] | None = None, stop_download: StopDownload | None = None
    ) -> None:
        self._finished.callback(
            {
                "txresponse": self._txresponse,
                "body": self._bodybuf.getvalue(),
                "flags": flags,
                "certificate": self._certificate,
                "ip_address": self._ip_address,
                "stop_download": stop_download,
            }
        )

    def connectionMade(self) -> None:
        assert self.transport
        if self._certificate is None:
            with suppress(AttributeError):
                self._certificate = ssl.Certificate(
                    self.transport._producer.getPeerCertificate()
                )

        if self._ip_address is None:
            self._ip_address = ipaddress.ip_address(
                self.transport._producer.getPeer().host
            )

        if self._tls_verbose_logging:
            connection = self.transport._producer.getHandle()
            hostname = urlparse_cached(self._request).hostname
            assert hostname is not None
            _log_ssl_conn_debug_info(hostname, connection)

    def dataReceived(self, data: bytes) -> None:
        # This maybe called several times after cancel was called with buffered data.
        if self._finished.called:
            return

        assert self.transport
        self._bodybuf.write(data)
        self._bytes_received += len(data)

        if stop_download := check_stop_download(
            signals.bytes_received, self._crawler, self._request, data=data
        ):
            self.transport.stopProducing()
            self.transport.loseConnection()
            self._finish_response(stop_download=stop_download)

        if self._maxsize and self._bytes_received > self._maxsize:
            logger.warning(
                get_maxsize_msg(
                    self._bytes_received, self._maxsize, self._request, expected=False
                )
            )
            # Clear buffer earlier to avoid keeping data in memory for a long time.
            self._bodybuf.truncate(0)
            self._finished.cancel()

        if (
            self._warnsize
            and self._bytes_received > self._warnsize
            and not self._reached_warnsize
        ):
            self._reached_warnsize = True
            logger.warning(
                get_warnsize_msg(
                    self._bytes_received, self._warnsize, self._request, expected=False
                )
            )

    def connectionLost(self, reason: Failure = connectionDone) -> None:
        if self._finished.called:
            return

        if reason.check(ResponseDone):
            self._finish_response()
            return

        if reason.check(PotentialDataLoss):
            self._finish_response(flags=["partial"])
            return

        if reason.check(ResponseFailed) and any(
            r.check(_DataLoss)
            for r in reason.value.reasons  # type: ignore[union-attr]
        ):
            if not self._fail_on_dataloss:
                self._finish_response(flags=["dataloss"])
                return

            exc = ResponseDataLossError()
            exc.__cause__ = reason.value
            reason = Failure(exc)

        self._finished.errback(reason)


class _LenientHTTPClientParser(HTTPClientParser):
    """Response parser that skips bad response header lines, those with no
    colon in them, instead of failing to parse the whole response.

    Some servers send such lines, and web browsers skip them and keep parsing
    the header lines that follow. See
    https://github.com/scrapy/scrapy/issues/210.
    """

    def lineReceived(self, line: bytes) -> None:
        # A copy of twisted.web._newclient.HTTPParser.lineReceived() where the
        # header name and value are only extracted from header lines that have
        # a colon.

        # Handle the normal CR LF case.
        if line[-1:] == b"\r":
            line = line[:-1]

        if self.state == STATUS:
            self.statusReceived(line)  # type: ignore[no-untyped-call]
            self.state = HEADER
            return

        # HEADER is the only other state in which lines are received, as the
        # parser switches to raw mode for the response body.
        if not line or line[0] not in b" \t":
            if self._partialHeader is not None:
                header = b"".join(self._partialHeader)
                if b":" in header:
                    name, value = header.split(b":", 1)
                    self.headerReceived(name, value.strip())  # type: ignore[no-untyped-call]
                else:
                    logger.debug(
                        f"Skipping the bad response header line {header!r}, as "
                        f"it has no colon."
                    )
            if not line:
                # Empty line means the header section is over.
                self.allHeadersReceived()  # type: ignore[no-untyped-call]
            else:
                # Line not beginning with LWS is another header.
                self._partialHeader = [line]
        else:
            # A line beginning with LWS is a continuation of a header begun on
            # a previous line.
            self._partialHeader.append(line)  # type: ignore[union-attr]


class _LenientHTTP11ClientProtocol(HTTP11ClientProtocol):
    """Protocol that parses responses with :class:`_LenientHTTPClientParser`."""

    def request(self, request: TxRequest) -> Deferred[IResponse]:
        d: Deferred[IResponse] = super().request(request)
        # HTTP11ClientProtocol.request() hardcodes the parser class, so the
        # only way to use a different one is to replace the class of the parser
        # object that it creates. This is safe because
        # _LenientHTTPClientParser defines no additional state. The parser is
        # always there because HTTPConnectionPool only reuses connections whose
        # protocol is in the QUIESCENT state, for which request() always
        # creates a parser.
        assert self._parser is not None
        self._parser.__class__ = _LenientHTTPClientParser
        return d


class _LenientHTTP11ClientFactory(_HTTP11ClientFactory):
    """Factory that builds :class:`_LenientHTTP11ClientProtocol` protocols."""

    noisy = False

    def buildProtocol(self, addr: IAddress | None) -> HTTP11ClientProtocol:
        return _LenientHTTP11ClientProtocol(self._quiescentCallback)  # type: ignore[no-untyped-call]
