import base64
import dataclasses
import logging
import mimetypes
import random
import re
import time
from abc import ABC, abstractmethod
from datetime import datetime
from ftplib import FTP
from io import BytesIO
from pathlib import Path
from posixpath import split
from shutil import rmtree
from tempfile import mkdtemp
from typing import Any
from unittest import mock
from unittest.mock import MagicMock

import attr
import pytest
from itemadapter import ItemAdapter
from twisted.internet.defer import Deferred
from twisted.python.failure import Failure

from scrapy.crawler import Crawler
from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.item import Field, Item
from scrapy.pipelines import files
from scrapy.pipelines.files import (
    FilesPipeline,
    FSFilesStore,
    FTPFilesStore,
    GCSFilesStore,
    S3FilesStore,
)
from scrapy.pipelines.media import FileException, _MediaRequestFiltered
from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer
from tests.utils.decorators import coroutine_test, inline_callbacks_test

from .utils.cloud import mock_google_cloud_storage
from .utils.media_pipelines import DUMMY_SPIDER_INFO, mocked_download_func


def get_ftp_content_and_delete(
    path: str,
    host: str,
    port: int,
    username: str,
    password: str,
    use_active_mode: bool = False,
) -> bytes:
    with FTP() as ftp:
        ftp.connect(host, port)
        ftp.login(username, password)
        if use_active_mode:
            ftp.set_pasv(False)
        ftp_data: list[bytes] = []

        def buffer_data(data: bytes) -> None:
            ftp_data.append(data)

        ftp.retrbinary(f"RETR {path}", buffer_data)
        dirname, filename = split(path)
        ftp.cwd(dirname)
        ftp.delete(filename)
    return b"".join(ftp_data)


class DeferredFSFilesStore(FSFilesStore):
    """A simple store with persist_file() returning a deferred."""

    def persist_file(self, path, buf, info, meta=None, headers=None):
        deferred: Deferred[None] = Deferred()
        # short-hand super() doesn't work in nested functions
        parent_persist_file = super().persist_file

        def cb():
            parent_persist_file(path, buf, info, meta=meta, headers=headers)
            deferred.callback(None)

        call_later(0.5, cb)
        return deferred


class TestFilesPipeline:
    def setup_method(self):
        self.tempdir = mkdtemp()
        self.pipeline = self._create_pipeline(FilesPipeline)

    def teardown_method(self):
        rmtree(self.tempdir)

    def _create_pipeline(
        self,
        pipeline_cls: type[FilesPipeline],
        settings: dict[str, Any] | None = None,
    ) -> FilesPipeline:
        crawler = get_crawler(
            DefaultSpider, {"FILES_STORE": self.tempdir, **(settings or {})}
        )
        crawler.spider = crawler._create_spider()
        crawler.engine = MagicMock(download_async=mocked_download_func)
        pipeline = build_from_crawler(pipeline_cls, crawler)
        pipeline.open_spider()
        return pipeline

    def test_file_path_query_parameters(self):
        file_path = self.pipeline.file_path

        req1 = Request("http://foo.bar/baz.txt?fizz")
        assert file_path(req1) == "full/a2b4913a62f65445aeae2bac08cd8c3b41d7195e.txt"

        req2 = Request("http://foo.bar/get_img.foo?file=photo.jpg")
        assert file_path(req2) == "full/7fc9461c9fd836515bea6983373097203a7d748e.jpg"

    def test_file_path(self):
        file_path = self.pipeline.file_path
        assert (
            file_path(Request("https://dev.mydeco.com/mydeco.pdf"))
            == "full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf"
        )
        assert (
            file_path(
                Request(
                    "http://www.maddiebrown.co.uk///catalogue-items//image_54642_12175_95307.txt"
                )
            )
            == "full/4ce274dd83db0368bafd7e406f382ae088e39219.txt"
        )
        assert (
            file_path(
                Request("https://dev.mydeco.com/two/dirs/with%20spaces%2Bsigns.doc")
            )
            == "full/94ccc495a17b9ac5d40e3eabf3afcb8c2c9b9e1a.doc"
        )
        assert (
            file_path(
                Request(
                    "http://www.dfsonline.co.uk/get_prod_image?img=status_0907_mdm.jpg"
                )
            )
            == "full/c67f916ff9d542e822dedf38f9fcb146d1faba78.jpg"
        )
        assert (
            file_path(Request("http://www.dorma.co.uk/images/product_details/2532/"))
            == "full/97ee6f8a46cbbb418ea91502fd24176865cf39b2"
        )
        assert (
            file_path(Request("http://www.dorma.co.uk/images/product_details/2532"))
            == "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1"
        )
        assert (
            file_path(
                Request("http://www.dorma.co.uk/images/product_details/2532"),
                response=Response("http://www.dorma.co.uk/images/product_details/2532"),
                info=object(),  # type: ignore[arg-type]
            )
            == "full/244e0dd7d96a3b7b01f54eded250c9e272577aa1"
        )
        assert (
            file_path(
                Request(
                    "http://www.dfsonline.co.uk/get_prod_image?img=status_0907_mdm.jpg.bohaha"
                )
            )
            == "full/e75f2fa260521b56f6b6a867447b8002d00b5841"
        )
        assert (
            file_path(
                Request(
                    "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
                                    //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y"
                )
            )
            == "full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png"
        )

    def test_fs_store(self):
        assert isinstance(self.pipeline.store, FSFilesStore)
        assert self.pipeline.store.basedir == self.tempdir

        path = "some/image/key.jpg"
        fullpath = Path(self.tempdir, "some", "image", "key.jpg")
        assert self.pipeline.store._get_filesystem_path(path) == fullpath

    @coroutine_test
    async def test_file_not_expired(self):
        item_url = "http://example.com/file.pdf"
        item = _create_item_with_files(item_url)
        with (
            mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
            mock.patch.object(
                FSFilesStore,
                "stat_file",
                return_value={"checksum": "abc", "last_modified": time.time()},
            ),
            mock.patch.object(
                FilesPipeline,
                "get_media_requests",
                return_value=[_prepare_request_object(item_url)],
            ),
        ):
            result = await self.pipeline.process_item(item)
        assert result["files"][0]["checksum"] == "abc"
        assert result["files"][0]["status"] == "uptodate"

    @coroutine_test
    async def test_file_expired(self):
        item_url = "http://example.com/file2.pdf"
        item = _create_item_with_files(item_url)
        with (
            mock.patch.object(
                FSFilesStore,
                "stat_file",
                return_value={
                    "checksum": "abc",
                    "last_modified": time.time()
                    - (self.pipeline.expires * 60 * 60 * 24 * 2),
                },
            ),
            mock.patch.object(
                FilesPipeline,
                "get_media_requests",
                return_value=[_prepare_request_object(item_url)],
            ),
            mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
        ):
            result = await self.pipeline.process_item(item)
        assert result["files"][0]["checksum"] != "abc"
        assert result["files"][0]["status"] == "downloaded"

    @coroutine_test
    async def test_file_cached(self):
        item_url = "http://example.com/file3.pdf"
        item = _create_item_with_files(item_url)
        with (
            mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
            mock.patch.object(
                FSFilesStore,
                "stat_file",
                return_value={
                    "checksum": "abc",
                    "last_modified": time.time()
                    - (self.pipeline.expires * 60 * 60 * 24 * 2),
                },
            ),
            mock.patch.object(
                FilesPipeline,
                "get_media_requests",
                return_value=[_prepare_request_object(item_url, flags=["cached"])],
            ),
        ):
            result = await self.pipeline.process_item(item)
        assert result["files"][0]["checksum"] != "abc"
        assert result["files"][0]["status"] == "cached"

    @coroutine_test
    async def test_file_stat_without_last_modified(self) -> None:
        """A stat result without a last modification time forces a download."""
        item_url = "http://example.com/file4.pdf"
        item = _create_item_with_files(item_url)
        with (
            mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
            mock.patch.object(
                FSFilesStore, "stat_file", return_value={"checksum": "abc"}
            ),
            mock.patch.object(
                FilesPipeline,
                "get_media_requests",
                return_value=[_prepare_request_object(item_url)],
            ),
        ):
            result = await self.pipeline.process_item(item)
        assert result["files"][0]["checksum"] != "abc"
        assert result["files"][0]["status"] == "downloaded"

    @coroutine_test
    async def test_file_empty_content(self, caplog: pytest.LogCaptureFixture) -> None:
        item_url = "http://example.com/empty.pdf"
        item = _create_item_with_files(item_url)
        request = Request(
            item_url, meta={"response": Response(item_url, status=200, body=b"")}
        )
        with (
            caplog.at_level(logging.WARNING),
            mock.patch.object(
                FilesPipeline, "get_media_requests", return_value=[request]
            ),
        ):
            result = await self.pipeline.process_item(item)
        assert result["files"] == []
        assert "File (empty-content): Empty file from" in caplog.text

    @coroutine_test
    async def test_file_downloaded_file_exception(
        self, caplog: pytest.LogCaptureFixture
    ) -> None:
        """A FileException from file_downloaded() is logged as a warning and
        kept as is."""

        class FailingFilesPipeline(FilesPipeline):
            def file_downloaded(self, response, request, info, *, item=None):
                raise FileException("boom")

        item_url = "http://example.com/file5.pdf"
        item = _create_item_with_files(item_url)
        pipeline = self._create_pipeline(FailingFilesPipeline)
        with (
            caplog.at_level(logging.WARNING),
            mock.patch.object(
                FilesPipeline,
                "get_media_requests",
                return_value=[_prepare_request_object(item_url)],
            ),
        ):
            result = await pipeline.process_item(item)
        assert result["files"] == []
        records = [
            r for r in caplog.records if "Error processing file" in r.getMessage()
        ]
        assert len(records) == 1
        assert records[0].levelname == "WARNING"
        assert "boom" in records[0].getMessage()

    @coroutine_test
    async def test_file_downloaded_unknown_error(
        self, caplog: pytest.LogCaptureFixture
    ) -> None:
        """Any other exception from file_downloaded() is logged as an error and
        reported as a FileException."""

        class FailingFilesPipeline(FilesPipeline):
            def file_downloaded(self, response, request, info, *, item=None):
                raise RuntimeError("boom")

        item_url = "http://example.com/file6.pdf"
        item = _create_item_with_files(item_url)
        pipeline = self._create_pipeline(FailingFilesPipeline)
        with (
            caplog.at_level(logging.WARNING),
            mock.patch.object(
                FilesPipeline,
                "get_media_requests",
                return_value=[_prepare_request_object(item_url)],
            ),
        ):
            result = await pipeline.process_item(item)
        assert result["files"] == []
        records = [
            r for r in caplog.records if "Error processing file" in r.getMessage()
        ]
        assert len(records) == 1
        assert records[0].levelname == "ERROR"
        exc_info = records[0].exc_info
        assert exc_info is not None
        assert exc_info[0] is RuntimeError

    @coroutine_test
    async def test_async_store(self) -> None:
        """Test that async persist_file() works and is awaited."""

        self.pipeline.store = DeferredFSFilesStore(self.tempdir)
        item_url = "http://example.com/file.pdf"
        item = _create_item_with_files(item_url)
        with (
            mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
            mock.patch.object(
                FilesPipeline,
                "get_media_requests",
                return_value=[_prepare_request_object(item_url)],
            ),
        ):
            result = await self.pipeline.process_item(item)
        assert result["files"][0]["status"] == "downloaded"
        assert result["files"][0]["checksum"]
        # check that the file was written by persist_file()
        path = Path(self.tempdir) / result["files"][0]["path"]
        assert path.exists()
        assert path.read_bytes() == b"data"

    def test_file_path_from_item(self):
        """
        Custom file path based on item data, overriding default implementation
        """

        class CustomFilesPipeline(FilesPipeline):
            def file_path(self, request, response=None, info=None, item=None) -> str:
                return f"full/{item.get('path')}"

        file_path = build_from_crawler(
            CustomFilesPipeline, get_crawler(None, {"FILES_STORE": self.tempdir})
        ).file_path
        item = {"path": "path-to-store-file"}
        request = Request("http://example.com")
        assert file_path(request, item=item) == "full/path-to-store-file"

    @coroutine_test
    async def test_file_path_from_response(self) -> None:
        """file_path() may build the path out of the response, e.g. to get the
        file extension from a response header, as long as FILES_EXPIRES is 0 to
        disable the up-to-date check, which runs before the download and hence
        cannot reach the same path."""

        class ContentTypeFilesPipeline(FilesPipeline):
            def file_path(self, request, response=None, info=None, *, item=None):
                path = super().file_path(request, response, info, item=item)
                if response is None:
                    return path
                content_type = response.headers["Content-Type"].decode()
                return path + (mimetypes.guess_extension(content_type) or "")

        item_url = "http://example.com/download?id=1"
        item = _create_item_with_files(item_url)
        pipeline = self._create_pipeline(ContentTypeFilesPipeline, {"FILES_EXPIRES": 0})
        request = _prepare_request_object(
            item_url, headers={"Content-Type": "application/pdf"}
        )
        with (
            mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
            # A fresh file at the response-less path is ignored thanks to
            # FILES_EXPIRES being 0.
            mock.patch.object(
                FSFilesStore,
                "stat_file",
                return_value={"checksum": "abc", "last_modified": time.time() - 1},
            ),
            mock.patch.object(
                FilesPipeline, "get_media_requests", return_value=[request]
            ),
        ):
            result = await pipeline.process_item(item)

        file_info = result["files"][0]
        assert file_info["status"] == "downloaded"
        assert file_info["path"].endswith(".pdf")
        assert (Path(self.tempdir) / file_info["path"]).read_bytes() == b"data"

    def test_media_failed_filtered_request(
        self, caplog: pytest.LogCaptureFixture
    ) -> None:
        """A filtered media request (IgnoreRequest) is reported as a
        _MediaRequestFiltered exception and logged at the DEBUG level, instead
        of as a download error with a traceback."""
        request = Request("http://example.com/file.pdf")
        reason = "Filtered offsite request to 'example.com'"
        failure = Failure(IgnoreRequest(reason))

        with (
            caplog.at_level(logging.DEBUG),
            pytest.raises(_MediaRequestFiltered, match=re.escape(reason)),
        ):
            self.pipeline.media_failed(failure, request, self.pipeline.spiderinfo)

        assert len(caplog.records) == 1
        record = caplog.records[0]
        assert record.levelname == "DEBUG"
        assert record.exc_info is None
        assert reason in record.getMessage()

    def test_media_failed_download_error(
        self, caplog: pytest.LogCaptureFixture
    ) -> None:
        """A genuine download error is reported as a FileException and logged as
        a warning."""
        request = Request("http://example.com/file.pdf")
        failure = Failure(Exception("boom"))

        with caplog.at_level(logging.WARNING), pytest.raises(FileException):
            self.pipeline.media_failed(failure, request, self.pipeline.spiderinfo)

        assert len(caplog.records) == 1
        assert caplog.records[0].levelname == "WARNING"

    @coroutine_test
    async def test_process_item_filtered_request(
        self, caplog: pytest.LogCaptureFixture
    ) -> None:
        """A filtered (e.g. offsite) media request is processed as a failed
        result without being logged as an error with a traceback."""
        item_url = "http://example.com/file.pdf"
        item = _create_item_with_files(item_url)
        request = Request(
            item_url,
            meta={
                "response": IgnoreRequest("Filtered offsite request to 'example.com'")
            },
        )
        with (
            caplog.at_level(logging.DEBUG),
            mock.patch.object(
                FilesPipeline, "get_media_requests", return_value=[request]
            ),
        ):
            result = await self.pipeline.process_item(item)

        assert result["files"] == []
        assert not any(r.levelname in ("WARNING", "ERROR") for r in caplog.records)
        assert any(
            "Filtered offsite request to 'example.com'" in r.getMessage()
            for r in caplog.records
        )

    @pytest.mark.parametrize(
        "bad_type",
        [
            "http://example.com/file.pdf",
            ("http://example.com/file.pdf",),
            {"url": "http://example.com/file.pdf"},
            123,
            None,
        ],
    )
    def test_rejects_non_list_file_urls(self, tmp_path, bad_type):
        pipeline = build_from_crawler(
            FilesPipeline, get_crawler(None, {"FILES_STORE": str(tmp_path)})
        )
        item = ItemWithFiles()
        item["file_urls"] = bad_type

        with pytest.raises(TypeError, match="file_urls must be a list of URLs"):
            list(pipeline.get_media_requests(item, None))  # type: ignore[arg-type]


class TestFilesPipelineFieldsMixin(ABC):
    @property
    @abstractmethod
    def item_class(self) -> Any:
        raise NotImplementedError

    def test_item_fields_default(self, tmp_path):
        url = "http://www.example.com/files/1.txt"
        item = self.item_class(name="item1", file_urls=[url])
        pipeline = build_from_crawler(
            FilesPipeline, get_crawler(None, {"FILES_STORE": tmp_path})
        )
        requests = list(pipeline.get_media_requests(item, None))  # type: ignore[arg-type]
        assert requests[0].url == url
        results = [(True, {"url": url})]
        item = pipeline.item_completed(results, item, None)  # type: ignore[arg-type]
        files = ItemAdapter(item).get("files")
        assert files == [results[0][1]]
        assert isinstance(item, self.item_class)

    def test_item_fields_override_settings(self, tmp_path):
        url = "http://www.example.com/files/1.txt"
        item = self.item_class(name="item1", custom_file_urls=[url])
        pipeline = build_from_crawler(
            FilesPipeline,
            get_crawler(
                None,
                {
                    "FILES_STORE": tmp_path,
                    "FILES_URLS_FIELD": "custom_file_urls",
                    "FILES_RESULT_FIELD": "custom_files",
                },
            ),
        )
        requests = list(pipeline.get_media_requests(item, None))  # type: ignore[arg-type]
        assert requests[0].url == url
        results = [(True, {"url": url})]
        item = pipeline.item_completed(results, item, None)  # type: ignore[arg-type]
        custom_files = ItemAdapter(item).get("custom_files")
        assert custom_files == [results[0][1]]
        assert isinstance(item, self.item_class)


class TestFilesPipelineFieldsDict(TestFilesPipelineFieldsMixin):
    item_class = dict


class FilesPipelineTestItem(Item):
    name = Field()
    # default fields
    file_urls = Field()
    files = Field()
    # overridden fields
    custom_file_urls = Field()
    custom_files = Field()


class TestFilesPipelineFieldsItem(TestFilesPipelineFieldsMixin):
    item_class = FilesPipelineTestItem


@dataclasses.dataclass
class FilesPipelineTestDataClass:
    name: str
    # default fields
    file_urls: list[str] = dataclasses.field(default_factory=list)
    files: list[dict[str, str]] = dataclasses.field(default_factory=list)
    # overridden fields
    custom_file_urls: list[str] = dataclasses.field(default_factory=list)
    custom_files: list[dict[str, str]] = dataclasses.field(default_factory=list)


class TestFilesPipelineFieldsDataClass(TestFilesPipelineFieldsMixin):
    item_class = FilesPipelineTestDataClass


@attr.s
class FilesPipelineTestAttrsItem:
    name = attr.ib(default="")
    # default fields
    file_urls: list[str] = attr.ib(default=list)
    files: list[dict[str, str]] = attr.ib(default=list)
    # overridden fields
    custom_file_urls: list[str] = attr.ib(default=list)
    custom_files: list[dict[str, str]] = attr.ib(default=list)


class TestFilesPipelineFieldsAttrsItem(TestFilesPipelineFieldsMixin):
    item_class = FilesPipelineTestAttrsItem


class TestFilesPipelineCustomSettings:
    default_cls_settings = {
        "EXPIRES": 90,
        "FILES_URLS_FIELD": "file_urls",
        "FILES_RESULT_FIELD": "files",
    }
    file_cls_attr_settings_map = {
        ("EXPIRES", "FILES_EXPIRES", "expires"),
        ("FILES_URLS_FIELD", "FILES_URLS_FIELD", "files_urls_field"),
        ("FILES_RESULT_FIELD", "FILES_RESULT_FIELD", "files_result_field"),
    }

    def _generate_fake_settings(
        self, tmp_path: Path, prefix: str | None = None
    ) -> dict[str, Any]:
        def random_string() -> str:
            return "".join([chr(random.randint(97, 123)) for _ in range(10)])

        settings = {
            "FILES_EXPIRES": random.randint(100, 1000),
            "FILES_URLS_FIELD": random_string(),
            "FILES_RESULT_FIELD": random_string(),
            "FILES_STORE": tmp_path,
        }
        if not prefix:
            return settings

        return {
            prefix.upper() + "_" + k if k != "FILES_STORE" else k: v
            for k, v in settings.items()
        }

    def _generate_fake_pipeline(self) -> type[FilesPipeline]:
        class UserDefinedFilePipeline(FilesPipeline):
            EXPIRES = 1001
            FILES_URLS_FIELD = "alfa"
            FILES_RESULT_FIELD = "beta"

        return UserDefinedFilePipeline

    def test_different_settings_for_different_instances(self, tmp_path):
        """
        If there are different instances with different settings they should keep
        different settings.
        """
        custom_settings = self._generate_fake_settings(tmp_path)
        another_pipeline = build_from_crawler(
            FilesPipeline, get_crawler(None, custom_settings)
        )
        one_pipeline = build_from_crawler(
            FilesPipeline, get_crawler(None, {"FILES_STORE": tmp_path})
        )
        for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
            default_value = self.default_cls_settings[pipe_attr]
            assert getattr(one_pipeline, pipe_attr) == default_value
            custom_value = custom_settings[settings_attr]
            assert default_value != custom_value
            assert getattr(another_pipeline, pipe_ins_attr) == custom_value

    def test_subclass_attributes_preserved_if_no_settings(self, tmp_path):
        """
        If subclasses override class attributes and there are no special settings those values should be kept.
        """
        pipe_cls = self._generate_fake_pipeline()
        pipe = build_from_crawler(
            pipe_cls, get_crawler(None, {"FILES_STORE": tmp_path})
        )
        for pipe_attr, _, pipe_ins_attr in self.file_cls_attr_settings_map:
            custom_value = getattr(pipe, pipe_ins_attr)
            assert custom_value != self.default_cls_settings[pipe_attr]
            assert getattr(pipe, pipe_ins_attr) == getattr(pipe, pipe_attr)

    def test_subclass_attrs_preserved_custom_settings(self, tmp_path):
        """
        If file settings are defined but they are not defined for subclass
        settings should be preserved.
        """
        pipeline_cls = self._generate_fake_pipeline()
        settings = self._generate_fake_settings(tmp_path)
        pipeline = build_from_crawler(pipeline_cls, get_crawler(None, settings))
        for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
            value = getattr(pipeline, pipe_ins_attr)
            setting_value = settings.get(settings_attr)
            assert value != self.default_cls_settings[pipe_attr]
            assert value == setting_value

    def test_no_custom_settings_for_subclasses(self, tmp_path):
        """
        If there are no settings for subclass and no subclass attributes, pipeline should use
        attributes of base class.
        """

        class UserDefinedFilesPipeline(FilesPipeline):
            pass

        user_pipeline = build_from_crawler(
            UserDefinedFilesPipeline, get_crawler(None, {"FILES_STORE": tmp_path})
        )
        for pipe_attr, _, pipe_ins_attr in self.file_cls_attr_settings_map:
            # Values from settings for custom pipeline should be set on pipeline instance.
            custom_value = self.default_cls_settings.get(pipe_attr.upper())
            assert getattr(user_pipeline, pipe_ins_attr) == custom_value

    def test_custom_settings_for_subclasses(self, tmp_path):
        """
        If there are custom settings for subclass and NO class attributes, pipeline should use custom
        settings.
        """

        class UserDefinedFilesPipeline(FilesPipeline):
            pass

        prefix = UserDefinedFilesPipeline.__name__.upper()
        settings = self._generate_fake_settings(tmp_path, prefix=prefix)
        user_pipeline = build_from_crawler(
            UserDefinedFilesPipeline, get_crawler(None, settings)
        )
        for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
            # Values from settings for custom pipeline should be set on pipeline instance.
            custom_value = settings.get(prefix + "_" + settings_attr)
            assert custom_value != self.default_cls_settings[pipe_attr]
            assert getattr(user_pipeline, pipe_inst_attr) == custom_value

    def test_custom_settings_and_class_attrs_for_subclasses(self, tmp_path):
        """
        If there are custom settings for subclass AND class attributes
        setting keys are preferred and override attributes.
        """
        pipeline_cls = self._generate_fake_pipeline()
        prefix = pipeline_cls.__name__.upper()
        settings = self._generate_fake_settings(tmp_path, prefix=prefix)
        user_pipeline = build_from_crawler(pipeline_cls, get_crawler(None, settings))
        for (
            pipe_cls_attr,
            settings_attr,
            pipe_inst_attr,
        ) in self.file_cls_attr_settings_map:
            custom_value = settings.get(prefix + "_" + settings_attr)
            assert custom_value != self.default_cls_settings[pipe_cls_attr]
            assert getattr(user_pipeline, pipe_inst_attr) == custom_value

    def test_cls_attrs_with_DEFAULT_prefix(self, tmp_path):
        class UserDefinedFilesPipeline(FilesPipeline):
            DEFAULT_FILES_RESULT_FIELD = "this"
            DEFAULT_FILES_URLS_FIELD = "that"

        pipeline = build_from_crawler(
            UserDefinedFilesPipeline, get_crawler(None, {"FILES_STORE": tmp_path})
        )
        assert (
            pipeline.files_result_field
            == UserDefinedFilesPipeline.DEFAULT_FILES_RESULT_FIELD
        )
        assert (
            pipeline.files_urls_field
            == UserDefinedFilesPipeline.DEFAULT_FILES_URLS_FIELD
        )

    def test_user_defined_subclass_default_key_names(self, tmp_path):
        """Test situation when user defines subclass of FilesPipeline,
        but uses attribute names for default pipeline (without prefixing
        them with pipeline class name).
        """
        settings = self._generate_fake_settings(tmp_path)

        class UserPipe(FilesPipeline):
            pass

        pipeline_cls = build_from_crawler(UserPipe, get_crawler(None, settings))

        for _, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
            expected_value = settings.get(settings_attr)
            assert getattr(pipeline_cls, pipe_inst_attr) == expected_value

    def test_file_pipeline_using_pathlike_objects(self, tmp_path):
        class CustomFilesPipelineWithPathLikeDir(FilesPipeline):
            def file_path(self, request, response=None, info=None, *, item=None) -> str:
                return str(Path("subdir") / Path(request.url).name)

        pipeline = build_from_crawler(
            CustomFilesPipelineWithPathLikeDir,
            get_crawler(None, {"FILES_STORE": tmp_path}),
        )
        request = Request("http://example.com/image01.jpg")
        assert pipeline.file_path(request) == str(Path("subdir/image01.jpg"))


class TestFSFilesStore:
    def test_constructor_with_pathlike_object(self, tmp_path: Path) -> None:
        assert FSFilesStore(tmp_path).basedir == str(tmp_path)

    def test_constructor_with_uri(self, tmp_path: Path) -> None:
        assert FSFilesStore(f"file://{tmp_path}").basedir == str(tmp_path)

    def test_stat_file(self, tmp_path: Path) -> None:
        store = FSFilesStore(tmp_path)
        store.persist_file("full/filename", BytesIO(b"data"), DUMMY_SPIDER_INFO)
        stat = store.stat_file("full/filename", DUMMY_SPIDER_INFO)
        assert stat["checksum"] == "8d777f385d3dfec8815d20f7496026dc"
        assert stat["last_modified"] == pytest.approx(time.time(), abs=60)

    def test_stat_missing_file(self, tmp_path: Path) -> None:
        store = FSFilesStore(tmp_path)
        assert store.stat_file("full/filename", DUMMY_SPIDER_INFO) == {}


@pytest.mark.requires_botocore
class TestS3FilesStore:
    @inline_callbacks_test
    def test_persist(self):
        bucket = "mybucket"
        key = "export.csv"
        uri = f"s3://{bucket}/{key}"
        buffer = mock.MagicMock()
        meta = {"foo": "bar"}
        path = ""
        content_type = "image/png"

        store = S3FilesStore(uri)
        from botocore.stub import Stubber  # noqa: PLC0415

        with Stubber(store.s3_client) as stub:
            stub.add_response(
                "put_object",
                expected_params={
                    "ACL": S3FilesStore.POLICY,
                    "Body": buffer,
                    "Bucket": bucket,
                    "CacheControl": S3FilesStore.HEADERS["Cache-Control"],
                    "ContentType": content_type,
                    "Key": key,
                    "Metadata": meta,
                },
                service_response={},
            )

            yield store.persist_file(
                path,
                buffer,
                info=DUMMY_SPIDER_INFO,
                meta=meta,
                headers={"Content-Type": content_type},
            )

            stub.assert_no_pending_responses()
            # The call to read does not happen with Stubber
            assert buffer.method_calls == [mock.call.seek(0)]

    @inline_callbacks_test
    def test_persist_without_headers(self):
        """Without custom headers only the default ones are sent."""
        bucket = "mybucket"
        key = "export.csv"
        buffer = mock.MagicMock()

        store = S3FilesStore(f"s3://{bucket}/{key}")
        from botocore.stub import Stubber  # noqa: PLC0415

        with Stubber(store.s3_client) as stub:
            stub.add_response(
                "put_object",
                expected_params={
                    "ACL": S3FilesStore.POLICY,
                    "Body": buffer,
                    "Bucket": bucket,
                    "CacheControl": S3FilesStore.HEADERS["Cache-Control"],
                    "Key": key,
                    "Metadata": {},
                },
                service_response={},
            )

            yield store.persist_file("", buffer, info=DUMMY_SPIDER_INFO)

            stub.assert_no_pending_responses()

    def test_missing_botocore(self):
        with (
            mock.patch(
                "scrapy.pipelines.files.is_botocore_available", return_value=False
            ),
            pytest.raises(NotConfigured, match="missing botocore library"),
        ):
            S3FilesStore("s3://mybucket/key")

    def test_wrong_uri_scheme(self):
        with pytest.raises(
            ValueError,
            match=re.escape(
                "Incorrect URI scheme in ftp://mybucket/key, expected 's3'"
            ),
        ):
            S3FilesStore("ftp://mybucket/key")

    def test_unsupported_header(self):
        store = S3FilesStore("s3://mybucket/key")
        with pytest.raises(
            TypeError, match='Header "X-Custom" is not supported by botocore'
        ):
            store._headers_to_botocore_kwargs({"X-Custom": "value"})

    @inline_callbacks_test
    def test_stat(self):
        bucket = "mybucket"
        key = "export.csv"
        uri = f"s3://{bucket}/{key}"
        checksum = "3187896a9657a28163abb31667df64c8"
        last_modified = datetime(2019, 12, 1)

        store = S3FilesStore(uri)
        from botocore.stub import Stubber  # noqa: PLC0415

        with Stubber(store.s3_client) as stub:
            stub.add_response(
                "head_object",
                expected_params={
                    "Bucket": bucket,
                    "Key": key,
                },
                service_response={
                    "ETag": f'"{checksum}"',
                    "LastModified": last_modified,
                },
            )

            file_stats = yield store.stat_file("", info=DUMMY_SPIDER_INFO)
            assert file_stats == {
                "checksum": checksum,
                "last_modified": last_modified.timestamp(),
            }

            stub.assert_no_pending_responses()

    def test_default_max_pool_connections(self) -> None:
        store = S3FilesStore("s3://mybucket/prefix/")
        config: Any = store.s3_client.meta.config
        assert config.max_pool_connections == 10

    @pytest.mark.parametrize(
        ("settings", "expected"),
        [
            ({}, 10),
            ({"REACTOR_THREADPOOL_MAXSIZE": 20}, 20),
            ({"AWS_MAX_POOL_CONNECTIONS": 30}, 30),
            ({"AWS_MAX_POOL_CONNECTIONS": 30, "REACTOR_THREADPOOL_MAXSIZE": 20}, 30),
        ],
    )
    def test_max_pool_connections(
        self, monkeypatch: pytest.MonkeyPatch, settings: dict[str, Any], expected: int
    ) -> None:
        # restores the value that FilesPipeline.from_crawler() sets on the class
        monkeypatch.setattr(S3FilesStore, "AWS_MAX_POOL_CONNECTIONS", None)
        crawler = get_crawler(
            settings_dict={"FILES_STORE": "s3://mybucket/prefix/", **settings}
        )
        store = build_from_crawler(FilesPipeline, crawler).store
        assert isinstance(store, S3FilesStore)
        config: Any = store.s3_client.meta.config
        assert config.max_pool_connections == expected


class TestGCSFilesStore:
    @staticmethod
    def build_gcs_files_store(
        *,
        permissions: tuple[str, ...] = (
            "storage.objects.get",
            "storage.objects.create",
        ),
    ) -> tuple[GCSFilesStore, Any, Any]:
        """Build a :class:`GCSFilesStore` mock.

        Returns ``(store, bucket_mock, blob_mock)``. Skips the test if
        google-cloud-storage is not installed. ``permissions`` is what
        ``Bucket.test_iam_permissions`` will return.
        """
        pytest.importorskip("google.cloud.storage")

        client_mock, bucket_mock, blob_mock = mock_google_cloud_storage()
        bucket_mock.test_iam_permissions.return_value = list(permissions)

        with mock.patch("google.cloud.storage.Client", return_value=client_mock):
            store = GCSFilesStore("gs://my_bucket/my_prefix/")
        return store, bucket_mock, blob_mock

    def test_init(self, caplog: pytest.LogCaptureFixture) -> None:
        with caplog.at_level(logging.WARNING):
            self.build_gcs_files_store()
        assert not caplog.records

    def test_get_perm_missing(self, caplog: pytest.LogCaptureFixture) -> None:
        self.build_gcs_files_store(permissions=("storage.objects.create",))
        assert (
            "No 'storage.objects.get' permission for GCS bucket my_bucket"
            in caplog.text
        )

    def test_create_perm_missing(self, caplog: pytest.LogCaptureFixture) -> None:
        self.build_gcs_files_store(permissions=("storage.objects.get",))
        assert (
            "No 'storage.objects.create' permission for GCS bucket my_bucket"
            in caplog.text
        )

    def test_update_stores(self, monkeypatch: pytest.MonkeyPatch) -> None:
        monkeypatch.setattr(GCSFilesStore, "GCS_PROJECT_ID", None)
        monkeypatch.setattr(GCSFilesStore, "POLICY", None)

        settings = Settings(
            {"GCS_PROJECT_ID": "my-project", "FILES_STORE_GCS_ACL": "publicRead"}
        )
        FilesPipeline._update_stores(settings)
        assert GCSFilesStore.GCS_PROJECT_ID == "my-project"
        assert GCSFilesStore.POLICY == "publicRead"

        # An empty FILES_STORE_GCS_ACL is normalised to None.
        settings = Settings({"GCS_PROJECT_ID": "my-project", "FILES_STORE_GCS_ACL": ""})
        FilesPipeline._update_stores(settings)
        assert GCSFilesStore.POLICY is None

    @coroutine_test
    async def test_persist(self) -> None:
        store, bucket, blob = self.build_gcs_files_store()
        await maybe_deferred_to_future(
            store.persist_file(
                "full/filename",
                BytesIO(b"hello"),
                info=DUMMY_SPIDER_INFO,
                meta={"foo": 1},
                headers={"Content-Type": "image/png"},
            )
        )
        bucket.blob.assert_called_once_with("my_prefix/full/filename")
        assert blob.cache_control == GCSFilesStore.CACHE_CONTROL
        assert blob.metadata == {"foo": "1"}
        blob.upload_from_string.assert_called_once_with(
            data=b"hello",
            content_type="image/png",
            predefined_acl=store.POLICY,
        )

    @coroutine_test
    async def test_persist_defaults(self) -> None:
        store, _, blob = self.build_gcs_files_store()
        await maybe_deferred_to_future(
            store.persist_file(
                "full/filename",
                BytesIO(b"hello"),
                info=DUMMY_SPIDER_INFO,
            )
        )
        blob.upload_from_string.assert_called_once_with(
            data=b"hello",
            content_type="application/octet-stream",
            predefined_acl=store.POLICY,
        )
        assert blob.metadata == {}

    @coroutine_test
    async def test_stat(self) -> None:
        store, bucket, blob = self.build_gcs_files_store()
        checksum = "cdcda85605e46d0af6110752770dce3c"
        blob.md5_hash = base64.b64encode(bytes.fromhex(checksum)).decode()
        updated = datetime(2019, 12, 1)
        blob.updated = updated
        bucket.get_blob.return_value = blob
        stat = await maybe_deferred_to_future(
            store.stat_file("full/filename", info=DUMMY_SPIDER_INFO)
        )
        bucket.get_blob.assert_called_once_with("my_prefix/full/filename")
        assert stat == {
            "checksum": checksum,
            "last_modified": time.mktime(updated.timetuple()),
        }

    @coroutine_test
    async def test_stat_missing_blob(self) -> None:
        store, bucket, _ = self.build_gcs_files_store()
        bucket.get_blob.return_value = None
        stat = await maybe_deferred_to_future(
            store.stat_file("full/filename", info=DUMMY_SPIDER_INFO)
        )
        assert stat == {}

    @coroutine_test
    async def test_blob_path_consistency(self) -> None:
        """Test to make sure that paths used to store files is the same as the one used to get
        already uploaded files.
        """
        store, bucket, _ = self.build_gcs_files_store()
        bucket.get_blob.return_value = None
        path = "full/my_data.txt"
        await maybe_deferred_to_future(
            store.persist_file(path, BytesIO(b""), info=DUMMY_SPIDER_INFO)
        )
        await maybe_deferred_to_future(store.stat_file(path, info=DUMMY_SPIDER_INFO))
        expected_blob_path = store.prefix + path
        bucket.blob.assert_called_with(expected_blob_path)
        bucket.get_blob.assert_called_with(expected_blob_path)


class TestFTPFileStore:
    @inline_callbacks_test
    def test_persist(self):
        data = b"TestFTPFilesStore: \xe2\x98\x83"
        buf = BytesIO(data)
        meta = {"foo": "bar"}
        path = "full/filename"
        with MockFTPServer() as ftp_server:
            # normally set via FilesPipeline.from_crawler()
            FTPFilesStore.FTP_USERNAME = "anonymous"
            FTPFilesStore.FTP_PASSWORD = "guest"

            store = FTPFilesStore(ftp_server.url("/"))
            empty_dict = yield store.stat_file(path, info=DUMMY_SPIDER_INFO)
            assert empty_dict == {}
            yield store.persist_file(
                path, buf, info=DUMMY_SPIDER_INFO, meta=meta, headers=None
            )
            stat = yield store.stat_file(path, info=DUMMY_SPIDER_INFO)
            assert "last_modified" in stat
            assert "checksum" in stat
            assert stat["checksum"] == "d113d66b2ec7258724a268bd88eef6b6"
            path = f"{store.basedir}/{path}"
            content = get_ftp_content_and_delete(
                path,
                store.host,
                store.port,
                store.username,
                store.password,
                bool(store.USE_ACTIVE_MODE),
            )
        assert data == content

    @inline_callbacks_test
    def test_persist_active_mode(self, monkeypatch: pytest.MonkeyPatch):
        data = b"active mode"
        path = "full/filename"
        monkeypatch.setattr(FTPFilesStore, "FTP_USERNAME", "anonymous")
        monkeypatch.setattr(FTPFilesStore, "FTP_PASSWORD", "guest")
        monkeypatch.setattr(FTPFilesStore, "USE_ACTIVE_MODE", True)
        with MockFTPServer() as ftp_server:
            store = FTPFilesStore(ftp_server.url("/"))
            yield store.persist_file(path, BytesIO(data), info=DUMMY_SPIDER_INFO)
            stat = yield store.stat_file(path, info=DUMMY_SPIDER_INFO)
        assert stat["checksum"] == "ff1575649a39a27c13faa0d37c84bab3"

    def test_wrong_uri_scheme(self):
        with pytest.raises(
            ValueError,
            match=re.escape(
                "Incorrect URI scheme in http://example.com/, expected 'ftp'"
            ),
        ):
            FTPFilesStore("http://example.com/")


class ItemWithFiles(Item):
    file_urls = Field()
    files = Field()


def _create_item_with_files(*files: str) -> ItemWithFiles:
    item = ItemWithFiles()
    item["file_urls"] = files
    return item


def _prepare_request_object(
    item_url: str,
    flags: list[str] | None = None,
    headers: dict[str, str] | None = None,
) -> Request:
    return Request(
        item_url,
        meta={
            "response": Response(
                item_url, status=200, body=b"data", flags=flags, headers=headers
            )
        },
    )


# this is separate from the one in test_pipeline_media.py to specifically test FilesPipeline subclasses
class TestBuildFromCrawler:
    def setup_method(self):
        self.tempdir = mkdtemp()
        self.crawler = get_crawler(None, {"FILES_STORE": self.tempdir})

    def teardown_method(self):
        rmtree(self.tempdir)

    def test_simple(self):
        class Pipeline(FilesPipeline):
            pass

        pipe = build_from_crawler(Pipeline, self.crawler)
        assert pipe.crawler == self.crawler
        assert pipe._fingerprinter
        assert pipe.store

    def test_has_from_crawler_and_init(self):
        class Pipeline(FilesPipeline):
            _from_crawler_called = False

            @classmethod
            def from_crawler(cls, crawler: Crawler) -> "Pipeline":
                settings = crawler.settings
                store_uri = settings["FILES_STORE"]
                o = cls(store_uri, crawler=crawler)
                o._from_crawler_called = True
                return o

        pipe = build_from_crawler(Pipeline, self.crawler)
        assert pipe.crawler == self.crawler
        assert pipe._fingerprinter
        assert pipe.store
        assert pipe._from_crawler_called


@pytest.mark.parametrize("store", [None, ""])
def test_files_pipeline_raises_notconfigured_when_files_store_invalid(store):
    settings = Settings()
    settings.clear()
    settings.set("FILES_STORE", store, priority="cmdline")
    crawler = get_crawler(settings_dict=dict(settings))

    with pytest.raises(NotConfigured):
        build_from_crawler(FilesPipeline, crawler)


def test_file_exception_deprecated_import():
    with pytest.warns(ScrapyDeprecationWarning, match="FileException"):
        assert files.FileException is FileException

    with pytest.raises(AttributeError):
        files.nonexistent
