diff --git a/sh_scrapy/commands/shub_image_info.py b/sh_scrapy/commands/shub_image_info.py index 70bd3db..cc95a17 100644 --- a/sh_scrapy/commands/shub_image_info.py +++ b/sh_scrapy/commands/shub_image_info.py @@ -45,7 +45,7 @@ def run(self, args, opts): result['metadata'] = {} for spider_name in result['spiders']: spider_cls = self.crawler_process.spider_loader.load(spider_name) - metadata_dict = get_spider_metadata(spider_cls) + metadata_dict = get_spider_metadata(spider_cls, normalize=True) try: # make sure it's serializable json.dumps(metadata_dict) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 329e4c6..749d844 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -343,3 +343,95 @@ class MySpider(Spider): if not SPIDER_METADATA_AVAILABLE: del expected["metadata"] assert data == expected + + +@pytest.mark.skipif(not SPIDER_METADATA_AVAILABLE, reason="scrapy-spider-metadata is not installed") +def test_image_info_args(tmp_path): + project_dir = create_project(tmp_path, spider_text=""" +from enum import Enum +from scrapy import Spider +from scrapy_spider_metadata import Args +from pydantic import BaseModel, Field + +class ToolEnum(Enum): + spanner = "spanner" + wrench = "wrench" + +class Parameters(BaseModel): + tool: ToolEnum = ToolEnum.spanner + +class MySpider(Args[Parameters], Spider): + name = "myspider" +""") + out, _ = call_command(project_dir, "shub-image-info") + data = json.loads(out) + expected = { + "project_type": "scrapy", + "spiders": ["myspider"], + "metadata": { + "myspider": { + "param_schema": { + "properties": { + "tool": { + "default": "spanner", + "enum": ["spanner", "wrench"], + "title": "Tool", + "type": "string", + }, + }, + "title": "Parameters", + "type": "object", + }, + }, + }, + } + if not SPIDER_METADATA_AVAILABLE: + del expected["metadata"] + assert data == expected + + +@pytest.mark.skipif(not SPIDER_METADATA_AVAILABLE, reason="scrapy-spider-metadata is not installed") +def test_image_info_args_metadata(tmp_path): + project_dir = create_project(tmp_path, spider_text=""" +from enum import Enum +from scrapy import Spider +from scrapy_spider_metadata import Args +from pydantic import BaseModel, Field + +class ToolEnum(Enum): + spanner = "spanner" + wrench = "wrench" + +class Parameters(BaseModel): + tool: ToolEnum = ToolEnum.spanner + +class MySpider(Args[Parameters], Spider): + name = "myspider" + metadata = {"foo": 42} +""") + out, _ = call_command(project_dir, "shub-image-info") + data = json.loads(out) + expected = { + "project_type": "scrapy", + "spiders": ["myspider"], + "metadata": { + "myspider": { + "foo": 42, + "param_schema": { + "properties": { + "tool": { + "default": "spanner", + "enum": ["spanner", "wrench"], + "title": "Tool", + "type": "string", + }, + }, + "title": "Parameters", + "type": "object", + }, + }, + }, + } + if not SPIDER_METADATA_AVAILABLE: + del expected["metadata"] + assert data == expected diff --git a/tox.ini b/tox.ini index c1b6641..e6b5829 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,8 @@ deps = hubstorage packaging py36-scrapy16: Scrapy==1.6 - scrapy-spider-metadata; python_version >= "3.8" + scrapy-spider-metadata>=0.1.1; python_version >= "3.8" + pydantic>=2; python_version >= "3.8" commands = pytest --verbose --cov=sh_scrapy --cov-report=term-missing --cov-report=html --cov-report=xml {posargs: sh_scrapy tests}