From 73d1e9e47171c904b629a35ebefc762cc7351b3b Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 7 Dec 2023 13:08:45 +0200 Subject: [PATCH 1/3] chore: type fixes --- ckan/cli/shell.py | 2 +- ckan/lib/helpers.py | 4 ++-- ckan/lib/pagination.py | 6 +++--- ckan/model/group.py | 10 +++++----- ckan/model/system.py | 3 +-- ckan/model/tag.py | 4 ++-- ckan/model/vocabulary.py | 2 +- ckanext/resourceproxy/blueprint.py | 1 - ckanext/tracking/middleware.py | 5 +++-- dev-requirements.txt | 25 ++++++++++++------------- package-lock.json | 12 ++++++------ pyproject.toml | 4 ++-- requirements.in | 24 ++++++++++++------------ requirements.txt | 24 ++++++++++++------------ setup.cfg | 2 +- test-infrastructure/docker-compose.yml | 2 +- 16 files changed, 64 insertions(+), 66 deletions(-) diff --git a/ckan/cli/shell.py b/ckan/cli/shell.py index 27c3bda7af9..f5a56b79611 100644 --- a/ckan/cli/shell.py +++ b/ckan/cli/shell.py @@ -30,7 +30,7 @@ def ipython(namespace: Mapping[str, Any], banner: str) -> None: c = Config() setattr(c.TerminalInteractiveShell, "banner2", banner) - IPython.start_ipython([], user_ns=namespace, config=c) # type: ignore + IPython.start_ipython([], user_ns=namespace, config=c) def python(namespace: Mapping[str, Any], banner: str) -> None: diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 88292063c4a..c71619b74f1 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -32,7 +32,7 @@ from flask import flash from flask import get_flashed_messages as _flask_get_flashed_messages from flask import redirect as _flask_redirect -from flask import _request_ctx_stack +from flask.globals import request_ctx from flask import url_for as _flask_default_url_for from werkzeug.routing import BuildError as FlaskRouteBuildError from ckan.lib import i18n @@ -311,7 +311,7 @@ def _get_auto_flask_context(): from ckan.config.middleware import _internal_test_request_context # This is a normal web request, there is a request context present - if _request_ctx_stack.top: + if request_ctx: return None # We are outside a web request. A test web application was created diff --git a/ckan/lib/pagination.py b/ckan/lib/pagination.py index ff6ab1442ae..1812a147d83 100644 --- a/ckan/lib/pagination.py +++ b/ckan/lib/pagination.py @@ -37,7 +37,7 @@ import re from string import Template -from typing import Any, Callable, Match, Optional, Sequence, List +from typing import Any, Callable, Match, Optional, Sequence, List, cast import dominate.tags as tags from markupsafe import Markup @@ -583,7 +583,7 @@ def _range(self, regexp_match: Match[str]): return self.separator.join(nav_items) - def _pagerlink(self, page: int, text: str): + def _pagerlink(self, page: int, text: str) -> Any: """ Create a URL that links to another page using url_for(). @@ -633,7 +633,7 @@ def _pagerlink(self, page: int, text: str): class Page(BasePage): def pager(self, *args: Any, **kwargs: Any) -> Markup: - with tags.div(cls=u"pagination-wrapper") as wrapper: + with cast(Any, tags.div(cls=u"pagination-wrapper")) as wrapper: tags.ul( "$link_previous ~2~ $link_next", cls="pagination justify-content-center" diff --git a/ckan/model/group.py b/ckan/model/group.py index 74d0f6233f1..3d305385194 100644 --- a/ckan/model/group.py +++ b/ckan/model/group.py @@ -136,7 +136,7 @@ def related_packages(self) -> list[_package.Package]: # TODO do we want to return all related packages or certain ones? return meta.Session.query(_package.Package).filter_by( id=self.table_id).all() - + def __str__(self): # refer to objects by name, not ID, to help debugging @@ -226,7 +226,7 @@ def set_approval_status(self, status: str) -> None: assert status in ["approved", "denied"] self.approval_status = status - def get_children_groups(self, type: str='group') -> list[Self]: + def get_children_groups(self, type: str='group') -> list[Group]: '''Returns the groups one level underneath this group in the hierarchy. ''' # The original intention of this method was to provide the full depth @@ -263,7 +263,7 @@ def get_children_group_hierarchy( id=self.id, type=type).all() return results - def get_parent_groups(self, type: str='group') -> list[Self]: + def get_parent_groups(self, type: str='group') -> list[Group]: '''Returns this group's parent groups. Returns a list. Will have max 1 value for organizations. @@ -279,7 +279,7 @@ def get_parent_groups(self, type: str='group') -> list[Self]: all() return result - def get_parent_group_hierarchy(self, type: str='group') -> list[Self]: + def get_parent_group_hierarchy(self, type: str='group') -> list[Group]: '''Returns this group's parent, parent's parent, parent's parent's parent etc.. Sorted with the top level parent first.''' result: list[Group] = meta.Session.query(Group).\ @@ -288,7 +288,7 @@ def get_parent_group_hierarchy(self, type: str='group') -> list[Self]: return result @classmethod - def get_top_level_groups(cls, type: str='group') -> list[Self]: + def get_top_level_groups(cls, type: str='group') -> list[Group]: '''Returns a list of the groups (of the specified type) which have no parent groups. Groups are sorted by title. ''' diff --git a/ckan/model/system.py b/ckan/model/system.py index 13baff832ff..8d0ca95f949 100644 --- a/ckan/model/system.py +++ b/ckan/model/system.py @@ -1,7 +1,6 @@ # encoding: utf-8 from typing import Optional -from typing_extensions import Self import ckan.model.domain_object as domain_object @@ -19,5 +18,5 @@ def purge(self) -> None: def by_name(cls, name: Optional[str], autoflush: bool = True, - for_update: bool = False) -> Optional[Self]: + for_update: bool = False) -> Optional["System"]: return System() diff --git a/ckan/model/tag.py b/ckan/model/tag.py index 3034f50696c..c32e191e67c 100644 --- a/ckan/model/tag.py +++ b/ckan/model/tag.py @@ -115,7 +115,7 @@ def by_name( @classmethod def get(cls, tag_id_or_name: str, - vocab_id_or_name: Optional[str]=None) -> Optional[Self]: + vocab_id_or_name: Optional[str]=None) -> Optional[Tag]: '''Return the tag with the given id or name, or None. By default only free tags (tags which do not belong to any vocabulary) @@ -184,7 +184,7 @@ def search_by_name( query = meta.Session.query(Tag) search_term = search_term.strip().lower() query = query.filter(Tag.name.contains(search_term)) - query: 'Query[Tag]' = query.distinct().join(Tag.package_tags) + query: 'Query[Self]' = query.distinct().join(Tag.package_tags) return query @classmethod diff --git a/ckan/model/vocabulary.py b/ckan/model/vocabulary.py index a5b41986881..fba2f00d016 100644 --- a/ckan/model/vocabulary.py +++ b/ckan/model/vocabulary.py @@ -36,7 +36,7 @@ def __init__(self, name: str) -> None: self.name = name @classmethod - def get(cls, id_or_name: str) -> Optional[Self]: + def get(cls, id_or_name: str) -> Optional[Vocabulary]: '''Return a Vocabulary object referenced by its id or name, or None if there is no vocabulary with the given id or name. ''' query = meta.Session.query(Vocabulary) diff --git a/ckanext/resourceproxy/blueprint.py b/ckanext/resourceproxy/blueprint.py index 557f5b5abe0..55e27193688 100644 --- a/ckanext/resourceproxy/blueprint.py +++ b/ckanext/resourceproxy/blueprint.py @@ -67,7 +67,6 @@ def proxy_resource(context: Context, data_dict: DataDict): r = requests.get(url, timeout=timeout, stream=True) response.headers[u'content-type'] = r.headers[u'content-type'] - response.charset = r.encoding or "utf-8" length = 0 chunk_size = config.get(u'ckan.resource_proxy.chunk_size') diff --git a/ckanext/tracking/middleware.py b/ckanext/tracking/middleware.py index ea6b854f255..c19209fc0a7 100644 --- a/ckanext/tracking/middleware.py +++ b/ckanext/tracking/middleware.py @@ -31,8 +31,9 @@ def track_request(response: Response) -> Response: request.environ.get('HTTP_ACCEPT_ENCODING', ''), ]) # raises a type error on python<3.9 - h = hashlib.new('md5', usedforsecurity=False) # type: ignore - key = h.update(key.encode()).hexdigest() + h = hashlib.new('md5', usedforsecurity=False) + h.update(key.encode()) + key = h.hexdigest() # store key/data here sql = '''INSERT INTO tracking_raw (user_key, url, tracking_type) diff --git a/dev-requirements.txt b/dev-requirements.txt index ffbe77339e9..ee357d5db4f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,25 +1,24 @@ # These are packages that are required by ckan developers - for running ckan in debug mode, running ckan tests, building the docs and to pip-compile the requirements.in file. beautifulsoup4==4.12.2 -cookiecutter==2.1.1 +cookiecutter==2.5.0 coveralls #Let Unpinned - Requires latest coveralls -docutils==0.18.1 # Needed until sphinx-rtd-theme 2.0 is out -Faker==19.6.2 +Faker==20.1.0 factory-boy==3.3.0 -flask-debugtoolbar==0.13.1 -freezegun==1.2.2 +flask-debugtoolbar==0.14.0 +freezegun==1.3.1 ipdb==0.13.13 pip-tools==7.3.0 -Pillow==10.0.1 -responses==0.23.3 -sphinx-rtd-theme==1.3.0 -sphinx==7.1.2 +Pillow==10.1.0 +responses==0.24.1 +sphinx-rtd-theme==2.0.0 +sphinx==7.2.6 toml==0.10.2 -towncrier==22.12.0 +towncrier==23.11.0 -pytest==7.4.2 +pytest==7.4.3 pytest-cov==4.1.0 -pytest-factoryboy==2.5.1 +pytest-factoryboy==2.6.0 pytest-freezegun==0.4.2 -pytest-rerunfailures==12.0 +pytest-rerunfailures==13.0 pytest-split==0.8.1 diff --git a/package-lock.json b/package-lock.json index 42506049adc..0e1a6272136 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3708,9 +3708,9 @@ } }, "node_modules/pyright": { - "version": "1.1.333", - "resolved": "https://registry.npmjs.org/pyright/-/pyright-1.1.333.tgz", - "integrity": "sha512-oFHXvzvg2cU1enatWqI76+sjSi+McsUBIJ9aR5W6p4kU9Rrgu+MYfXX5aHEPaZE64Vz3sNbG7IelXuAVEkyk/A==", + "version": "1.1.339", + "resolved": "https://registry.npmjs.org/pyright/-/pyright-1.1.339.tgz", + "integrity": "sha512-YHa58uKBcBal5E35DMWhHnHDoaH0OHoM90VvV+CYus4Z7pTPKFWLgl+mfH3ufmgOzxkmvW0LM0hmDOJaYJYfcA==", "dev": true, "bin": { "pyright": "index.js", @@ -8656,9 +8656,9 @@ "dev": true }, "pyright": { - "version": "1.1.333", - "resolved": "https://registry.npmjs.org/pyright/-/pyright-1.1.333.tgz", - "integrity": "sha512-oFHXvzvg2cU1enatWqI76+sjSi+McsUBIJ9aR5W6p4kU9Rrgu+MYfXX5aHEPaZE64Vz3sNbG7IelXuAVEkyk/A==", + "version": "1.1.339", + "resolved": "https://registry.npmjs.org/pyright/-/pyright-1.1.339.tgz", + "integrity": "sha512-YHa58uKBcBal5E35DMWhHnHDoaH0OHoM90VvV+CYus4Z7pTPKFWLgl+mfH3ufmgOzxkmvW0LM0hmDOJaYJYfcA==", "dev": true, "requires": { "fsevents": "~2.3.2" diff --git a/pyproject.toml b/pyproject.toml index 1f8528e8960..af56560f43f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,10 +47,10 @@ markers = [ ] testpaths = ["ckan", "ckanext"] -addopts = "--strict-markers --pdbcls=IPython.terminal.debugger:TerminalPdb -p no:ckan" +addopts = "--strict-markers --pdbcls=IPython.terminal.debugger:TerminalPdb -p no:ckan --ckan-ini=test-core.ini" [tool.pyright] -pythonVersion = "3.8" +pythonVersion = "3.9" include = ["ckan", "ckanext"] exclude = [ "**/test*", diff --git a/requirements.in b/requirements.in index cb72ffb2f18..a2ecbb5a222 100644 --- a/requirements.in +++ b/requirements.in @@ -1,21 +1,21 @@ # The file contains the direct ckan requirements (python3). # Use pip-compile to create a requirements.txt file from this -alembic==1.12.0 -Babel==2.12.1 +alembic==1.13.0 +Babel==2.13.1 Beaker==1.12.1 -bleach==6.0.0 -blinker==1.6.2 +bleach==6.1.0 +blinker==1.7.0 certifi>=2023.7.22 click==8.1.7 -dominate==2.8.0 +dominate==2.9.0 feedgen==0.9.0 -Flask==2.3.3 -Flask-Babel==3.1.0 -Flask-Login==0.6.2 -Flask-WTF==1.1.1 +Flask==3.0.0 +Flask-Babel==4.0.0 +Flask-Login==0.6.3 +Flask-WTF==1.2.1 greenlet>=2.0.2 Jinja2==3.1.2 -Markdown==3.4.4 +Markdown==3.5.1 passlib==1.7.4 polib==1.2.0 psycopg2==2.9.7 @@ -28,11 +28,11 @@ PyUtilib==6.0.0 pyyaml==6.0.1 requests==2.31.0 rq==1.15.1 -simplejson==3.19.1 +simplejson==3.19.2 SQLAlchemy[mypy]==1.4.49 sqlparse==0.4.4 typing_extensions==4.8.0 tzlocal==5.0.1 webassets==2.0 -Werkzeug[watchdog]==2.3.7 +Werkzeug[watchdog]==3.0.1 zope.interface==6.0 diff --git a/requirements.txt b/requirements.txt index 6d7f8188ce1..af477d1595d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,17 +4,17 @@ # # pip-compile requirements.in # -alembic==1.12.0 +alembic==1.13.0 # via -r requirements.in -babel==2.12.1 +babel==2.13.1 # via # -r requirements.in # flask-babel beaker==1.12.1 # via -r requirements.in -bleach==6.0.0 +bleach==6.1.0 # via -r requirements.in -blinker==1.6.2 +blinker==1.7.0 # via # -r requirements.in # flask @@ -31,21 +31,21 @@ click==8.1.7 # rq deprecated==1.2.13 # via redis -dominate==2.8.0 +dominate==2.9.0 # via -r requirements.in feedgen==0.9.0 # via -r requirements.in -flask==2.3.3 +flask==3.0.0 # via # -r requirements.in # flask-babel # flask-login # flask-wtf -flask-babel==3.1.0 +flask-babel==4.0.0 # via -r requirements.in -flask-login==0.6.2 +flask-login==0.6.3 # via -r requirements.in -flask-wtf==1.1.1 +flask-wtf==1.2.1 # via -r requirements.in greenlet==2.0.2 # via @@ -70,7 +70,7 @@ lxml==4.9.1 # via feedgen mako==1.2.2 # via alembic -markdown==3.4.4 +markdown==3.5.1 # via -r requirements.in markupsafe==2.1.1 # via @@ -120,7 +120,7 @@ requests==2.31.0 # pysolr rq==1.15.1 # via -r requirements.in -simplejson==3.19.1 +simplejson==3.19.2 # via -r requirements.in six==1.16.0 # via @@ -154,7 +154,7 @@ webassets==2.0 # via -r requirements.in webencodings==0.5.1 # via bleach -werkzeug[watchdog]==2.3.7 +werkzeug[watchdog]==3.0.1 # via # -r requirements.in # flask diff --git a/setup.cfg b/setup.cfg index e711a8fb863..336928e0815 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,9 +18,9 @@ classifiers = Development Status :: 5 - Production/Stable License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+) Programming Language :: Python - Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 [options] python_requires = >= 3.9 diff --git a/test-infrastructure/docker-compose.yml b/test-infrastructure/docker-compose.yml index 634a4022d29..f386ccd683d 100644 --- a/test-infrastructure/docker-compose.yml +++ b/test-infrastructure/docker-compose.yml @@ -24,7 +24,7 @@ services: image: "ckan/ckan-solr:master" ckan: - image: python:3.8-bullseye + image: python:3.9-bullseye networks: - default environment: From 6ad50c303fb87ad53d2133b7d46a79e3882cf2fd Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 7 Dec 2023 13:33:53 +0200 Subject: [PATCH 2/3] chore: remove unused self --- ckan/model/vocabulary.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ckan/model/vocabulary.py b/ckan/model/vocabulary.py index fba2f00d016..2e1a77ace51 100644 --- a/ckan/model/vocabulary.py +++ b/ckan/model/vocabulary.py @@ -4,7 +4,6 @@ from typing import Optional, TYPE_CHECKING from sqlalchemy import types, Column, Table from sqlalchemy.orm import Mapped -from typing_extensions import Self import ckan.model.meta as meta import ckan.model.types as _types From 4b45b13d98135a7a37ff3984c929ff2d88343a72 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Fri, 8 Dec 2023 13:32:55 +0200 Subject: [PATCH 3/3] chore: use function instead of direct context check --- ckan/lib/helpers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index c71619b74f1..1cb49e59148 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -29,10 +29,9 @@ from markdown import markdown from bleach import clean as bleach_clean, ALLOWED_TAGS, ALLOWED_ATTRIBUTES from ckan.common import asbool, config, current_user -from flask import flash +from flask import flash, has_request_context from flask import get_flashed_messages as _flask_get_flashed_messages from flask import redirect as _flask_redirect -from flask.globals import request_ctx from flask import url_for as _flask_default_url_for from werkzeug.routing import BuildError as FlaskRouteBuildError from ckan.lib import i18n @@ -311,7 +310,7 @@ def _get_auto_flask_context(): from ckan.config.middleware import _internal_test_request_context # This is a normal web request, there is a request context present - if request_ctx: + if has_request_context(): return None # We are outside a web request. A test web application was created