Complete pip-tools setup (#5508)

.pre-commit-config.yaml

@@ -31,7 +31,7 @@ repos:
       ^docs/[^/]*\.svg$
   - id: requirements-txt-fixer
     exclude: >-
-      ^requirements/dev.txt$
+      ^requirements/(cython|dev|doc-spelling|lint).txt$
   - id: trailing-whitespace
   - id: file-contents-sorter
     files: |

CHANGES/5486.misc

@@ -0,0 +1 @@
+Complete pip-tools setup.

Makefile

@@ -9,6 +9,9 @@ CS := $(wildcard aiohttp/*.c)
 PYS := $(wildcard aiohttp/*.py)
 REQS := $(wildcard requirements/*.txt)
 ALLS := $(sort $(CYS) $(CS) $(PYS) $(REQS))
+IN := doc-spelling lint cython dev
+REQIN := $(foreach fname,$(IN),requirements/$(fname).in)
+
 
 .PHONY: all
 all: test
@@ -45,9 +48,11 @@ endif
 # Enumerate intermediate files to don't remove them automatically.
 .SECONDARY: $(call to-hash,$(ALLS))
 
+.update-pip:
+	@pip install -U 'pip'
 
-.install-cython: $(call to-hash,requirements/cython.txt)
-	pip install -r requirements/cython.txt
+.install-cython: .update-pip $(call to-hash,requirements/cython.txt)
+	@pip install -r requirements/cython.txt
 	@touch .install-cython
 
 aiohttp/_find_header.c: $(call to-hash,aiohttp/hdrs.py ./tools/gen.py)
@@ -62,7 +67,7 @@ aiohttp/%.c: aiohttp/%.pyx $(call to-hash,$(CYS)) aiohttp/_find_header.c
 cythonize: .install-cython $(PYXS:.pyx=.c)
 
 .install-deps: .install-cython $(PYXS:.pyx=.c) $(call to-hash,$(CYS) $(REQS))
-	pip install -r requirements/dev.txt
+	@pip install -r requirements/dev.txt
 	@touch .install-deps
 
 .PHONY: lint
@@ -135,17 +140,14 @@ doc:
 doc-spelling:
 	@make -C docs spelling SPHINXOPTS="-W -E"
 
-.update-pip:
-	@pip install -U 'pip'
-
 .PHONY: compile-deps
 compile-deps: .update-pip
 	@pip install pip-tools
-	@pip-compile --allow-unsafe -q requirements/dev.in
+	@$(foreach fname,$(REQIN),pip-compile --allow-unsafe -q $(fname);)
 
 .PHONY: install
 install: .update-pip
-	@pip install -r requirements/dev.in -c requirements/dev.txt
+	@pip install -r requirements/dev.txt
 
 .PHONY: install-dev
 install-dev: .develop

requirements/base.txt

@@ -12,6 +12,8 @@ chardet==4.0.0
 frozenlist==1.1.1
 gunicorn==20.0.4
 idna-ssl==1.1.0; python_version<"3.7"
+typing==3.7.4.3; python_version<"3.7"
 typing_extensions==3.7.4.3
 uvloop==0.14.0; platform_system!="Windows" and implementation_name=="cpython" and python_version<"3.9" # MagicStack/uvloop#14
 yarl==1.6.3
+zipp==3.4.1; python_version<"3.7"

requirements/cython.in

@@ -0,0 +1,3 @@
+-r multidict.txt
+cython==0.29.21
+typing_extensions==3.7.4.3  # required for parsing aiohttp/hdrs.py by tools/gen.py

requirements/cython.txt

@@ -1,3 +1,12 @@
--r multidict.txt
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --allow-unsafe requirements/cython.in
+#
 cython==0.29.21
-typing_extensions==3.7.4.3  # required for parsing aiohttp/hdrs.py by tools/gen.py
+    # via -r requirements/cython.in
+multidict==5.1.0
+    # via -r requirements/multidict.txt
+typing_extensions==3.7.4.3
+    # via -r requirements/cython.in

requirements/dev.in

@@ -1,4 +1,4 @@
--r lint.txt
+-r lint.in
 -r test.txt
 -r doc.txt
 cherry_picker==1.3.2; python_version>="3.6"

requirements/dev.txt

@@ -20,6 +20,8 @@ async-generator==1.10
     # via -r requirements/base.txt
 async-timeout==4.0.0a3
     # via -r requirements/base.txt
+asynctest==0.13.0 ; python_version < "3.8"
+    # via -r requirements/base.txt
 attrs==20.3.0
     # via
     #   -r requirements/base.txt
@@ -28,7 +30,7 @@ attrs==20.3.0
 babel==2.9.0
     # via sphinx
 black==20.8b1 ; implementation_name == "cpython"
-    # via -r requirements/lint.txt
+    # via -r requirements/lint.in
 blockdiag==2.0.1
     # via sphinxcontrib-blockdiag
 brotli==1.0.9
@@ -58,21 +60,26 @@ coverage==5.3.1
     # via
     #   -r requirements/test.txt
     #   pytest-cov
-cryptography==3.3.1
+cryptography==3.3.1 ; platform_machine != "i686" and python_version < "3.9"
     # via
+    #   -r requirements/test.txt
     #   pyjwt
     #   trustme
+dataclasses==0.8 ; python_version < "3.7"
+    # via
+    #   -r requirements/lint.in
+    #   black
 distlib==0.3.1
     # via virtualenv
 docutils==0.16
     # via sphinx
 filelock==3.0.12
     # via virtualenv
 flake8-pyi==20.10.0
-    # via -r requirements/lint.txt
+    # via -r requirements/lint.in
 flake8==3.8.4
     # via
-    #   -r requirements/lint.txt
+    #   -r requirements/lint.in
     #   flake8-pyi
 freezegun==1.0.0
     # via -r requirements/test.txt
@@ -88,19 +95,35 @@ gunicorn==20.0.4
     # via -r requirements/base.txt
 identify==1.5.12
     # via pre-commit
+idna-ssl==1.1.0 ; python_version < "3.7"
+    # via -r requirements/base.txt
 idna==2.10
     # via
+    #   idna-ssl
     #   requests
     #   trustme
     #   yarl
 imagesize==1.2.0
     # via sphinx
+importlib-metadata==3.7.0 ; python_version < "3.8"
+    # via
+    #   -r requirements/lint.in
+    #   flake8
+    #   pluggy
+    #   pre-commit
+    #   pytest
+    #   virtualenv
+importlib-resources==5.1.2 ; python_version < "3.9"
+    # via
+    #   -r requirements/lint.in
+    #   pre-commit
+    #   virtualenv
 incremental==17.5.0
     # via towncrier
 iniconfig==1.1.1
     # via pytest
-isort==5.6.4
-    # via -r requirements/lint.txt
+isort==5.7.0
+    # via -r requirements/lint.in
 jinja2==2.11.2
     # via
     #   sphinx
@@ -120,7 +143,7 @@ mypy-extensions==0.4.3 ; implementation_name == "cpython"
     #   mypy
 mypy==0.790 ; implementation_name == "cpython"
     # via
-    #   -r requirements/lint.txt
+    #   -r requirements/lint.in
     #   -r requirements/test.txt
 nodeenv==1.5.0
     # via pre-commit
@@ -135,7 +158,7 @@ pillow==8.1.0
 pluggy==0.13.1
     # via pytest
 pre-commit==2.9.3
-    # via -r requirements/lint.txt
+    # via -r requirements/lint.in
 py==1.10.0
     # via pytest
 pycares==3.1.1
@@ -162,7 +185,7 @@ pytest-mock==3.5.1
     # via -r requirements/test.txt
 pytest==6.1.2
     # via
-    #   -r requirements/lint.txt
+    #   -r requirements/lint.in
     #   -r requirements/test.txt
     #   pytest-cov
     #   pytest-mock
@@ -191,7 +214,7 @@ six==1.15.0
     #   virtualenv
 snowballstemmer==2.0.0
     # via sphinx
-sphinx==3.5.0
+sphinx==3.5.1
     # via
     #   -r requirements/doc.txt
     #   sphinxcontrib-asyncio
@@ -223,26 +246,40 @@ towncrier==19.2.0
     # via -r requirements/doc.txt
 trustme==0.7.0 ; platform_machine != "i686"
     # via -r requirements/test.txt
-typed-ast==1.4.2
+typed-ast==1.4.2 ; implementation_name == "cpython"
     # via
+    #   -r requirements/lint.in
     #   black
     #   mypy
 typing-extensions==3.7.4.3
     # via
     #   -r requirements/base.txt
     #   async-timeout
     #   black
+    #   importlib-metadata
     #   mypy
+    #   yarl
+typing==3.7.4.3 ; python_version < "3.7"
+    # via
+    #   -r requirements/base.txt
+    #   aiodns
 uritemplate==3.0.1
     # via gidgethub
 urllib3==1.26.2
     # via requests
+uvloop==0.14.0 ; platform_system != "Windows" and implementation_name == "cpython" and python_version < "3.9"
+    # via -r requirements/base.txt
 virtualenv==20.3.1
     # via pre-commit
 webcolors==1.11.1
     # via blockdiag
 yarl==1.6.3
     # via -r requirements/base.txt
+zipp==3.4.1 ; python_version < "3.7"
+    # via
+    #   -r requirements/base.txt
+    #   importlib-metadata
+    #   importlib-resources
 
 # The following packages are considered to be unsafe in a requirements file:
 setuptools==51.3.1

requirements/doc-spelling.in

@@ -0,0 +1,2 @@
+-r doc.txt
+sphinxcontrib-spelling==7.1.0; platform_system!="Windows"  # We only use it in Travis CI

requirements/doc-spelling.txt

@@ -1,2 +1,98 @@
--r doc.txt
-sphinxcontrib-spelling==7.1.0; platform_system!="Windows"  # We only use it in Travis CI
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --allow-unsafe requirements/doc-spelling.in
+#
+aiohttp-theme==0.1.6
+    # via -r requirements/doc.txt
+alabaster==0.7.12
+    # via sphinx
+babel==2.9.0
+    # via sphinx
+blockdiag==2.0.1
+    # via sphinxcontrib-blockdiag
+certifi==2020.12.5
+    # via requests
+chardet==4.0.0
+    # via requests
+click==7.1.2
+    # via towncrier
+docutils==0.16
+    # via sphinx
+funcparserlib==0.3.6
+    # via blockdiag
+idna==2.10
+    # via requests
+imagesize==1.2.0
+    # via sphinx
+importlib-metadata==3.7.2
+    # via sphinxcontrib-spelling
+incremental==21.3.0
+    # via towncrier
+jinja2==2.11.3
+    # via
+    #   sphinx
+    #   towncrier
+markupsafe==1.1.1
+    # via jinja2
+packaging==20.9
+    # via sphinx
+pillow==8.1.2
+    # via blockdiag
+pyenchant==3.2.0
+    # via sphinxcontrib-spelling
+pygments==2.8.0
+    # via
+    #   -r requirements/doc.txt
+    #   sphinx
+pyparsing==2.4.7
+    # via packaging
+pytz==2021.1
+    # via babel
+requests==2.25.1
+    # via sphinx
+snowballstemmer==2.1.0
+    # via sphinx
+sphinx==3.5.1
+    # via
+    #   -r requirements/doc.txt
+    #   sphinxcontrib-asyncio
+    #   sphinxcontrib-blockdiag
+    #   sphinxcontrib-spelling
+sphinxcontrib-applehelp==1.0.2
+    # via sphinx
+sphinxcontrib-asyncio==0.3.0
+    # via -r requirements/doc.txt
+sphinxcontrib-blockdiag==2.0.0
+    # via -r requirements/doc.txt
+sphinxcontrib-devhelp==1.0.2
+    # via sphinx
+sphinxcontrib-htmlhelp==1.0.3
+    # via sphinx
+sphinxcontrib-jsmath==1.0.1
+    # via sphinx
+sphinxcontrib-qthelp==1.0.3
+    # via sphinx
+sphinxcontrib-serializinghtml==1.1.4
+    # via sphinx
+sphinxcontrib-spelling==7.1.0 ; platform_system != "Windows"
+    # via -r requirements/doc-spelling.in
+toml==0.10.2
+    # via towncrier
+towncrier==19.2.0
+    # via -r requirements/doc.txt
+typing-extensions==3.7.4.3
+    # via importlib-metadata
+urllib3==1.26.3
+    # via requests
+webcolors==1.11.1
+    # via blockdiag
+zipp==3.4.1
+    # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+setuptools==54.1.1
+    # via
+    #   blockdiag
+    #   sphinx

requirements/lint.in

@@ -0,0 +1,11 @@
+black==20.8b1; implementation_name=="cpython"
+dataclasses==0.8; python_version < "3.7"
+flake8==3.8.4
+flake8-pyi==20.10.0
+importlib-metadata==3.7.0; python_version < "3.8"
+importlib-resources; python_version < "3.9"
+isort==5.7.0
+mypy==0.790; implementation_name=="cpython"
+pre-commit==2.9.3
+pytest==6.1.2
+typed-ast==1.4.2; implementation_name=="cpython"