Better checksum calculation (#5183) (#5186)

Co-authored-by: Andrew Svetlov <[email protected]>

.github/workflows/ci.yml

@@ -115,27 +115,16 @@ jobs:
         path: ${{ steps.pip-cache.outputs.dir }}
         restore-keys: |
             pip-ci-${{ runner.os }}-${{ matrix.pyver }}-{{ matrix.no-extensions }}-
-    - name: Install cython
-      if: ${{ matrix.no-extensions == '' }}
-      uses: py-actions/py-dependency-install@v2
-      with:
-        path: requirements/cython.txt
     - name: Cythonize
       if: ${{ matrix.no-extensions == '' }}
       run: |
         make cythonize
-    - name: Install dependencies
-      uses: py-actions/py-dependency-install@v2
-      with:
-        path: requirements/test.txt
-      env:
-        AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }}
     - name: Run unittests
       env:
         COLOR: 'yes'
         AIOHTTP_NO_EXTENSIONS: ${{ matrix.no-extensions }}
       run: |
-        python -m pytest tests -vv
+        make vvtest
         python -m coverage xml
     - name: Upload coverage
       uses: codecov/codecov-action@v1
@@ -168,10 +157,6 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
-    - name: Install cython
-      uses: py-actions/py-dependency-install@v2
-      with:
-        path: requirements/cython.txt
     - name: Cythonize
       run: |
         make cythonize
@@ -210,11 +195,6 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
-    - name: Install cython
-      if: ${{ matrix.no-extensions == '' }}
-      uses: py-actions/py-dependency-install@v2
-      with:
-        path: requirements/cython.txt
     - name: Cythonize
       if: ${{ matrix.no-extensions == '' }}
       run: |
@@ -255,11 +235,6 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.pyver }}
-    - name: Install cython
-      if: ${{ matrix.no-extensions == '' }}
-      uses: py-actions/py-dependency-install@v2
-      with:
-        path: requirements/cython.txt
     - name: Cythonize
       if: ${{ matrix.no-extensions == '' }}
       run: |

.gitignore

@@ -20,6 +20,7 @@
 .envrc
 .flake
 .gitconfig
+.hash
 .idea
 .install-cython
 .install-deps

Makefile

@@ -1,47 +1,67 @@
 # Some simple testing tasks (sorry, UNIX only).
 
-to-md5 = $1 $(addsuffix .md5,$1)
+to-hash-one = $(dir $1).hash/$(addsuffix .hash,$(notdir $1))
+to-hash = $(foreach fname,$1,$(call to-hash-one,$(fname)))
 
-CYS = $(wildcard aiohttp/*.pyx) $(wildcard aiohttp/*.pyi)  $(wildcard aiohttp/*.pxd)
-PYXS = $(wildcard aiohttp/*.pyx)
-CS = $(wildcard aiohttp/*.c)
-PYS = $(wildcard aiohttp/*.py)
-REQS = $(wildcard requirements/*.txt)
-SRC = aiohttp examples tests setup.py
+CYS := $(wildcard aiohttp/*.pyx) $(wildcard aiohttp/*.pyi)  $(wildcard aiohttp/*.pxd)
+PYXS := $(wildcard aiohttp/*.pyx)
+CS := $(wildcard aiohttp/*.c)
+PYS := $(wildcard aiohttp/*.py)
+REQS := $(wildcard requirements/*.txt)
+ALLS := $(sort $(CYS) $(CS) $(PYS) $(REQS))
 
 .PHONY: all
 all: test
 
-# Recipe from https://www.cmcrossroads.com/article/rebuilding-when-files-checksum-changes
-%.md5: FORCE
-	@$(if $(filter-out $(shell cat $@ 2>/dev/null),$(shell md5sum $*)),md5sum $* > $@)
+tst:
+	@echo $(call to-hash,requirements/cython.txt)
+	@echo $(call to-hash,aiohttp/%.pyx)
+
 
+# Recipe from https://www.cmcrossroads.com/article/rebuilding-when-files-checksum-changes
 FORCE:
 
+# check_sum.py works perfectly fine but slow when called for every file from $(ALLS)
+# (perhaps even several times for each file).
+# That is why much less readable but faster solution exists
+ifneq (, $(shell which sha256sum))
+%.hash: FORCE
+	$(eval $@_ABS := $(abspath $@))
+	$(eval $@_NAME := $($@_ABS))
+	$(eval $@_HASHDIR := $(dir $($@_ABS)))
+	$(eval $@_TMP := $($@_HASHDIR)../$(notdir $($@_ABS)))
+	$(eval $@_ORIG := $(subst /.hash/../,/,$(basename $($@_TMP))))
+	@#echo ==== $($@_ABS) $($@_HASHDIR) $($@_NAME) $($@_TMP) $($@_ORIG)
+	@if ! (sha256sum --check $($@_ABS) 1>/dev/null 2>/dev/null); then \
+	  mkdir -p $($@_HASHDIR); \
+	  echo re-hash $($@_ORIG); \
+	  sha256sum $($@_ORIG) > $($@_ABS); \
+	fi
+else
+%.hash: FORCE
+	@./tools/check_sum.py $@ # --debug
+endif
+
 # Enumerate intermediate files to don't remove them automatically.
-# The target must exist, no need to execute it.
-.PHONY: _keep-intermediate-files
-_keep-intermediate-files: $(addsuffix .md5,$(CYS))\
-                         $(addsuffix .md5,$(CS))\
-                         $(addsuffix .md5,$(PYS))\
-                         $(addsuffix .md5,$(REQS))
-
-.install-cython: $(call to-md5,requirements/cython.txt)
+.SECONDARY: $(call to-hash,$(ALLS))
+
+
+.install-cython: $(call to-hash,requirements/cython.txt)
 	pip install -r requirements/cython.txt
 	@touch .install-cython
 
-aiohttp/_find_header.c: $(call to-md5,aiohttp/hdrs.py)
+aiohttp/_find_header.c: $(call to-hash,aiohttp/hdrs.py ./tools/gen.py)
 	./tools/gen.py
 
 # _find_headers generator creates _headers.pyi as well
-aiohttp/%.c: $(call to-md5,aiohttp/%.pyx) aiohttp/_find_header.c
+aiohttp/%.c: aiohttp/%.pyx $(call to-hash,$(CYS)) aiohttp/_find_header.c
 	cython -3 -o $@ $< -I aiohttp
 
 
 .PHONY: cythonize
 cythonize: .install-cython $(PYXS:.pyx=.c)
 
-.install-deps: .install-cython $(PYXS:.pyx=.c) $(call to-md5,$(CYS) $(REQS))
+.install-deps: .install-cython $(PYXS:.pyx=.c) $(call to-hash,$(CYS) $(REQS))
 	pip install -r requirements/dev.txt
 	@touch .install-deps
 
@@ -56,7 +76,7 @@ fmt format:
 mypy:
 	mypy aiohttp
 
-.develop: .install-deps $(call to-md5,$(PYS) $(CYS) $(CS))
+.develop: .install-deps $(call to-hash,$(PYS) $(CYS) $(CS))
 	pip install -e .
 	@touch .develop
 
@@ -68,16 +88,23 @@ test: .develop
 vtest: .develop
 	@pytest -s -v
 
+.PHONY: vvtest
+vvtest: .develop
+	@pytest -vv
+
 .PHONY: clean
 clean:
 	@rm -rf `find . -name __pycache__`
+	@rm -rf `find . -name .hash`
+	@rm -rf `find . -name .md5`  # old styling
 	@rm -f `find . -type f -name '*.py[co]' `
 	@rm -f `find . -type f -name '*~' `
 	@rm -f `find . -type f -name '.*~' `
 	@rm -f `find . -type f -name '@*' `
 	@rm -f `find . -type f -name '#*#' `
 	@rm -f `find . -type f -name '*.orig' `
 	@rm -f `find . -type f -name '*.rej' `
+	@rm -f `find . -type f -name '*.md5' `  # old styling
 	@rm -f .coverage
 	@rm -rf htmlcov
 	@rm -rf build

requirements/lint.txt

@@ -1,4 +1,4 @@
-black==20.8b1; python_version >= "3.6"
+black==20.8b1; implementation_name=="cpython"
 flake8==3.8.4
 flake8-pyi==20.10.0
 isort==5.6.4

tools/check_sum.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+import argparse
+import hashlib
+import pathlib
+import sys
+
+PARSER = argparse.ArgumentParser(
+    description="Helper for check file hashes in Makefile instead of bare timestamps"
+)
+PARSER.add_argument("dst", metavar="DST", type=pathlib.Path)
+PARSER.add_argument("-d", "--debug", action="store_true", default=False)
+
+
+def main(argv):
+    args = PARSER.parse_args(argv)
+    dst = args.dst
+    assert dst.suffix == ".hash"
+    dirname = dst.parent
+    if dirname.name != ".hash":
+        if args.debug:
+            print(f"Invalid name {dst} -> dirname {dirname}", file=sys.stderr)
+        return 0
+    dirname.mkdir(exist_ok=True)
+    src_dir = dirname.parent
+    src_name = dst.stem  # drop .hash
+    full_src = src_dir / src_name
+    hasher = hashlib.sha256()
+    try:
+        hasher.update(full_src.read_bytes())
+    except OSError:
+        if args.debug:
+            print(f"Cannot open {full_src}", file=sys.stderr)
+        return 0
+    src_hash = hasher.hexdigest()
+    if dst.exists():
+        dst_hash = dst.read_text()
+    else:
+        dst_hash = ""
+    if src_hash != dst_hash:
+        dst.write_text(src_hash)
+        print(f"re-hash {src_hash}")
+    else:
+        if args.debug:
+            print(f"Skip {src_hash} checksum, up-to-date")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))