From 6778b9a743a708aaa102b9c90663439388a7317a Mon Sep 17 00:00:00 2001 From: Jimmy Christensen Date: Sat, 26 Oct 2024 00:07:36 +0200 Subject: [PATCH] Add script for converting imports in grpc python modules and remove 2to3 (#1557) **Link the Issue(s) this Pull Request is related to.** This is to fix #1555 **Summarize your change.** Replaces 2to3 with a simple script that adds "from ." in front of pb2 imports. This is done to support newer versions of python where 2to3 has been removed. --- ci/build_sphinx_docs.sh | 2 +- ci/fix_compiled_proto.py | 26 ++++++++++++++++++++++++ ci/python_coverage_report.sh | 4 ++-- ci/run_python_lint.sh | 4 ++-- ci/run_python_tests.sh | 16 +++++++-------- connectors/prometheus_metrics/Dockerfile | 2 +- cueadmin/Dockerfile | 2 +- cuegui/Dockerfile | 2 +- cuesubmit/Dockerfile | 2 +- proto/README.md | 10 ++++----- pycue/Dockerfile | 2 +- pyoutline/Dockerfile | 2 +- requirements.txt | 1 - rqd/Dockerfile | 2 +- sandbox/install-client-sources.sh | 2 +- 15 files changed, 51 insertions(+), 28 deletions(-) create mode 100644 ci/fix_compiled_proto.py diff --git a/ci/build_sphinx_docs.sh b/ci/build_sphinx_docs.sh index 9f207d51e..b39c68fc2 100755 --- a/ci/build_sphinx_docs.sh +++ b/ci/build_sphinx_docs.sh @@ -10,7 +10,7 @@ python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto # Fix imports to work in both Python 2 and 3. See # for more info. -2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +python ci/fix_compiled_proto.py pycue/opencue/compiled_proto # Build the docs and treat warnings as errors ~/.local/bin/sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html diff --git a/ci/fix_compiled_proto.py b/ci/fix_compiled_proto.py new file mode 100644 index 000000000..0a5534803 --- /dev/null +++ b/ci/fix_compiled_proto.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +"""Script that makes the imports in the generated compiled_proto python files relative. + +""" +import os +import re +import sys +import glob + +PYTHON_SCRIPT_PATH = sys.argv[1] + +if os.path.isdir(PYTHON_SCRIPT_PATH): + pattern = re.compile(r"^import \w+ as \w+_pb2") + for filepath in glob.glob(os.path.join(PYTHON_SCRIPT_PATH, "*_pb2*.py")): + filedata = [] + with open(filepath) as f: + for line in f.readlines(): + match = pattern.match(line) + if match is not None: + line = f"from . {line}" + filedata.append(line.strip("\n")) + with open(filepath, "w") as f: + f.write("\n".join(filedata)) +else: + print("Argument is not a directory") diff --git a/ci/python_coverage_report.sh b/ci/python_coverage_report.sh index e0c65328f..4477c8d44 100755 --- a/ci/python_coverage_report.sh +++ b/ci/python_coverage_report.sh @@ -9,8 +9,8 @@ python -m pip install coverage pytest-xvfb # Protos need to have their Python code generated in order for tests to pass. python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto -2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py -2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py +python ci/fix_compiled_proto.py pycue/opencue/compiled_proto +python ci/fix_compiled_proto.py rqd/rqd/compiled_proto # Run coverage for each component individually, but append it all into the same report. python -m coverage run --source=pycue/opencue/,pycue/FileSequence/ --omit=pycue/opencue/compiled_proto/* pycue/tests/test_suite.py diff --git a/ci/run_python_lint.sh b/ci/run_python_lint.sh index bd86c9188..ba98d2b6c 100755 --- a/ci/run_python_lint.sh +++ b/ci/run_python_lint.sh @@ -13,8 +13,8 @@ python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc # Fix imports to work in both Python 2 and 3. See # for more info. -2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py -2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py +python ci/fix_compiled_proto.py pycue/opencue/compiled_proto +python ci/fix_compiled_proto.py rqd/rqd/compiled_proto echo "Running lint for pycue/..." cd pycue diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh index 4e1b0212b..edb0322cc 100755 --- a/ci/run_python_tests.sh +++ b/ci/run_python_tests.sh @@ -19,14 +19,14 @@ python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc # Fix imports to work in both Python 2 and 3. See # for more info. -2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py -2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py - -python3 -m unittest discover -s pycue/tests -t pycue -p "*.py" -PYTHONPATH=pycue python3 -m unittest discover -s pyoutline/tests -t pyoutline -p "*.py" -PYTHONPATH=pycue python3 -m unittest discover -s cueadmin/tests -t cueadmin -p "*.py" -PYTHONPATH=pycue:pyoutline python3 -m unittest discover -s cuesubmit/tests -t cuesubmit -p "*.py" -python3 -m unittest discover -s rqd/tests -t rqd -p "*.py" +python ci/fix_compiled_proto.py pycue/opencue/compiled_proto +python ci/fix_compiled_proto.py rqd/rqd/compiled_proto + +python -m unittest discover -s pycue/tests -t pycue -p "*.py" +PYTHONPATH=pycue python -m unittest discover -s pyoutline/tests -t pyoutline -p "*.py" +PYTHONPATH=pycue python -m unittest discover -s cueadmin/tests -t cueadmin -p "*.py" +PYTHONPATH=pycue:pyoutline python -m unittest discover -s cuesubmit/tests -t cuesubmit -p "*.py" +python -m unittest discover -s rqd/tests -t rqd -p "*.py" # Xvfb no longer supports Python 2. diff --git a/connectors/prometheus_metrics/Dockerfile b/connectors/prometheus_metrics/Dockerfile index f710dce0c..357b5a0bf 100644 --- a/connectors/prometheus_metrics/Dockerfile +++ b/connectors/prometheus_metrics/Dockerfile @@ -48,7 +48,7 @@ RUN python -m grpc_tools.protoc \ # Fix imports to work in both Python 2 and 3. See # for more info. -RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto RUN cd pycue && python setup.py install diff --git a/cueadmin/Dockerfile b/cueadmin/Dockerfile index 20ff77c1e..dd1359edc 100644 --- a/cueadmin/Dockerfile +++ b/cueadmin/Dockerfile @@ -24,7 +24,7 @@ RUN python3 -m grpc_tools.protoc \ # Fix imports to work in both Python 2 and 3. See # for more info. -RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto COPY cueadmin/README.md ./cueadmin/ COPY cueadmin/setup.py ./cueadmin/ diff --git a/cuegui/Dockerfile b/cuegui/Dockerfile index 3e6630804..6e53c7f04 100644 --- a/cuegui/Dockerfile +++ b/cuegui/Dockerfile @@ -54,7 +54,7 @@ RUN python3.6 -m grpc_tools.protoc \ # Fix imports to work in both Python 2 and 3. See # for more info. -RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto COPY cuegui/README.md ./cuegui/ COPY cuegui/setup.py ./cuegui/ diff --git a/cuesubmit/Dockerfile b/cuesubmit/Dockerfile index eb2a4902a..47c615f7c 100644 --- a/cuesubmit/Dockerfile +++ b/cuesubmit/Dockerfile @@ -41,7 +41,7 @@ RUN python3.6 -m grpc_tools.protoc \ # Fix imports to work in both Python 2 and 3. See # for more info. -RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto COPY pyoutline/README.md ./pyoutline/ COPY pyoutline/setup.py ./pyoutline/ diff --git a/proto/README.md b/proto/README.md index 0b039b462..904a112e1 100644 --- a/proto/README.md +++ b/proto/README.md @@ -14,15 +14,14 @@ To generate: ```sh python -m grpc_tools.protoc -I=. --python_out=../rqd/rqd/compiled_proto --grpc_python_out=../rqd/rqd/compiled_proto ./*.proto -2to3 -wn -f import ../rqd/rqd/compiled_proto/*_pb2*.py +python ../ci/fix_compiled_proto.py ../rqd/rqd/compiled_proto ``` For Windows (Powershell): ```powershell python -m grpc_tools.protoc --proto_path=. --python_out=../rqd/rqd/compiled_proto --grpc_python_out=../rqd/rqd/compiled_proto (ls *.proto).Name -cd ..\rqd\rqd\compiled_proto\ -2to3 -wn -f import (ls *_pb2*.py).Name +python ../ci/fix_compiled_proto.py ../rqd/rqd/compiled_proto ``` @@ -32,15 +31,14 @@ To generate: ```sh python -m grpc_tools.protoc -I=. --python_out=../pycue/opencue/compiled_proto --grpc_python_out=../pycue/opencue/compiled_proto ./*.proto -2to3 -wn -f import ../pycue/opencue/compiled_proto/*_pb2*.py +python ../ci/fix_compiled_proto.py ../pycue/opencue/compiled_proto ``` For Windows (Powershell): ```powershell python -m grpc_tools.protoc --proto_path=. --python_out=../pycue/opencue/compiled_proto --grpc_python_out=../pycue/opencue/compiled_proto (ls *.proto).Name -cd ..\pycue\opencue\compiled_proto\ -2to3 -wn -f import (ls *_pb2*.py).Name +python ../ci/fix_compiled_proto.py ../pycue/opencue/compiled_proto ``` diff --git a/pycue/Dockerfile b/pycue/Dockerfile index 9698e94bc..c61d3cf16 100644 --- a/pycue/Dockerfile +++ b/pycue/Dockerfile @@ -25,7 +25,7 @@ RUN python3 -m grpc_tools.protoc \ # Fix imports to work in both Python 2 and 3. See # for more info. -RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +RUN python ci/fix_compiled_proto.py pycue/opencue/compiled_proto COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)" | tee VERSION diff --git a/pyoutline/Dockerfile b/pyoutline/Dockerfile index bc7155daf..6937a5584 100644 --- a/pyoutline/Dockerfile +++ b/pyoutline/Dockerfile @@ -24,7 +24,7 @@ RUN python3 -m grpc_tools.protoc \ # Fix imports to work in both Python 2 and 3. See # for more info. -RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +RUN python ci/fix_compiled_proto.py pycue/opencue/compiled_proto COPY pyoutline/README.md ./pyoutline/ COPY pyoutline/setup.py ./pyoutline/ diff --git a/requirements.txt b/requirements.txt index cceee9237..8af83f243 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -2to3==1.0 enum34==1.1.6 future==1.0.0 grpcio==1.48.2;python_version<"3.7" diff --git a/rqd/Dockerfile b/rqd/Dockerfile index c3a0a0dc1..6847ad10a 100644 --- a/rqd/Dockerfile +++ b/rqd/Dockerfile @@ -35,7 +35,7 @@ RUN python3.9 -m grpc_tools.protoc \ # Fix imports to work in both Python 2 and 3. See # for more info. -RUN 2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py +RUN python ci/fix_compiled_proto.py rqd/rqd/compiled_proto COPY VERSION.in VERSIO[N] ./ RUN test -e VERSION || echo "$(cat VERSION.in)" | tee VERSION diff --git a/sandbox/install-client-sources.sh b/sandbox/install-client-sources.sh index 7e15ed018..80ed1f39e 100755 --- a/sandbox/install-client-sources.sh +++ b/sandbox/install-client-sources.sh @@ -10,7 +10,7 @@ python -m grpc_tools.protoc -I=. \ --python_out=../pycue/opencue/compiled_proto \ --grpc_python_out=../pycue/opencue/compiled_proto ./*.proto cd .. -2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py +python ../ci/fix_compiled_proto.py pycue/opencue/compiled_proto # Install all client packages. pip install pycue/ pyoutline/ cueadmin/ cuesubmit/ cuegui/