Add script for converting imports in grpc python modules and remove 2…

…to3 (AcademySoftwareFoundation#1557) **Link the Issue(s) this Pull Request is related to.** This is to fix AcademySoftwareFoundation#1555 **Summarize your change.** Replaces 2to3 with a simple script that adds "from ." in front of pb2 imports. This is done to support newer versions of python where 2to3 has been removed.
DiegoTavares · Oct 25, 2024 · 6778b9a · 6778b9a
1 parent f7244dd
commit 6778b9a
Show file tree

Hide file tree

Showing 15 changed files with 51 additions and 28 deletions.
diff --git a/ci/build_sphinx_docs.sh b/ci/build_sphinx_docs.sh
@@ -10,7 +10,7 @@ python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 # Build the docs and treat warnings as errors
 ~/.local/bin/sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html
diff --git a/ci/fix_compiled_proto.py b/ci/fix_compiled_proto.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+
+"""Script that makes the imports in the generated compiled_proto python files relative.
+
+"""
+import os
+import re
+import sys
+import glob
+
+PYTHON_SCRIPT_PATH = sys.argv[1]
+
+if os.path.isdir(PYTHON_SCRIPT_PATH):
+    pattern = re.compile(r"^import \w+ as \w+_pb2")
+    for filepath in glob.glob(os.path.join(PYTHON_SCRIPT_PATH, "*_pb2*.py")):
+        filedata = []
+        with open(filepath) as f:
+            for line in f.readlines():
+                match = pattern.match(line)
+                if match is not None:
+                    line = f"from . {line}"
+                filedata.append(line.strip("\n"))
+        with open(filepath, "w") as f:
+            f.write("\n".join(filedata))
+else:
+    print("Argument is not a directory")
diff --git a/ci/python_coverage_report.sh b/ci/python_coverage_report.sh
@@ -9,8 +9,8 @@ python -m pip install coverage pytest-xvfb
 # Protos need to have their Python code generated in order for tests to pass.
 python -m grpc_tools.protoc -I=proto/ --python_out=pycue/opencue/compiled_proto --grpc_python_out=pycue/opencue/compiled_proto proto/*.proto
 python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc_python_out=rqd/rqd/compiled_proto proto/*.proto
-2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
-2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
+python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
+python ci/fix_compiled_proto.py rqd/rqd/compiled_proto
 
 # Run coverage for each component individually, but append it all into the same report.
 python -m coverage run --source=pycue/opencue/,pycue/FileSequence/ --omit=pycue/opencue/compiled_proto/* pycue/tests/test_suite.py

diff --git a/ci/run_python_lint.sh b/ci/run_python_lint.sh
@@ -13,8 +13,8 @@ python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
-2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
+python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
+python ci/fix_compiled_proto.py rqd/rqd/compiled_proto
 
 echo "Running lint for pycue/..."
 cd pycue

diff --git a/ci/run_python_tests.sh b/ci/run_python_tests.sh
@@ -19,14 +19,14 @@ python -m grpc_tools.protoc -I=proto/ --python_out=rqd/rqd/compiled_proto --grpc
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
-2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
-
-python3 -m unittest discover -s pycue/tests -t pycue -p "*.py"
-PYTHONPATH=pycue python3 -m unittest discover -s pyoutline/tests -t pyoutline -p "*.py"
-PYTHONPATH=pycue python3 -m unittest discover -s cueadmin/tests -t cueadmin -p "*.py"
-PYTHONPATH=pycue:pyoutline python3 -m unittest discover -s cuesubmit/tests -t cuesubmit -p "*.py"
-python3 -m unittest discover -s rqd/tests -t rqd -p "*.py"
+python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
+python ci/fix_compiled_proto.py rqd/rqd/compiled_proto
+
+python -m unittest discover -s pycue/tests -t pycue -p "*.py"
+PYTHONPATH=pycue python -m unittest discover -s pyoutline/tests -t pyoutline -p "*.py"
+PYTHONPATH=pycue python -m unittest discover -s cueadmin/tests -t cueadmin -p "*.py"
+PYTHONPATH=pycue:pyoutline python -m unittest discover -s cuesubmit/tests -t cuesubmit -p "*.py"
+python -m unittest discover -s rqd/tests -t rqd -p "*.py"
 
 
 # Xvfb no longer supports Python 2.

diff --git a/connectors/prometheus_metrics/Dockerfile b/connectors/prometheus_metrics/Dockerfile
@@ -48,7 +48,7 @@ RUN python -m grpc_tools.protoc \
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 RUN cd pycue && python setup.py install
 

diff --git a/cueadmin/Dockerfile b/cueadmin/Dockerfile
@@ -24,7 +24,7 @@ RUN python3 -m grpc_tools.protoc \
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 COPY cueadmin/README.md ./cueadmin/
 COPY cueadmin/setup.py ./cueadmin/

diff --git a/cuegui/Dockerfile b/cuegui/Dockerfile
@@ -54,7 +54,7 @@ RUN python3.6 -m grpc_tools.protoc \
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 COPY cuegui/README.md ./cuegui/
 COPY cuegui/setup.py ./cuegui/

diff --git a/cuesubmit/Dockerfile b/cuesubmit/Dockerfile
@@ -41,7 +41,7 @@ RUN python3.6 -m grpc_tools.protoc \
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+RUN python3 ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 COPY pyoutline/README.md ./pyoutline/
 COPY pyoutline/setup.py ./pyoutline/

diff --git a/proto/README.md b/proto/README.md
@@ -14,15 +14,14 @@ To generate:
 
 ```sh
 python -m grpc_tools.protoc -I=. --python_out=../rqd/rqd/compiled_proto --grpc_python_out=../rqd/rqd/compiled_proto ./*.proto
-2to3 -wn -f import ../rqd/rqd/compiled_proto/*_pb2*.py
+python ../ci/fix_compiled_proto.py ../rqd/rqd/compiled_proto
 ```
 
 For Windows (Powershell):
 
 ```powershell
 python -m grpc_tools.protoc --proto_path=. --python_out=../rqd/rqd/compiled_proto --grpc_python_out=../rqd/rqd/compiled_proto (ls *.proto).Name
-cd ..\rqd\rqd\compiled_proto\
-2to3 -wn -f import (ls *_pb2*.py).Name
+python ../ci/fix_compiled_proto.py ../rqd/rqd/compiled_proto
 ```
 
 
@@ -32,15 +31,14 @@ To generate:
 
 ```sh
 python -m grpc_tools.protoc -I=. --python_out=../pycue/opencue/compiled_proto --grpc_python_out=../pycue/opencue/compiled_proto ./*.proto
-2to3 -wn -f import ../pycue/opencue/compiled_proto/*_pb2*.py
+python ../ci/fix_compiled_proto.py ../pycue/opencue/compiled_proto
 ```
 
 For Windows (Powershell):
 
 ```powershell
 python -m grpc_tools.protoc --proto_path=. --python_out=../pycue/opencue/compiled_proto --grpc_python_out=../pycue/opencue/compiled_proto (ls *.proto).Name
-cd ..\pycue\opencue\compiled_proto\
-2to3 -wn -f import (ls *_pb2*.py).Name
+python ../ci/fix_compiled_proto.py ../pycue/opencue/compiled_proto
 ```
 
 
diff --git a/pycue/Dockerfile b/pycue/Dockerfile
@@ -25,7 +25,7 @@ RUN python3 -m grpc_tools.protoc \
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+RUN python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 COPY VERSION.in VERSIO[N] ./
 RUN test -e VERSION || echo "$(cat VERSION.in)" | tee VERSION

diff --git a/pyoutline/Dockerfile b/pyoutline/Dockerfile
@@ -24,7 +24,7 @@ RUN python3 -m grpc_tools.protoc \
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+RUN python ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 COPY pyoutline/README.md ./pyoutline/
 COPY pyoutline/setup.py ./pyoutline/

diff --git a/requirements.txt b/requirements.txt
@@ -1,4 +1,3 @@
-2to3==1.0
 enum34==1.1.6
 future==1.0.0
 grpcio==1.48.2;python_version<"3.7"

diff --git a/rqd/Dockerfile b/rqd/Dockerfile
@@ -35,7 +35,7 @@ RUN python3.9 -m grpc_tools.protoc \
 
 # Fix imports to work in both Python 2 and 3. See
 # <https://github.com/protocolbuffers/protobuf/issues/1491> for more info.
-RUN 2to3 -wn -f import rqd/rqd/compiled_proto/*_pb2*.py
+RUN python ci/fix_compiled_proto.py rqd/rqd/compiled_proto
 
 COPY VERSION.in VERSIO[N] ./
 RUN test -e VERSION || echo "$(cat VERSION.in)" | tee VERSION

diff --git a/sandbox/install-client-sources.sh b/sandbox/install-client-sources.sh
@@ -10,7 +10,7 @@ python -m grpc_tools.protoc -I=. \
   --python_out=../pycue/opencue/compiled_proto \
   --grpc_python_out=../pycue/opencue/compiled_proto ./*.proto
 cd ..
-2to3 -wn -f import pycue/opencue/compiled_proto/*_pb2*.py
+python ../ci/fix_compiled_proto.py pycue/opencue/compiled_proto
 
 # Install all client packages.
 pip install pycue/ pyoutline/ cueadmin/ cuesubmit/ cuegui/