From d2984f5242ed2250ad1c11b6164e2e8e11e2a804 Mon Sep 17 00:00:00 2001 From: alrex Date: Mon, 27 Sep 2021 13:28:32 -0700 Subject: [PATCH] remove need to clone core repo (#678) --- .flake8 | 1 - .github/workflows/test.yml | 15 ------ .gitignore | 2 - .isort.cfg | 2 +- CONTRIBUTING.md | 14 +++--- README.md | 8 +--- docs/conf.py | 8 +--- eachdist.ini | 2 - gen-requirements.txt | 2 +- scripts/generate_instrumentation_bootstrap.py | 39 +++++++++------ tox.ini | 48 +++++++++++-------- 11 files changed, 64 insertions(+), 77 deletions(-) diff --git a/.flake8 b/.flake8 index e1a2de4f93..3910a27f34 100644 --- a/.flake8 +++ b/.flake8 @@ -19,6 +19,5 @@ exclude = exporter/opentelemetry-exporter-jaeger/build/* docs/examples/opentelemetry-example-app/src/opentelemetry_example_app/grpc/gen/ docs/examples/opentelemetry-example-app/build/* - opentelemetry-python-core/ opentelemetry-proto/build/* opentelemetry-proto/src/opentelemetry/proto/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e0d1ea9b4d..3c200d28a6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,12 +28,6 @@ jobs: steps: - name: Checkout Contrib Repo @ SHA - ${{ github.sha }} uses: actions/checkout@v2 - - name: Checkout Core Repo @ SHA - ${{ env.CORE_REPO_SHA }} - uses: actions/checkout@v2 - with: - repository: open-telemetry/opentelemetry-python - ref: ${{ env.CORE_REPO_SHA }} - path: opentelemetry-python-core - name: Set up Python ${{ env[matrix.python-version] }} uses: actions/setup-python@v2 with: @@ -84,12 +78,6 @@ jobs: steps: - name: Checkout Contrib Repo @ SHA - ${{ github.sha }} uses: actions/checkout@v2 - - name: Checkout Core Repo @ SHA ${{ env.CORE_REPO_SHA }} - uses: actions/checkout@v2 - with: - repository: open-telemetry/opentelemetry-python - ref: ${{ env.CORE_REPO_SHA }} - path: opentelemetry-python-core - name: Set up Python 3.9 uses: actions/setup-python@v2 with: @@ -109,6 +97,3 @@ jobs: key: v4-misc-tox-cache-${{ matrix.tox-environment }}-${{ hashFiles('tox.ini', 'dev-requirements.txt', 'gen-requirements.txt', 'docs-requirements.txt') }} - name: run tox run: tox -e ${{ matrix.tox-environment }} - - name: Ensure generated code is up to date - if: matrix.tox-environment == 'generate' - run: git diff --exit-code || (echo 'Generated code is out of date, please run "tox -e generate" and commit the changes in this PR.' && exit 1) diff --git a/.gitignore b/.gitignore index 29118d428e..c359e13727 100644 --- a/.gitignore +++ b/.gitignore @@ -23,8 +23,6 @@ lib64 __pycache__ venv*/ .venv*/ -opentelemetry-python-core*/ -/opentelemetry-python-core # Installer logs pip-log.txt diff --git a/.isort.cfg b/.isort.cfg index 8e5e4ac012..afe42d3d41 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -14,6 +14,6 @@ profile=black ; docs: https://github.com/timothycrosley/isort#multi-line-output-modes multi_line_output=3 skip=target -skip_glob=**/gen/*,.venv*/*,venv*/*,opentelemetry-python-core/*,.tox/* +skip_glob=**/gen/*,.venv*/*,venv*/*,.tox/* known_first_party=opentelemetry known_third_party=psutil,pytest,redis,redis_opentracing diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fee9958f39..8d4dab8d14 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -156,16 +156,16 @@ For a deeper discussion, see: https://github.com/open-telemetry/opentelemetry-sp ## Running Tests Locally 1. Go to your Contrib repo directory. `git clone git@github.com:open-telemetry/opentelemetry-python-contrib.git && cd opentelemetry-python-contrib`. -2. Clone the [OpenTelemetry Python](https://github.com/open-telemetry/opentelemetry-python) Python Core repo to a folder named `opentelemetry-python-core`. `git clone https://github.com/open-telemetry/opentelemetry-python.git opentelemetry-python-core`. -3. Make sure you have `tox` installed. `pip install tox`. -4. Run `tox` without any arguments to run tests for all the packages. Read more about [tox](https://tox.readthedocs.io/en/latest/). +2. Make sure you have `tox` installed. `pip install tox`. +3. Run `tox` without any arguments to run tests for all the packages. Read more about [tox](https://tox.readthedocs.io/en/latest/). ### Testing against a different Core repo branch/commit -1. Change directory to the repo that was cloned above. `cd opentelemetry-python-core`. -2. Move the head of this repo to the SHA hash you want your tests to use. The current SHA hash can be found in `.github/workflows/test.yml` file under the `opentelemetry-python-contrib` directory. For example, currently it is `1a12fa0d681e37c1fda9cb8d46212ff3bbf6b76a`. So use `git fetch && git checkout `. -3. Go back to the root directory. `cd ../`. -4. With `tox` installed, run tests for a package. (e.g. `tox -e test-instrumentation-flask`.) +Some of the tox targets install packages from the [OpenTelemetry Python Core Repository](https://github.com/open-telemetry/opentelemetry-python) via pip. The version of the packages installed defaults to the main branch in that repository when tox is run locally. It is possible to install packages tagged with a specific git commit hash by setting an environment variable before running tox as per the following example: + +CORE_REPO_SHA=c49ad57bfe35cfc69bfa863d74058ca9bec55fc3 tox + +The continuation integration overrides that environment variable with as per the configuration [here](https://github.com/open-telemetry/opentelemetry-python-contrib/blob/main/.github/workflows/test.yml#L9). ## Style Guide diff --git a/README.md b/README.md index 78e4e3aa29..f923fa4228 100644 --- a/README.md +++ b/README.md @@ -116,12 +116,8 @@ Maintainers ([@open-telemetry/python-maintainers](https://github.com/orgs/open-t 1. Go to your Contrib repo directory. `cd ~/git/opentelemetry-python-contrib`. 2. Create a virtual env in your Contrib repo directory. `python3 -m venv my_test_venv`. 3. Activate your virtual env. `source my_test_venv/bin/activate`. -4. Clone the [OpenTelemetry Python](https://github.com/open-telemetry/opentelemetry-python) Python Core repo to a folder named `opentelemetry-python-core`. `git clone git@github.com:open-telemetry/opentelemetry-python.git opentelemetry-python-core`. -5. Change directory to the repo that was just cloned. `cd opentelemetry-python-core`. -6. Move the head of this repo to the hash you want your tests to use. This is currently the SHA `47483865854c7adae7455f8441dab7f814f4ce2a` as seen in `.github/workflows/test.yml`. Use `git fetch && git checkout 47483865854c7adae7455f8441dab7f814f4ce2a`. -7. Go back to the root directory. `cd ../`. -8. Make sure you have `tox` installed. `pip install tox`. -9. Run tests for a package. (e.g. `tox -e test-instrumentation-flask`.) +4. Make sure you have `tox` installed. `pip install tox`. +5. Run tests for a package. (e.g. `tox -e test-instrumentation-flask`.) ### Thanks to all the people who already contributed! diff --git a/docs/conf.py b/docs/conf.py index 83b7bb6aaa..b58bc6c52d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,11 +24,7 @@ settings.configure() -source_dirs = [ - os.path.abspath( - "../opentelemetry-python-core/opentelemetry-instrumentation/src/" - ), -] +source_dirs = [] exp = "../exporter" exp_dirs = [ @@ -51,7 +47,7 @@ if isdir(join(sdk_ext, f)) ] -sys.path[:0] = source_dirs + exp_dirs + instr_dirs + sdk_ext_dirs +sys.path[:0] = exp_dirs + instr_dirs + sdk_ext_dirs # -- Project information ----------------------------------------------------- diff --git a/eachdist.ini b/eachdist.ini index 375a80a9c7..3cf737021e 100644 --- a/eachdist.ini +++ b/eachdist.ini @@ -3,8 +3,6 @@ [DEFAULT] ignore= _template - reference - opentelemetry-python-core sortfirst= util/opentelemetry-util-http diff --git a/gen-requirements.txt b/gen-requirements.txt index edb4ed125b..d4a52e0da8 100644 --- a/gen-requirements.txt +++ b/gen-requirements.txt @@ -3,4 +3,4 @@ astor==0.8.1 jinja2~=2.7 isort black - +requests diff --git a/scripts/generate_instrumentation_bootstrap.py b/scripts/generate_instrumentation_bootstrap.py index a1c2d69adc..c5832d0eb9 100755 --- a/scripts/generate_instrumentation_bootstrap.py +++ b/scripts/generate_instrumentation_bootstrap.py @@ -15,18 +15,17 @@ # limitations under the License. import ast +import filecmp import logging import os import subprocess import sys +import tempfile import astor import pkg_resources -from otel_packaging import ( - get_instrumentation_packages, - root_path, - scripts_path, -) +import requests +from otel_packaging import get_instrumentation_packages, scripts_path logging.basicConfig(level=logging.INFO) logger = logging.getLogger("instrumentation_list_generator") @@ -50,16 +49,16 @@ default_instrumentations = [] """ +tmpdir = tempfile.TemporaryDirectory() # pylint: disable=R1732 +gen_path = os.path.join(tmpdir.name, "new.py",) -gen_path = os.path.join( - root_path, - "opentelemetry-python-core", - "opentelemetry-instrumentation", - "src", - "opentelemetry", - "instrumentation", - "bootstrap_gen.py", -) +current_path = os.path.join(tmpdir.name, "current.py",) + +core_repo = os.getenv("CORE_REPO_SHA", "main") +url = f"https://raw.githubusercontent.com/open-telemetry/opentelemetry-python/{core_repo}/opentelemetry-instrumentation/src/opentelemetry/instrumentation/bootstrap_gen.py" +r = requests.get(url, allow_redirects=True) +with open(current_path, "wb") as output: + output.write(r.content) def main(): @@ -99,7 +98,7 @@ def main(): "scripts/eachdist.py", "format", "--path", - "opentelemetry-python-core/opentelemetry-instrumentation", + tmpdir.name, ], check=True, ) @@ -107,5 +106,15 @@ def main(): logger.info("generated %s", gen_path) +def compare(): + if not filecmp.cmp(current_path, gen_path): + logger.info( + 'Generated code is out of date, please run "tox -e generate" and commit bootstrap_gen.py to core repo.' + ) + os.replace(gen_path, "bootstrap_gen.py") + sys.exit(1) + + if __name__ == "__main__": main() + compare() diff --git a/tox.ini b/tox.ini index 274a3f9264..4ac5447d83 100644 --- a/tox.ini +++ b/tox.ini @@ -184,6 +184,12 @@ deps = ; FIXME: add coverage testing ; FIXME: add mypy testing +setenv = + ; override CORE_REPO_SHA via env variable when testing other branches/commits than main + ; i.e: CORE_REPO_SHA=dde62cebffe519c35875af6d06fae053b3be65ec tox -e + CORE_REPO_SHA={env:CORE_REPO_SHA:"main"} + CORE_REPO="git+https://github.com/open-telemetry/opentelemetry-python.git@{env:CORE_REPO_SHA}" + changedir = test-instrumentation-aiohttp-client: instrumentation/opentelemetry-instrumentation-aiohttp-client/tests test-instrumentation-aiopg: instrumentation/opentelemetry-instrumentation-aiopg/tests @@ -228,11 +234,11 @@ commands_pre = py3{6,7,8,9}: python -m pip install -U pip setuptools wheel ; Install common packages for all the tests. These are not needed in all the ; cases but it saves a lot of boilerplate in this file. - test: pip install {toxinidir}/opentelemetry-python-core/opentelemetry-api[test] - test: pip install {toxinidir}/opentelemetry-python-core/opentelemetry-semantic-conventions[test] - test: pip install {toxinidir}/opentelemetry-python-core/opentelemetry-instrumentation[test] - test: pip install {toxinidir}/opentelemetry-python-core/opentelemetry-sdk[test] - test: pip install {toxinidir}/opentelemetry-python-core/tests/util[test] + test: pip install "opentelemetry-api[test] @ {env:CORE_REPO}#egg=opentelemetry-api&subdirectory=opentelemetry-api" + test: pip install "opentelemetry-semantic-conventions[test] @ {env:CORE_REPO}#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions" + test: pip install "opentelemetry-instrumentation[test] @ {env:CORE_REPO}#egg=opentelemetry-instrumentation&subdirectory=opentelemetry-instrumentation" + test: pip install "opentelemetry-sdk[test] @ {env:CORE_REPO}#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk" + test: pip install "opentelemetry-test[test] @ {env:CORE_REPO}#egg=opentelemetry-test&subdirectory=tests/util" celery: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-celery[test] @@ -299,7 +305,7 @@ commands_pre = sqlalchemy{11,14}: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-sqlalchemy[test] - elasticsearch{2,5,6}: pip install {toxinidir}/opentelemetry-python-core/opentelemetry-instrumentation {toxinidir}/instrumentation/opentelemetry-instrumentation-elasticsearch[test] + elasticsearch{2,5,6}: pip install "{env:CORE_REPO}#egg=opentelemetry-instrumentation&subdirectory=opentelemetry-instrumentation" {toxinidir}/instrumentation/opentelemetry-instrumentation-elasticsearch[test] httpx: pip install {toxinidir}/instrumentation/opentelemetry-instrumentation-httpx[test] @@ -323,10 +329,10 @@ deps = pytest commands_pre = - python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-api - python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-semantic-conventions - python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-instrumentation - python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-sdk + python -m pip install "{env:CORE_REPO}#egg=opentelemetry-api&subdirectory=opentelemetry-api" + python -m pip install "{env:CORE_REPO}#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions" + python -m pip install "{env:CORE_REPO}#egg=opentelemetry-instrumentation&subdirectory=opentelemetry-instrumentation" + python -m pip install "{env:CORE_REPO}#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk" python -m pip install {toxinidir}/util/opentelemetry-util-http changedir = docs @@ -348,11 +354,11 @@ deps = httpretty commands_pre = - python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-api - python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-semantic-conventions - python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-instrumentation[test] - python -m pip install {toxinidir}/opentelemetry-python-core/opentelemetry-sdk - python -m pip install {toxinidir}/opentelemetry-python-core/tests/util + python -m pip install "{env:CORE_REPO}#egg=opentelemetry-api&subdirectory=opentelemetry-api" + python -m pip install "{env:CORE_REPO}#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions" + python -m pip install "{env:CORE_REPO}#egg=opentelemetry-instrumentation&subdirectory=opentelemetry-instrumentation" + python -m pip install "{env:CORE_REPO}#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk" + python -m pip install "{env:CORE_REPO}#egg=opentelemetry-test&subdirectory=tests/util" python -m pip install -e {toxinidir}/util/opentelemetry-util-http[test] python -m pip install -e {toxinidir}/instrumentation/opentelemetry-instrumentation-wsgi[test] python -m pip install -e {toxinidir}/instrumentation/opentelemetry-instrumentation-dbapi[test] @@ -417,11 +423,11 @@ changedir = tests/opentelemetry-docker-tests/tests commands_pre = - pip install -e {toxinidir}/opentelemetry-python-core/opentelemetry-api \ - -e {toxinidir}/opentelemetry-python-core/opentelemetry-semantic-conventions \ - -e {toxinidir}/opentelemetry-python-core/opentelemetry-instrumentation \ - -e {toxinidir}/opentelemetry-python-core/opentelemetry-sdk \ - -e {toxinidir}/opentelemetry-python-core/tests/util \ + pip install "{env:CORE_REPO}#egg=opentelemetry-api&subdirectory=opentelemetry-api" \ + "{env:CORE_REPO}#egg=opentelemetry-semantic-conventions&subdirectory=opentelemetry-semantic-conventions" \ + "{env:CORE_REPO}#egg=opentelemetry-instrumentation&subdirectory=opentelemetry-instrumentation" \ + "{env:CORE_REPO}#egg=opentelemetry-sdk&subdirectory=opentelemetry-sdk" \ + "{env:CORE_REPO}#egg=opentelemetry-test&subdirectory=tests/util" \ -e {toxinidir}/instrumentation/opentelemetry-instrumentation-asyncpg \ -e {toxinidir}/instrumentation/opentelemetry-instrumentation-celery \ -e {toxinidir}/instrumentation/opentelemetry-instrumentation-dbapi \ @@ -432,7 +438,7 @@ commands_pre = -e {toxinidir}/instrumentation/opentelemetry-instrumentation-sqlalchemy \ -e {toxinidir}/instrumentation/opentelemetry-instrumentation-aiopg \ -e {toxinidir}/instrumentation/opentelemetry-instrumentation-redis \ - -e {toxinidir}/opentelemetry-python-core/exporter/opentelemetry-exporter-opencensus + "{env:CORE_REPO}#egg=opentelemetry-exporter-opencensus&subdirectory=exporter/opentelemetry-exporter-opencensus" docker-compose up -d python check_availability.py