Skip to content

Commit

Permalink
Multiprocessing Configure
Browse files Browse the repository at this point in the history
1. Add configure-time parallelism of builds by running discovery of types for each namespace in a separate process.
2. Add cmake example and documentation
3. Enable manifest output (json)

After this change the following will be added:
1. Add build-time parallelism by running code generation for each namespace in separate processes.
2. Enable use of a manifest as an input instead of dynamically discovering type dependencies.
3. Enable `-j n` argument to manually disable or limit parallelism.
4. Disable parallelism automatically if there is only one namespace.
  • Loading branch information
thirtytwobits committed Dec 7, 2024
1 parent 2bdc442 commit 8b67b5f
Show file tree
Hide file tree
Showing 39 changed files with 1,490 additions and 343 deletions.
2 changes: 1 addition & 1 deletion .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "Python dev environment",
"image": "ghcr.io/opencyphal/toxic:tx22.4.2",
"image": "ghcr.io/opencyphal/toxic:tx22.4.3",
"workspaceFolder": "/workspace",
"workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=delegated",
"mounts": [
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ jobs:
release:
if: ${{ github.event_name == 'release' && !github.event.release.prerelease }}
runs-on: ubuntu-latest
container: ghcr.io/opencyphal/toxic:tx22.4.2
container: ghcr.io/opencyphal/toxic:tx22.4.3
steps:
- uses: actions/checkout@v4
with:
Expand All @@ -25,7 +25,7 @@ jobs:
- name: test-nnvg
run: tox -e py311-nnvg
- name: test-doctest
run: tox -e py311-doctest,py310-rstdoctest
run: tox -e py311-doctest,py311-rstdoctest
- name: test-pytest
run: tox -e py311-test
- name: package
Expand Down
26 changes: 13 additions & 13 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,19 +13,19 @@ on:
jobs:
test:
runs-on: ubuntu-latest
container: ghcr.io/opencyphal/toxic:tx22.4.2
container: ghcr.io/opencyphal/toxic:tx22.4.3
steps:
- uses: actions/checkout@v4
with:
submodules: true
- name: lint
run: tox -e lint
- name: test-nnvg
run: tox -e py310-nnvg
run: tox -e py311-nnvg
- name: test-doctest
run: tox -e py310-doctest,py310-rstdoctest
run: tox -e py311-doctest,py311-rstdoctest
- name: test-pytest
run: tox -e py310-test
run: tox -e py311-test
- name: test-report
run: tox -e report
- name: package
Expand All @@ -39,7 +39,7 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: xunit-results
path: .tox/py310-test/tmp/xunit-result.xml
path: .tox/py311-test/tmp/xunit-result.xml
- name: upload-package
uses: actions/upload-artifact@v4
with:
Expand All @@ -54,7 +54,7 @@ jobs:
with:
# Disabling shallow clone is recommended for improving relevancy of reporting
fetch-depth: 0
- name: download-converage-reports
- name: download-coverage-reports
uses: actions/download-artifact@v4
with:
name: coverage-reports
Expand All @@ -63,7 +63,7 @@ jobs:
uses: actions/download-artifact@v4
with:
name: xunit-results
path: .tox/py310-test/tmp/
path: .tox/py311-test/tmp/
- name: set-environment
run: |
echo NUNAVUT_MAJOR_MINOR_VERSION=$(./.github/verify.py --major-minor-version-only) >> $GITHUB_ENV
Expand All @@ -83,7 +83,7 @@ jobs:
-Dsonar.projectVersion=${{ env.NUNAVUT_MAJOR_MINOR_VERSION }}
-Dsonar.python.version=python3.11
-Dsonar.python.coverage.reportPaths=.tox/report/tmp/coverage.xml
-Dsonar.python.xunit.reportPath=.tox/py310-test/tmp/xunit-result.xml
-Dsonar.python.xunit.reportPath=.tox/py311-test/tmp/xunit-result.xml
- name: report-pr
if: ${{ github.event_name == 'pull_request' }}
uses: sonarsource/sonarcloud-github-action@master
Expand All @@ -97,12 +97,12 @@ jobs:
-Dsonar.projectVersion=${{ env.NUNAVUT_MAJOR_MINOR_VERSION }}
-Dsonar.python.version=python3.11
-Dsonar.python.coverage.reportPaths=.tox/report/tmp/coverage.xml
-Dsonar.python.xunit.reportPath=.tox/py310-test/tmp/xunit-result.xml
-Dsonar.python.xunit.reportPath=.tox/py311-test/tmp/xunit-result.xml
compat-test-python3-mac:
strategy:
matrix:
python3-version: ['11','12']
python3-version: ['11','12', '13']
python3-platform: ['macos-latest']
runs-on: ${{ matrix.python3-platform }}
needs: test
Expand All @@ -122,9 +122,9 @@ jobs:
compat-test-python3-ubuntu:
strategy:
matrix:
python3-version: ['8', '9', '10', '11', '12']
python3-version: ['8', '9', '10', '11', '12', '13']
runs-on: ubuntu-latest
container: ghcr.io/opencyphal/toxic:tx22.4.2
container: ghcr.io/opencyphal/toxic:tx22.4.3
needs: test
steps:
- uses: actions/checkout@v4
Expand Down Expand Up @@ -262,7 +262,7 @@ jobs:
language-verification-python:
runs-on: ubuntu-latest
needs: test
container: ghcr.io/opencyphal/toxic:tx22.4.2
container: ghcr.io/opencyphal/toxic:tx22.4.3
steps:
- uses: actions/checkout@v4
with:
Expand Down
4 changes: 2 additions & 2 deletions .readthedocs.yaml
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
version: 2
build:
os: ubuntu-22.04
os: ubuntu-lts-latest
tools:
python: "3.11"
python: "latest"
sphinx:
configuration: conf.py
python:
Expand Down
1 change: 0 additions & 1 deletion .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
"request": "launch",
"module": "pytest",
"args": [
"--no-cov",
"--keep-generated",
"--rootdir=${workspaceFolder}",
"${file}"
Expand Down
1 change: 1 addition & 0 deletions .vscode/nunavut-words.txt
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ postprocessor
postprocessors
roadmap
rtype
scsv
Sriram
tobytes
transcompilation
Expand Down
1 change: 1 addition & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@
"files.insertFinalNewline": true,
"files.trimFinalNewlines": true,
"files.trimTrailingWhitespace": true,
"reflow.preferredLineLength": 120,
"testMate.cpp.test.advancedExecutables": [
{
"pattern": "verification/build_vscode/**/suite/{test,Test,TEST}_*",
Expand Down
13 changes: 13 additions & 0 deletions .vscode/tasks.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,19 @@
"isDefault": true
}
},
{
"label": "doc build (tox)",
"type": "shell",
"command": "tox",
"args": [
"-e",
"docs"
],
"group": {
"kind": "build",
},
"problemMatcher": []
},
{
"label": "verify c native32",
"type": "shell",
Expand Down
6 changes: 3 additions & 3 deletions CONTRIBUTING.rst
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,8 @@ To run the full suite of `tox`_ tests locally you'll need docker. Once you have
and running do::

git submodule update --init --recursive
docker pull ghcr.io/opencyphal/toxic:tx22.4.2
docker run --rm -v $PWD:/repo ghcr.io/opencyphal/toxic:tx22.4.2 tox
docker pull ghcr.io/opencyphal/toxic:tx22.4.3
docker run --rm -v $PWD:/repo ghcr.io/opencyphal/toxic:tx22.4.3 tox

To run a limited suite using only locally available interpreters directly on your host machine,
skip the docker invocations and use ``tox run -s``.
Expand Down Expand Up @@ -198,7 +198,7 @@ Building The Docs
We rely on `read the docs`_ to build our documentation from github but we also verify this build
as part of our tox build. This means you can view a local copy after completing a full, successful
test run (See `Running The Tests`_) or do
:code:`docker run --rm -t -v $PWD:/repo ghcr.io/opencyphal/toxic:tx22.4.2 /bin/sh -c "tox run -e docs"` to build
:code:`docker run --rm -t -v $PWD:/repo ghcr.io/opencyphal/toxic:tx22.4.3 /bin/sh -c "tox run -e docs"` to build
the docs target. You can open the index.html under ``.tox_{host platform}/docs/tmp/index.html`` or run a local
web-server::

Expand Down
Loading

0 comments on commit 8b67b5f

Please sign in to comment.