API docs for the whole stack via doxygen (backport #278) (#284)

* API docs for the whole stack via doxygen (#278)

* Use all repositories for doxygen

* Switch API also for multi-version

* Convert bash scripts to python

* Fix check_links

* Add branch_version for every repo

* Update .gitignore

* Abort only if subrepos have uncommited changes

* Checkout correct branches for multiversion-api

* Fix ci-format.yml

* Update exclude patterns

* Readd delete_sub_repos

* Add debug output

* Add pycache to gitignore

* Fix multiversion (PR) workflow

* Fix ref

* Fix ref

* Fix refs for PR workflow

* Fix add_sub_repos

* Add debug output

* Delete sub-repos at the correct time

* Try to set the correct branch also for PRs

* Fix multiversion-PR with sphinx

* Fix api_multi_version

* Add ROS version to doxygen output

* Last text updates

* Run wf only if relevant settings have changed

* Apply suggestions from code review

Co-authored-by: Sai Kishor Kothakota <[email protected]>

---------

Co-authored-by: Sai Kishor Kothakota <[email protected]>

* Fix deploy defines for humble

---------

Co-authored-by: Christoph Fröhlich <[email protected]>
Co-authored-by: Sai Kishor Kothakota <[email protected]>
Co-authored-by: Christoph Froehlich <[email protected]>

.github/workflows/ci-format.yml

@@ -24,7 +24,7 @@ jobs:
           reviewer-stats
         path: ~/reviews
     - name: Copy stats
-      run: ./make_help_scripts/add_pr_stats
+      run: ./make_help_scripts/add_pr_stats.py
     - uses: pre-commit/[email protected]
       with:
         extra_args: --all-files --hook-stage manual

.github/workflows/sphinx-check-page-multiversion.yml

@@ -0,0 +1,53 @@
+name: "Build Multiversion Page"
+on:
+  workflow_dispatch:
+    inputs:
+      BASE_BRANCH:
+        description: 'Chose branch to run on'
+        required: true
+        default: 'master'
+        type: choice
+        options:
+        - humble
+        - iron
+        - master
+  pull_request:
+    branches:
+      - master
+    paths:
+    - 'make_help_scripts/**'
+    - 'conf.py'
+
+jobs:
+  build-multiversion:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+        cache: 'pip'
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install --upgrade --requirement requirements.txt
+      shell: bash
+    - name: Install generate_parameter_library
+      run: |
+        cd
+        git clone https://github.com/PickNikRobotics/generate_parameter_library.git
+        cd generate_parameter_library/generate_parameter_library_py/
+        python -m pip install .
+      shell: bash
+    - name: Install doxygen and graphviz
+      run: sudo apt-get install -y doxygen graphviz
+    - name: Build multiversion with API
+      env:
+        BASE_BRANCH: ${{ github.event.inputs.BASE_BRANCH }}
+        BASE_BRANCH_PR: ${{ github.event.pull_request.head.ref }}
+      run: |
+        git config --local user.email "[email protected]"
+        git config --local user.name "GitHub Action"
+        make multiversion-with-api

.gitignore

@@ -1,9 +1,16 @@
 # ignore pages build by sphinx
 _build/*
 
+# ignore python cache
+**/__pycache__/
+
 # ignore other docs that are included either by checkout or symlink
 doc/ros2_control
 doc/ros2_control_demos
 doc/ros2_controllers
 doc/gazebo_ros2_control
 doc/gz_ros2_control
+doc/control_msgs/
+doc/control_toolbox/
+doc/kinematics_interface/
+doc/realtime_tools/

Makefile

@@ -30,80 +30,80 @@ html-with-api: Makefile
 	@echo Step 1: Creating html files
 	$(SPHINXBUILD) $(SPHINXOPTS) $(SOURCEDIR) $(BUILDDIR)/html
 	@echo Step 2: Building API
-	./make_help_scripts/create_api
+	./make_help_scripts/create_api.py
 
 html-all-subrepos: Makefile
 	@echo Single html file without API
 	@echo Step 1: Cloning all subrepositories
-	./make_help_scripts/add_sub_repos
+	./make_help_scripts/add_sub_repos.py
 	@echo Step 2: Building documentation
 	$(SPHINXBUILD) $(SPHINXOPTS) $(SOURCEDIR) $(BUILDDIR)/html
 	@echo Step 3: Deleting subrepositories in doc/ folder
-	./make_help_scripts/delete_sub_repos
+	./make_help_scripts/delete_sub_repos.py
 
 html-all-subrepos-with-errors: Makefile
 	@echo Single html file without API
 	@echo Step 1: Cloning all subrepositories
-	./make_help_scripts/add_sub_repos
+	./make_help_scripts/add_sub_repos.py
 	@echo Step 2: Building documentation
 	$(SPHINXBUILD) $(SPHINXOPTS) -W --keep-going $(SOURCEDIR) $(BUILDDIR)/html
 	@echo Step 3: Deleting subrepositories in doc/ folder
-	./make_help_scripts/delete_sub_repos
+	./make_help_scripts/delete_sub_repos.py
 
 html-all-subrepos-with-api: Makefile
 	@echo Single html file with API
 	@echo Step 1: Cloning all subrepositories
-	./make_help_scripts/add_sub_repos
+	./make_help_scripts/add_sub_repos.py
 	@echo Step 2: Building documentation
 	$(SPHINXBUILD) $(SPHINXOPTS) $(SOURCEDIR) $(BUILDDIR)/html
-	@echo Step 3: Deleting subrepositories in doc/ folder
-	./make_help_scripts/delete_sub_repos
-	@echo Step 4: Building API
-	./make_help_scripts/create_api
+	@echo Step 3: Building API
+	./make_help_scripts/create_api.py
+	@echo Step 4: Deleting subrepositories in doc/ folder
+	./make_help_scripts/delete_sub_repos.py
 
 linkcheck-all-subrepos-with-api: Makefile
 	@echo Single version html file with API and linkcheck
 	@echo Step 1: Cloning all subrepositories
-	./make_help_scripts/add_sub_repos
+	./make_help_scripts/add_sub_repos.py
 	@echo Step 2: Building API
-	./make_help_scripts/create_api
-	@echo Step 3: Cloning all subrepositories again
-	./make_help_scripts/add_sub_repos
-	@echo Step 4: Check links
-	./make_help_scripts/check_links $(BUILDDIR)
+	./make_help_scripts/create_api.py
+	@echo Step 3: Check links
+	./make_help_scripts/check_links.py $(BUILDDIR)/html
+	@echo Step 4: Deleting subrepositories in doc/ folder
+	./make_help_scripts/delete_sub_repos.py
 
 multiversion: Makefile
 	@echo Building multi version documentation without API
 	@echo Step 1: Creating temporary commits
-	./make_help_scripts/add_tmp_commits
+	./make_help_scripts/add_tmp_commits.py
 	@echo Step 2: Build multi version documentation
 	sphinx-multiversion $(SPHINXOPTS) $(SOURCEDIR) $(BUILDDIR)/html
 	@echo Step 3: Deleting temporary commits
-	./make_help_scripts/delete_tmp_commits
+	./make_help_scripts/delete_tmp_commits.py
 	@echo Step 4: Create correct index
 	@echo "<html><head><meta http-equiv=\"refresh\" content=\"0; url=master/index.html\" /></head></html>" > "$(BUILDDIR)"/html/index.html
 
 multiversion-with-errors: Makefile
 	@echo Building multi version documentation without API
 	@echo Step 1: Creating temporary commits
-	./make_help_scripts/add_tmp_commits
+	./make_help_scripts/add_tmp_commits.py
 	@echo Step 2: Build multi version documentation
 	sphinx-multiversion $(SPHINXOPTS) -W --keep-going $(SOURCEDIR) $(BUILDDIR)/html
 	@echo Step 3: Deleting temporary commits
-	./make_help_scripts/delete_tmp_commits
+	./make_help_scripts/delete_tmp_commits.py
 	@echo Step 4: Create correct index
 	@echo "<html><head><meta http-equiv=\"refresh\" content=\"0; url=master/index.html\" /></head></html>" > "$(BUILDDIR)"/html/index.html
 
 multiversion-with-api: Makefile
 	@echo Building multi version documentation with API
 	@echo Step 1: Creating temporary commits
-	./make_help_scripts/add_tmp_commits
+	./make_help_scripts/add_tmp_commits.py
 	@echo Step 2: Build multi version documentation
 	sphinx-multiversion $(SPHINXOPTS) $(SOURCEDIR) $(BUILDDIR)/html
 	@echo Step 3: Deleting temporary commits
-	./make_help_scripts/delete_tmp_commits
+	./make_help_scripts/delete_tmp_commits.py
 	@echo Step 4: Building multiverison API
-	./make_help_scripts/create_api_multi_version
+	./make_help_scripts/create_api_multi_version.py
 	@echo Step 5: Create correct index
 	@echo "<html><head><meta http-equiv=\"refresh\" content=\"0; url=master/index.html\" /></head></html>" > "$(BUILDDIR)"/html/index.html
 

conf.py

@@ -67,10 +67,14 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-# exclude index.rst files for metapackages for rosdoc2
+# exclude index.rst files from packages/metapackages for rosdoc2
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store",
                     "**/CHANGELOG.rst", "**/README.rst",
                     "doc/ros2_control/ros2_control/**.rst",
+                    "doc/realtime_tools/doc/**.rst",
+                    "doc/control_msgs/doc/**.rst",
+                    "doc/control_toolbox/doc/**.rst",
+                    "doc/kinematics_interface/**.rst",
                     "doc/ros2_controllers/ros2_controllers/**.rst"]
 
 # The name of the Pygments (syntax highlighting) style to use.
@@ -183,10 +187,18 @@
 # Drop any source link suffix
 html_sourcelink_suffix = ""
 
-
-# Add branches you want to whtielist here.
-smv_branch_whitelist = r"^(foxy|galactic|humble|rolling)$"
-smv_released_pattern = r"^refs/(heads|remotes/[^/]+)/(foxy|galactic|rolling).*$"
+# branch from which is started to checkout other branches
+# TODO(anyone) use make_help_scripts/deploy_defines.py to set this variable
+if os.environ.get('BASE_BRANCH_PR') is not None:
+  base_branch = os.environ.get('BASE_BRANCH_PR')
+elif os.environ.get('BASE_BRANCH') is not None:
+  base_branch = os.environ.get('BASE_BRANCH')
+else:
+  base_branch = "humble"
+
+# Add branches you want to whitelist here.
+smv_branch_whitelist = r"^(foxy|galactic|"+ base_branch + r")$"
+smv_released_pattern = r"^refs/(heads|remotes/[^/]+)/(foxy|galactic|humble|iron).*$"
 smv_remote_whitelist = r"^(origin)$"
 smv_latest_version = "humble"
 smv_eol_versions = []
@@ -311,7 +323,16 @@ def smv_rewrite_configs(app, config):
     # to rewrite the various configuration items with the current version.
     if app.config.smv_current_version != "":
         branch_distro = {
-            "master": "rolling",
+            base_branch: "rolling",
+            "iron": "iron",
+            "humble": "humble",
+            "foxy": "foxy",
+            "galactic": "galactic"
+        }
+        # this map is used to match branches of control.ros.org to REPOS_FILE_BRANCH macro
+        subrepo_branch = {
+            base_branch: "master",
+            "iron": "iron",
             "humble": "humble",
             "foxy": "foxy",
             "galactic": "galactic"

doc/api_list/api_list.rst

@@ -3,8 +3,12 @@
 API Documentation
 =================
 
+ros2_control stack
+#############################
 API documentation for the whole framework is parsed by doxygen and can be found `here <../api/index.html>`_.
 
+Per-Package API Documentation
+#############################
 In the following, you can find links to the per-package API documentation published on docs.ros.org.
 
 ros2_control

doc/api_list/mainpage.md

@@ -2,7 +2,7 @@ This is the API documentation of the ros2_control framework including the follow
 
 * [ros2_control][ros2_control] - the main interfaces and components of the framework;
 * [ros2_controllers][ros2_controllers] - widely used controllers, such as forward command controller, joint trajectory controller, differential drive controller;
-* [ros2_control_demos][ros2_control_demos] - examples implementations of common use-cases for a smooth start;
+* [ros2_control_demos][ros2_control_demos] - example implementations of common use-cases for a smooth start;
 * [control_toolbox][control_toolbox] - some widely-used control theory implementations (e.g. PID) used by controllers;
 * [realtime_tools][realtime_tools] - general toolkit for realtime support, e.g., realtime buffers and publishers;
 * [control_msgs][control_msgs] - common messages.

index.rst

@@ -35,13 +35,13 @@ The ros2_control framework consists of the following Github repositories:
 * `ros2_controllers`_ - widely used controllers, such as forward command controller, joint trajectory controller, differential drive controller;
 * `control_toolbox`_ - some widely-used control theory implementations (e.g. PID) used by controllers;
 * `realtime_tools`_ - general toolkit for realtime support, e.g., realtime buffers and publishers;
-* `control_msgs`_ - common messages.
-* `kinematics_interface`_ - for using C++ kinematics frameworks.
+* `control_msgs`_ - common messages;
+* `kinematics_interface`_ - for using C++ kinematics frameworks;
 
 
 Additionally, there are following (unreleased) packages are relevant for getting-started and project management:
 
-* `ros2_control_demos`_ - examples implementations of common use-cases for a smooth start;
+* `ros2_control_demos`_ - example implementations of common use-cases for a smooth start;
 * `roadmap`_ - planning and design docs for the project.
 
 Development Organisation and Communication

make_help_scripts/add_pr_stats.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+# Copyright (c) 2023 ros2_control maintainers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from deploy_defines import add_pr_stats_file
+
+add_pr_stats_file()

make_help_scripts/add_sub_repos.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# Copyright (c) 2023 ros2_control maintainers
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+import deploy_defines
+
+def add_sub_repositories(base_branch):
+    # checkout a base for defined starting point
+    print(f"base_branch: {base_branch}")
+    print(f"base_version: {deploy_defines.branch_version[base_branch]}")
+    os.chdir(deploy_defines.base_dir)
+    for repo_name, repo_details in deploy_defines.repos.items():
+        repo_path = os.path.join("doc", repo_name)
+        branch = repo_details["branch_version"][deploy_defines.branch_version[base_branch]]
+        if not os.path.isdir(repo_path):
+            print(f"Create {repo_path} and checkout {branch} branch")
+            subprocess.run(["git", "clone", "-b", branch, repo_details["url"], repo_path], check=True)
+        else:
+            print(f"Update {repo_path} and checkout {branch} branch")
+            os.chdir(repo_path)
+            subprocess.run(["git", "fetch", "origin"], check=True)
+            subprocess.run(["git", "checkout", branch], check=True)
+            subprocess.run(["git", "pull"], check=True)
+            os.chdir(deploy_defines.base_dir)
+        if repo_details["pr"]:
+            PR = repo_details["pr"]
+            print(f"checkout PR: {PR}")
+            os.chdir(repo_path)
+            subprocess.run(["git", "fetch", "origin", f"{PR}:PR"], check=True)
+            subprocess.run(["git", "checkout", "PR"], check=True)
+            os.chdir(deploy_defines.base_dir)
+
+if __name__ == "__main__":
+    add_sub_repositories(deploy_defines.base_branch)
+    deploy_defines.add_pr_stats_file()