diff --git a/.github/workflows/docker-readme.yml b/.github/workflows/docker-readme.yml index 29dd787d638e..9e156e835056 100644 --- a/.github/workflows/docker-readme.yml +++ b/.github/workflows/docker-readme.yml @@ -24,7 +24,7 @@ jobs: list-files: "json" filters: | readme: - - 'src/docker/**/README.md' + - added|modified: 'src/docker/**/README.md' update: if: ${{ needs.collect.outputs.readme_files != '' && toJson(fromJson(needs.collect.outputs.readme_files)) != '[]' }} diff --git a/.github/workflows/update_translations.yml b/.github/workflows/update_translations.yml index 9419f4aaef25..7685aecd5661 100644 --- a/.github/workflows/update_translations.yml +++ b/.github/workflows/update_translations.yml @@ -66,6 +66,7 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} branch: '${{ env.branch-name }}' + force_with_lease: true - name: Create Pull Request if: steps.calculate_diff.outputs.additions > 228 && steps.calculate_diff.outputs.deletions > 60 diff --git a/datasets/doc/source/recommended-fl-datasets.rst b/datasets/doc/source/recommended-fl-datasets.rst new file mode 100644 index 000000000000..92479bd0542a --- /dev/null +++ b/datasets/doc/source/recommended-fl-datasets.rst @@ -0,0 +1,167 @@ +Recommended FL Datasets +======================= + +This page lists the recommended datasets for federated learning research, which can be +used with Flower Datasets ``flwr-datasets``. To learn about the library, see the +`quickstart tutorial `_ . To +see the full FL example with Flower and Flower Datasets open the `quickstart-pytorch +`_. + +.. note:: + + All datasets from `HuggingFace Hub `_ can be used with our library. This page presents just a set of datasets we collected that you might find useful. + +For more information about any dataset, visit its page by clicking the dataset name. For more information how to use the + +Image Datasets +-------------- + +.. list-table:: Image Datasets + :widths: 40 40 20 + :header-rows: 1 + + * - Name + - Size + - Image Shape + * - `ylecun/mnist `_ + - train 60k; + test 10k + - 28x28 + * - `uoft-cs/cifar10 `_ + - train 50k; + test 10k + - 32x32x3 + * - `uoft-cs/cifar100 `_ + - train 50k; + test 10k + - 32x32x3 + * - `zalando-datasets/fashion_mnist `_ + - train 60k; + test 10k + - 28x28 + * - `flwrlabs/femnist `_ + - train 814k + - 28x28 + * - `zh-plus/tiny-imagenet `_ + - train 100k; + valid 10k + - 64x64x3 + * - `flwrlabs/usps `_ + - train 7.3k; + test 2k + - 16x16 + * - `flwrlabs/pacs `_ + - train 10k + - 227x227 + * - `flwrlabs/cinic10 `_ + - train 90k; + valid 90k; + test 90k + - 32x32x3 + * - `flwrlabs/caltech101 `_ + - train 8.7k + - varies + * - `flwrlabs/office-home `_ + - train 15.6k + - varies + * - `flwrlabs/fed-isic2019 `_ + - train 18.6k; + test 4.7k + - varies + * - `ufldl-stanford/svhn `_ + - train 73.3k; + test 26k; + extra 531k + - 32x32x3 + * - `sasha/dog-food `_ + - train 2.1k; + test 0.9k + - varies + * - `Mike0307/MNIST-M `_ + - train 59k; + test 9k + - 32x32 + +Audio Datasets +-------------- + +.. list-table:: Audio Datasets + :widths: 35 30 15 + :header-rows: 1 + + * - Name + - Size + - Subset + * - `google/speech_commands `_ + - train 64.7k + - v0.01 + * - `google/speech_commands `_ + - train 105.8k + - v0.02 + * - `flwrlabs/ambient-acoustic-context `_ + - train 70.3k + - + * - `fixie-ai/common_voice_17_0 `_ + - varies + - 14 versions + * - `fixie-ai/librispeech_asr `_ + - varies + - clean/other + +Tabular Datasets +---------------- + +.. list-table:: Tabular Datasets + :widths: 35 30 + :header-rows: 1 + + * - Name + - Size + * - `scikit-learn/adult-census-income `_ + - train 32.6k + * - `jlh/uci-mushrooms `_ + - train 8.1k + * - `scikit-learn/iris `_ + - train 150 + +Text Datasets +------------- + +.. list-table:: Text Datasets + :widths: 40 30 30 + :header-rows: 1 + + * - Name + - Size + - Category + * - `sentiment140 `_ + - train 1.6M; + test 0.5k + - Sentiment + * - `google-research-datasets/mbpp `_ + - full 974; sanitized 427 + - General + * - `openai/openai_humaneval `_ + - test 164 + - General + * - `lukaemon/mmlu `_ + - varies + - General + * - `takala/financial_phrasebank `_ + - train 4.8k + - Financial + * - `pauri32/fiqa-2018 `_ + - train 0.9k; validation 0.1k; test 0.2k + - Financial + * - `zeroshot/twitter-financial-news-sentiment `_ + - train 9.5k; validation 2.4k + - Financial + * - `bigbio/pubmed_qa `_ + - train 2M; validation 11k + - Medical + * - `openlifescienceai/medmcqa `_ + - train 183k; validation 4.3k; test 6.2k + - Medical + * - `bigbio/med_qa `_ + - train 10.1k; test 1.3k; validation 1.3k + - Medical diff --git a/datasets/flwr_datasets/__init__.py b/datasets/flwr_datasets/__init__.py index bd68fa43c606..dfae804046bc 100644 --- a/datasets/flwr_datasets/__init__.py +++ b/datasets/flwr_datasets/__init__.py @@ -15,7 +15,7 @@ """Flower Datasets main package.""" -from flwr_datasets import partitioner, preprocessor +from flwr_datasets import metrics, partitioner, preprocessor from flwr_datasets import utils as utils from flwr_datasets import visualization from flwr_datasets.common.version import package_version as _package_version diff --git a/dev/prepare-release-changelog.sh b/dev/prepare-release-changelog.sh deleted file mode 100755 index 3f2a2ae325e9..000000000000 --- a/dev/prepare-release-changelog.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -e -cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/../ - -# Get the current date in the format YYYY-MM-DD -current_date=$(date +"%Y-%m-%d") - -tags=$(git tag --sort=-v:refname) -new_version=$1 -old_version=$(echo "$tags" | sed -n '1p') - -shortlog=$(git shortlog "$old_version"..main -s | grep -vEi '(\(|\[)bot(\)|\])' | awk '{name = substr($0, index($0, $2)); printf "%s`%s`", sep, name; sep=", "} END {print ""}') - -token="" -thanks="\n### Thanks to our contributors\n\nWe would like to give our special thanks to all the contributors who made the new version of Flower possible (in \`git shortlog\` order):\n\n$shortlog $token" - -# Check if the token exists in the markdown file -if ! grep -q "$token" doc/source/ref-changelog.md; then - # If the token does not exist in the markdown file, append the new content after the version - awk -v version="$new_version" -v date="$current_date" -v text="$thanks" \ - '{ if ($0 ~ "## Unreleased") print "## " version " (" date ")\n" text; else print $0 }' doc/source/ref-changelog.md > temp.md && mv temp.md doc/source/ref-changelog.md -else - # If the token exists, replace the line containing the token with the new shortlog - awk -v token="$token" -v newlog="$shortlog $token" '{ if ($0 ~ token) print newlog; else print $0 }' doc/source/ref-changelog.md > temp.md && mv temp.md doc/source/ref-changelog.md -fi diff --git a/dev/update_changelog.py b/dev/update_changelog.py new file mode 100644 index 000000000000..0b4359d90e13 --- /dev/null +++ b/dev/update_changelog.py @@ -0,0 +1,297 @@ +# mypy: ignore-errors +# Copyright 2023 Flower Labs GmbH. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Update the changelog using PR titles.""" + + +import pathlib +import re + +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib +from datetime import date +from sys import argv +from typing import Optional + +from github import Github +from github.PullRequest import PullRequest +from github.Repository import Repository +from github.Tag import Tag + +REPO_NAME = "adap/flower" +CHANGELOG_FILE = "doc/source/ref-changelog.md" +CHANGELOG_SECTION_HEADER = "### Changelog entry" + +# Load the TOML configuration +with (pathlib.Path(__file__).parent.resolve() / "changelog_config.toml").open( + "rb" +) as file: + CONFIG = tomllib.load(file) + +# Extract types, project, and scope from the config +TYPES = "|".join(CONFIG["type"]) +PROJECTS = "|".join(CONFIG["project"]) + "|\\*" +SCOPE = CONFIG["scope"] +ALLOWED_VERBS = CONFIG["allowed_verbs"] + +# Construct the pattern +PATTERN_TEMPLATE = CONFIG["pattern_template"] +PATTERN = PATTERN_TEMPLATE.format(types=TYPES, projects=PROJECTS, scope=SCOPE) + + +def _get_latest_tag(gh_api: Github) -> tuple[Repository, Optional[Tag]]: + """Retrieve the latest tag from the GitHub repository.""" + repo = gh_api.get_repo(REPO_NAME) + tags = repo.get_tags() + return repo, tags[0] if tags.totalCount > 0 else None + + +def _add_shortlog(new_version: str, shortlog: str) -> None: + """Update the markdown file with the new version or update existing logs.""" + token = f"" + entry = ( + "\n### Thanks to our contributors\n\n" + "We would like to give our special thanks to all the contributors " + "who made the new version of Flower possible " + f"(in `git shortlog` order):\n\n{shortlog} {token}" + ) + current_date = date.today() + + with open(CHANGELOG_FILE, encoding="utf-8") as file: + content = file.readlines() + + token_exists = any(token in line for line in content) + + with open(CHANGELOG_FILE, "w", encoding="utf-8") as file: + for line in content: + if token in line: + token_exists = True + file.write(line) + elif "## Unreleased" in line and not token_exists: + # Add the new entry under "## Unreleased" + file.write(f"## {new_version} ({current_date})\n{entry}\n") + token_exists = True + else: + file.write(line) + + +def _get_pull_requests_since_tag( + repo: Repository, tag: Tag +) -> tuple[str, set[PullRequest]]: + """Get a list of pull requests merged into the main branch since a given tag.""" + commit_shas = set() + contributors = set() + prs = set() + + for commit in repo.compare(tag.commit.sha, "main").commits: + commit_shas.add(commit.sha) + if commit.author.name is None: + continue + if "[bot]" in commit.author.name: + continue + contributors.add(commit.author.name) + + for pr_info in repo.get_pulls( + state="closed", sort="created", direction="desc", base="main" + ): + if pr_info.merge_commit_sha in commit_shas: + prs.add(pr_info) + if len(prs) == len(commit_shas): + break + + shortlog = ", ".join([f"`{name}`" for name in sorted(contributors)]) + return shortlog, prs + + +def _format_pr_reference(title: str, number: int, url: str) -> str: + """Format a pull request reference as a markdown list item.""" + parts = title.strip().replace("*", "").split("`") + formatted_parts = [] + + for i, part in enumerate(parts): + if i % 2 == 0: + # Even index parts are normal text, ensure we do not add extra bold if empty + if part.strip(): + formatted_parts.append(f"**{part.strip()}**") + else: + formatted_parts.append("") + else: + # Odd index parts are inline code + formatted_parts.append(f"`{part.strip()}`") + + # Join parts with spaces but avoid extra spaces + formatted_title = " ".join(filter(None, formatted_parts)) + return f"- {formatted_title} ([#{number}]({url}))" + + +def _extract_changelog_entry( + pr_info: PullRequest, +) -> dict[str, str]: + """Extract the changelog entry from a pull request's body.""" + # Use regex search to find matches + match = re.search(PATTERN, pr_info.title) + if match: + # Extract components from the regex groups + pr_type = match.group(1) + pr_project = match.group(2) + pr_scope = match.group(3) # Correctly capture optional sub-scope + pr_subject = match.group( + 4 + ) # Capture subject starting with uppercase and no terminal period + return { + "type": pr_type, + "project": pr_project, + "scope": pr_scope, + "subject": pr_subject, + } + + return { + "type": "unknown", + "project": "unknown", + "scope": "unknown", + "subject": "unknown", + } + + +def _update_changelog(prs: set[PullRequest]) -> bool: + """Update the changelog file with entries from provided pull requests.""" + breaking_changes = False + unknown_changes = False + + with open(CHANGELOG_FILE, "r+", encoding="utf-8") as file: + content = file.read() + unreleased_index = content.find("## Unreleased") + + if unreleased_index == -1: + print("Unreleased header not found in the changelog.") + return False + + # Find the end of the Unreleased section + next_header_index = content.find("## ", unreleased_index + 1) + next_header_index = ( + next_header_index if next_header_index != -1 else len(content) + ) + + for pr_info in prs: + parsed_title = _extract_changelog_entry(pr_info) + + # Skip if PR should be skipped or already in changelog + if ( + parsed_title.get("scope", "unknown") == "skip" + or f"#{pr_info.number}]" in content + ): + continue + + pr_type = parsed_title.get("type", "unknown") + if pr_type == "feat": + insert_content_index = content.find("### What", unreleased_index + 1) + elif pr_type == "docs": + insert_content_index = content.find( + "### Documentation improvements", unreleased_index + 1 + ) + elif pr_type == "break": + breaking_changes = True + insert_content_index = content.find( + "### Incompatible changes", unreleased_index + 1 + ) + elif pr_type in {"ci", "fix", "refactor"}: + insert_content_index = content.find( + "### Other changes", unreleased_index + 1 + ) + else: + unknown_changes = True + insert_content_index = unreleased_index + + pr_reference = _format_pr_reference( + pr_info.title, pr_info.number, pr_info.html_url + ) + + content = _insert_entry_no_desc( + content, + pr_reference, + insert_content_index, + ) + + next_header_index = content.find("## ", unreleased_index + 1) + next_header_index = ( + next_header_index if next_header_index != -1 else len(content) + ) + + if unknown_changes: + content = _insert_entry_no_desc( + content, + "### Unknown changes", + unreleased_index, + ) + + if not breaking_changes: + content = _insert_entry_no_desc( + content, + "None", + content.find("### Incompatible changes", unreleased_index + 1), + ) + + # Finalize content update + file.seek(0) + file.write(content) + file.truncate() + return True + + +def _insert_entry_no_desc( + content: str, pr_reference: str, unreleased_index: int +) -> str: + """Insert a changelog entry for a pull request with no specific description.""" + insert_index = content.find("\n", unreleased_index) + 1 + content = ( + content[:insert_index] + "\n" + pr_reference + "\n" + content[insert_index:] + ) + return content + + +def _bump_minor_version(tag: Tag) -> Optional[str]: + """Bump the minor version of the tag.""" + match = re.match(r"v(\d+)\.(\d+)\.(\d+)", tag.name) + if match is None: + return None + major, minor, _ = [int(x) for x in match.groups()] + # Increment the minor version and reset patch version + new_version = f"v{major}.{minor + 1}.0" + return new_version + + +def main() -> None: + """Update changelog using the descriptions of PRs since the latest tag.""" + # Initialize GitHub Client with provided token (as argument) + gh_api = Github(argv[1]) + repo, latest_tag = _get_latest_tag(gh_api) + if not latest_tag: + print("No tags found in the repository.") + return + + shortlog, prs = _get_pull_requests_since_tag(repo, latest_tag) + if _update_changelog(prs): + new_version = _bump_minor_version(latest_tag) + if not new_version: + print("Wrong tag format.") + return + _add_shortlog(new_version, shortlog) + print("Changelog updated succesfully.") + + +if __name__ == "__main__": + main() diff --git a/doc/source/contributor-how-to-release-flower.rst b/doc/source/contributor-how-to-release-flower.rst index fafc02cab64c..44982ab765ab 100644 --- a/doc/source/contributor-how-to-release-flower.rst +++ b/doc/source/contributor-how-to-release-flower.rst @@ -10,9 +10,9 @@ During the release The version number of a release is stated in ``pyproject.toml``. To release a new version of Flower, the following things need to happen (in that order): -1. Run ``python3 src/py/flwr_tool/update_changelog.py `` in order to add - every new change to the changelog (feel free to make manual changes to the changelog - afterwards until it looks good). +1. Run ``python3 ./dev/update_changelog.py `` in order to add every new + change to the changelog (feel free to make manual changes to the changelog afterwards + until it looks good). 2. Once the changelog has been updated with all the changes, run ``./dev/prepare-release-changelog.sh v``, where ```` is the version stated in ``pyproject.toml`` (notice the ``v`` added before it). This will diff --git a/doc/source/docker/tutorial-deploy-on-multiple-machines.rst b/doc/source/docker/tutorial-deploy-on-multiple-machines.rst index ca71a0d44f2e..ffe0b090af9e 100644 --- a/doc/source/docker/tutorial-deploy-on-multiple-machines.rst +++ b/doc/source/docker/tutorial-deploy-on-multiple-machines.rst @@ -38,8 +38,9 @@ Step 1: Set Up 1. Clone the Flower repository and change to the ``distributed`` directory: .. code-block:: bash + :substitutions: - $ git clone --depth=1 https://github.com/adap/flower.git + $ git clone --depth=1 --branch v|stable_flwr_version| https://github.com/adap/flower.git $ cd flower/src/docker/distributed 2. Get the IP address from the remote machine and save it for later. diff --git a/doc/source/docker/tutorial-quickstart-docker-compose.rst b/doc/source/docker/tutorial-quickstart-docker-compose.rst index 63aaf8309aec..3c0a6463e50e 100644 --- a/doc/source/docker/tutorial-quickstart-docker-compose.rst +++ b/doc/source/docker/tutorial-quickstart-docker-compose.rst @@ -24,8 +24,9 @@ Step 1: Set Up 1. Clone the Docker Compose ``complete`` directory: .. code-block:: bash + :substitutions: - $ git clone --depth=1 https://github.com/adap/flower.git _tmp \ + $ git clone --depth=1 --branch v|stable_flwr_version| https://github.com/adap/flower.git _tmp \ && mv _tmp/src/docker/complete . \ && rm -rf _tmp && cd complete diff --git a/doc/source/ref-changelog.md b/doc/source/ref-changelog.md index e9a84d344d59..7916435bc2fd 100644 --- a/doc/source/ref-changelog.md +++ b/doc/source/ref-changelog.md @@ -2,6 +2,14 @@ ## Unreleased +### What's new? + +### Other changes + +### Documentation improvements + +### Incompatible changes + ## v1.13.0 (2024-11-20) ### Thanks to our contributors diff --git a/src/docker/clientapp/README.md b/src/docker/clientapp/README.md index a610de66eeae..e2f6f35d1769 100644 --- a/src/docker/clientapp/README.md +++ b/src/docker/clientapp/README.md @@ -21,8 +21,11 @@ - `unstable` - points to the last successful build of the `main` branch -- `nightly`, `.dev` e.g. `1.13.0.dev20241014` +- `nightly`, `.dev` e.g. `1.14.0.dev20241120` - uses Python 3.11 and Ubuntu 24.04 +- `1.13.0`, `1.13.0-py3.11-ubuntu24.04` +- `1.13.0-py3.10-ubuntu24.04` +- `1.13.0-py3.9-ubuntu24.04` - `1.12.0`, `1.12.0-py3.11-ubuntu24.04` - `1.12.0-py3.10-ubuntu24.04` - `1.12.0-py3.9-ubuntu24.04` diff --git a/src/docker/serverapp/README.md b/src/docker/serverapp/README.md index 110712fe3bfd..bcba0f3ef889 100644 --- a/src/docker/serverapp/README.md +++ b/src/docker/serverapp/README.md @@ -21,8 +21,11 @@ - `unstable` - points to the last successful build of the `main` branch -- `nightly`, `.dev` e.g. `1.13.0.dev20241014` +- `nightly`, `.dev` e.g. `1.14.0.dev20241120` - uses Python 3.11 and Ubuntu 24.04 +- `1.13.0`, `1.13.0-py3.11-ubuntu24.04` +- `1.13.0-py3.10-ubuntu24.04` +- `1.13.0-py3.9-ubuntu24.04` - `1.12.0`, `1.12.0-py3.11-ubuntu24.04` - `1.12.0-py3.10-ubuntu24.04` - `1.12.0-py3.9-ubuntu24.04` diff --git a/src/docker/superexec/Dockerfile b/src/docker/superexec/Dockerfile deleted file mode 100644 index 9e4cc722921e..000000000000 --- a/src/docker/superexec/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2024 Flower Labs GmbH. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -ARG BASE_REPOSITORY=flwr/base -ARG BASE_IMAGE -FROM $BASE_REPOSITORY:$BASE_IMAGE - -ENTRYPOINT ["flower-superexec"] diff --git a/src/docker/superexec/README.md b/src/docker/superexec/README.md deleted file mode 100644 index 074ee3e0b072..000000000000 --- a/src/docker/superexec/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Deprecation Notice - -Starting with Flower version 1.13.0, the `flwr/superexec` Docker image has been deprecated because the SuperExec has been removed in Flower 1.13 (to be precise: the functionalities of the SuperExec have been merged into the SuperLink). -Its functionality has been merged into [`flwr/superlink`](https://hub.docker.com/r/flwr/superlink) image. - -To migrate your Docker setup to the latest version, please follow the -[Quickstart with Docker](https://flower.ai/docs/framework/docker/tutorial-quickstart-docker.html) guide. - -# Flower SuperExec - -

- - Flower Website - -

- -## Quick reference - -- **Learn more:**
- [Quickstart with Docker](https://flower.ai/docs/framework/docker/tutorial-quickstart-docker.html) and [Quickstart with Docker Compose](https://flower.ai/docs/framework/docker/tutorial-quickstart-docker-compose.html) - -- **Where to get help:**
- [Flower Discuss](https://discuss.flower.ai), [Slack](https://flower.ai/join-slack) or [GitHub](https://github.com/adap/flower) - -- **Supported architectures:**
- `amd64`, `arm64v8` - -## Supported tags - -- `unstable` - - points to the last successful build of the `main` branch -- `nightly`, `.dev` e.g. `1.13.0.dev20241014` - - uses Python 3.11 and Ubuntu 24.04 -- `1.12.0`, `1.12.0-py3.11-ubuntu24.04` -- `1.12.0-py3.10-ubuntu24.04` -- `1.12.0-py3.9-ubuntu24.04` -- `1.11.1`, `1.11.1-py3.11-ubuntu22.04` -- `1.11.1-py3.10-ubuntu22.04` -- `1.11.1-py3.9-ubuntu22.04` -- `1.11.1-py3.8-ubuntu22.04` -- `1.11.0`, `1.11.0-py3.11-ubuntu22.04` -- `1.11.0-py3.10-ubuntu22.04` -- `1.11.0-py3.9-ubuntu22.04` -- `1.11.0-py3.8-ubuntu22.04` -- `1.10.0`, `1.10.0-py3.11-ubuntu22.04` -- `1.10.0-py3.10-ubuntu22.04` -- `1.10.0-py3.9-ubuntu22.04` -- `1.10.0-py3.8-ubuntu22.04` diff --git a/src/docker/superlink/README.md b/src/docker/superlink/README.md index af03ce1c8054..444003964a6b 100644 --- a/src/docker/superlink/README.md +++ b/src/docker/superlink/README.md @@ -21,8 +21,10 @@ - `unstable` - points to the last successful build of the `main` branch -- `nightly`, `.dev` e.g. `1.13.0.dev20241014` +- `nightly`, `.dev` e.g. `1.14.0.dev20241120` - uses Python 3.11 and Ubuntu 24.04 +- `1.13.0`, `1.13.0-py3.11-alpine3.19` +- `1.13.0-py3.11-ubuntu24.04` - `1.12.0`, `1.12.0-py3.11-alpine3.19` - `1.12.0-py3.11-ubuntu24.04` - `1.11.1`, `1.11.1-py3.11-alpine3.19` diff --git a/src/docker/supernode/README.md b/src/docker/supernode/README.md index 493f98cc78e4..c6b207ffe7cf 100644 --- a/src/docker/supernode/README.md +++ b/src/docker/supernode/README.md @@ -21,8 +21,12 @@ - `unstable` - points to the last successful build of the `main` branch -- `nightly`, `.dev` e.g. `1.13.0.dev20241014` +- `nightly`, `.dev` e.g. `1.14.0.dev20241120` - uses Python 3.11 and Ubuntu 24.04 +- `1.13.0`, `1.13.0-py3.11-alpine3.19` +- `1.13.0-py3.11-ubuntu24.04` +- `1.13.0-py3.10-ubuntu24.04` +- `1.13.0-py3.9-ubuntu24.04` - `1.12.0`, `1.12.0-py3.11-alpine3.19` - `1.12.0-py3.11-ubuntu24.04` - `1.12.0-py3.10-ubuntu24.04` diff --git a/src/py/flwr_tool/update_changelog.py b/src/py/flwr_tool/update_changelog.py deleted file mode 100644 index e3cffff7e36c..000000000000 --- a/src/py/flwr_tool/update_changelog.py +++ /dev/null @@ -1,243 +0,0 @@ -# mypy: ignore-errors -# Copyright 2023 Flower Labs GmbH. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""This module is used to update the changelog.""" - - -import re -from sys import argv - -from github import Github - -REPO_NAME = "adap/flower" -CHANGELOG_FILE = "doc/source/ref-changelog.md" -CHANGELOG_SECTION_HEADER = "### Changelog entry" - - -def _get_latest_tag(gh_api): - """Retrieve the latest tag from the GitHub repository.""" - repo = gh_api.get_repo(REPO_NAME) - tags = repo.get_tags() - return tags[0] if tags.totalCount > 0 else None - - -def _get_pull_requests_since_tag(gh_api, tag): - """Get a list of pull requests merged into the main branch since a given tag.""" - repo = gh_api.get_repo(REPO_NAME) - commits = {commit.sha for commit in repo.compare(tag.commit.sha, "main").commits} - prs = set() - for pr_info in repo.get_pulls( - state="closed", sort="created", direction="desc", base="main" - ): - if pr_info.merge_commit_sha in commits: - prs.add(pr_info) - if len(prs) == len(commits): - break - return prs - - -def _format_pr_reference(title, number, url): - """Format a pull request reference as a markdown list item.""" - return f"- **{title.replace('*', '')}** ([#{number}]({url}))" - - -def _extract_changelog_entry(pr_info): - """Extract the changelog entry from a pull request's body.""" - if not pr_info.body: - return None, "general" - - entry_match = re.search( - f"{CHANGELOG_SECTION_HEADER}(.+?)(?=##|$)", pr_info.body, re.DOTALL - ) - if not entry_match: - return None, None - - entry_text = entry_match.group(1).strip() - - # Remove markdown comments - entry_text = re.sub(r"", "", entry_text, flags=re.DOTALL).strip() - - token_markers = { - "general": "", - "skip": "", - "baselines": "", - "examples": "", - "sdk": "", - "simulations": "", - } - - # Find the token based on the presence of its marker in entry_text - token = next( - (token for token, marker in token_markers.items() if marker in entry_text), None - ) - - return entry_text, token - - -def _update_changelog(prs): - """Update the changelog file with entries from provided pull requests.""" - with open(CHANGELOG_FILE, "r+", encoding="utf-8") as file: - content = file.read() - unreleased_index = content.find("## Unreleased") - - if unreleased_index == -1: - print("Unreleased header not found in the changelog.") - return - - # Find the end of the Unreleased section - next_header_index = content.find("##", unreleased_index + 1) - next_header_index = ( - next_header_index if next_header_index != -1 else len(content) - ) - - for pr_info in prs: - pr_entry_text, category = _extract_changelog_entry(pr_info) - - # Skip if PR should be skipped or already in changelog - if category == "skip" or f"#{pr_info.number}]" in content: - continue - - pr_reference = _format_pr_reference( - pr_info.title, pr_info.number, pr_info.html_url - ) - - # Process based on category - if category in ["general", "baselines", "examples", "sdk", "simulations"]: - entry_title = _get_category_title(category) - content = _update_entry( - content, - entry_title, - pr_info, - unreleased_index, - next_header_index, - ) - - elif pr_entry_text: - content = _insert_new_entry( - content, pr_info, pr_reference, pr_entry_text, unreleased_index - ) - - else: - content = _insert_entry_no_desc(content, pr_reference, unreleased_index) - - next_header_index = content.find("##", unreleased_index + 1) - next_header_index = ( - next_header_index if next_header_index != -1 else len(content) - ) - - # Finalize content update - file.seek(0) - file.write(content) - file.truncate() - - print("Changelog updated.") - - -def _get_category_title(category): - """Get the title of a changelog section based on its category.""" - headers = { - "general": "General improvements", - "baselines": "General updates to Flower Baselines", - "examples": "General updates to Flower Examples", - "sdk": "General updates to Flower SDKs", - "simulations": "General updates to Flower Simulations", - } - return headers.get(category, "") - - -def _update_entry( - content, category_title, pr_info, unreleased_index, next_header_index -): - """Update a specific section in the changelog content.""" - if ( - section_index := content.find( - category_title, unreleased_index, next_header_index - ) - ) != -1: - newline_index = content.find("\n", section_index) - closing_parenthesis_index = content.rfind(")", unreleased_index, newline_index) - updated_entry = f", [{pr_info.number}]({pr_info.html_url})" - content = ( - content[:closing_parenthesis_index] - + updated_entry - + content[closing_parenthesis_index:] - ) - else: - new_section = ( - f"\n- **{category_title}** ([#{pr_info.number}]({pr_info.html_url}))\n" - ) - insert_index = content.find("\n", unreleased_index) + 1 - content = content[:insert_index] + new_section + content[insert_index:] - return content - - -def _insert_new_entry(content, pr_info, pr_reference, pr_entry_text, unreleased_index): - """Insert a new entry into the changelog.""" - if (existing_entry_start := content.find(pr_entry_text)) != -1: - pr_ref_end = content.rfind("\n", 0, existing_entry_start) - updated_entry = ( - f"{content[pr_ref_end]}\n, [{pr_info.number}]({pr_info.html_url})" - ) - content = content[:pr_ref_end] + updated_entry + content[existing_entry_start:] - else: - insert_index = content.find("\n", unreleased_index) + 1 - - # Split the pr_entry_text into paragraphs - paragraphs = pr_entry_text.split("\n") - - # Indent each paragraph - indented_paragraphs = [ - " " + paragraph if paragraph else paragraph for paragraph in paragraphs - ] - - # Join the paragraphs back together, ensuring each is separated by a newline - indented_pr_entry_text = "\n".join(indented_paragraphs) - - content = ( - content[:insert_index] - + "\n" - + pr_reference - + "\n\n" - + indented_pr_entry_text - + "\n" - + content[insert_index:] - ) - return content - - -def _insert_entry_no_desc(content, pr_reference, unreleased_index): - """Insert a changelog entry for a pull request with no specific description.""" - insert_index = content.find("\n", unreleased_index) + 1 - content = ( - content[:insert_index] + "\n" + pr_reference + "\n" + content[insert_index:] - ) - return content - - -def main(): - """Update changelog using the descriptions of PRs since the latest tag.""" - # Initialize GitHub Client with provided token (as argument) - gh_api = Github(argv[1]) - latest_tag = _get_latest_tag(gh_api) - if not latest_tag: - print("No tags found in the repository.") - return - - prs = _get_pull_requests_since_tag(gh_api, latest_tag) - _update_changelog(prs) - - -if __name__ == "__main__": - main()