From 26564b7be943f3b37fe53a5600d89aa76e13529a Mon Sep 17 00:00:00 2001 From: Usman Rashid Date: Wed, 24 Jul 2024 16:42:00 +1200 Subject: [PATCH] Updated check_module_versions script --- .github/check_module_versions.py | 321 +++++++++++++++++++++++++++++++ .github/update_checks.py | 271 -------------------------- CHANGELOG.md | 2 +- 3 files changed, 322 insertions(+), 272 deletions(-) create mode 100755 .github/check_module_versions.py delete mode 100755 .github/update_checks.py diff --git a/.github/check_module_versions.py b/.github/check_module_versions.py new file mode 100755 index 00000000..fe039efc --- /dev/null +++ b/.github/check_module_versions.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 + +import colorlog +from urllib.parse import quote_plus +from functools import cmp_to_key +from pathlib import Path + +import subprocess +import argparse +import requests +import logging +import semver +import json +import re +import os + +PIPELINE_REPO = "plant-food-research-open/assemblyqc" + + +def get_logger(): + formatter = colorlog.ColoredFormatter( + "%(log_color)s%(levelname)-8s%(reset)s %(blue)s%(message)s", + datefmt=None, + reset=True, + log_colors={ + "DEBUG": "cyan", + "INFO": "green", + "WARNING": "yellow", + "ERROR": "red", + "CRITICAL": "red,bg_white", + }, + secondary_log_colors={}, + style="%", + ) + + handler = colorlog.StreamHandler() + handler.setFormatter(formatter) + + logger = colorlog.getLogger("") + logger.addHandler(handler) + return logger + + +LOGGER = get_logger() + + +def first(from_list): + if from_list == []: + return None + + return from_list[0] + + +def extract_semvar(from_version_str): + version_numbers = first( + re.findall(r"^[vV]?(\d+)\.?(\d*)\.?(\d*)$", from_version_str) + ) + if version_numbers == None: + return None + + major_ver = int(version_numbers[0]) + minor_ver = int(f"{version_numbers[1]}") if version_numbers[1] != "" else 0 + patch_ver = int(f"{version_numbers[2]}") if version_numbers[2] != "" else 0 + + return f"{major_ver}.{minor_ver}.{patch_ver}" + + +def check_version_status(git_repo_from_meta): + + host = git_repo_from_meta["host"] + org = git_repo_from_meta["org"] + repo = git_repo_from_meta["repo"] + current_ver = git_repo_from_meta["semver"] + + sem_versions = [] + if host == "github": + r = subprocess.run( + " ".join( + [ + "gh api", + '-H "Accept: application/vnd.github+json"', + '-H "X-GitHub-Api-Version: 2022-11-28"', + f"/repos/{org}/{repo}/tags", + ] + ), + shell=True, + capture_output=True, + text=True, + ) + + if r.returncode != 0: + LOGGER.warning(f"Failed to get tags for {org}/{repo}: {r.stderr}") + return None + + response_data = json.loads(r.stdout) + + elif host == "gitlab": + encoded = quote_plus(f"{org}/{repo}") + r = requests.get( + f"https://gitlab.com/api/v4/projects/{encoded}/repository/tags" + ) + + if r.status_code != 200: + LOGGER.warning(f"Failed to get tags for {org}/{repo}: {r.json()}") + return None + response_data = r.json() + + else: + raise f"{host} is not supported!" + + available_versions = [x["name"] for x in response_data] + + if available_versions == []: + LOGGER.warning(f"No versions available for {host}/{org}/{repo}") + return None + + LOGGER.debug(f"Available versions for {host}/{org}/{repo}: {available_versions}") + + sem_versions = [extract_semvar(x["name"]) for x in response_data] + sem_versions = [v for v in sem_versions if v != None] + if sem_versions == []: + LOGGER.warning(f"{host}/{org}/{repo} versions do not conform to semver") + return None + + newer_vers = [v for v in sem_versions if semver.compare(v, current_ver) > 0] + + if newer_vers == []: + LOGGER.info( + f"{host}/{org}/{repo} does not have a new version compared to {current_ver}, first/last {available_versions[0]}/{available_versions[-1]}" + ) + return None + + latest_version = max(newer_vers, key=cmp_to_key(semver.compare)) + + LOGGER.info( + f"{host}/{org}/{repo} has a new version {latest_version} compared to {current_ver}" + ) + + return {**git_repo_from_meta, "latest_version": latest_version} + + +def get_new_versions_from_meta_paths(): + + module_meta_paths = ( + subprocess.run( + "find ./modules -name meta.yml", + shell=True, + capture_output=True, + text=True, + ) + .stdout.strip() + .split("\n") + ) + + git_repos_from_meta = [] + for meta_path in module_meta_paths: + meta_text = Path(meta_path).read_text() + main_path = f"{os.path.dirname(meta_path)}/main.nf" + main_text = Path(main_path).read_text() + repo = first( + re.findall( + r"\s+tool_dev_url:\s+\"?https://(github|gitlab).com/([\w-]+)/([\w-]+)\"?", + meta_text, + ) + ) + + if repo == None: + LOGGER.warning(f"No repo found in {meta_path}") + continue + + LOGGER.debug(f"{meta_path} repo: {repo}") + + if "mulled-v2" in main_text: + LOGGER.warning(f"Mulled container found in {main_path}") + continue + + version = first(re.findall(rf".*/([\w-]+):v?([\.0-9]+)", main_text)) + + if version == None: + LOGGER.warning(f"No version found in {main_path}") + continue + + LOGGER.debug(f"{main_path} version: {version}") + + semver_version = extract_semvar(version[1]) + + if semver_version == None: + LOGGER.warning( + f"Version {version} from {main_path} does not conform to semver" + ) + continue + + repo_data = { + "meta.yml": meta_path, + "main.nf": main_path, + "host": repo[0], + "org": repo[1], + "repo": repo[2], + "tool": version[0], + "semver": semver_version, + } + + LOGGER.debug(f"{meta_path} version data: {repo_data}") + + git_repos_from_meta.append(repo_data) + + with_latest = [check_version_status(r) for r in git_repos_from_meta] + filtered = [v for v in with_latest if v != None] + grouped = {} + + for v in filtered: + key = f"{v['host']}/{v['org']}/{v['repo']}" + if key not in grouped.keys(): + grouped[key] = {} + grouped[key]["latest_version"] = v["latest_version"] + grouped[key]["modules"] = [] + grouped[key]["modules"].append(v) + continue + + grouped[key]["modules"].append(v) + grouped[key]["latest_version"] = semver.max_ver( + grouped[key]["latest_version"], v["latest_version"] + ) + + return grouped + + +def get_repo_issue_titles(repo): + r = subprocess.run( + " ".join( + [ + "gh api", + '-H "Accept: application/vnd.github+json"', + '-H "X-GitHub-Api-Version: 2022-11-28"', + f"/repos/{repo}/issues", + ] + ), + shell=True, + capture_output=True, + text=True, + ) + + if r.returncode != 0: + LOGGER.error(f"Failed to get issues for {repo}: {r.stderr}") + exit(1) + + response_data = json.loads(r.stdout) + + update_issues = [ + x["title"] for x in response_data if x["title"].startswith("[UPDATE]") + ] + + return update_issues + + +def register_issue(host_repo_name, issue_title, tool_data): + + first_module = tool_data["modules"][0] + host, org, repo = (first_module["host"], first_module["org"], first_module["repo"]) + + module_paths = "\n".join( + [f"- {module['main.nf']}" for module in tool_data["modules"]] + ) + + issue_data = [ + "gh api", + "--method POST", + '-H "Accept: application/vnd.github+json"', + '-H "X-GitHub-Api-Version: 2022-11-28"', + f"/repos/{host_repo_name}/issues", + f'-f "title={issue_title}"', + f'-f "body=- https://{host}.com/{org}/{repo}\n{module_paths}"', + '-f "labels[]=enhancement"', + ] + + issue_post = subprocess.run( + " ".join(issue_data), + shell=True, + capture_output=True, + text=True, + ) + + if issue_post.returncode == 0: + LOGGER.info(f"Submitted issue: {issue_title}") + else: + LOGGER.warning(f"Failed to submitted issue: {issue_title}: {issue_post.stderr}") + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + + parser.add_argument( + "-v", "--verbose", action="store_true", help="Enable verbose mode" + ) + parser.add_argument( + "-d", "--dry-run", action="store_true", help="Enable dry-run mode" + ) + args = parser.parse_args() + + if args.verbose: + LOGGER.setLevel(level=logging.DEBUG) + else: + LOGGER.setLevel(level=logging.INFO) + + git_repos_from_meta = get_new_versions_from_meta_paths() + update_issue_titles = get_repo_issue_titles(PIPELINE_REPO) + + for key, tool_data in git_repos_from_meta.items(): + + new_issue_title = f"[UPDATE] {key.upper()} -> {tool_data['latest_version']}" + + if new_issue_title in update_issue_titles: + LOGGER.info(f"{new_issue_title} has already been raised!") + continue + + if args.dry_run: + LOGGER.info(f"Dry run new issue: {new_issue_title}") + continue + + register_issue(PIPELINE_REPO, new_issue_title, tool_data) diff --git a/.github/update_checks.py b/.github/update_checks.py deleted file mode 100755 index 2cf5d4e4..00000000 --- a/.github/update_checks.py +++ /dev/null @@ -1,271 +0,0 @@ -#!/usr/bin/env python3 - -import colorlog -from urllib.parse import quote_plus -from functools import cmp_to_key -from pathlib import Path - -import subprocess -import argparse -import requests -import logging -import semver -import json -import re -import os - -PIPELINE_REPO = "plant-food-research-open/assemblyqc" - - -def get_logger(): - formatter = colorlog.ColoredFormatter( - "%(log_color)s%(levelname)-8s%(reset)s %(blue)s%(message)s", - datefmt=None, - reset=True, - log_colors={ - "DEBUG": "cyan", - "INFO": "green", - "WARNING": "yellow", - "ERROR": "red", - "CRITICAL": "red,bg_white", - }, - secondary_log_colors={}, - style="%", - ) - - handler = colorlog.StreamHandler() - handler.setFormatter(formatter) - - logger = colorlog.getLogger("") - logger.addHandler(handler) - return logger - - -LOGGER = get_logger() - - -def first(from_list): - if from_list == []: - return None - - return from_list[0] - - -def flatten(xss): - return [x for xs in xss for x in xs] - - -def extract_semvar(from_version_str): - version_numbers = first( - re.findall(r"^[vV]?(\d+)\.?(\d*)\.?(\d*)$", from_version_str) - ) - if version_numbers == None: - return None - - minor_ver = f".{version_numbers[1]}" if version_numbers[1] != "" else ".0" - patch_ver = f".{version_numbers[2]}" if version_numbers[2] != "" else ".0" - - return f"{version_numbers[0]}{minor_ver}{patch_ver}" - - -def check_version_status(repo_tuple): - git, org, repo, tag = repo_tuple - - query_ver = extract_semvar(tag) - - if query_ver == None: - LOGGER.warning(f"{repo_tuple} does not conform to semver") - return None - - sem_versions = [] - if git == "github": - r = subprocess.run( - " ".join( - [ - "gh api", - '-H "Accept: application/vnd.github+json"', - '-H "X-GitHub-Api-Version: 2022-11-28"', - f"/repos/{org}/{repo}/tags", - ] - ), - shell=True, - capture_output=True, - text=True, - ) - - if r.returncode != 0: - LOGGER.warning(f"Failed to get tags for {org}/{repo}: {r.stderr}") - return None - response_data = json.loads(r.stdout) - - elif git == "gitlab": - encoded = quote_plus(f"{org}/{repo}") - r = requests.get( - f"https://gitlab.com/api/v4/projects/{encoded}/repository/tags" - ) - - if r.status_code != 200: - LOGGER.warning(f"Failed to get tags for {org}/{repo}: {r.json()}") - return None - response_data = r.json() - - else: - raise f"{git} is not supported!" - - available_versions = [x["name"] for x in response_data] - LOGGER.debug(f"Available versions for {repo_tuple}: {available_versions}") - - sem_versions = [extract_semvar(x["name"]) for x in response_data] - sem_versions = [v for v in sem_versions if v != None] - if sem_versions == []: - LOGGER.warning(f"{repo_tuple} versions do not conform to semver") - return None - - newer_vers = [v for v in sem_versions if semver.compare(v, query_ver) > 0] - - if newer_vers == []: - LOGGER.debug(f"{repo_tuple} does not have a new version") - return None - - return (git, org, repo, tag, max(newer_vers, key=cmp_to_key(semver.compare))) - - -def get_new_versions_from_meta_paths() -> list[tuple[str]]: - - module_meta_paths = ( - subprocess.run( - "find ./modules -name meta.yml", - shell=True, - capture_output=True, - text=True, - ) - .stdout.strip() - .split("\n") - ) - - git_repos_by_meta = [] - for meta_path in module_meta_paths: - meta_text = Path(meta_path).read_text() - main_text = Path(f"{os.path.dirname(meta_path)}/main.nf").read_text() - repos = re.findall( - r"\s+tool_dev_url: \"?https://(github|gitlab).com/([\w-]+)/([\w-]+)\"?", - meta_text, - ) - versions = [ - first(re.findall(rf".*/{repo[2]}:([\.0-9]+)", main_text)) for repo in repos - ] - - repo_ver = [ - (repo[0], repo[1], repo[2], version) - for (repo, version) in zip(repos, versions) - if version != None - ] - - if repo_ver == []: - continue - - git_repos_by_meta.append(repo_ver) - - git_repos_by_meta = sorted(list(set(flatten(git_repos_by_meta)))) - - new_versions = [check_version_status(r) for r in git_repos_by_meta] - - return sorted( - list(set([(v[0], v[1], v[2], v[4]) for v in new_versions if v != None])) - ) - - -def get_repo_issues(repo): - r = subprocess.run( - " ".join( - [ - "gh api", - '-H "Accept: application/vnd.github+json"', - '-H "X-GitHub-Api-Version: 2022-11-28"', - f"/repos/{repo}/issues", - ] - ), - shell=True, - capture_output=True, - text=True, - ) - - if r.returncode != 0: - LOGGER.error(f"Failed to get issues for {repo}: {r.stderr}") - exit(1) - - response_data = json.loads(r.stdout) - - update_issues = [ - first(re.findall(r"\[UPDATE\] (\w+)/([\w-]+)/([\w-]+) -> (.*)", x["title"])) - for x in response_data - if x["title"].startswith("[UPDATE]") - ] - - return update_issues - - -def register_issue(host_repo_name, tool_data: list[str]): - - issue_data = [ - "gh api", - "--method POST", - '-H "Accept: application/vnd.github+json"', - '-H "X-GitHub-Api-Version: 2022-11-28"', - f"/repos/{host_repo_name}/issues", - f'-f "title=[UPDATE] {tool_data[0]}/{tool_data[1]}/{tool_data[2]} -> {tool_data[3]}"', - f'-f "body=- https://{tool_data[0].lower()}.com/{tool_data[1].lower()}/{tool_data[2].lower()}"', - '-f "labels[]=enhancement"', - ] - - issue_post = subprocess.run( - " ".join(issue_data), - shell=True, - capture_output=True, - text=True, - ) - - if issue_post.returncode == 0: - LOGGER.info(f"Submitted issue for: {tool_data}") - else: - LOGGER.warning(f"Failed to submitted issue for: {tool_data}") - - -if __name__ == "__main__": - - parser = argparse.ArgumentParser() - - parser.add_argument( - "-v", "--verbose", action="store_true", help="Enable verbose mode" - ) - parser.add_argument( - "-d", "--dry-run", action="store_true", help="Enable dry-run mode" - ) - args = parser.parse_args() - - if args.verbose: - LOGGER.setLevel(level=logging.DEBUG) - else: - LOGGER.setLevel(level=logging.INFO) - - new_versions = get_new_versions_from_meta_paths() - repo_issues = get_repo_issues(PIPELINE_REPO) - - new_issues = set( - [(v[0].upper(), v[1].upper(), v[2].upper(), v[3].upper()) for v in new_versions] - ) - set(repo_issues) - - new_issues = sorted(list(new_issues)) - - if new_issues == []: - LOGGER.info("No updates are available!") - exit(0) - - LOGGER.info(f"Going to submit following updates: {new_issues}") - - if args.dry_run: - LOGGER.info(f"Exiting with dry-run") - exit(0) - - for issue in new_issues: - register_issue(PIPELINE_REPO, issue) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3645d085..e54e608d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## v2.1.0+dev - [23-July-2024] +## v2.1.0+dev - [24-July-2024] ### `Added`