From a470a88f14070a70ec9cbbe436d177636a143e25 Mon Sep 17 00:00:00 2001 From: Or Shoval Date: Thu, 4 Nov 2021 14:31:45 +0200 Subject: [PATCH] Introduce create ticket script The script automates github issue to Jita ticket mirroring. Signed-off-by: Or Shoval --- .gitignore | 130 +---------------------------------------- Dockerfile | 10 ++++ LICENSE | 2 +- README.md | 57 +++++++++++++++++- config.py | 35 +++++++++++ githublib.py | 102 ++++++++++++++++++++++++++++++++ jiralib.py | 66 +++++++++++++++++++++ main.py | 53 +++++++++++++++++ manifests/cronjob.yaml | 32 ++++++++++ manifests/gitpod.yaml | 22 +++++++ ticket.py | 46 +++++++++++++++ tox.ini | 62 ++++++++++++++++++++ 12 files changed, 487 insertions(+), 130 deletions(-) create mode 100644 Dockerfile create mode 100755 config.py create mode 100755 githublib.py create mode 100755 jiralib.py create mode 100755 main.py create mode 100644 manifests/cronjob.yaml create mode 100644 manifests/gitpod.yaml create mode 100755 ticket.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index b6e4761..443eb7b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,129 +1,3 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ +__pycache__ +secret.txt .tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5cb72af --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM fedora:32 + +RUN dnf install -y python3 git pip \ + && dnf clean all \ + && rm -rf /var/cache/yum + +RUN pip install requests jira + +CMD [“echo”, “Hello World”] + diff --git a/LICENSE b/LICENSE index 261eeb9..ea282d6 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright [2021] [Red Hat, Inc] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 121e13a..d65300b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,57 @@ # github2jira -Scrap github issues and create Jira tickets +github2jira automates mirroring of github issues to Jira tickets. + +The tool scans github for issues that match the desired criteria, +and for each one of them creates a Jira ticket (unless it already exists). + +## One time configuration +1. Create github token https://github.com/settings/tokens, refer it as `GITHUB_TOKEN` +2. Make sure you have a Jira bot access (either a user:pass or user:token), refer as `JIRA_USERNAME`,`JIRA_TOKEN` +3. Get your Jira project id, refer as `JIRA_PROJECT_ID` +`curl -s -u JIRA_USERNAME:JIRA_TOKEN -X GET -H "Content-Type: application/json" /rest/api/latest/project/ | jq .id` + +## Running manually + +1. export the following envvars: +``` +export JIRA_SERVER=<..> # for example https://nmstate.atlassian.net +export JIRA_PROJECT=<..> # name of the Jira project (ticket names are JIRA_PROJECT-#) +export JIRA_PROJECT_ID=<..> # see "One time configuration" section +export JIRA_COMPONENT=<..> # which component to set in the created tickets +export GITHUB_OWNER=<..> # the x of https://github.com/x/y +export GITHUB_REPO=<..> # the y of https://github.com/x/y +export GITHUB_LABEL=<..> # which label to filter + +export JIRA_USERNAME=<..> # see "One time configuration" section +export JIRA_TOKEN=<..> # see "One time configuration" section +export GITHUB_TOKEN=<..> # see "One time configuration" section +``` + +2. Run `./main.py` in order to fetch github issues and create a ticket for them + +### Additional settings + +`dryrun`: Use `./main.py --dryrun` in order to run the tool in dryrun mode. +dryrun mode will fetch github issues, and report what Jira tickets it would create, +but without creating them. + +## Running as k8s payload + +In order to have a fully automated mirroring process, +it is suggested to run the tool as a cron jon. + +One of the methods to achieve it, is to run it as k8s CronJob payload. + +### One time configuration: Build docker image for the script + +1. From the project folder, run `docker build -f Dockerfile -t .` +once its done, push it to your image repository, or rename and push to a local registry. + +### Deploy as k8s payload + +1. Create secret.txt with the exports from the section above (include the export command). + +2. Create a configmap for the txt file +`kubectl create configmap git-token --from-file=secret.txt` + +3. Deploy either a pod or a CronJob (see manifests folder). diff --git a/config.py b/config.py new file mode 100755 index 0000000..dfb4c0b --- /dev/null +++ b/config.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# +# This file is part of the github2jira project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2021 Red Hat, Inc. +# + +import os + + +class Config: + def __init__(self, var_names): + self.var_names = var_names + self.vars = {} + + def Load(self): + for var_name in self.var_names: + value = os.getenv(var_name) + if value is None: + print(f"Error: cant find {var_name}") + return False + self.vars[var_name] = value + return True diff --git a/githublib.py b/githublib.py new file mode 100755 index 0000000..7d19ad0 --- /dev/null +++ b/githublib.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# +# This file is part of the github2jira project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2021 Red Hat, Inc. +# + +import time +import requests + +from datetime import datetime + +from config import Config + +SECONDS_PER_WEEK = 604800 +# max github pages to process +GITHUB_MAX_PAGES = 20 +# process upto x weeks back +MAX_DELTA_WEEKS = 4 + + +class GithubConfig(Config): + def __init__(self): + super().__init__( + ["GITHUB_TOKEN", "GITHUB_OWNER", "GITHUB_REPO", "GITHUB_LABEL"] + ) + + +class Issue: + def __init__(self, issue): + self.issue = issue + + def filter(self, expected_label): + if expected_label == "": + return True + for label in self.issue["labels"]: + if label["name"] == expected_label: + return True + return False + + def age_relevant(self, max_delta): + epoch_time_now = int(time.time()) + TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" + timestamp = self.issue["created_at"] + epoch = int(datetime.strptime(timestamp, TIME_FORMAT).timestamp()) + return (epoch_time_now - epoch) < (max_delta * SECONDS_PER_WEEK) + + @property + def repo(self): + return self.issue["html_url"].split("/")[4] + + @property + def id(self): + return self.issue["number"] + + @property + def url(self): + return self.issue["html_url"] + + @property + def title(self): + return self.issue["title"] + + +class Github: + def __init__(self, cfg): + self.owner = cfg.vars["GITHUB_OWNER"] + self.repo = cfg.vars["GITHUB_REPO"] + self.expected_label = cfg.vars["GITHUB_LABEL"] + self.query_url = f"https://api.github.com/repos/{self.owner}/{self.repo}/issues" + self.headers = {"Authorization": f"token {cfg.vars['GITHUB_TOKEN']}"} + + def issues(self): + for page in range(1, GITHUB_MAX_PAGES): + params = {"state": "open", "page": page, "per_page": "100"} + r = requests.get(self.query_url, headers=self.headers, params=params) + issues = r.json() + + if len(issues) == 0: + return + + for element in issues: + issue = Issue(element) + if "pull" in issue.url: + continue + + if issue.filter(self.expected_label) and issue.age_relevant( + MAX_DELTA_WEEKS + ): + yield (issue) diff --git a/jiralib.py b/jiralib.py new file mode 100755 index 0000000..3f0f8eb --- /dev/null +++ b/jiralib.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# +# This file is part of the github2jira project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2021 Red Hat, Inc. +# + +from jira import JIRA + +from config import Config + + +class JiraConfig(Config): + def __init__(self): + super().__init__( + [ + "JIRA_SERVER", + "JIRA_USERNAME", + "JIRA_TOKEN", + "JIRA_PROJECT", + "JIRA_PROJECT_ID", + "JIRA_COMPONENT", + ] + ) + + +class Jira: + def __init__(self, cfg): + self.project = cfg.vars["JIRA_PROJECT"] + self.project_id = cfg.vars["JIRA_PROJECT_ID"] + self.server = cfg.vars["JIRA_SERVER"] + self.component = cfg.vars["JIRA_COMPONENT"] + + jiraOptions = {"server": self.server} + self.jira = JIRA( + options=jiraOptions, + basic_auth=(cfg.vars["JIRA_USERNAME"], cfg.vars["JIRA_TOKEN"]), + ) + + def issue_exists(self, issue): + query = f'project={self.project} AND text ~ "GITHUB:{issue.repo}-{issue.id}"' + issues = self.jira.search_issues(query) + return len(issues) != 0 + + def create_issue(self, issue): + issue_dict = dict() + issue_dict["project"] = dict({"id": self.project_id}) + issue_dict["summary"] = f"[GITHUB:{issue.repo}-{issue.id}] {issue.title}" + issue_dict["description"] = issue.url + issue_dict["issuetype"] = dict({"name": "Task"}) + if self.component != "": + issue_dict["components"] = [dict({"name": self.component})] + + return self.jira.create_issue(issue_dict) diff --git a/main.py b/main.py new file mode 100755 index 0000000..102b928 --- /dev/null +++ b/main.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# +# This file is part of the github2jira project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2021 Red Hat, Inc. +# + +import sys + +import argparse + +from ticket import Ticket +from jiralib import JiraConfig +from githublib import Github, GithubConfig + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--dryrun", default=False, action="store_true") + args = parser.parse_args() + + if args.dryrun: + print("INFO: dryrun mode enabled") + + github_config = GithubConfig() + assert github_config.Load() + github = Github(github_config) + + jira_config = JiraConfig() + assert jira_config.Load() + ticket = Ticket(jira_config, args.dryrun) + + for issue in github.issues(): + ticket.create(issue) + if ticket.flood_protection_reached(): + print("Flood protection limit reached, exiting") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/manifests/cronjob.yaml b/manifests/cronjob.yaml new file mode 100644 index 0000000..e1ccc5d --- /dev/null +++ b/manifests/cronjob.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cron-github +spec: + schedule: "0 */1 * * *" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + containers: + - name: github + image: quay.io/oshoval/github:latest + command: + - /bin/sh + - -ce + - | + source /app/secret.txt + git clone https://github.com/oshoval/github2jira.git + ./github2jira/main.py + volumeMounts: + - name: configs + mountPath: /app/secret.txt + subPath: secret.txt + restartPolicy: Never + volumes: + - name: configs + configMap: + name: git-token diff --git a/manifests/gitpod.yaml b/manifests/gitpod.yaml new file mode 100644 index 0000000..54edcd0 --- /dev/null +++ b/manifests/gitpod.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: github + namespace: default +spec: + containers: + - image: quay.io/oshoval/github:latest + name: github + command: + - /bin/bash + - -c + - sleep infinity + volumeMounts: + - name: configs + mountPath: /app/secret.txt + subPath: secret.txt + volumes: + - name: configs + configMap: + name: git-token diff --git a/ticket.py b/ticket.py new file mode 100755 index 0000000..940b015 --- /dev/null +++ b/ticket.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# +# This file is part of the github2jira project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2021 Red Hat, Inc. +# + +from jiralib import Jira + +# how many tickets can be opened on each cycle +FLOOD_PROTECTION_LIMIT = 3 + + +class Ticket: + def __init__(self, jira_config, dryrun): + self.dryrun = dryrun + self.issues_created = 0 + self.jira = Jira(jira_config) + + def create(self, issue): + if not self.jira.issue_exists(issue): + if not self.dryrun: + created_issue = self.jira.create_issue(issue) + issue_url = f"{self.jira.server}/browse/{created_issue}" + print(f"Created issue {issue_url} for {issue.url}") + else: + print(f"Dry Run Created issue for {issue.url}") + + self.issues_created += 1 + else: + print("Issue for", issue.url, "already exists") + + def flood_protection_reached(self): + return self.issues_created == FLOOD_PROTECTION_LIMIT diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..1a90c96 --- /dev/null +++ b/tox.ini @@ -0,0 +1,62 @@ +[tox] +envlist = black, flake8, pylint, yamllint, py36, py38 +skip_missing_interpreters = True + +[testenv:black] +skip_install = true +basepython = python3.6 +changedir = {toxinidir} +deps = + black==21.6b0 +# style configured via pyproject.toml +commands = + black \ + --check \ + --diff \ + {posargs} \ + ./ + +[testenv:flake8] +basepython = python3.6 +skip_install = true +changedir = {toxinidir} +deps = + flake8==3.7.9 +commands = + flake8 \ + --statistics {posargs} \ + main.py \ + config.py \ + githublib.py \ + jiralib.py \ + ticket.py + +[testenv:pylint] +basepython = python3.6 +sitepackages = true +skip_install = true +changedir = {toxinidir} +deps = + pylint==2.4.4 +commands = + pylint \ + --errors-only \ + {posargs} \ + main.py \ + config.py \ + githublib.py \ + jiralib.py \ + ticket.py + +[testenv:yamllint] +basepython = python3.6 +skip_install = true +changedir = {toxinidir} +deps = + yamllint==1.23.0 +commands = + yamllint manifests/ + +[flake8] +show_source = True +max-line-length=90