diff --git a/.gitignore b/.gitignore index b6e4761..85bf552 100644 --- a/.gitignore +++ b/.gitignore @@ -1,129 +1,3 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ +status.cfg +__pycache__ +secret.txt \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0098c80 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM fedora + +# Install packages +RUN dnf install -y python3 \ + git \ + pip + +RUN pip install requests +RUN pip install jira + +CMD [“echo”, “Hello World!”] + diff --git a/LICENSE b/LICENSE index 261eeb9..d17e899 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright [2021] [The KubeVirt Authors] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 121e13a..22d0c3c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,77 @@ # github2jira Scrap github issues and create Jira tickets + +1. Create github token https://github.com/settings/tokens +2. `export GITHUB_TOKEN="your_token"` +3. Run `./create_ticket.py` in order to list sig/network issues + +# Build docker image for the script + +1. Run `docker build -f Dockerfile -t quay.io/oshoval/github:latest .` +once its done, push it to quay, or rename and push a local registry. + +# Run as k8s payload + +1. Create secret.txt with the following contents +`export GITHUB_TOKEN=` + +2. Create a configmap for the txt file +`kubectl create configmap git-token --from-file=secret.txt` + +3. Create a pod (good for testing) +``` +apiVersion: v1 +kind: Pod +metadata: + name: github + namespace: default +spec: + containers: + - image: quay.io/oshoval/github:latest + name: github + command: + - /bin/bash + - -c + - sleep infinity + volumeMounts: + - name: configs + mountPath: /app/secret.txt + subPath: secret.txt + volumes: + - name: configs + configMap: + name: git-token +``` + +or a CronJob +``` +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cron-github +spec: + schedule: "0 */1 * * *" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + containers: + - name: github + image: quay.io/oshoval/github:latest + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - date; source /app/secret.txt; ./github2jira/create_ticket.py + volumeMounts: + - name: configs + mountPath: /app/secret.txt + subPath: secret.txt + restartPolicy: Never + volumes: + - name: configs + configMap: + name: git-token +``` diff --git a/create_ticket.py b/create_ticket.py new file mode 100755 index 0000000..982eacf --- /dev/null +++ b/create_ticket.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 + +import os +import json +import sys +from datetime import datetime +import time +import argparse + +from jiralib import Jira +from githublib import Github + +# process upto this x weeks back +max_delta_weeks = 2 +SECONDS_PER_WEEK = 604800 +# how many tickets can be opened on each cycle +flood_protection_limit = 3 + +debug = 0 +dry_run = False + +def check_time(epoch_time_now, created_at, max_delta): + epoch = int(datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%SZ").timestamp()) + return epoch_time_now - epoch < max_delta + +def process_issue(jira, github, issue): + html_url = issue["html_url"] + issue_id = issue["number"] + title = issue["title"] + + if "pull" in html_url: + return False + + is_network = False + for label in issue["labels"]: + if label["name"] == "sig/network": + is_network = True + break + + if is_network == False: + return False + + epoch_time_now = int(time.time()) + if jira.issue_exists(github.repo, issue_id) == False: + if check_time(epoch_time_now, issue["created_at"], max_delta_weeks*SECONDS_PER_WEEK) == False: + return False + + if dry_run == False: + created_issue = jira.create_issue(github.repo, issue_id, title, issue["html_url"]) + print(f'Created issue {jira.server}/browse/{created_issue} for {issue["html_url"]}') + else: + print(f'Dry Run Created issue for {issue["html_url"]}') + return True + else: + print("Issue for", issue["html_url"], "already exists") + + return False + +def loop(jira, github): + issues_created = 0 + + for page in range(1, 20): + issues = github.get_issues(page) + if len(issues) == 0: + break + + if debug: + print(json.dumps(issues, sort_keys=True, indent=4)) + + for issue in issues: + res = process_issue(jira, github, issue) + if res == True: + issues_created += 1 + if issues_created == flood_protection_limit: + print("Flood protection limit reached, exiting") + sys.exit(0) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('server', help='jira server') + parser.add_argument('username', help='jira username') + parser.add_argument('project', help='jira project') + parser.add_argument('project_id', help='jira project id') + parser.add_argument('owner', help='github owner') + parser.add_argument('repo', help='github repo') + args = parser.parse_args() + + if dry_run == True: + print("Dry run enabled") + + jira = Jira(args.server, args.username, args.project, args.project_id) + github = Github(args.owner, args.repo) + + loop(jira, github) + +if __name__ == '__main__': + main() diff --git a/cronjob.yaml b/cronjob.yaml new file mode 100644 index 0000000..b1c3d0c --- /dev/null +++ b/cronjob.yaml @@ -0,0 +1,32 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cron-github +spec: + schedule: "*/1 * * * *" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + containers: + - name: github + image: quay.io/oshoval/github:latest + command: + - /bin/sh + - -ce + - | + source /app/secret.txt + ls -l + git clone https://github.com/oshoval/github2jira.git + ./github2jira/create_ticket.py + volumeMounts: + - name: configs + mountPath: /app/secret.txt + subPath: secret.txt + restartPolicy: Never + volumes: + - name: configs + configMap: + name: git-token diff --git a/githublib.py b/githublib.py new file mode 100755 index 0000000..9726374 --- /dev/null +++ b/githublib.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +import os +import sys +import requests +import json + +class Github: + def __init__(self, owner, repo): + self.owner = owner + self.repo = repo + + token = os.getenv('GITHUB_TOKEN') + if token == None: + print("Error: cant find GITHUB_TOKEN") + sys.exit(1) + + self.query_url = f"https://api.github.com/repos/{owner}/{repo}/issues" + self.headers = {'Authorization': f'token {token}'} + + def get_issues(self, page): + params = { + "state": "open", "page" : page, "per_page": "100" + } + + r = requests.get(self.query_url, headers=self.headers, params=params) + data = r.json() + return data + +def main(): + print("githublib self test") + github = Github("kubevirt", "kubevirt") + data = github.get_issues(1) + print(json.dumps(data[0], sort_keys=True, indent=4)) + +if __name__ == '__main__': + main() diff --git a/gitpod.yaml b/gitpod.yaml new file mode 100644 index 0000000..a6f68ee --- /dev/null +++ b/gitpod.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: github + namespace: default +spec: + containers: + - image: quay.io/oshoval/github:latest + name: github + command: + - /bin/bash + - -c + - sleep infinity + volumeMounts: + - name: configs + mountPath: /app/secret.txt + subPath: secret.txt + volumes: + - name: configs + configMap: + name: git-token diff --git a/jiralib.py b/jiralib.py new file mode 100755 index 0000000..0dcb369 --- /dev/null +++ b/jiralib.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +import os +import sys +import argparse + +from jira import JIRA + +class Jira: + def __init__(self, server, email, project, project_id): + self.project = project + self.project_id = project_id + self.server = server + + jira_token = os.getenv('JIRA_TOKEN') + if jira_token == None: + print("Error: cant find JIRA_TOKEN") + sys.exit(1) + + jiraOptions = {'server': self.server} + self.jira = JIRA(options = jiraOptions, basic_auth = (email, jira_token)) + + def issue_exists(self, repo, id): + query = f'project={self.project} AND text ~ "GITHUB:{repo}-{id}"' + issues = self.jira.search_issues(query) + return len(issues) != 0 + + def create_issue(self, repo, issue_id, title, body): + issue_dict=dict() + issue_dict['project']=dict({'id':self.project_id}) + issue_dict['summary']=f"[GITHUB:{repo}-{issue_id}] {title}" + issue_dict['description']=body + issue_dict['issuetype']=dict({'name':'Task'}) + issue_dict['components']=[dict({'name':'CNV Network'})] + + issue = self.jira.create_issue(issue_dict) + return issue + +def main(): + print("jiralib self test") + + parser = argparse.ArgumentParser() + parser.add_argument('server', help='jira server') + parser.add_argument('username', help='jira username') + parser.add_argument('project', help='jira project') + parser.add_argument('project_id', help='jira project id') + args = parser.parse_args() + + jira = Jira(args.server, args.username, args.project, args.project_id) + res = jira.issue_exists("kubevirt", "123") + if not res: + raise AssertionError() + print("OK") + +if __name__ == '__main__': + main()