diff --git a/sig-security-tooling/cve-feed/hack/fetch-cve-feed.sh b/sig-security-tooling/cve-feed/hack/fetch-cve-feed.sh index 0e8fb29..f52a330 100755 --- a/sig-security-tooling/cve-feed/hack/fetch-cve-feed.sh +++ b/sig-security-tooling/cve-feed/hack/fetch-cve-feed.sh @@ -17,17 +17,18 @@ set -o nounset set -o errexit set -o pipefail -#install python-pip3 +# install python-pip3 apt-get update apt-get install -y python3-pip -#install requests module +# install requests module pip3 install requests -#python script to generate official-cve-feed.json -python3 fetch-official-cve-feed.py +# python script to generate official-cve-feed.json +# tee duplicates the output from the script to stdout for logs and the JSON file +python3 fetch-official-cve-feed.py | tee official_cve_feed.json -#function to calculate the hash value of official-cve-feed.json +# function to calculate the hash value of official-cve-feed.json calculate_hash(){ if command -v shasum >/dev/null 2>&1; then cat "$@" | shasum -a 256 | cut -d' ' -f1 @@ -39,12 +40,13 @@ calculate_hash(){ fi } -#check if official-cve-feed.json blob exists in the bucket +# check if official-cve-feed.json blob exists in the bucket set -e EXIT_CODE=0 gsutil ls gs://k8s-cve-feed/official-cve-feed.json >/dev/null 2>&1 || EXIT_CODE=$? -#fetch the hash value of existing official-cve-feed.json json, if differs then upload the new cve feed data to the existing blob. +# fetch the hash value of existing official-cve-feed.json json, if differs then +# upload the new cve feed data to the existing blob. if [[ $EXIT_CODE -eq 1 ]]; then gsutil cp official-cve-feed.json gs://k8s-cve-feed calculate_hash official-cve-feed.json > cve-feed-hash diff --git a/sig-security-tooling/cve-feed/hack/fetch-official-cve-feed.py b/sig-security-tooling/cve-feed/hack/fetch-official-cve-feed.py index 7b6f7ac..5a56e0e 100755 --- a/sig-security-tooling/cve-feed/hack/fetch-official-cve-feed.py +++ b/sig-security-tooling/cve-feed/hack/fetch-official-cve-feed.py @@ -16,6 +16,7 @@ import json import requests +from datetime import datetime url = 'https://api.github.com/search/issues?q=is:issue+label:official-cve-feed+\ state:closed+repo:kubernetes/kubernetes&per_page=100' @@ -33,21 +34,39 @@ res = requests.get(res.links['next']['url'], headers=headers) gh_items.extend(res.json()['items']) -cve_list = [] +feed_envelope = { + 'version': 'https://jsonfeed.org/version/1.1', + 'title': 'Auto-refreshing Official CVE Feed', + 'home_page_url': 'https://kubernetes.io', + 'feed_url': 'https://kubernetes.io/docs/reference/issues-security/official-cve-feed/index.json', + 'description': 'Auto-refreshing official CVE feed for Kubernetes repository', + 'authors': [ + { + 'name': 'Kubernetes Community', + 'url': 'https://www.kubernetes.dev' + } + ], + '_kubernetes_io': None, + 'items': None, +} +# format the timestamp the same way as GitHub RFC 3339 timestamps, with only seconds and not milli and microseconds. +root_kubernetes_io = {'updated_at': datetime.utcnow().isoformat(sep='T', timespec='seconds') + 'Z'} +feed_envelope['_kubernetes_io'] = root_kubernetes_io +cve_list = [] for item in gh_items: # These keys respects the item jsonfeed spec https://www.jsonfeed.org/version/1.1/ - cve = {'url': None, 'id': None, 'summary': None, 'external_url': None, - 'content_text': None, '_kubernetes_io': None, 'date_published': None} + cve = {'content_text': None, 'date_published': None, 'external_url': None, + 'id': None,'summary': None, 'url': None, '_kubernetes_io': None} # This is a custom extension - kubernetes_io = {'google_group_url': None, 'issue_number': None} - cve['_kubernetes_io'] = kubernetes_io + item_kubernetes_io = {'google_group_url': None, 'issue_number': None} + cve['_kubernetes_io'] = item_kubernetes_io cve['url'] = item['html_url'] cve['_kubernetes_io']['issue_number'] = item['number'] cve['content_text'] = item['body'] cve['date_published'] = item['created_at'] - # This because some CVEs were titled "CVE-XXXX-XXXX - Something" instead of + # This is because some CVEs were titled "CVE-XXXX-XXXX - Something" instead of # "CVE-XXXX-XXXX: Something" on GitHub (see https://github.com/kubernetes/kubernetes/issues/60813). title = item['title'].replace(' -', ':') # This splits the CVE into its ID and the description/name, however some are in the following forms: @@ -64,8 +83,6 @@ cve['_kubernetes_io']['google_group_url'] = f'https://groups.google.com/g/kubernetes-announce/search?q={cve_id}' cve_list.append(cve) -cves = json.dumps(cve_list, sort_keys=True, indent=4) -print(cves) -# write the final cve list to official_cve_feed.json -with open('official-cve-feed.json', 'w') as cvejson: - cvejson.write(cves) +feed_envelope['items'] = cve_list +json_feed = json.dumps(feed_envelope, sort_keys=False, indent=4) +print(json_feed)