Add JSON feed envelope and root last_updated field

Simplify the output by printing to STDOUT and using tee to duplicate the
STDOUT in a file in the bash script. Fix typos in comments and add
explanations about usage of tee

Co-authored-by: Pushkar Joglekar <[email protected]>

sig-security-tooling/cve-feed/hack/fetch-cve-feed.sh

@@ -17,17 +17,18 @@ set -o nounset
 set -o errexit
 set -o pipefail
 
-#install python-pip3
+# install python-pip3
 apt-get update
 apt-get install -y python3-pip
 
-#install requests module
+# install requests module
 pip3 install requests
 
-#python script to generate official-cve-feed.json 
-python3 fetch-official-cve-feed.py
+# python script to generate official-cve-feed.json 
+# tee duplicates the output from the script to stdout for logs and the JSON file
+python3 fetch-official-cve-feed.py | tee official_cve_feed.json
 
-#function to calculate the hash value of official-cve-feed.json 
+# function to calculate the hash value of official-cve-feed.json 
 calculate_hash(){
     if command -v shasum >/dev/null 2>&1; then
     cat "$@" | shasum -a 256 | cut -d' ' -f1
@@ -39,12 +40,13 @@ calculate_hash(){
     fi
 }
 
-#check if official-cve-feed.json blob exists in the bucket
+# check if official-cve-feed.json blob exists in the bucket
 set -e
 EXIT_CODE=0
 gsutil ls gs://k8s-cve-feed/official-cve-feed.json >/dev/null 2>&1 || EXIT_CODE=$?
 
-#fetch the hash value of existing official-cve-feed.json json, if differs then upload the new cve feed data to the existing blob.
+# fetch the hash value of existing official-cve-feed.json json, if differs then
+# upload the new cve feed data to the existing blob.
 if [[ $EXIT_CODE -eq 1  ]]; then 
     gsutil cp official-cve-feed.json gs://k8s-cve-feed
     calculate_hash official-cve-feed.json > cve-feed-hash

sig-security-tooling/cve-feed/hack/fetch-official-cve-feed.py

@@ -16,6 +16,7 @@
 
 import json
 import requests
+from datetime import datetime
 
 url = 'https://api.github.com/search/issues?q=is:issue+label:official-cve-feed+\
 state:closed+repo:kubernetes/kubernetes&per_page=100'
@@ -33,21 +34,39 @@
     res = requests.get(res.links['next']['url'], headers=headers)
     gh_items.extend(res.json()['items'])
 
-cve_list = []
+feed_envelope = {
+    'version': 'https://jsonfeed.org/version/1.1',
+    'title': 'Auto-refreshing Official CVE Feed',
+    'home_page_url': 'https://kubernetes.io',
+    'feed_url': 'https://kubernetes.io/docs/reference/issues-security/official-cve-feed/index.json',
+    'description': 'Auto-refreshing official CVE feed for Kubernetes repository',
+    'authors': [
+        {
+            'name': 'Kubernetes Community',
+            'url': 'https://www.kubernetes.dev'
+        }
+    ],
+    '_kubernetes_io': None,
+    'items': None,
+}
+# format the timestamp the same way as GitHub RFC 3339 timestamps, with only seconds and not milli and microseconds.
+root_kubernetes_io = {'updated_at': datetime.utcnow().isoformat(sep='T', timespec='seconds') + 'Z'}
+feed_envelope['_kubernetes_io'] = root_kubernetes_io
 
+cve_list = []
 for item in gh_items:
     # These keys respects the item jsonfeed spec https://www.jsonfeed.org/version/1.1/
-    cve = {'url': None, 'id': None, 'summary': None, 'external_url': None,
-    'content_text': None, '_kubernetes_io': None, 'date_published': None}
+    cve = {'content_text': None, 'date_published': None, 'external_url': None,
+    'id': None,'summary': None, 'url': None, '_kubernetes_io': None}
     # This is a custom extension
-    kubernetes_io = {'google_group_url': None, 'issue_number': None}
-    cve['_kubernetes_io'] = kubernetes_io
+    item_kubernetes_io = {'google_group_url': None, 'issue_number': None}
+    cve['_kubernetes_io'] = item_kubernetes_io
 
     cve['url'] = item['html_url']
     cve['_kubernetes_io']['issue_number'] = item['number']
     cve['content_text'] = item['body']
     cve['date_published'] = item['created_at']
-    # This because some CVEs were titled "CVE-XXXX-XXXX - Something" instead of
+    # This is because some CVEs were titled "CVE-XXXX-XXXX - Something" instead of
     # "CVE-XXXX-XXXX: Something" on GitHub (see https://github.com/kubernetes/kubernetes/issues/60813).
     title = item['title'].replace(' -', ':')
     # This splits the CVE into its ID and the description/name, however some are in the following forms:
@@ -64,8 +83,6 @@
             cve['_kubernetes_io']['google_group_url'] = f'https://groups.google.com/g/kubernetes-announce/search?q={cve_id}'
     cve_list.append(cve)
 
-cves = json.dumps(cve_list, sort_keys=True, indent=4)
-print(cves)
-# write the final cve list to official_cve_feed.json
-with open('official-cve-feed.json', 'w') as cvejson:
-    cvejson.write(cves)
+feed_envelope['items'] = cve_list
+json_feed = json.dumps(feed_envelope, sort_keys=False, indent=4)
+print(json_feed)