SECURESIGN-1476 | Add the Redis backfill job to Ansible collection

molecule/default/vars/podman.yml

@@ -9,6 +9,8 @@ tas_single_node_podman:
       mirror: quay.io/securesign/trillian-logsigner
     - location: registry.redhat.io/rhtas/rekor-server-rhel9
       mirror: quay.io/securesign/rekor-server
+    - location: registry.redhat.io/rhtas/rekor-backfill-redis-rhel9
+      mirror: quay.io/securesign/rekor-backfill-redis
     - location: registry.redhat.io/rhtas/certificate-transparency-rhel9
       mirror: quay.io/securesign/certificate-transparency-go
     - location: registry.redhat.io/rhtas/trillian-redis-rhel9

roles/tas_single_node/README.md

@@ -19,6 +19,7 @@ Deploy the [RHTAS](https://docs.redhat.com/en/documentation/red_hat_trusted_arti
 |---|---|---|---|
 | tas_single_node_podman_network | Name of the Podman network for containers to use. | str |  `rhtas`  |
 | tas_single_node_rekor_redis | Details on the Redis connection for Rekor. You can set this to a custom Redis instance. | dict of 'tas_single_node_rekor_redis' options |  `{'database_deploy': True, 'redis': {'host': 'rekor-redis-pod', 'port': 6379, 'password': 'password'}}`  |
+| tas_single_node_backfill_redis | Configuration options for the backfill redis job. | dict of 'tas_single_node_backfill_redis' options |  `{'enabled': True, 'schedule': '*-*-* 00:00:00'}`  |
 | tas_single_node_trillian | Details on the database connection for Trillian. You can set this to a custom MySQL or MariaDB instance. | dict of 'tas_single_node_trillian' options |  `{'database_deploy': True, 'mysql': {'user': 'mysql', 'root_password': 'rootpassword', 'password': 'password', 'database': 'trillian', 'host': 'trillian-mysql-pod', 'port': 3306}}`  |
 | tas_single_node_rekor_public_key_retries | The number of attempts to retrieve the Rekor public key when constructing the trust root. | int |  `5`  |
 | tas_single_node_rekor_public_key_delay | The number of seconds to wait before retrying the retrieval of the Rekor public key when constructing the trust root. | int |  `10`  |
@@ -61,6 +62,13 @@ Deploy the [RHTAS](https://docs.redhat.com/en/documentation/red_hat_trusted_arti
 | port | The Redis host port number. | int | no |  |
 | password | The Redis password. | str | no |  |
 
+#### Options for main > tas_single_node_backfill_redis
+
+|Option|Description|Type|Required|Default|
+|---|---|---|---|---|
+| enabled | Enable or disable the backfill redis job. | bool | no |  |
+| schedule | Schedule the backfill redis job should follow. | str | no |  |
+
 #### Options for main > tas_single_node_trillian
 
 |Option|Description|Type|Required|Default|

roles/tas_single_node/defaults/main.yml

@@ -12,6 +12,10 @@ tas_single_node_rekor_redis:
     port: 6379
     password: password
 
+tas_single_node_backfill_redis:
+  enabled: true
+  schedule: "*-*-* 00:00:00"
+
 tas_single_node_trillian:
   database_deploy: true
   mysql:
@@ -66,6 +70,8 @@ tas_single_node_ctlog_image:
   "registry.redhat.io/rhtas/certificate-transparency-rhel9@sha256:31e7318a9b19ed04ef0f25949f1f1709d293b532316b27a06f83fa5174547b17"
 tas_single_node_rekor_redis_image:
   "registry.redhat.io/rhtas/trillian-redis-rhel9@sha256:18820b1fbdbc2cc3e917822974910332d937b03cfe781628bd986fd6a5ee318e"
+tas_single_node_backfill_redis_image:
+  "registry.redhat.io/rhtas/rekor-backfill-redis-rhel9@sha256:6aa3ca40e0f9e32a0a211a930b21ff009b83e46609bfa5bb328979e4799d13c7"
 tas_single_node_trillian_db_image:
   "registry.redhat.io/rhtas/trillian-database-rhel9@sha256:501612745e63e5504017079388bec191ffacf00ffdebde7be6ca5b8e4fd9d323"
 tas_single_node_tuf_image:

roles/tas_single_node/meta/argument_specs.yml

@@ -62,6 +62,25 @@ argument_specs:
                 type: "str"
                 required: false
                 version_added: "1.1.0"
+      tas_single_node_backfill_redis:
+        description: "Configuration options for the backfill redis job."
+        type: "dict"
+        required: false
+        version_added: "1.1.1"
+        default:
+          enabled: true
+          schedule: "*-*-* 00:00:00"
+        options:
+          enabled:
+            description: "Enable or disable the backfill redis job."
+            type: "bool"
+            required: false
+            version_added: "1.1.1"
+          schedule:
+            description: "Schedule the backfill redis job should follow."
+            type: "str"
+            required: false
+            version_added: "1.1.1"
       tas_single_node_trillian:
         description: "Details on the database connection for Trillian. You can set this to a custom MySQL or MariaDB instance."
         type: "dict"

roles/tas_single_node/tasks/podman.yml

@@ -45,6 +45,7 @@
       "{{ tas_single_node_rekor_enabled }}",
       "{{ tas_single_node_ctlog_enabled }}",
       "{{ tas_single_node_rekor_enabled and tas_single_node_rekor_redis.database_deploy }}",
+      "{{ tas_single_node_rekor_enabled and tas_single_node_rekor_redis.database_deploy and tas_single_node_backfill_redis.enabled }}",
       "{{ tas_single_node_trillian_enabled and tas_single_node_trillian.database_deploy }}",
       "{{ tas_single_node_tuf_enabled }}",
       "{{ tas_single_node_tuf_enabled }}",
@@ -62,6 +63,7 @@
     - "{{ tas_single_node_rekor_server_image }}"
     - "{{ tas_single_node_ctlog_image }}"
     - "{{ tas_single_node_rekor_redis_image }}"
+    - "{{ tas_single_node_backfill_redis_image }}"
     - "{{ tas_single_node_trillian_db_image }}"
     - "{{ tas_single_node_tuf_image }}"
     - "{{ tas_single_node_http_server_image }}"

roles/tas_single_node/tasks/podman/rekor.yml

@@ -69,3 +69,29 @@
       configmap_changed: "{{ configmap_result.changed }}"
       secret: "{{ tas_single_node_rekor_secret }}"
       secret_changed: "{{ secret_result.changed }}"
+
+- name: Deploy backfill Redis job
+  ansible.builtin.include_tasks: podman/install_manifest.yml
+  vars:
+    podman_spec:
+      state: started
+      systemd_file: backfill_redis
+      network: "{{ tas_single_node_podman_network }}"
+      kube_file_content: "{{ lookup('template', 'manifests/rekor/backfill_redis.j2') | from_yaml }}"
+  when: tas_single_node_backfill_redis.enabled
+
+- name: Copy backfill_redis.timer file to server
+  ansible.builtin.template:
+    src: systemd/backfill_redis.timer.j2
+    dest: "{{ tas_single_node_systemd_directory }}/backfill_redis.timer"
+    mode: "0644"
+  when: tas_single_node_backfill_redis.enabled
+  register: copy_timer_systemd_file
+
+- name: Enable and start backfill_redis.timer
+  ansible.builtin.systemd:
+    name: backfill_redis.timer
+    enabled: true
+    state: started
+    daemon_reload: "{{ copy_timer_systemd_file.changed }}"
+  when: tas_single_node_backfill_redis.enabled

roles/tas_single_node/templates/manifests/rekor/backfill_redis.j2

@@ -0,0 +1,75 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: backfill-redis
+  namespace: backfill-redis
+  labels:
+    app.component: backfill-redis
+    app.instance: backfill-redis
+    app.name: backfill-redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.component: backfill-redis
+      app.instance: backfill-redis
+      app.name: backfill-redis
+  template:
+    metadata:
+      labels:
+        app.component: backfill-redis
+        app.instance: backfill-redis
+        app.name: backfill-redis
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: backfill-redis
+          image: "{{ tas_single_node_backfill_redis_image }}"
+          command:
+            - /bin/sh
+            - -c
+          args:
+            - |
+              set +x
+              echo "info: Querying Rekor for current log size..."
+              endIndex=$(curl -sS http://{{ tas_single_node_rekor_server_pod }}-pod:{{ tas_single_node_rekor_server_port_http }}/api/v1/log | sed -E 's/.*"treeSize":([0-9]+).*/\1/')
+              endIndex=$((endIndex-1))
+              echo "info: Fetched endIndex from Rekor; endIndex = ${endIndex}"
+
+              if [ "${endIndex}" -lt 0 ]; then
+                echo "info: no rekor entries found"
+                exit 0
+              fi
+
+              echo "info: Checking Redis for last filled index..."
+              startIndex=$(redis-cli \
+                -h {{ tas_single_node_rekor_redis.redis.host }} \
+                -p {{ tas_single_node_rekor_redis.redis.port }} \
+{% if tas_single_node_rekor_redis.redis.password != "" %}
+                -a "{{ tas_single_node_rekor_redis.redis.password }}" \
+{% endif %}
+                GET last_filled_index)
+
+              if [ -z "$startIndex" ]; then
+                startIndex=0
+              fi
+              echo "info: Retrieved startIndex from Redis; startIndex = ${startIndex}"
+
+              echo "info: Executing backfill-redis from ${startIndex} to ${endIndex}"
+              backfill-redis \
+                --redis-hostname={{ tas_single_node_rekor_redis.redis.host }} \
+                --redis-port={{ tas_single_node_rekor_redis.redis.port }} \
+{% if tas_single_node_rekor_redis.redis.password != "" %}
+                --redis-password="{{ tas_single_node_rekor_redis.redis.password }}" \
+{% endif %}
+                --rekor-address=http://{{ tas_single_node_rekor_server_pod }}-pod:{{ tas_single_node_rekor_server_port_http }} \
+                --start="${startIndex}" --end="${endIndex}"
+
+              echo "info: Updating last_filled_index in Redis to $((endIndex + 1))"
+              redis-cli \
+                -h {{ tas_single_node_rekor_redis.redis.host }} \
+                -p {{ tas_single_node_rekor_redis.redis.port }} \
+{% if tas_single_node_rekor_redis.redis.password != "" %}
+                -a "{{ tas_single_node_rekor_redis.redis.password }}" \
+{% endif %}
+                SET last_filled_index "$((endIndex + 1))"

roles/tas_single_node/templates/systemd/backfill_redis.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Runs The backfill redis job
+
+[Timer]
+OnCalendar={{ tas_single_node_backfill_redis.schedule }}
+Persistent=true
+Unit=backfill_redis.service
+
+[Install]
+WantedBy=timers.target

roles/tas_single_node/vars/main.yml

@@ -18,6 +18,10 @@ tas_single_node_rekor_templates:
   - manifests/rekor/redis-server.j2
   - manifests/rekor/rekor-server.j2
 
+tas_single_node_backfill_redis:
+  enabled: true
+  schedule: "*-*-* 00:00:00"
+
 # Individual service enablement
 tas_single_node_trillian_enabled: true
 tas_single_node_rekor_enabled: true

vm-testing/dev-images.sed

@@ -5,7 +5,6 @@ s#registry.redhat.io/rhtas/fulcio-rhel9#quay.io/securesign/fulcio-server#
 s#registry.redhat.io/rhtas/trillian-redis-rhel9#quay.io/securesign/trillian-redis#
 s#registry.redhat.io/rhtas/rekor-server-rhel9#quay.io/securesign/rekor-server#
 s#registry.redhat.io/rhtas/rekor-search-ui-rhel9#quay.io/securesign/rekor-search-ui#
-s#registry.redhat.io/rhtas/rekor-backfill-redis-rhel9#quay.io/securesign/trillian-redis#
 s#registry.redhat.io/rhtas/tuf-server-rhel9#quay.io/securesign/scaffold-tuf-server#
 s#registry.redhat.io/rhtas/tuffer-rhel9#quay.io/securesign/tuffer#
 s#registry.redhat.io/rhtas/certificate-transparency-rhel9#quay.io/securesign/certificate-transparency-go#
@@ -15,3 +14,4 @@ s#registry.redhat.io/rhtas/segment-reporting-rhel9#quay.io/securesign/segment-ba
 s#registry.redhat.io/rhtas/timestamp-authority-rhel9#quay.io/securesign/timestamp-authority#
 s#registry.redhat.io/rhtas/trillian-createtree-rhel9#quay.io/securesign/trillian-createtree#
 s#registry.redhat.io/rhtas/client-server-rhel9#quay.io/securesign/client-server#
+s#registry.redhat.io/rhtas/rekor-backfill-redis-rhel9#quay.io/securesign/rekor-backfill-redis#