add-backfill-redis-job

roles/tas_single_node/README.md

@@ -19,6 +19,7 @@ Deploy the [RHTAS](https://docs.redhat.com/en/documentation/red_hat_trusted_arti
 |---|---|---|---|
 | tas_single_node_podman_network | Name of the Podman network for containers to use. | str |  `rhtas`  |
 | tas_single_node_rekor_redis | Details on the Redis connection for Rekor. You can set this to a custom Redis instance. | dict of 'tas_single_node_rekor_redis' options |  `{'database_deploy': True, 'redis': {'host': 'rekor-redis-pod', 'port': 6379, 'password': 'password'}}`  |
+| tas_single_node_backfill_redis_enabled | Enable or disable the backfill redis job | bool |  `True`  |
 | tas_single_node_trillian | Details on the database connection for Trillian. You can set this to a custom MySQL or MariaDB instance. | dict of 'tas_single_node_trillian' options |  `{'database_deploy': True, 'mysql': {'user': 'mysql', 'root_password': 'rootpassword', 'password': 'password', 'database': 'trillian', 'host': 'trillian-mysql-pod', 'port': 3306}}`  |
 | tas_single_node_rekor_public_key_retries | The number of attempts to retrieve the Rekor public key when constructing the trust root. | int |  `5`  |
 | tas_single_node_rekor_public_key_delay | The number of seconds to wait before retrying the retrieval of the Rekor public key when constructing the trust root. | int |  `10`  |

roles/tas_single_node/defaults/main.yml

@@ -12,6 +12,8 @@ tas_single_node_rekor_redis:
     port: 6379
     password: password
 
+tas_single_node_backfill_redis_enabled: true
+
 tas_single_node_trillian:
   database_deploy: true
   mysql:
@@ -66,6 +68,8 @@ tas_single_node_ctlog_image:
   "registry.redhat.io/rhtas/certificate-transparency-rhel9@sha256:31e7318a9b19ed04ef0f25949f1f1709d293b532316b27a06f83fa5174547b17"
 tas_single_node_rekor_redis_image:
   "registry.redhat.io/rhtas/trillian-redis-rhel9@sha256:18820b1fbdbc2cc3e917822974910332d937b03cfe781628bd986fd6a5ee318e"
+tas_single_node_backfill_redis_image:
+  "registry.redhat.io/rhtas/rekor-backfill-redis-rhel9@sha256:c5995c88063bd9875ae61c299bcf549002fcde724aab09807c70934e73daf356"
 tas_single_node_trillian_db_image:
   "registry.redhat.io/rhtas/trillian-database-rhel9@sha256:501612745e63e5504017079388bec191ffacf00ffdebde7be6ca5b8e4fd9d323"
 tas_single_node_tuf_image:

roles/tas_single_node/meta/argument_specs.yml

@@ -62,6 +62,12 @@ argument_specs:
                 type: "str"
                 required: false
                 version_added: "1.1.0"
+      tas_single_node_backfill_redis_enabled:
+        description: "Enable or disable the backfill redis job"
+        type: "bool"
+        required: false
+        version_added: "1.1.1"
+        default: true
       tas_single_node_trillian:
         description: "Details on the database connection for Trillian. You can set this to a custom MySQL or MariaDB instance."
         type: "dict"

roles/tas_single_node/tasks/podman.yml

@@ -45,6 +45,7 @@
       "{{ tas_single_node_rekor_enabled }}",
       "{{ tas_single_node_ctlog_enabled }}",
       "{{ tas_single_node_rekor_enabled and tas_single_node_rekor_redis.database_deploy }}",
+      "{{ tas_single_node_rekor_enabled and tas_single_node_rekor_redis.database_deploy and tas_single_node_backfill_redis_enabled }}",
       "{{ tas_single_node_trillian_enabled and tas_single_node_trillian.database_deploy }}",
       "{{ tas_single_node_tuf_enabled }}",
       "{{ tas_single_node_trillian_enabled }}",
@@ -61,6 +62,7 @@
     - "{{ tas_single_node_rekor_server_image }}"
     - "{{ tas_single_node_ctlog_image }}"
     - "{{ tas_single_node_rekor_redis_image }}"
+    - "{{ tas_single_node_backfill_redis_image }}"
     - "{{ tas_single_node_trillian_db_image }}"
     - "{{ tas_single_node_tuf_image }}"
     - "{{ tas_single_node_trillian_netcat_image }}"

roles/tas_single_node/tasks/podman/rekor.yml

@@ -69,3 +69,29 @@
       configmap_changed: "{{ configmap_result.changed }}"
       secret: "{{ tas_single_node_rekor_secret }}"
       secret_changed: "{{ secret_result.changed }}"
+
+- name: Deploy backfill Redis job
+  ansible.builtin.include_tasks: podman/install_manifest.yml
+  vars:
+    podman_spec:
+      state: started
+      systemd_file: backfill_redis
+      network: "{{ tas_single_node_podman_network }}"
+      kube_file_content: "{{ lookup('template', 'manifests/rekor/backfill_redis.j2') | from_yaml }}"
+  when: tas_single_node_backfill_redis_enabled
+
+- name: Copy backfill_redis.timer file to server
+  ansible.builtin.template:
+    src: systemd/backfill_redis.timer.j2
+    dest: "{{ tas_single_node_systemd_directory }}/backfill_redis.timer"
+    mode: "0644"
+  when: tas_single_node_backfill_redis_enabled
+  register: copy_timer_systemd_file
+
+- name: Enable and start backfill_redis.timer
+  ansible.builtin.systemd:
+    name: backfill_redis.timer
+    enabled: true
+    state: started
+    daemon_reload: "{{ copy_timer_systemd_file.changed }}"
+  when: tas_single_node_backfill_redis_enabled

roles/tas_single_node/templates/manifests/rekor/backfill_redis.j2

@@ -0,0 +1,44 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: backfill-redis
+  namespace: backfill-redis
+  labels:
+    app.component: backfill-redis
+    app.instance: backfill-redis
+    app.name: backfill-redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.component: backfill-redis
+      app.instance: backfill-redis
+      app.name: backfill-redis
+  template:
+    metadata:
+      labels:
+        app.component: backfill-redis
+        app.instance: backfill-redis
+        app.name: backfill-redis
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: backfill-redis
+          image: "{{ tas_single_node_backfill_redis_image }}"
+          command:
+            - /bin/sh
+            - -c
+          args:
+            - >
+              endIndex=$(curl -sS http://{{ tas_single_node_rekor_server_pod }}-pod:{{ tas_single_node_rekor_server_port_http }}/api/v1/log | sed -E 's/.*"treeSize":([0-9]+).*/\1/');
+              endIndex=$((endIndex-1));
+              if [ "${endIndex}" -lt 0 ]; then
+                echo "info: no rekor entries found";
+                exit 0;
+              fi;
+              backfill-redis 
+              --hostname={{ tas_single_node_rekor_redis.redis.host }}
+              --port={{ tas_single_node_rekor_redis.redis.port }}
+              --password="{{ tas_single_node_rekor_redis.redis.password }}"
+              --rekor-address=http://{{ tas_single_node_rekor_server_pod }}-pod:{{ tas_single_node_rekor_server_port_http }}
+              --enable-redis-index-resume=true --end=${endIndex};

roles/tas_single_node/templates/systemd/backfill_redis.timer.j2

@@ -0,0 +1,10 @@
+[Unit]
+Description=Runs The backfill redis job
+
+[Timer]
+OnCalendar=*-*-* 00:00:00
+Persistent=true
+Unit=backfill_redis.service
+
+[Install]
+WantedBy=timers.target

vm-testing/dev-images.sed

@@ -5,7 +5,6 @@ s#registry.redhat.io/rhtas/fulcio-rhel9#quay.io/securesign/fulcio-server#
 s#registry.redhat.io/rhtas/trillian-redis-rhel9#quay.io/securesign/trillian-redis#
 s#registry.redhat.io/rhtas/rekor-server-rhel9#quay.io/securesign/rekor-server#
 s#registry.redhat.io/rhtas/rekor-search-ui-rhel9#quay.io/securesign/rekor-search-ui#
-s#registry.redhat.io/rhtas/rekor-backfill-redis-rhel9#quay.io/securesign/trillian-redis#
 s#registry.redhat.io/rhtas/tuf-server-rhel9#quay.io/securesign/scaffold-tuf-server#
 s#registry.redhat.io/rhtas/certificate-transparency-rhel9#quay.io/securesign/certificate-transparency-go#
 s#registry.redhat.io/rhtas/client-server-cg-rhel9#quay.io/securesign/cli-client-server-cg#
@@ -14,3 +13,4 @@ s#registry.redhat.io/rhtas/segment-reporting-rhel9#quay.io/securesign/segment-ba
 s#registry.redhat.io/rhtas/timestamp-authority-rhel9#quay.io/securesign/timestamp-authority#
 s#registry.redhat.io/rhtas/trillian-createtree-rhel9#quay.io/securesign/trillian-createtree#
 s#registry.redhat.io/rhtas/client-server-rhel9#quay.io/securesign/client-server#
+s#registry.redhat.io/rhtas/rekor-backfill-redis-rhel9#quay.io/securesign/rekor-backfill-redis#