add-backfill-redis-job

roles/tas_single_node/defaults/main.yml

@@ -7,10 +7,19 @@ tas_single_node_podman_network: rhtas
 
 tas_single_node_rekor_redis:
   database_deploy: true
+  backfill_redis: true
   redis:
     host: rekor-redis-pod
     port: 6379
     password: password
+  backfill_redis:
+    enabled: true
+    schedule:
+      minute: "0"
+      hour: "0"
+      day: "*"
+      month: "*"
+      weekday: "*"
 
 tas_single_node_trillian:
   database_deploy: true
@@ -65,6 +74,8 @@ tas_single_node_ctlog_image:
   "registry.redhat.io/rhtas/certificate-transparency-rhel9@sha256:31e7318a9b19ed04ef0f25949f1f1709d293b532316b27a06f83fa5174547b17"
 tas_single_node_rekor_redis_image:
   "registry.redhat.io/rhtas/trillian-redis-rhel9@sha256:18820b1fbdbc2cc3e917822974910332d937b03cfe781628bd986fd6a5ee318e"
+tas_single_node_backfill_redis_image:
+  "registry.redhat.io/rhtas/rekor-backfill-redis-rhel9@sha256:c5995c88063bd9875ae61c299bcf549002fcde724aab09807c70934e73daf356"
 tas_single_node_trillian_db_image:
   "registry.redhat.io/rhtas/trillian-database-rhel9@sha256:501612745e63e5504017079388bec191ffacf00ffdebde7be6ca5b8e4fd9d323"
 tas_single_node_tuf_image:

roles/tas_single_node/meta/argument_specs.yml

@@ -35,6 +35,14 @@ argument_specs:
             host: rekor-redis-pod
             port: 6379
             password: password
+          backfill_redis:
+            enabled: true
+            schedule:
+              minute: "0"
+              hour: "0"
+              day: "*"
+              month: "*"
+              weekday: "*"
         options:
           database_deploy:
             description: "Whether or not to deploy Redis."
@@ -62,6 +70,53 @@ argument_specs:
                 type: "str"
                 required: false
                 version_added: "1.1.0"
+          backfill_redis:
+            description: "Backfill Redis configuration"
+            type: "dict"
+            required: false
+            version_added: "1.1.1"
+            options:
+              enabled:
+                description: "Whether or not to deploy the backfill Redis cron job."
+                type: "bool"
+                required: false
+                version_added: "1.1.1"
+              schedule:
+                description: "Cron schedule for the backfill Redis job."
+                type: "dict"
+                required: false
+                version_added: "1.1.1"
+                options:
+                  minute:
+                    description: "Specifies the minute(s) of each hour when the cron job should run (0-59)."
+                    type: "str"
+                    required: false
+                    default: "*"
+                    version_added: "1.1.1"
+                  hour:
+                    description: "Specifies the hour(s) of each day when the cron job should run (0-23)."
+                    type: "str"
+                    required: false
+                    default: "*"
+                    version_added: "1.1.1"
+                  day:
+                    description: "Specifies the day(s) of the month when the cron job should run (1-31)."
+                    type: "str"
+                    required: false
+                    default: "*"
+                    version_added: "1.1.1"
+                  month:
+                    description: "Specifies the month(s) when the cron job should run (1-12 or month names)."
+                    type: "str"
+                    required: false
+                    default: "*"
+                    version_added: "1.1.1"
+                  weekday:
+                    description: "Specifies the day(s) of the week when the cron job should run (0-6, where 0=Sunday)."
+                    type: "str"
+                    required: false
+                    default: "*"
+                    version_added: "1.1.1"
       tas_single_node_trillian:
         description: "Details on the database connection for Trillian. You can set this to a custom MySQL or MariaDB instance."
         type: "dict"

roles/tas_single_node/tasks/podman.yml

@@ -45,6 +45,7 @@
       "{{ tas_single_node_rekor_enabled }}",
       "{{ tas_single_node_ctlog_enabled }}",
       "{{ tas_single_node_rekor_enabled and tas_single_node_rekor_redis.database_deploy }}",
+      "{{ tas_single_node_rekor_enabled and tas_single_node_rekor_redis.database_deploy and tas_single_node_rekor_redis.backfill_redis.enabled }}",
       "{{ tas_single_node_trillian_enabled and tas_single_node_trillian.database_deploy }}",
       "{{ tas_single_node_tuf_enabled }}",
       "{{ tas_single_node_trillian_enabled }}",
@@ -61,6 +62,7 @@
     - "{{ tas_single_node_rekor_server_image }}"
     - "{{ tas_single_node_ctlog_image }}"
     - "{{ tas_single_node_rekor_redis_image }}"
+    - "{{ tas_single_node_backfill_redis_image }}"
     - "{{ tas_single_node_trillian_db_image }}"
     - "{{ tas_single_node_tuf_image }}"
     - "{{ tas_single_node_trillian_netcat_image }}"

roles/tas_single_node/tasks/podman/rekor.yml

@@ -67,3 +67,33 @@
       configmap_changed: "{{ configmap_result.changed }}"
       secret: "{{ tas_single_node_rekor_secret }}"
       secret_changed: "{{ secret_result.changed }}"
+
+- name: Get RHTAS network details
+  ansible.builtin.command:
+    cmd: "podman network inspect {{ tas_single_node_podman_network }}"
+  register: tas_podman_network_results
+  changed_when: false
+  when: tas_single_node_rekor_redis.backfill_redis.enabled
+
+- name: Set DNS Resolver
+  ansible.builtin.set_fact:
+    dns_resolver: "{{ (tas_podman_network_results.stdout | from_json)[0].subnets[0].gateway }}"
+  when: tas_single_node_rekor_redis.backfill_redis.enabled
+
+- name: Deploy backfill_redis script
+  ansible.builtin.template:
+    src: manifests/rekor/backfill_redis.sh.j2
+    dest: /usr/local/bin/backfill_redis.sh
+    mode: '0755'
+  when: tas_single_node_rekor_redis.backfill_redis.enabled
+
+- name: Create Backfill Redis Cronjob
+  ansible.builtin.cron:
+    name: "backfill_redis"
+    minute: "{{ tas_single_node_rekor_redis.backfill_redis.schedule.minute }}"
+    hour: "{{ tas_single_node_rekor_redis.backfill_redis.schedule.hour }}"
+    day: "{{ tas_single_node_rekor_redis.backfill_redis.schedule.day }}"
+    month: "{{ tas_single_node_rekor_redis.backfill_redis.schedule.month }}"
+    weekday: "{{ tas_single_node_rekor_redis.backfill_redis.schedule.weekday }}"
+    job: "/usr/local/bin/backfill_redis.sh"
+  when: tas_single_node_rekor_redis.backfill_redis.enabled

roles/tas_single_node/templates/manifests/rekor/backfill_redis.sh.j2

@@ -0,0 +1,38 @@
+#!/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+LAST_INDEX_FILE="/tmp/rekor_last_index"
+
+endIndex=$(curl -sS http://localhost:3001/api/v1/log | jq -r '.treeSize - 1')
+if [ "$endIndex" -lt 0 ]; then
+  echo "info: no rekor entries found"
+  exit 0
+fi
+
+if [ -f "$LAST_INDEX_FILE" ]; then
+  startIndex=$(cat "$LAST_INDEX_FILE")
+else
+  startIndex=0
+fi
+
+if [ "$startIndex" -gt "$endIndex" ]; then
+  startIndex=0
+fi
+
+if [ "$startIndex" -le "$endIndex" ]; then
+  podman run --rm --name backfill_redis \
+    -v "{{ tas_single_node_certs_dir }}/ingress-rekor.pem:/usr/local/share/ca-certificates/ingress-rekor.pem:ro" \
+    -e SSL_CERT_FILE=/usr/local/share/ca-certificates/ingress-rekor.pem \
+    --add-host="rekor.{{ tas_single_node_base_hostname }}:{{ dns_resolver }}" \
+    --add-host="{{ tas_single_node_rekor_redis.redis.host }}:{{ dns_resolver }}" \
+    "{{ tas_single_node_backfill_redis_image }}" \
+    --hostname="{{ tas_single_node_rekor_redis.redis.host }}" \
+    --password="{{ tas_single_node_rekor_redis.redis.password }}" \
+    --port="{{ tas_single_node_rekor_redis.redis.port }}" \
+    --rekor-address="http://rekor.{{ tas_single_node_base_hostname }}" \
+    --start="$startIndex" --end="$endIndex"
+
+  echo "$endIndex" > "$LAST_INDEX_FILE"
+else
+  echo "info: no new entries to process"
+fi

vm-testing/dev-images.sed

@@ -5,7 +5,6 @@ s#registry.redhat.io/rhtas/fulcio-rhel9#quay.io/securesign/fulcio-server#
 s#registry.redhat.io/rhtas/trillian-redis-rhel9#quay.io/securesign/trillian-redis#
 s#registry.redhat.io/rhtas/rekor-server-rhel9#quay.io/securesign/rekor-server#
 s#registry.redhat.io/rhtas/rekor-search-ui-rhel9#quay.io/securesign/rekor-search-ui#
-s#registry.redhat.io/rhtas/rekor-backfill-redis-rhel9#quay.io/securesign/trillian-redis#
 s#registry.redhat.io/rhtas/tuf-server-rhel9#quay.io/securesign/scaffold-tuf-server#
 s#registry.redhat.io/rhtas/certificate-transparency-rhel9#quay.io/securesign/certificate-transparency-go#
 s#registry.redhat.io/rhtas/client-server-cg-rhel9#quay.io/securesign/cli-client-server-cg#
@@ -14,3 +13,4 @@ s#registry.redhat.io/rhtas/segment-reporting-rhel9#quay.io/securesign/segment-ba
 s#registry.redhat.io/rhtas/timestamp-authority-rhel9#quay.io/securesign/timestamp-authority#
 s#registry.redhat.io/rhtas/trillian-createtree-rhel9#quay.io/securesign/trillian-createtree#
 s#registry.redhat.io/rhtas/client-server-rhel9#quay.io/securesign/client-server#
+s#registry.redhat.io/rhtas/rekor-backfill-redis-rhel9#quay.io/securesign/rekor-backfill-redis#