[COST-5170] - POC on XL manifest reports

project-koku · Dec 3, 2024 · 678f71f · 678f71f
1 parent 3d8ded0
commit 678f71f
Show file tree

Hide file tree

Showing 30 changed files with 170 additions and 3 deletions.
diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml
@@ -683,6 +683,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -868,6 +870,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -1353,6 +1357,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -1535,6 +1541,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -1723,6 +1731,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -1911,6 +1921,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -2101,6 +2113,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -2293,6 +2307,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -2485,6 +2501,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -2675,6 +2693,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -2865,6 +2885,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -3055,6 +3077,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -3247,6 +3271,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -3443,6 +3469,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -3639,6 +3667,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -3833,6 +3863,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -4021,6 +4053,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -4209,6 +4243,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -4397,6 +4433,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -4589,6 +4627,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -4781,6 +4821,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -4975,6 +5017,8 @@ objects:
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER
@@ -5895,6 +5939,10 @@ parameters:
 - displayName: Provider count for batch polling
   name: POLLING_BATCH_SIZE
   value: "100"
+- description: Minimum number of reports before provider classed as XL
+  displayName: Min report count for large providers
+  name: XL_REPORT_COUNT
+  value: "100"
 - displayName: Timer to prevent triggering tasks while still processing in days
   name: PROCESSING_WAIT_TIMER
   value: "3"

diff --git a/deploy/kustomize/base/base.yaml b/deploy/kustomize/base/base.yaml
@@ -531,6 +531,10 @@ parameters:
 - displayName: Provider count for batch polling
   name: POLLING_BATCH_SIZE
   value: "100"
+- description: Minimum number of reports before provider classed as XL
+  displayName: Min report count for large providers
+  name: XL_REPORT_COUNT
+  value: "100"
 - displayName: Timer to prevent triggering tasks while still processing in days
   name: PROCESSING_WAIT_TIMER
   value: "3"

diff --git a/deploy/kustomize/patches/masu.yaml b/deploy/kustomize/patches/masu.yaml
@@ -106,6 +106,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/scheduler.yaml b/deploy/kustomize/patches/scheduler.yaml
@@ -103,6 +103,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-celery.yaml b/deploy/kustomize/patches/worker-celery.yaml
@@ -93,6 +93,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-cost-model-penalty.yaml b/deploy/kustomize/patches/worker-cost-model-penalty.yaml
@@ -93,6 +93,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-cost-model-xl.yaml b/deploy/kustomize/patches/worker-cost-model-xl.yaml
@@ -93,6 +93,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-cost-model.yaml b/deploy/kustomize/patches/worker-cost-model.yaml
@@ -93,6 +93,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-download-penalty.yaml b/deploy/kustomize/patches/worker-download-penalty.yaml
@@ -95,6 +95,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-download-xl.yaml b/deploy/kustomize/patches/worker-download-xl.yaml
@@ -95,6 +95,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-download.yaml b/deploy/kustomize/patches/worker-download.yaml
@@ -95,6 +95,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-hcs.yaml b/deploy/kustomize/patches/worker-hcs.yaml
@@ -95,6 +95,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-ocp-penalty.yaml b/deploy/kustomize/patches/worker-ocp-penalty.yaml
@@ -93,6 +93,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-ocp-xl.yaml b/deploy/kustomize/patches/worker-ocp-xl.yaml
@@ -93,6 +93,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-ocp.yaml b/deploy/kustomize/patches/worker-ocp.yaml
@@ -93,6 +93,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-priority-penalty.yaml b/deploy/kustomize/patches/worker-priority-penalty.yaml
@@ -95,6 +95,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-priority-xl.yaml b/deploy/kustomize/patches/worker-priority-xl.yaml
@@ -95,6 +95,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-priority.yaml b/deploy/kustomize/patches/worker-priority.yaml
@@ -95,6 +95,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-refresh-penalty.yaml b/deploy/kustomize/patches/worker-refresh-penalty.yaml
@@ -93,6 +93,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-refresh-xl.yaml b/deploy/kustomize/patches/worker-refresh-xl.yaml
@@ -93,6 +93,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-refresh.yaml b/deploy/kustomize/patches/worker-refresh.yaml
@@ -93,6 +93,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER

diff --git a/deploy/kustomize/patches/worker-summary-penalty.yaml b/deploy/kustomize/patches/worker-summary-penalty.yaml
@@ -93,6 +93,8 @@
           value: ${POLLING_TIMER}
         - name: POLLING_BATCH_SIZE
           value: ${POLLING_BATCH_SIZE}
+        - name: XL_REPORT_COUNT
+          value: ${XL_REPORT_COUNT}
         - name: PROCESSING_WAIT_TIMER
           value: ${PROCESSING_WAIT_TIMER}
         - name: LARGE_PROCESSING_WAIT_TIMER