From bf08f39c2500a7e1542755d1070f76a43ea016f7 Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Wed, 18 Dec 2024 15:48:58 +0100
Subject: [PATCH 01/19] [opensearch-logs] adding audit user for audit logs

---
 .../templates/config/_internal_users.yml.tpl  |  6 ++++++
 .../templates/config/_roles.yml.tpl           | 20 +++++++++++++++++++
 .../templates/config/_roles_mapping.yml.tpl   |  5 +++++
 3 files changed, 31 insertions(+)

diff --git a/system/opensearch-logs/templates/config/_internal_users.yml.tpl b/system/opensearch-logs/templates/config/_internal_users.yml.tpl
index 501f7da3d8..08ce9b9af4 100644
--- a/system/opensearch-logs/templates/config/_internal_users.yml.tpl
+++ b/system/opensearch-logs/templates/config/_internal_users.yml.tpl
@@ -38,6 +38,12 @@ otel:
   backend_roles:
   - "otel"
 
+audit:
+  hash: "{{ .Values.users.audit.nohash }}"
+  reserved: true
+  backend_roles:
+  - "audit"
+
 otellogs:
   hash: "{{ .Values.users.otellogs.nohash }}"
   reserved: true
diff --git a/system/opensearch-logs/templates/config/_roles.yml.tpl b/system/opensearch-logs/templates/config/_roles.yml.tpl
index b94137542a..0c8d668819 100644
--- a/system/opensearch-logs/templates/config/_roles.yml.tpl
+++ b/system/opensearch-logs/templates/config/_roles.yml.tpl
@@ -249,6 +249,26 @@ compute:
     - "indices:admin/create"
     - "indices:data/write/bulk*"
     - "indices:data/write/index"
+audit:
+  reserved: false
+  cluster_permissions:
+  - "cluster_monitor"
+  - "cluster_composite_ops"
+  - "cluster:admin/ingest/pipeline/put"
+  - "cluster:admin/ingest/pipeline/get"
+  - "indices:admin/template/get"
+  - "cluster_manage_index_templates"
+  - "cluster:admin/opensearch/ml/predict"
+  index_permissions:
+  - index_patterns:
+    - "audit-*"
+    allowed_actions:
+    - "indices:admin/template/get"
+    - "indices:admin/template/put"
+    - "indices:admin/mapping/put"
+    - "indices:admin/create"
+    - "indices:data/write/bulk*"
+    - "indices:data/write/index"
 otel:
   reserved: false
   cluster_permissions:
diff --git a/system/opensearch-logs/templates/config/_roles_mapping.yml.tpl b/system/opensearch-logs/templates/config/_roles_mapping.yml.tpl
index f7177dd9b0..4e25c9be65 100644
--- a/system/opensearch-logs/templates/config/_roles_mapping.yml.tpl
+++ b/system/opensearch-logs/templates/config/_roles_mapping.yml.tpl
@@ -32,6 +32,11 @@ greenhouse:
   users:
   - "greenhouse"
 
+audit:
+  reserved: false
+  users:
+  - "audit"
+
 jump:
   reserved: false
   users:

From 7fde4d8c83213895b0ba4981b872a2c06f5a4318 Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Fri, 20 Dec 2024 10:25:11 +0100
Subject: [PATCH 02/19] [opensearch-logs] testing new index pattern names

---
 .../config/_install-index-pattern.sh.tpl      | 61 ++++++++++++++++---
 1 file changed, 52 insertions(+), 9 deletions(-)

diff --git a/system/opensearch-logs/templates/config/_install-index-pattern.sh.tpl b/system/opensearch-logs/templates/config/_install-index-pattern.sh.tpl
index a5e199bb08..016cc0d3f9 100644
--- a/system/opensearch-logs/templates/config/_install-index-pattern.sh.tpl
+++ b/system/opensearch-logs/templates/config/_install-index-pattern.sh.tpl
@@ -1,16 +1,59 @@
 #!/bin/bash
+export BASIC_AUTH_HEADER=${ADMIN_USER}:${ADMIN_PASSWORD}
 
-# 0. Check for index policy
-for i in $(curl -s -u ${ADMIN_USER}:${ADMIN_PASSWORD} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|awk -F- '{ print $1 }'|sort|uniq|grep -v "\."|grep -v "index")
+
+
+#  Creating aliases for all indexes, because logstash-* is also selecting datastreams besides the logstash-2024... indexes.
+for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|awk -F- '{ print $1 }'|sort|uniq|grep -v "\."|grep -v "index")
   do
+    #Creating an alias for all standard indexes, which are not datastreams to mitigate the issue with indexes, where for example storage-* is selecting the index and also the datastream, which shows up in dashboards as duplicate entries
     echo "using index $i from Opensearch-Logs"
+    export ALIAS_EXISTS=`curl -s -i -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases/${i}"|grep "content-length"|awk -F: '{ print $2 }'|tr -d '[:space:]'`
+    if [[ "$ALIAS_EXISTS" -gt 0 ]]
+     then
+      echo "Alias and dashboard index pattern for index ${i} already exists. Nothing to do."
+    else
+      echo "setting OpenSearch dashboard index mapping for index $i"
+      curl -s -XPOST --header "content-type: application/JSON" -u  ${BASIC_AUTH_HEADER}"${CLUSTER_HOST}/_aliases" -H "osd-xsrf: true" -d "{ \"actions\": [ { \"add\": { \"index\": \"${i}-2*\", \"alias\": \"${i}\" } } ] }"
+    fi
+    echo "Deleting old index pattern based on index-* format"
+    export DASHBOARD_PATTERN=`curl -s --header "content-type: application/JSON" --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*"|grep "content-length"|awk -F: '{ print $2 }'|tr -d '[:space:]'`
+    if [[ "$DASHBOARD_PATTERN" -gt 0 ]]
+    then
+      echo "Old dashboard pattern exists for for index ${i}, it will be removed"
+      curl -s -XDELETE -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*"
+    else
+     echo "No old dashboard pattern for index $i"
+    fi
+done
+
+# Dashboard index pattern for all available aliases, which are not datastreams
+for i in $(curl -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases?v"|grep -v "\-ds"|grep -v "^\."|awk '{ print $1 }'|uniq)
+  do
+    echo "using alias $i from Opensearch-Logs"
+    echo "Setting OpenSearch dashboard index mapping for alias $i"
+    curl -s --header "content-type: application/JSON" --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}"
+    if [ $? -eq 0 ]
+    then
+      echo "index pattern for alias ${i} already exists in Opensearch dashboard, nothing to do"
+    else
+      echo "INFO: creating index-pattern in Dashboards for datastream alias $i"
+      curl -s -XPOST --header "content-type: application/JSON" -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}" -H "osd-xsrf: true" -d "{ \"attributes\": { \"title\": \"${i}\", \"timeFieldName\": \"@timestamp\" } }"
+    fi
+done
+
+
+# Dashboard index pattern for all available datastreams
+for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases?v"|grep "\-ds"|awk '{ print $1 }'|uniq)
+  do
+    echo "using datastream $i from Opensearch-Logs"
     echo "setting OpenSearch dashboard index mapping for index $i"
-    curl --header "content-type: application/JSON" --fail -XGET -u ${ADMIN_USER}:${ADMIN_PASSWORD} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*"
-  if [ $? -eq 0 ]
+    curl -s --header "content-type: application/JSON" --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}"
+    if [ $? -eq 0 ]
     then
-      echo "index ${i} already exists in Opensearch dashboard"
-  else
-    echo "INFO: creating index-pattern in Dashboards for $i logs"
-    curl -XPOST --header "content-type: application/JSON" -u ${ADMIN_USER}:${ADMIN_PASSWORD} "https://${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*" -H "osd-xsrf: true" -d "{ \"attributes\": { \"title\": \"${i}-*\", \"timeFieldName\": \"@timestamp\" } }"
-  fi
+        echo "index ${i} already exists in Opensearch dashboard"
+    else
+      echo "INFO: creating index-pattern in Dashboards for datastream alias $i"
+      curl -s -XPOST --header "content-type: application/JSON" -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}" -H "osd-xsrf: true" -d "{ \"attributes\": { \"title\": \"${i}\", \"timeFieldName\": \"@timestamp\" } }"
+    fi
 done

From f6851dfe1369180334230f781474d1ae8aa8a56c Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Fri, 20 Dec 2024 10:54:21 +0100
Subject: [PATCH 03/19] [opensearch-logs] remove unused config

---
 .../config/_install-dashboard-pattern.sh.tpl  | 54 +++++++++++++----
 .../config/_install-index-pattern.sh.tpl      | 59 -------------------
 .../install-dashboard-pattern-job.yaml        |  1 -
 3 files changed, 42 insertions(+), 72 deletions(-)
 delete mode 100644 system/opensearch-logs/templates/config/_install-index-pattern.sh.tpl

diff --git a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
index 350074ecb1..016cc0d3f9 100644
--- a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
+++ b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
@@ -1,29 +1,59 @@
 #!/bin/bash
+export BASIC_AUTH_HEADER=${ADMIN_USER}:${ADMIN_PASSWORD}
 
-# 0. Check for index policy
-for i in $(curl -s -u ${ADMIN_USER}:${ADMIN_PASSWORD} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|awk -F- '{ print $1 }'|sort|uniq|grep -v "\."|grep -v "index")
+
+
+#  Creating aliases for all indexes, because logstash-* is also selecting datastreams besides the logstash-2024... indexes.
+for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|awk -F- '{ print $1 }'|sort|uniq|grep -v "\."|grep -v "index")
   do
+    #Creating an alias for all standard indexes, which are not datastreams to mitigate the issue with indexes, where for example storage-* is selecting the index and also the datastream, which shows up in dashboards as duplicate entries
     echo "using index $i from Opensearch-Logs"
-    echo "setting OpenSearch dashboard index mapping for index $i"
-    curl --header "content-type: application/JSON" --fail -XGET -u ${ADMIN_USER}:${ADMIN_PASSWORD} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*"
-    if [ $? -eq 0 ]; then
-       echo "index ${i} already exists in Opensearch dashboard"
+    export ALIAS_EXISTS=`curl -s -i -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases/${i}"|grep "content-length"|awk -F: '{ print $2 }'|tr -d '[:space:]'`
+    if [[ "$ALIAS_EXISTS" -gt 0 ]]
+     then
+      echo "Alias and dashboard index pattern for index ${i} already exists. Nothing to do."
+    else
+      echo "setting OpenSearch dashboard index mapping for index $i"
+      curl -s -XPOST --header "content-type: application/JSON" -u  ${BASIC_AUTH_HEADER}"${CLUSTER_HOST}/_aliases" -H "osd-xsrf: true" -d "{ \"actions\": [ { \"add\": { \"index\": \"${i}-2*\", \"alias\": \"${i}\" } } ] }"
+    fi
+    echo "Deleting old index pattern based on index-* format"
+    export DASHBOARD_PATTERN=`curl -s --header "content-type: application/JSON" --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*"|grep "content-length"|awk -F: '{ print $2 }'|tr -d '[:space:]'`
+    if [[ "$DASHBOARD_PATTERN" -gt 0 ]]
+    then
+      echo "Old dashboard pattern exists for for index ${i}, it will be removed"
+      curl -s -XDELETE -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*"
+    else
+     echo "No old dashboard pattern for index $i"
+    fi
+done
+
+# Dashboard index pattern for all available aliases, which are not datastreams
+for i in $(curl -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases?v"|grep -v "\-ds"|grep -v "^\."|awk '{ print $1 }'|uniq)
+  do
+    echo "using alias $i from Opensearch-Logs"
+    echo "Setting OpenSearch dashboard index mapping for alias $i"
+    curl -s --header "content-type: application/JSON" --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}"
+    if [ $? -eq 0 ]
+    then
+      echo "index pattern for alias ${i} already exists in Opensearch dashboard, nothing to do"
     else
-      echo "INFO: creating index-pattern in Dashboards for $i logs"
-      curl -XPOST --header "content-type: application/JSON" -u ${ADMIN_USER}:${ADMIN_PASSWORD} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*" -H "osd-xsrf: true" -d "{ \"attributes\": { \"title\": \"${i}-*\", \"timeFieldName\": \"@timestamp\" } }"
+      echo "INFO: creating index-pattern in Dashboards for datastream alias $i"
+      curl -s -XPOST --header "content-type: application/JSON" -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}" -H "osd-xsrf: true" -d "{ \"attributes\": { \"title\": \"${i}\", \"timeFieldName\": \"@timestamp\" } }"
     fi
 done
 
+
 # Dashboard index pattern for all available datastreams
-for i in $(curl -s -u ${ADMIN_USER}:${ADMIN_PASSWORD} "${CLUSTER_HOST}/_cat/aliases?v"|grep "\-ds"|awk '{ print $1 }'|uniq)
+for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases?v"|grep "\-ds"|awk '{ print $1 }'|uniq)
   do
     echo "using datastream $i from Opensearch-Logs"
     echo "setting OpenSearch dashboard index mapping for index $i"
-    curl --header "content-type: application/JSON" --fail -XGET -u ${ADMIN_USER}:${ADMIN_PASSWORD} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}"
-    if [ $? -eq 0 ]; then
+    curl -s --header "content-type: application/JSON" --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}"
+    if [ $? -eq 0 ]
+    then
         echo "index ${i} already exists in Opensearch dashboard"
     else
       echo "INFO: creating index-pattern in Dashboards for datastream alias $i"
-      curl -XPOST --header "content-type: application/JSON" -u ${ADMIN_USER}:${ADMIN_PASSWORD} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}" -H "osd-xsrf: true" -d "{ \"attributes\": { \"title\": \"${i}\", \"timeFieldName\": \"@timestamp\" } }"
+      curl -s -XPOST --header "content-type: application/JSON" -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}" -H "osd-xsrf: true" -d "{ \"attributes\": { \"title\": \"${i}\", \"timeFieldName\": \"@timestamp\" } }"
     fi
 done
diff --git a/system/opensearch-logs/templates/config/_install-index-pattern.sh.tpl b/system/opensearch-logs/templates/config/_install-index-pattern.sh.tpl
deleted file mode 100644
index 016cc0d3f9..0000000000
--- a/system/opensearch-logs/templates/config/_install-index-pattern.sh.tpl
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/bin/bash
-export BASIC_AUTH_HEADER=${ADMIN_USER}:${ADMIN_PASSWORD}
-
-
-
-#  Creating aliases for all indexes, because logstash-* is also selecting datastreams besides the logstash-2024... indexes.
-for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|awk -F- '{ print $1 }'|sort|uniq|grep -v "\."|grep -v "index")
-  do
-    #Creating an alias for all standard indexes, which are not datastreams to mitigate the issue with indexes, where for example storage-* is selecting the index and also the datastream, which shows up in dashboards as duplicate entries
-    echo "using index $i from Opensearch-Logs"
-    export ALIAS_EXISTS=`curl -s -i -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases/${i}"|grep "content-length"|awk -F: '{ print $2 }'|tr -d '[:space:]'`
-    if [[ "$ALIAS_EXISTS" -gt 0 ]]
-     then
-      echo "Alias and dashboard index pattern for index ${i} already exists. Nothing to do."
-    else
-      echo "setting OpenSearch dashboard index mapping for index $i"
-      curl -s -XPOST --header "content-type: application/JSON" -u  ${BASIC_AUTH_HEADER}"${CLUSTER_HOST}/_aliases" -H "osd-xsrf: true" -d "{ \"actions\": [ { \"add\": { \"index\": \"${i}-2*\", \"alias\": \"${i}\" } } ] }"
-    fi
-    echo "Deleting old index pattern based on index-* format"
-    export DASHBOARD_PATTERN=`curl -s --header "content-type: application/JSON" --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*"|grep "content-length"|awk -F: '{ print $2 }'|tr -d '[:space:]'`
-    if [[ "$DASHBOARD_PATTERN" -gt 0 ]]
-    then
-      echo "Old dashboard pattern exists for for index ${i}, it will be removed"
-      curl -s -XDELETE -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*"
-    else
-     echo "No old dashboard pattern for index $i"
-    fi
-done
-
-# Dashboard index pattern for all available aliases, which are not datastreams
-for i in $(curl -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases?v"|grep -v "\-ds"|grep -v "^\."|awk '{ print $1 }'|uniq)
-  do
-    echo "using alias $i from Opensearch-Logs"
-    echo "Setting OpenSearch dashboard index mapping for alias $i"
-    curl -s --header "content-type: application/JSON" --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}"
-    if [ $? -eq 0 ]
-    then
-      echo "index pattern for alias ${i} already exists in Opensearch dashboard, nothing to do"
-    else
-      echo "INFO: creating index-pattern in Dashboards for datastream alias $i"
-      curl -s -XPOST --header "content-type: application/JSON" -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}" -H "osd-xsrf: true" -d "{ \"attributes\": { \"title\": \"${i}\", \"timeFieldName\": \"@timestamp\" } }"
-    fi
-done
-
-
-# Dashboard index pattern for all available datastreams
-for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases?v"|grep "\-ds"|awk '{ print $1 }'|uniq)
-  do
-    echo "using datastream $i from Opensearch-Logs"
-    echo "setting OpenSearch dashboard index mapping for index $i"
-    curl -s --header "content-type: application/JSON" --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}"
-    if [ $? -eq 0 ]
-    then
-        echo "index ${i} already exists in Opensearch dashboard"
-    else
-      echo "INFO: creating index-pattern in Dashboards for datastream alias $i"
-      curl -s -XPOST --header "content-type: application/JSON" -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}" -H "osd-xsrf: true" -d "{ \"attributes\": { \"title\": \"${i}\", \"timeFieldName\": \"@timestamp\" } }"
-    fi
-done
diff --git a/system/opensearch-logs/templates/install-dashboard-pattern-job.yaml b/system/opensearch-logs/templates/install-dashboard-pattern-job.yaml
index 3836eb7619..75a777aac3 100644
--- a/system/opensearch-logs/templates/install-dashboard-pattern-job.yaml
+++ b/system/opensearch-logs/templates/install-dashboard-pattern-job.yaml
@@ -14,7 +14,6 @@ metadata:
     # job is considered part of the release.
     "helm.sh/hook": post-install,post-upgrade
     "helm.sh/hook-weight": "-5"
-    "helm.sh/hook-delete-policy": hook-succeeded
 spec:
   template:
     metadata:

From 34622176e1e17ada64b07160f11e5ebb4ebda345 Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Fri, 20 Dec 2024 11:01:27 +0100
Subject: [PATCH 04/19] [opensearch-logs] fix typo in dashboard pattern script

---
 .../templates/config/_install-dashboard-pattern.sh.tpl        | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
index 016cc0d3f9..2520928821 100644
--- a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
+++ b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
@@ -1,8 +1,6 @@
 #!/bin/bash
 export BASIC_AUTH_HEADER=${ADMIN_USER}:${ADMIN_PASSWORD}
 
-
-
 #  Creating aliases for all indexes, because logstash-* is also selecting datastreams besides the logstash-2024... indexes.
 for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|awk -F- '{ print $1 }'|sort|uniq|grep -v "\."|grep -v "index")
   do
@@ -14,7 +12,7 @@ for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/indices?v"|awk
       echo "Alias and dashboard index pattern for index ${i} already exists. Nothing to do."
     else
       echo "setting OpenSearch dashboard index mapping for index $i"
-      curl -s -XPOST --header "content-type: application/JSON" -u  ${BASIC_AUTH_HEADER}"${CLUSTER_HOST}/_aliases" -H "osd-xsrf: true" -d "{ \"actions\": [ { \"add\": { \"index\": \"${i}-2*\", \"alias\": \"${i}\" } } ] }"
+      curl -s -XPOST --header "content-type: application/JSON" -u  ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_aliases" -H "osd-xsrf: true" -d "{ \"actions\": [ { \"add\": { \"index\": \"${i}-2*\", \"alias\": \"${i}\" } } ] }"
     fi
     echo "Deleting old index pattern based on index-* format"
     export DASHBOARD_PATTERN=`curl -s --header "content-type: application/JSON" --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*"|grep "content-length"|awk -F: '{ print $2 }'|tr -d '[:space:]'`

From 5ac07f4366bbc4cb0800a0513166d8f96b6e48d5 Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Fri, 20 Dec 2024 11:24:36 +0100
Subject: [PATCH 05/19] [opensearch-logs] fix alias creation for indexes with
 more than one dash

---
 .../templates/config/_install-dashboard-pattern.sh.tpl          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
index 2520928821..4ea9d4f00b 100644
--- a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
+++ b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
@@ -2,7 +2,7 @@
 export BASIC_AUTH_HEADER=${ADMIN_USER}:${ADMIN_PASSWORD}
 
 #  Creating aliases for all indexes, because logstash-* is also selecting datastreams besides the logstash-2024... indexes.
-for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|awk -F- '{ print $1 }'|sort|uniq|grep -v "\."|grep -v "index")
+for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|grep -v "^\."|sort|uniq|tr -d '0-9.'|awk '{print substr($0,1,length($0)-1)}'|uniq)
   do
     #Creating an alias for all standard indexes, which are not datastreams to mitigate the issue with indexes, where for example storage-* is selecting the index and also the datastream, which shows up in dashboards as duplicate entries
     echo "using index $i from Opensearch-Logs"

From f4dfa414b9c5ad504dbf01acd73b1a204318c38e Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Fri, 20 Dec 2024 11:39:25 +0100
Subject: [PATCH 06/19] [opensearch-logs] remove date from aliases

---
 .../templates/config/_install-dashboard-pattern.sh.tpl          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
index 4ea9d4f00b..3a9bd33f19 100644
--- a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
+++ b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
@@ -2,7 +2,7 @@
 export BASIC_AUTH_HEADER=${ADMIN_USER}:${ADMIN_PASSWORD}
 
 #  Creating aliases for all indexes, because logstash-* is also selecting datastreams besides the logstash-2024... indexes.
-for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|grep -v "^\."|sort|uniq|tr -d '0-9.'|awk '{print substr($0,1,length($0)-1)}'|uniq)
+for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|grep -v "^\."|sort|sed 's/-[0-9].*\.[0-9].*\.[0-9].*$//'|uniq)
   do
     #Creating an alias for all standard indexes, which are not datastreams to mitigate the issue with indexes, where for example storage-* is selecting the index and also the datastream, which shows up in dashboards as duplicate entries
     echo "using index $i from Opensearch-Logs"

From 1dfddb3cf25deef56a5328aa74cd8e0a727178b6 Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Fri, 20 Dec 2024 11:43:44 +0100
Subject: [PATCH 07/19] [opensearch-logs] remove index from alias

---
 .../templates/config/_install-dashboard-pattern.sh.tpl          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
index 3a9bd33f19..899538c302 100644
--- a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
+++ b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
@@ -2,7 +2,7 @@
 export BASIC_AUTH_HEADER=${ADMIN_USER}:${ADMIN_PASSWORD}
 
 #  Creating aliases for all indexes, because logstash-* is also selecting datastreams besides the logstash-2024... indexes.
-for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|grep -v "^\."|sort|sed 's/-[0-9].*\.[0-9].*\.[0-9].*$//'|uniq)
+for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|grep -v "^\."|sort|sed 's/-[0-9].*\.[0-9].*\.[0-9].*$//'|uniq|grep -v index)
   do
     #Creating an alias for all standard indexes, which are not datastreams to mitigate the issue with indexes, where for example storage-* is selecting the index and also the datastream, which shows up in dashboards as duplicate entries
     echo "using index $i from Opensearch-Logs"

From 848addbb26a6ebc69578c7c9421fc947726545c5 Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Mon, 23 Dec 2024 09:00:46 +0100
Subject: [PATCH 08/19] [opensearch-logs] mv dashboard/alias script to cron

---
 .../config/_install-dashboard-pattern.sh.tpl    |  9 ++-------
 ... => cron-install-dashboard-pattern-job.yaml} | 17 ++++-------------
 2 files changed, 6 insertions(+), 20 deletions(-)
 rename system/opensearch-logs/templates/{install-dashboard-pattern-job.yaml => cron-install-dashboard-pattern-job.yaml} (75%)

diff --git a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
index 899538c302..228e8ba5e3 100644
--- a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
+++ b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
@@ -7,13 +7,8 @@ for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/indices?v"|awk
     #Creating an alias for all standard indexes, which are not datastreams to mitigate the issue with indexes, where for example storage-* is selecting the index and also the datastream, which shows up in dashboards as duplicate entries
     echo "using index $i from Opensearch-Logs"
     export ALIAS_EXISTS=`curl -s -i -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases/${i}"|grep "content-length"|awk -F: '{ print $2 }'|tr -d '[:space:]'`
-    if [[ "$ALIAS_EXISTS" -gt 0 ]]
-     then
-      echo "Alias and dashboard index pattern for index ${i} already exists. Nothing to do."
-    else
-      echo "setting OpenSearch dashboard index mapping for index $i"
-      curl -s -XPOST --header "content-type: application/JSON" -u  ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_aliases" -H "osd-xsrf: true" -d "{ \"actions\": [ { \"add\": { \"index\": \"${i}-2*\", \"alias\": \"${i}\" } } ] }"
-    fi
+    echo "Creating for updating alias $i, because alias setting is only valid for indexes, which were created before the alias creation timestamp"
+    curl -s -XPOST --header "content-type: application/JSON" -u  ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_aliases" -H "osd-xsrf: true" -d "{ \"actions\": [ { \"add\": { \"index\": \"${i}-2*\", \"alias\": \"${i}\" } } ] }"
     echo "Deleting old index pattern based on index-* format"
     export DASHBOARD_PATTERN=`curl -s --header "content-type: application/JSON" --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*"|grep "content-length"|awk -F: '{ print $2 }'|tr -d '[:space:]'`
     if [[ "$DASHBOARD_PATTERN" -gt 0 ]]
diff --git a/system/opensearch-logs/templates/install-dashboard-pattern-job.yaml b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
similarity index 75%
rename from system/opensearch-logs/templates/install-dashboard-pattern-job.yaml
rename to system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
index 75a777aac3..beadbfdedc 100644
--- a/system/opensearch-logs/templates/install-dashboard-pattern-job.yaml
+++ b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
@@ -1,5 +1,5 @@
 apiVersion: batch/v1
-kind: Job
+kind: CronJob
 metadata:
   name: "install-dashboard-pattern"
   labels:
@@ -9,20 +9,11 @@ metadata:
     helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
     ccloud/service: logs
     ccloud/support-group: observability
-  annotations:
-    # This is what defines this resource as a hook. Without this line, the
-    # job is considered part of the release.
-    "helm.sh/hook": post-install,post-upgrade
-    "helm.sh/hook-weight": "-5"
 spec:
-  template:
-    metadata:
-      name: "install-dashboard-pattern"
-      labels:
-        app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
-        app.kubernetes.io/instance: {{ .Release.Name | quote }}
-        helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+  schedule: "30 0,6,12,18 * * *"
+  jobTemplate:
     spec:
+      backoffLimit: 3
       restartPolicy: Never
       containers:
       - name: install-dashboard-pattern 

From 93eac60fc66af90c51297bbc36e6e5c706cc3f9d Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Mon, 23 Dec 2024 09:07:13 +0100
Subject: [PATCH 09/19] [opensearch-logs] fix formatting

---
 .../cron-install-dashboard-pattern-job.yaml   | 61 ++++++++++---------
 1 file changed, 32 insertions(+), 29 deletions(-)

diff --git a/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
index beadbfdedc..2ba4e9f41b 100644
--- a/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
+++ b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
@@ -9,37 +9,40 @@ metadata:
     helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
     ccloud/service: logs
     ccloud/support-group: observability
+spec.jobTemplate.spec.template.spec.restartPolicy
 spec:
   schedule: "30 0,6,12,18 * * *"
   jobTemplate:
     spec:
       backoffLimit: 3
-      restartPolicy: Never
-      containers:
-      - name: install-dashboard-pattern 
-        image: "{{ .Values.global.registry }}/unified-kubernetes-toolbox:latest"
-        command: ["/bin/bash", "/scripts/install-dashboard-pattern.sh"]
-        env:
-          - name: ADMIN_USER
-            valueFrom:
-              secretKeyRef:
-                name: cron-secrets
-                key: ADMIN_USER
-          - name: ADMIN_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: cron-secrets
-                key: ADMIN_PASSWORD
-          - name: CLUSTER_HOST
-            value: "https://opensearch-logs-client.{{ .Values.global.clusterType }}.{{ .Values.global.region }}.{{ .Values.global.tld }}:{{ .Values.httpPort }}"
-          - name: DASHBOARD_HOST
-            value: "https://logs.{{ .Values.global.region }}.{{ .Values.global.tld }}"
-        volumeMounts:
-        - mountPath: /scripts/install-dashboard-pattern.sh
-          name: security-config
-          subPath: install-dashboard-pattern.sh
-      volumes:
-      - name: security-config
-        secret:
-          defaultMode: 420
-          secretName: security-config
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: install-dashboard-pattern 
+            image: "{{ .Values.global.registry }}/unified-kubernetes-toolbox:latest"
+            command: ["/bin/bash", "/scripts/install-dashboard-pattern.sh"]
+            env:
+              - name: ADMIN_USER
+                valueFrom:
+                  secretKeyRef:
+                    name: cron-secrets
+                    key: ADMIN_USER
+              - name: ADMIN_PASSWORD
+                valueFrom:
+                  secretKeyRef:
+                    name: cron-secrets
+                    key: ADMIN_PASSWORD
+              - name: CLUSTER_HOST
+                value: "https://opensearch-logs-client.{{ .Values.global.clusterType }}.{{ .Values.global.region }}.{{ .Values.global.tld }}:{{ .Values.httpPort }}"
+              - name: DASHBOARD_HOST
+                value: "https://logs.{{ .Values.global.region }}.{{ .Values.global.tld }}"
+            volumeMounts:
+            - mountPath: /scripts/install-dashboard-pattern.sh
+              name: security-config
+              subPath: install-dashboard-pattern.sh
+          volumes:
+          - name: security-config
+            secret:
+              defaultMode: 420
+              secretName: security-config

From b74b958df77db51826dba7c3835fa5a06d7268ac Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Mon, 23 Dec 2024 09:09:39 +0100
Subject: [PATCH 10/19] [opensearch-logs] fix formatting

---
 .../templates/cron-install-dashboard-pattern-job.yaml            | 1 -
 1 file changed, 1 deletion(-)

diff --git a/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
index 2ba4e9f41b..f52e372c4f 100644
--- a/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
+++ b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
@@ -9,7 +9,6 @@ metadata:
     helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
     ccloud/service: logs
     ccloud/support-group: observability
-spec.jobTemplate.spec.template.spec.restartPolicy
 spec:
   schedule: "30 0,6,12,18 * * *"
   jobTemplate:

From 73d23d1bb87aee3caa105a0a69fac42631ee04a5 Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Mon, 23 Dec 2024 09:24:21 +0100
Subject: [PATCH 11/19] [opensearch-logs] testing cron

---
 .../templates/cron-install-dashboard-pattern-job.yaml           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
index f52e372c4f..c6db9ec429 100644
--- a/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
+++ b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
@@ -10,7 +10,7 @@ metadata:
     ccloud/service: logs
     ccloud/support-group: observability
 spec:
-  schedule: "30 0,6,12,18 * * *"
+  schedule: "30 0,6,8,12,18 * * *"
   jobTemplate:
     spec:
       backoffLimit: 3

From 44d1c1e1f38f660c93976b52a15951d9a593581f Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Mon, 23 Dec 2024 09:42:49 +0100
Subject: [PATCH 12/19] [opensearch-logs] fix alias creation for indexes with
 more than one dash

---
 .../templates/config/_install-dashboard-pattern.sh.tpl          | 2 +-
 .../templates/cron-install-dashboard-pattern-job.yaml           | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
index 228e8ba5e3..103d3d681f 100644
--- a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
+++ b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
@@ -2,7 +2,7 @@
 export BASIC_AUTH_HEADER=${ADMIN_USER}:${ADMIN_PASSWORD}
 
 #  Creating aliases for all indexes, because logstash-* is also selecting datastreams besides the logstash-2024... indexes.
-for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|grep -v "^\."|sort|sed 's/-[0-9].*\.[0-9].*\.[0-9].*$//'|uniq|grep -v index)
+for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|grep -v "^\."|sort|sed 's/-[0-9].*\.[0-9].*\.[0-9].*$//'|uniq|grep -v index|grep -v "alerts-other"|grep -v deployments|grep -v maillog|grep -v ss4o|grep -v sample)
   do
     #Creating an alias for all standard indexes, which are not datastreams to mitigate the issue with indexes, where for example storage-* is selecting the index and also the datastream, which shows up in dashboards as duplicate entries
     echo "using index $i from Opensearch-Logs"
diff --git a/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
index c6db9ec429..493ea446bd 100644
--- a/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
+++ b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
@@ -10,7 +10,7 @@ metadata:
     ccloud/service: logs
     ccloud/support-group: observability
 spec:
-  schedule: "30 0,6,8,12,18 * * *"
+  schedule: "30,50 0,6,8,12,18 * * *"
   jobTemplate:
     spec:
       backoffLimit: 3

From 43f28155c1e8d6f1702667f0e8e01dc4ed24d72d Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Mon, 23 Dec 2024 10:45:23 +0100
Subject: [PATCH 13/19] [opensearch-logs] remove test time from cron

---
 .../templates/cron-install-dashboard-pattern-job.yaml           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
index 493ea446bd..c6db9ec429 100644
--- a/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
+++ b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
@@ -10,7 +10,7 @@ metadata:
     ccloud/service: logs
     ccloud/support-group: observability
 spec:
-  schedule: "30,50 0,6,8,12,18 * * *"
+  schedule: "30 0,6,8,12,18 * * *"
   jobTemplate:
     spec:
       backoffLimit: 3

From 47933f77be2e72e54cd1cf6ddc2e3875841aeb11 Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Wed, 18 Dec 2024 15:48:58 +0100
Subject: [PATCH 14/19] [opensearch-logs] adding audit user for audit logs, job
 for alias and dashboard patterns

---
 .../config/_install-dashboard-pattern.sh.tpl  | 47 ++++++++++++----
 .../config/_install-index-pattern.sh.tpl      | 16 ------
 .../templates/config/_internal_users.yml.tpl  |  6 ++
 .../templates/config/_roles.yml.tpl           | 20 +++++++
 .../templates/config/_roles_mapping.yml.tpl   |  5 ++
 .../cron-install-dashboard-pattern-job.yaml   | 47 ++++++++++++++++
 .../install-dashboard-pattern-job.yaml        | 55 -------------------
 7 files changed, 113 insertions(+), 83 deletions(-)
 delete mode 100644 system/opensearch-logs/templates/config/_install-index-pattern.sh.tpl
 create mode 100644 system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
 delete mode 100644 system/opensearch-logs/templates/install-dashboard-pattern-job.yaml

diff --git a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
index 350074ecb1..103d3d681f 100644
--- a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
+++ b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
@@ -1,29 +1,52 @@
 #!/bin/bash
+export BASIC_AUTH_HEADER=${ADMIN_USER}:${ADMIN_PASSWORD}
 
-# 0. Check for index policy
-for i in $(curl -s -u ${ADMIN_USER}:${ADMIN_PASSWORD} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|awk -F- '{ print $1 }'|sort|uniq|grep -v "\."|grep -v "index")
+#  Creating aliases for all indexes, because logstash-* is also selecting datastreams besides the logstash-2024... indexes.
+for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|grep -v "^\."|sort|sed 's/-[0-9].*\.[0-9].*\.[0-9].*$//'|uniq|grep -v index|grep -v "alerts-other"|grep -v deployments|grep -v maillog|grep -v ss4o|grep -v sample)
   do
+    #Creating an alias for all standard indexes, which are not datastreams to mitigate the issue with indexes, where for example storage-* is selecting the index and also the datastream, which shows up in dashboards as duplicate entries
     echo "using index $i from Opensearch-Logs"
-    echo "setting OpenSearch dashboard index mapping for index $i"
-    curl --header "content-type: application/JSON" --fail -XGET -u ${ADMIN_USER}:${ADMIN_PASSWORD} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*"
-    if [ $? -eq 0 ]; then
-       echo "index ${i} already exists in Opensearch dashboard"
+    export ALIAS_EXISTS=`curl -s -i -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases/${i}"|grep "content-length"|awk -F: '{ print $2 }'|tr -d '[:space:]'`
+    echo "Creating for updating alias $i, because alias setting is only valid for indexes, which were created before the alias creation timestamp"
+    curl -s -XPOST --header "content-type: application/JSON" -u  ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_aliases" -H "osd-xsrf: true" -d "{ \"actions\": [ { \"add\": { \"index\": \"${i}-2*\", \"alias\": \"${i}\" } } ] }"
+    echo "Deleting old index pattern based on index-* format"
+    export DASHBOARD_PATTERN=`curl -s --header "content-type: application/JSON" --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*"|grep "content-length"|awk -F: '{ print $2 }'|tr -d '[:space:]'`
+    if [[ "$DASHBOARD_PATTERN" -gt 0 ]]
+    then
+      echo "Old dashboard pattern exists for for index ${i}, it will be removed"
+      curl -s -XDELETE -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*"
+    else
+     echo "No old dashboard pattern for index $i"
+    fi
+done
+
+# Dashboard index pattern for all available aliases, which are not datastreams
+for i in $(curl -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases?v"|grep -v "\-ds"|grep -v "^\."|awk '{ print $1 }'|uniq)
+  do
+    echo "using alias $i from Opensearch-Logs"
+    echo "Setting OpenSearch dashboard index mapping for alias $i"
+    curl -s --header "content-type: application/JSON" --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}"
+    if [ $? -eq 0 ]
+    then
+      echo "index pattern for alias ${i} already exists in Opensearch dashboard, nothing to do"
     else
-      echo "INFO: creating index-pattern in Dashboards for $i logs"
-      curl -XPOST --header "content-type: application/JSON" -u ${ADMIN_USER}:${ADMIN_PASSWORD} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*" -H "osd-xsrf: true" -d "{ \"attributes\": { \"title\": \"${i}-*\", \"timeFieldName\": \"@timestamp\" } }"
+      echo "INFO: creating index-pattern in Dashboards for datastream alias $i"
+      curl -s -XPOST --header "content-type: application/JSON" -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}" -H "osd-xsrf: true" -d "{ \"attributes\": { \"title\": \"${i}\", \"timeFieldName\": \"@timestamp\" } }"
     fi
 done
 
+
 # Dashboard index pattern for all available datastreams
-for i in $(curl -s -u ${ADMIN_USER}:${ADMIN_PASSWORD} "${CLUSTER_HOST}/_cat/aliases?v"|grep "\-ds"|awk '{ print $1 }'|uniq)
+for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases?v"|grep "\-ds"|awk '{ print $1 }'|uniq)
   do
     echo "using datastream $i from Opensearch-Logs"
     echo "setting OpenSearch dashboard index mapping for index $i"
-    curl --header "content-type: application/JSON" --fail -XGET -u ${ADMIN_USER}:${ADMIN_PASSWORD} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}"
-    if [ $? -eq 0 ]; then
+    curl -s --header "content-type: application/JSON" --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}"
+    if [ $? -eq 0 ]
+    then
         echo "index ${i} already exists in Opensearch dashboard"
     else
       echo "INFO: creating index-pattern in Dashboards for datastream alias $i"
-      curl -XPOST --header "content-type: application/JSON" -u ${ADMIN_USER}:${ADMIN_PASSWORD} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}" -H "osd-xsrf: true" -d "{ \"attributes\": { \"title\": \"${i}\", \"timeFieldName\": \"@timestamp\" } }"
+      curl -s -XPOST --header "content-type: application/JSON" -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}" -H "osd-xsrf: true" -d "{ \"attributes\": { \"title\": \"${i}\", \"timeFieldName\": \"@timestamp\" } }"
     fi
 done
diff --git a/system/opensearch-logs/templates/config/_install-index-pattern.sh.tpl b/system/opensearch-logs/templates/config/_install-index-pattern.sh.tpl
deleted file mode 100644
index a5e199bb08..0000000000
--- a/system/opensearch-logs/templates/config/_install-index-pattern.sh.tpl
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-
-# 0. Check for index policy
-for i in $(curl -s -u ${ADMIN_USER}:${ADMIN_PASSWORD} "${CLUSTER_HOST}/_cat/indices?v"|awk '{ print $3 }'|awk -F- '{ print $1 }'|sort|uniq|grep -v "\."|grep -v "index")
-  do
-    echo "using index $i from Opensearch-Logs"
-    echo "setting OpenSearch dashboard index mapping for index $i"
-    curl --header "content-type: application/JSON" --fail -XGET -u ${ADMIN_USER}:${ADMIN_PASSWORD} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*"
-  if [ $? -eq 0 ]
-    then
-      echo "index ${i} already exists in Opensearch dashboard"
-  else
-    echo "INFO: creating index-pattern in Dashboards for $i logs"
-    curl -XPOST --header "content-type: application/JSON" -u ${ADMIN_USER}:${ADMIN_PASSWORD} "https://${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}-*" -H "osd-xsrf: true" -d "{ \"attributes\": { \"title\": \"${i}-*\", \"timeFieldName\": \"@timestamp\" } }"
-  fi
-done
diff --git a/system/opensearch-logs/templates/config/_internal_users.yml.tpl b/system/opensearch-logs/templates/config/_internal_users.yml.tpl
index 501f7da3d8..08ce9b9af4 100644
--- a/system/opensearch-logs/templates/config/_internal_users.yml.tpl
+++ b/system/opensearch-logs/templates/config/_internal_users.yml.tpl
@@ -38,6 +38,12 @@ otel:
   backend_roles:
   - "otel"
 
+audit:
+  hash: "{{ .Values.users.audit.nohash }}"
+  reserved: true
+  backend_roles:
+  - "audit"
+
 otellogs:
   hash: "{{ .Values.users.otellogs.nohash }}"
   reserved: true
diff --git a/system/opensearch-logs/templates/config/_roles.yml.tpl b/system/opensearch-logs/templates/config/_roles.yml.tpl
index b94137542a..0c8d668819 100644
--- a/system/opensearch-logs/templates/config/_roles.yml.tpl
+++ b/system/opensearch-logs/templates/config/_roles.yml.tpl
@@ -249,6 +249,26 @@ compute:
     - "indices:admin/create"
     - "indices:data/write/bulk*"
     - "indices:data/write/index"
+audit:
+  reserved: false
+  cluster_permissions:
+  - "cluster_monitor"
+  - "cluster_composite_ops"
+  - "cluster:admin/ingest/pipeline/put"
+  - "cluster:admin/ingest/pipeline/get"
+  - "indices:admin/template/get"
+  - "cluster_manage_index_templates"
+  - "cluster:admin/opensearch/ml/predict"
+  index_permissions:
+  - index_patterns:
+    - "audit-*"
+    allowed_actions:
+    - "indices:admin/template/get"
+    - "indices:admin/template/put"
+    - "indices:admin/mapping/put"
+    - "indices:admin/create"
+    - "indices:data/write/bulk*"
+    - "indices:data/write/index"
 otel:
   reserved: false
   cluster_permissions:
diff --git a/system/opensearch-logs/templates/config/_roles_mapping.yml.tpl b/system/opensearch-logs/templates/config/_roles_mapping.yml.tpl
index f7177dd9b0..4e25c9be65 100644
--- a/system/opensearch-logs/templates/config/_roles_mapping.yml.tpl
+++ b/system/opensearch-logs/templates/config/_roles_mapping.yml.tpl
@@ -32,6 +32,11 @@ greenhouse:
   users:
   - "greenhouse"
 
+audit:
+  reserved: false
+  users:
+  - "audit"
+
 jump:
   reserved: false
   users:
diff --git a/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
new file mode 100644
index 0000000000..c6db9ec429
--- /dev/null
+++ b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
@@ -0,0 +1,47 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: "install-dashboard-pattern"
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    app.kubernetes.io/version: {{ .Chart.AppVersion }}
+    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    ccloud/service: logs
+    ccloud/support-group: observability
+spec:
+  schedule: "30 0,6,8,12,18 * * *"
+  jobTemplate:
+    spec:
+      backoffLimit: 3
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+          - name: install-dashboard-pattern 
+            image: "{{ .Values.global.registry }}/unified-kubernetes-toolbox:latest"
+            command: ["/bin/bash", "/scripts/install-dashboard-pattern.sh"]
+            env:
+              - name: ADMIN_USER
+                valueFrom:
+                  secretKeyRef:
+                    name: cron-secrets
+                    key: ADMIN_USER
+              - name: ADMIN_PASSWORD
+                valueFrom:
+                  secretKeyRef:
+                    name: cron-secrets
+                    key: ADMIN_PASSWORD
+              - name: CLUSTER_HOST
+                value: "https://opensearch-logs-client.{{ .Values.global.clusterType }}.{{ .Values.global.region }}.{{ .Values.global.tld }}:{{ .Values.httpPort }}"
+              - name: DASHBOARD_HOST
+                value: "https://logs.{{ .Values.global.region }}.{{ .Values.global.tld }}"
+            volumeMounts:
+            - mountPath: /scripts/install-dashboard-pattern.sh
+              name: security-config
+              subPath: install-dashboard-pattern.sh
+          volumes:
+          - name: security-config
+            secret:
+              defaultMode: 420
+              secretName: security-config
diff --git a/system/opensearch-logs/templates/install-dashboard-pattern-job.yaml b/system/opensearch-logs/templates/install-dashboard-pattern-job.yaml
deleted file mode 100644
index 3836eb7619..0000000000
--- a/system/opensearch-logs/templates/install-dashboard-pattern-job.yaml
+++ /dev/null
@@ -1,55 +0,0 @@
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: "install-dashboard-pattern"
-  labels:
-    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
-    app.kubernetes.io/instance: {{ .Release.Name | quote }}
-    app.kubernetes.io/version: {{ .Chart.AppVersion }}
-    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
-    ccloud/service: logs
-    ccloud/support-group: observability
-  annotations:
-    # This is what defines this resource as a hook. Without this line, the
-    # job is considered part of the release.
-    "helm.sh/hook": post-install,post-upgrade
-    "helm.sh/hook-weight": "-5"
-    "helm.sh/hook-delete-policy": hook-succeeded
-spec:
-  template:
-    metadata:
-      name: "install-dashboard-pattern"
-      labels:
-        app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
-        app.kubernetes.io/instance: {{ .Release.Name | quote }}
-        helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
-    spec:
-      restartPolicy: Never
-      containers:
-      - name: install-dashboard-pattern 
-        image: "{{ .Values.global.registry }}/unified-kubernetes-toolbox:latest"
-        command: ["/bin/bash", "/scripts/install-dashboard-pattern.sh"]
-        env:
-          - name: ADMIN_USER
-            valueFrom:
-              secretKeyRef:
-                name: cron-secrets
-                key: ADMIN_USER
-          - name: ADMIN_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: cron-secrets
-                key: ADMIN_PASSWORD
-          - name: CLUSTER_HOST
-            value: "https://opensearch-logs-client.{{ .Values.global.clusterType }}.{{ .Values.global.region }}.{{ .Values.global.tld }}:{{ .Values.httpPort }}"
-          - name: DASHBOARD_HOST
-            value: "https://logs.{{ .Values.global.region }}.{{ .Values.global.tld }}"
-        volumeMounts:
-        - mountPath: /scripts/install-dashboard-pattern.sh
-          name: security-config
-          subPath: install-dashboard-pattern.sh
-      volumes:
-      - name: security-config
-        secret:
-          defaultMode: 420
-          secretName: security-config

From 3fb04bdfe7879cd6e81374a027fc7da48f406c95 Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Thu, 2 Jan 2025 11:22:52 +0100
Subject: [PATCH 15/19] [opensearch-logs] fix datastream list

---
 .../templates/config/_install-dashboard-pattern.sh.tpl        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
index 103d3d681f..633d52b3a2 100644
--- a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
+++ b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
@@ -37,11 +37,11 @@ done
 
 
 # Dashboard index pattern for all available datastreams
-for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases?v"|grep "\-ds"|awk '{ print $1 }'|uniq)
+for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases?v"|grep "\-ds"|awk '{ print $1 }'|sort|uniq)
   do
     echo "using datastream $i from Opensearch-Logs"
     echo "setting OpenSearch dashboard index mapping for index $i"
-    curl -s --header "content-type: application/JSON" --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}"
+    curl -s --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}"
     if [ $? -eq 0 ]
     then
         echo "index ${i} already exists in Opensearch dashboard"

From fdad6878344c243e617fd4bc69ed0a4e299c5dde Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Thu, 2 Jan 2025 12:33:07 +0100
Subject: [PATCH 16/19] [opensearch-logs] adding second data user

---
 .../templates/config/_internal_users.yml.tpl                | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/system/opensearch-logs/templates/config/_internal_users.yml.tpl b/system/opensearch-logs/templates/config/_internal_users.yml.tpl
index 08ce9b9af4..d3ca294c16 100644
--- a/system/opensearch-logs/templates/config/_internal_users.yml.tpl
+++ b/system/opensearch-logs/templates/config/_internal_users.yml.tpl
@@ -14,6 +14,12 @@ data:
   backend_roles:
   - "data"
 
+data2:
+  hash: "{{ .Values.users.data2.nohash }}"
+  reserved: true
+  backend_roles:
+  - "data"
+
 greenhouse:
   hash: "{{ .Values.users.greenhouse.nohash }}"
   reserved: true

From f730ef3b61c21f4fc7cc08ee576a5c18a92fe675 Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Thu, 2 Jan 2025 13:52:31 +0100
Subject: [PATCH 17/19] [opensearch-logs] testing index pattern creation

---
 .../templates/config/_install-dashboard-pattern.sh.tpl        | 4 ++--
 .../templates/cron-install-dashboard-pattern-job.yaml         | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
index 633d52b3a2..14b3ae92aa 100644
--- a/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
+++ b/system/opensearch-logs/templates/config/_install-dashboard-pattern.sh.tpl
@@ -21,11 +21,11 @@ for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/indices?v"|awk
 done
 
 # Dashboard index pattern for all available aliases, which are not datastreams
-for i in $(curl -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases?v"|grep -v "\-ds"|grep -v "^\."|awk '{ print $1 }'|uniq)
+for i in $(curl -s -u ${BASIC_AUTH_HEADER} "${CLUSTER_HOST}/_cat/aliases?v"|grep -v "\-ds"|grep -v "^\."|awk '{ print $1 }'|sort|uniq)
   do
     echo "using alias $i from Opensearch-Logs"
     echo "Setting OpenSearch dashboard index mapping for alias $i"
-    curl -s --header "content-type: application/JSON" --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}"
+    curl -s --fail -XGET -u ${BASIC_AUTH_HEADER} "${DASHBOARD_HOST}/api/saved_objects/index-pattern/${i}"
     if [ $? -eq 0 ]
     then
       echo "index pattern for alias ${i} already exists in Opensearch dashboard, nothing to do"
diff --git a/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
index c6db9ec429..3b1cef92e3 100644
--- a/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
+++ b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
@@ -10,7 +10,7 @@ metadata:
     ccloud/service: logs
     ccloud/support-group: observability
 spec:
-  schedule: "30 0,6,8,12,18 * * *"
+  schedule: "30 0,6,8,12,13,18 * * *"
   jobTemplate:
     spec:
       backoffLimit: 3

From fb4d9e1f85bbaa92b3b2cee97d62bfe16194e09c Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Thu, 2 Jan 2025 14:34:02 +0100
Subject: [PATCH 18/19] [opensearch-logs] remove unused cron entry

---
 .../templates/cron-install-dashboard-pattern-job.yaml           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
index 3b1cef92e3..c6db9ec429 100644
--- a/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
+++ b/system/opensearch-logs/templates/cron-install-dashboard-pattern-job.yaml
@@ -10,7 +10,7 @@ metadata:
     ccloud/service: logs
     ccloud/support-group: observability
 spec:
-  schedule: "30 0,6,8,12,13,18 * * *"
+  schedule: "30 0,6,8,12,18 * * *"
   jobTemplate:
     spec:
       backoffLimit: 3

From ec861b103e833afe6869ac5ab8ed70e17ef83937 Mon Sep 17 00:00:00 2001
From: Olaf Heydorn <olaf.heydorn@sap.com>
Date: Thu, 2 Jan 2025 16:29:05 +0100
Subject: [PATCH 19/19] [opensearch-logs] second user for otel

---
 .../templates/config/_internal_users.yml.tpl                | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/system/opensearch-logs/templates/config/_internal_users.yml.tpl b/system/opensearch-logs/templates/config/_internal_users.yml.tpl
index d3ca294c16..c50141825a 100644
--- a/system/opensearch-logs/templates/config/_internal_users.yml.tpl
+++ b/system/opensearch-logs/templates/config/_internal_users.yml.tpl
@@ -44,6 +44,12 @@ otel:
   backend_roles:
   - "otel"
 
+otel2:
+  hash: "{{ .Values.users.otel2.nohash }}"
+  reserved: true
+  backend_roles:
+  - "otel"
+
 audit:
   hash: "{{ .Values.users.audit.nohash }}"
   reserved: true