Merge pull request #17 from canonical/replace-yq-by-python-yaml

[DPE-2264] Replaced yq processing with python yaml

rockcraft.yaml

@@ -34,9 +34,10 @@ parts:
     plugin: nil
     stage-snaps:
       - opensearch/2/candidate
-      - yq
     stage-packages:
       - curl
+      - python3
+      - python3-yaml
       - util-linux
     override-prime: |
       craftctl default
@@ -53,8 +54,8 @@ parts:
       dpkg-query -f '${db:Status-Abbrev},${binary:Package},${Version},${source:Package},${Source:Version}\n' -W > ${rocks}/dpkg.query
       
       ## for snap packages
-      cp snap.yq/manifest.yaml ${rocks}
-      cp snap.yq/snapcraft.yaml ${rocks}
+      cp snap.opensearch/manifest.yaml ${rocks}
+      cp snap.opensearch/snapcraft.yaml ${rocks}
       
 
   non-root-user:
@@ -77,5 +78,7 @@ parts:
     source: scripts
     organize:
       start.sh: bin/start.sh
+      set_conf.py: bin/set_conf.py
     stage:
       - bin/start.sh
+      - bin/set_conf.py

scripts/set_conf.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+import ast
+import uuid
+import yaml
+
+from argparse import ArgumentParser, Namespace
+from io import StringIO
+from typing import Any, Dict
+
+
+def parse_args() -> Namespace:
+    parser = ArgumentParser()
+    parser.add_argument("-f", "--file", help="configuration file path")
+    parser.add_argument("-k", "--key", help="the config key to set")
+    parser.add_argument("-v", "--value", help="value of the config entry to set")
+    return parser.parse_args()
+
+
+def load(name: str) -> Dict[str, Any]:
+    with open(name, mode="r") as f:
+        lines = f.read().splitlines()
+
+        # when the yaml doc is empty, parser has no way of knowing it's a yaml doc
+        random_id = uuid.uuid4().hex
+        lines.append(f"{random_id}: {random_id}")
+
+        yaml_doc = yaml.safe_load(StringIO("\n".join(lines)))
+        del yaml_doc[random_id]
+
+        return yaml_doc
+
+
+def write(name: str, yaml_doc: Dict[str, Any]) -> None:
+    with open(name, mode="w") as f:
+        yaml.dump(yaml_doc, f)
+
+
+if __name__ == "__main__":
+    args = parse_args()
+
+    data = load(args.file)
+
+    if args.value.startswith("[") and args.value.endswith("]"):
+        # convert a string represented List into an actual python list
+        data[args.key] = ast.literal_eval(args.value)
+    else:
+        data[args.key] = args.value
+
+    write(args.file, data)

scripts/start.sh

@@ -12,59 +12,10 @@ SEED_HOSTS="${SEED_HOSTS:-}"
 
 function set_yaml_prop() {
     local target_file="${1}"
-    local full_key_path="${2}"
+    local key="${2}"
     local value="${3}"
-    local append="${4:-"no"}"
-    local split_array_content="${5:-"yes"}"
 
-    operator="="
-
-    # allow appending
-    if [ "${append}" == "yes" ]; then
-        operator="+="
-    fi
-
-    # traversal must be done through the "/" separator to allow for "." in key names
-    IFS='/' read -r -a keys <<< "${full_key_path}"
-
-    expression=""
-    for key in "${keys[@]}"
-    do
-        prefix=""
-        suffix=""
-        if [[ "${key}" != [* ]]; then
-            prefix=".\""
-            suffix="\""
-        fi
-        expression="${expression}${prefix}${key}${suffix}"
-    done
-
-    # yq fails serializing values starting with or containing special characters so they must be wrapped in double quotes
-    # so, wrap any non number
-    if [[ "${value}" == [* ]]; then
-        value=${value:1:-1}
-
-        if [ "${split_array_content}" == "yes" ]; then
-            IFS=',' read -r -a arr_elts <<< "${value}"
-
-            value=""
-            for key in "${arr_elts[@]}"
-            do
-                key=$(echo -e "${key}" | tr -d '[:space:]')
-                if ! [[ ${key} =~ ^[0-9]+$ ]] && ! [[ ${key} =~ ^\".*\"$ ]]; then
-                    key="\"${key}\""
-                fi
-                value="${value}${key},"
-            done
-            value="[${value:0:-1}]"
-        else
-            value="[${value}]"
-        fi
-    elif ! [[ "${value}" =~ ^[0-9]+$ ]]  && ! [[ ${value} =~ ^\".*\"$ ]]; then
-       value="\"${value}\""
-    fi
-
-    /usr/bin/yq -i "${expression} ${operator} ${value}" "${target_file}"
+    /usr/bin/python3 /usr/bin/set_conf.py --file "${target_file}" --key "${key}" --value "${value}"
 }
 
 function network_host() {