Create a specialized database container (#1726)

* Add db initialization to database container
* Add database to build components
* Update GitHub Actions to build & update database component
* Add test build of database container to GitHub Actions workflow

.github/actions/combine-deploy-update/action.yml

@@ -33,6 +33,11 @@ runs:
     - name: Deploy updated images
       run: echo "Update images with version ${{ inputs.image_tag }}"
       shell: bash
+    - name: Update database
+      run: kubectl --context ${{ inputs.kube_context }}
+        set image deployment/database
+        database="${{ inputs.image_registry }}${{ inputs.image_registry_alias }}/combine_database:${{ inputs.image_tag }}"
+      shell: bash
     - name: Update frontend
       run: kubectl --context ${{ inputs.kube_context }}
         set image deployment/frontend

.github/workflows/database.yml

@@ -0,0 +1,24 @@
+name: database
+
+on:
+  pull_request:
+    branches: [master]
+
+jobs:
+  docker_build:
+    if: ${{ github.event.type }} == "PullRequest"
+    runs-on: ubuntu-latest
+    steps:
+      # For subfolders, currently a full checkout is required.
+      # See: https://github.com/marketplace/actions/build-and-push-docker-images#path-context
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Build database image
+        run: |
+          deploy/scripts/build.py --components database
+        shell: bash
+      - name: Image digest
+        run: |
+          docker image inspect combine_database:latest -f '{{json .Id}}'
+        shell: bash

.github/workflows/deploy_qa.yml

@@ -8,7 +8,7 @@ jobs:
   build:
     strategy:
       matrix:
-        component: [frontend, backend, maintenance]
+        component: [frontend, backend, maintenance, database]
     runs-on: ubuntu-latest
     outputs:
       image_tag: ${{ steps.build_combine.outputs.image_tag }}

.gitignore

@@ -76,6 +76,7 @@ venv
 # Intermediate and output files for Semantic Domain import scripts
 !deploy/scripts/semantic_domains/xml/*.xml
 deploy/scripts/semantic_domains/json/*.json
+database/semantic_domains/*
 
 # Kubernetes Configuration files
 **/site_files/

database/Dockerfile

@@ -0,0 +1,18 @@
+FROM mongo:5.0
+
+WORKDIR /app
+
+RUN mkdir /data/semantic-domains
+
+# Copy semantic domain import files
+COPY semantic_domains/* /data/semantic-domains/
+
+# from https://hub.docker.com/_/mongo
+# Initializing a fresh instance
+# When a container is started for the first time it will execute files
+# with extensions .sh and .js that are found in /docker-entrypoint-initdb.d.
+# Files will be executed in alphabetical order. .js files will be executed
+# by mongosh (mongo on versions below 6) using the database specified by
+# the MONGO_INITDB_DATABASE variable, if it is present, or test otherwise.
+# You may also switch databases within the .js script.
+COPY init/* /docker-entrypoint-initdb.d/

database/init/update-semantic-domains.sh

@@ -0,0 +1,4 @@
+#! /usr/bin/bash
+
+mongoimport -d CombineDatabase -c SemanticDomainTree /data/semantic-domains/tree.json --mode=merge --upsertFields=id,guid,lang
+mongoimport -d CombineDatabase -c SemanticDomains /data/semantic-domains/nodes.json --mode=merge --upsertFields=id,guid,lang

deploy/helm/thecombine/charts/database/templates/_helpers.tpl

@@ -0,0 +1,25 @@
+{{/* Build container image name */}}
+{{- define "database.containerImage" -}}
+  {{- if .Values.global.imageRegistry }}
+    {{- $registry := .Values.global.imageRegistry }}
+    {{- if contains "awsEcr" .Values.global.imageRegistry }}
+      {{- $registry = printf "%s.dkr.ecr.%s.amazonaws.com" .Values.global.awsAccount .Values.global.awsDefaultRegion }}
+    {{- end }}
+    {{- printf "%s/%s:%s" $registry .Values.imageName .Values.global.imageTag }}
+  {{- else }}
+    {{- printf "%s:%s" .Values.imageName .Values.global.imageTag }}
+  {{- end }}
+{{- end }}
+
+{{/* Get the Image Pull Policy */}}
+{{- define "database.imagePullPolicy" }}
+  {{- if .Values.global.imagePullPolicy }}
+    {{- print .Values.global.imagePullPolicy }}
+  {{- else }}
+    {{- if eq .Values.global.imageTag "latest" }}
+      {{- print "Always" }}
+    {{- else }}
+      {{- print "IfNotPresent" }}
+    {{- end }}
+  {{- end }}
+{{- end }}

deploy/helm/thecombine/charts/database/templates/database.yaml

@@ -38,8 +38,8 @@ spec:
         combine-component: database
     spec:
       containers:
-        - image: mongo:{{ .Values.mongoImageTag }}
-          imagePullPolicy: IfNotPresent
+        - image: {{ template "database.containerImage" . }}
+          imagePullPolicy: {{ template "database.imagePullPolicy" . }}
           name: database
           ports:
             - containerPort: 27017

deploy/helm/thecombine/charts/database/values.yaml

@@ -2,5 +2,13 @@
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
 
-mongoImageTag: "5.0"
+global:
+  # Update strategy should be "Recreate" or "Rolling Update"
+  updateStrategy: Recreate
+  pullSecretName: "None"
+  imageTag: "latest"
+  # Define the image registry to use (may be blank for local images)
+  imageRegistry: ""
+
 persistentVolumeSize: 16Gi
+imageName: combine_database

deploy/scripts/build.py

@@ -17,17 +17,20 @@
 import subprocess
 import sys
 import time
-from typing import Dict, List, Optional
+from typing import Callable, Dict, List, Optional
 
 from app_release import get_release, set_release
 from enum_types import JobStatus
+from sem_dom_import import generate_semantic_domains
 from streamfile import StreamFile
 
 
 @dataclass(frozen=True)
 class BuildSpec:
     dir: Path
     name: str
+    pre_build: Callable[[], None]
+    post_build: Callable[[], None]
 
 
 @dataclass
@@ -118,6 +121,41 @@ def check_jobs(self) -> JobStatus:
 """Absolute path to the checked out repository."""
 
 
+# Pre-build functions for the different build components
+def build_semantic_domains() -> None:
+    """Create the semantic domain definition files."""
+    source_dir = project_dir / "deploy" / "scripts" / "semantic_domains" / "xml"
+    output_dir = project_dir / "database" / "semantic_domains"
+    generate_semantic_domains(list(source_dir.glob("*.xml")), output_dir)
+
+
+def create_release_file() -> None:
+    """Create the release.js file to be built into the Frontend."""
+    release_file = project_dir / "public" / "scripts" / "release.js"
+    set_release(get_release(), release_file)
+
+
+def rm_release_file() -> None:
+    """Remove release.js file if it exists."""
+    release_file = project_dir / "public" / "scripts" / "release.js"
+    if release_file.exists():
+        release_file.unlink()
+
+
+def no_op() -> None:
+    pass
+
+
+# Create a dictionary to look up the build spec from
+# a component name
+build_specs: Dict[str, BuildSpec] = {
+    "backend": BuildSpec(project_dir / "Backend", "backend", no_op, no_op),
+    "database": BuildSpec(project_dir / "database", "database", build_semantic_domains, no_op),
+    "maintenance": BuildSpec(project_dir / "maintenance", "maint", no_op, no_op),
+    "frontend": BuildSpec(project_dir, "frontend", create_release_file, rm_release_file),
+}
+
+
 def get_image_name(repo: Optional[str], component: str, tag: Optional[str]) -> str:
     """Build the image name from the repo, the component, and the image tag."""
     tag_str = ""
@@ -137,7 +175,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--components",
         nargs="*",
-        choices=["frontend", "backend", "maintenance"],
+        choices=build_specs.keys(),
         help="Combine components to build.",
     )
     parser.add_argument(
@@ -180,13 +218,6 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Print extra debugging information.",
     )
-    group.add_argument(
-        "--info",
-        "-i",
-        action="store_true",
-        help="Print info when jobs start and stop in addition to job output.",
-    )
-    # Transform --output-mode argument to an enumerated type
     return parser.parse_args()
 
 
@@ -198,10 +229,10 @@ def main() -> None:
     # independent of the logging facility
     if args.debug:
         log_level = logging.DEBUG
-    elif args.info:
-        log_level = logging.INFO
-    else:
+    elif args.quiet:
         log_level = logging.WARNING
+    else:
+        log_level = logging.INFO
     logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level)
 
     # Setup required build engine - docker or nerdctl
@@ -224,25 +255,13 @@ def main() -> None:
     if args.components is not None:
         to_do = args.components
     else:
-        to_do = ["backend", "frontend", "maintenance"]
-
-    # Create a dictionary to look up the build spec from
-    # a component name
-    build_specs: Dict[str, BuildSpec] = {
-        "backend": BuildSpec(project_dir / "Backend", "backend"),
-        "maintenance": BuildSpec(project_dir / "maintenance", "maint"),
-        "frontend": BuildSpec(project_dir, "frontend"),
-    }
-
-    # Create the version file
-    release_file = project_dir / "public" / "scripts" / "release.js"
-
-    set_release(get_release(), release_file)
+        to_do = build_specs.keys()
 
     # Create the set of jobs to be run for all components
     job_set: Dict[str, JobQueue] = {}
     for component in to_do:
         spec = build_specs[component]
+        spec.pre_build()
         image_name = get_image_name(args.repo, spec.name, args.tag)
         job_set[component] = JobQueue(component)
         job_set[component].add_job(
@@ -279,9 +298,10 @@ def main() -> None:
         if not running_jobs:
             break
         time.sleep(5.0)
-    # Remove the version file
-    if release_file.exists():
-        release_file.unlink()
+    # Run the post_build cleanup functions
+    for component in to_do:
+        build_specs[component].post_build()
+
     # Print job summary if output mode is ALL
     if not args.quiet:
         logging.info("Job Summary:")

deploy/scripts/sem_dom_import.py

@@ -210,7 +210,7 @@ def get_sem_doms(node: ElementTree.Element, parent: SemDomTreeMap, prev: SemDomM
         elif field.tag == "Abbreviation":
             for abbrev_node in field:
                 lang, id_text = get_auni_text(abbrev_node)
-                logging.info(f"id[{lang}]='{id_text}'")
+                logging.debug(f"id[{lang}]='{id_text}'")
                 domain_set[lang].id = id_text
         elif field.tag == "Description":
             for descr_node in field:
@@ -265,17 +265,8 @@ def write_json(output_dir: Path) -> None:
                 file.write(f"{domain_tree[lang][id].to_json()}\n")
 
 
-def main() -> None:
-    args = parse_args()
-    # setup logging levels
-    if args.debug:
-        log_level = logging.DEBUG
-    elif args.verbose:
-        log_level = logging.INFO
-    else:
-        log_level = logging.WARNING
-    logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level)
-    for xmlfile in args.input_files:
+def generate_semantic_domains(input_files: List[Path], output_dir: Path) -> None:
+    for xmlfile in input_files:
         logging.info(f"Parsing {xmlfile}")
         tree = ElementTree.parse(xmlfile)
         root = tree.getroot()
@@ -299,7 +290,20 @@ def main() -> None:
         logging.info(f"Number of {lang} Domains: {len(domain_nodes[lang])}")
     for lang in domain_tree:
         logging.info(f"Number of {lang} Tree Nodes: {len(domain_tree[lang])}")
-    write_json(args.output_dir)
+    write_json(output_dir)
+
+
+def main() -> None:
+    args = parse_args()
+    # setup logging levels
+    if args.debug:
+        log_level = logging.DEBUG
+    elif args.verbose:
+        log_level = logging.INFO
+    else:
+        log_level = logging.WARNING
+    logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level)
+    generate_semantic_domains(args.input_files, args.output_dir)
 
 
 if __name__ == "__main__":