From 32a5277ecdba04239b90b9cbb1b22bd5be70efdd Mon Sep 17 00:00:00 2001 From: Jim Grady Date: Wed, 7 Sep 2022 09:57:56 -0400 Subject: [PATCH] Create a specialized database container (#1726) * Add db initialization to database container * Add database to build components * Update GitHub Actions to build & update database component * Add test build of database container to GitHub Actions workflow --- .../actions/combine-deploy-update/action.yml | 5 ++ .github/workflows/database.yml | 24 ++++++ .github/workflows/deploy_qa.yml | 2 +- .gitignore | 1 + database/Dockerfile | 18 +++++ database/init/update-semantic-domains.sh | 4 + .../charts/database/templates/_helpers.tpl | 25 ++++++ .../charts/database/templates/database.yaml | 4 +- .../thecombine/charts/database/values.yaml | 10 ++- deploy/scripts/build.py | 78 ++++++++++++------- deploy/scripts/sem_dom_import.py | 30 +++---- 11 files changed, 155 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/database.yml create mode 100644 database/Dockerfile create mode 100755 database/init/update-semantic-domains.sh create mode 100644 deploy/helm/thecombine/charts/database/templates/_helpers.tpl diff --git a/.github/actions/combine-deploy-update/action.yml b/.github/actions/combine-deploy-update/action.yml index 47378beed4..9a9c50fade 100644 --- a/.github/actions/combine-deploy-update/action.yml +++ b/.github/actions/combine-deploy-update/action.yml @@ -33,6 +33,11 @@ runs: - name: Deploy updated images run: echo "Update images with version ${{ inputs.image_tag }}" shell: bash + - name: Update database + run: kubectl --context ${{ inputs.kube_context }} + set image deployment/database + database="${{ inputs.image_registry }}${{ inputs.image_registry_alias }}/combine_database:${{ inputs.image_tag }}" + shell: bash - name: Update frontend run: kubectl --context ${{ inputs.kube_context }} set image deployment/frontend diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml new file mode 100644 index 0000000000..803ab14f8a --- /dev/null +++ b/.github/workflows/database.yml @@ -0,0 +1,24 @@ +name: database + +on: + pull_request: + branches: [master] + +jobs: + docker_build: + if: ${{ github.event.type }} == "PullRequest" + runs-on: ubuntu-latest + steps: + # For subfolders, currently a full checkout is required. + # See: https://github.com/marketplace/actions/build-and-push-docker-images#path-context + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Build database image + run: | + deploy/scripts/build.py --components database + shell: bash + - name: Image digest + run: | + docker image inspect combine_database:latest -f '{{json .Id}}' + shell: bash diff --git a/.github/workflows/deploy_qa.yml b/.github/workflows/deploy_qa.yml index 2ee9fcb61e..1fdda8a222 100644 --- a/.github/workflows/deploy_qa.yml +++ b/.github/workflows/deploy_qa.yml @@ -8,7 +8,7 @@ jobs: build: strategy: matrix: - component: [frontend, backend, maintenance] + component: [frontend, backend, maintenance, database] runs-on: ubuntu-latest outputs: image_tag: ${{ steps.build_combine.outputs.image_tag }} diff --git a/.gitignore b/.gitignore index f9f2e5bd62..cfb4f0f655 100644 --- a/.gitignore +++ b/.gitignore @@ -76,6 +76,7 @@ venv # Intermediate and output files for Semantic Domain import scripts !deploy/scripts/semantic_domains/xml/*.xml deploy/scripts/semantic_domains/json/*.json +database/semantic_domains/* # Kubernetes Configuration files **/site_files/ diff --git a/database/Dockerfile b/database/Dockerfile new file mode 100644 index 0000000000..16068ab191 --- /dev/null +++ b/database/Dockerfile @@ -0,0 +1,18 @@ +FROM mongo:5.0 + +WORKDIR /app + +RUN mkdir /data/semantic-domains + +# Copy semantic domain import files +COPY semantic_domains/* /data/semantic-domains/ + +# from https://hub.docker.com/_/mongo +# Initializing a fresh instance +# When a container is started for the first time it will execute files +# with extensions .sh and .js that are found in /docker-entrypoint-initdb.d. +# Files will be executed in alphabetical order. .js files will be executed +# by mongosh (mongo on versions below 6) using the database specified by +# the MONGO_INITDB_DATABASE variable, if it is present, or test otherwise. +# You may also switch databases within the .js script. +COPY init/* /docker-entrypoint-initdb.d/ diff --git a/database/init/update-semantic-domains.sh b/database/init/update-semantic-domains.sh new file mode 100755 index 0000000000..275bc7c89f --- /dev/null +++ b/database/init/update-semantic-domains.sh @@ -0,0 +1,4 @@ +#! /usr/bin/bash + +mongoimport -d CombineDatabase -c SemanticDomainTree /data/semantic-domains/tree.json --mode=merge --upsertFields=id,guid,lang +mongoimport -d CombineDatabase -c SemanticDomains /data/semantic-domains/nodes.json --mode=merge --upsertFields=id,guid,lang diff --git a/deploy/helm/thecombine/charts/database/templates/_helpers.tpl b/deploy/helm/thecombine/charts/database/templates/_helpers.tpl new file mode 100644 index 0000000000..a46caa8a17 --- /dev/null +++ b/deploy/helm/thecombine/charts/database/templates/_helpers.tpl @@ -0,0 +1,25 @@ +{{/* Build container image name */}} +{{- define "database.containerImage" -}} + {{- if .Values.global.imageRegistry }} + {{- $registry := .Values.global.imageRegistry }} + {{- if contains "awsEcr" .Values.global.imageRegistry }} + {{- $registry = printf "%s.dkr.ecr.%s.amazonaws.com" .Values.global.awsAccount .Values.global.awsDefaultRegion }} + {{- end }} + {{- printf "%s/%s:%s" $registry .Values.imageName .Values.global.imageTag }} + {{- else }} + {{- printf "%s:%s" .Values.imageName .Values.global.imageTag }} + {{- end }} +{{- end }} + +{{/* Get the Image Pull Policy */}} +{{- define "database.imagePullPolicy" }} + {{- if .Values.global.imagePullPolicy }} + {{- print .Values.global.imagePullPolicy }} + {{- else }} + {{- if eq .Values.global.imageTag "latest" }} + {{- print "Always" }} + {{- else }} + {{- print "IfNotPresent" }} + {{- end }} + {{- end }} +{{- end }} diff --git a/deploy/helm/thecombine/charts/database/templates/database.yaml b/deploy/helm/thecombine/charts/database/templates/database.yaml index 6c0f9497dc..71bbf013f4 100644 --- a/deploy/helm/thecombine/charts/database/templates/database.yaml +++ b/deploy/helm/thecombine/charts/database/templates/database.yaml @@ -38,8 +38,8 @@ spec: combine-component: database spec: containers: - - image: mongo:{{ .Values.mongoImageTag }} - imagePullPolicy: IfNotPresent + - image: {{ template "database.containerImage" . }} + imagePullPolicy: {{ template "database.imagePullPolicy" . }} name: database ports: - containerPort: 27017 diff --git a/deploy/helm/thecombine/charts/database/values.yaml b/deploy/helm/thecombine/charts/database/values.yaml index 74ebd2a243..8076f898c5 100644 --- a/deploy/helm/thecombine/charts/database/values.yaml +++ b/deploy/helm/thecombine/charts/database/values.yaml @@ -2,5 +2,13 @@ # This is a YAML-formatted file. # Declare variables to be passed into your templates. -mongoImageTag: "5.0" +global: + # Update strategy should be "Recreate" or "Rolling Update" + updateStrategy: Recreate + pullSecretName: "None" + imageTag: "latest" + # Define the image registry to use (may be blank for local images) + imageRegistry: "" + persistentVolumeSize: 16Gi +imageName: combine_database diff --git a/deploy/scripts/build.py b/deploy/scripts/build.py index a8c7761cc4..5bac70ba29 100755 --- a/deploy/scripts/build.py +++ b/deploy/scripts/build.py @@ -17,10 +17,11 @@ import subprocess import sys import time -from typing import Dict, List, Optional +from typing import Callable, Dict, List, Optional from app_release import get_release, set_release from enum_types import JobStatus +from sem_dom_import import generate_semantic_domains from streamfile import StreamFile @@ -28,6 +29,8 @@ class BuildSpec: dir: Path name: str + pre_build: Callable[[], None] + post_build: Callable[[], None] @dataclass @@ -118,6 +121,41 @@ def check_jobs(self) -> JobStatus: """Absolute path to the checked out repository.""" +# Pre-build functions for the different build components +def build_semantic_domains() -> None: + """Create the semantic domain definition files.""" + source_dir = project_dir / "deploy" / "scripts" / "semantic_domains" / "xml" + output_dir = project_dir / "database" / "semantic_domains" + generate_semantic_domains(list(source_dir.glob("*.xml")), output_dir) + + +def create_release_file() -> None: + """Create the release.js file to be built into the Frontend.""" + release_file = project_dir / "public" / "scripts" / "release.js" + set_release(get_release(), release_file) + + +def rm_release_file() -> None: + """Remove release.js file if it exists.""" + release_file = project_dir / "public" / "scripts" / "release.js" + if release_file.exists(): + release_file.unlink() + + +def no_op() -> None: + pass + + +# Create a dictionary to look up the build spec from +# a component name +build_specs: Dict[str, BuildSpec] = { + "backend": BuildSpec(project_dir / "Backend", "backend", no_op, no_op), + "database": BuildSpec(project_dir / "database", "database", build_semantic_domains, no_op), + "maintenance": BuildSpec(project_dir / "maintenance", "maint", no_op, no_op), + "frontend": BuildSpec(project_dir, "frontend", create_release_file, rm_release_file), +} + + def get_image_name(repo: Optional[str], component: str, tag: Optional[str]) -> str: """Build the image name from the repo, the component, and the image tag.""" tag_str = "" @@ -137,7 +175,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--components", nargs="*", - choices=["frontend", "backend", "maintenance"], + choices=build_specs.keys(), help="Combine components to build.", ) parser.add_argument( @@ -180,13 +218,6 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Print extra debugging information.", ) - group.add_argument( - "--info", - "-i", - action="store_true", - help="Print info when jobs start and stop in addition to job output.", - ) - # Transform --output-mode argument to an enumerated type return parser.parse_args() @@ -198,10 +229,10 @@ def main() -> None: # independent of the logging facility if args.debug: log_level = logging.DEBUG - elif args.info: - log_level = logging.INFO - else: + elif args.quiet: log_level = logging.WARNING + else: + log_level = logging.INFO logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) # Setup required build engine - docker or nerdctl @@ -224,25 +255,13 @@ def main() -> None: if args.components is not None: to_do = args.components else: - to_do = ["backend", "frontend", "maintenance"] - - # Create a dictionary to look up the build spec from - # a component name - build_specs: Dict[str, BuildSpec] = { - "backend": BuildSpec(project_dir / "Backend", "backend"), - "maintenance": BuildSpec(project_dir / "maintenance", "maint"), - "frontend": BuildSpec(project_dir, "frontend"), - } - - # Create the version file - release_file = project_dir / "public" / "scripts" / "release.js" - - set_release(get_release(), release_file) + to_do = build_specs.keys() # Create the set of jobs to be run for all components job_set: Dict[str, JobQueue] = {} for component in to_do: spec = build_specs[component] + spec.pre_build() image_name = get_image_name(args.repo, spec.name, args.tag) job_set[component] = JobQueue(component) job_set[component].add_job( @@ -279,9 +298,10 @@ def main() -> None: if not running_jobs: break time.sleep(5.0) - # Remove the version file - if release_file.exists(): - release_file.unlink() + # Run the post_build cleanup functions + for component in to_do: + build_specs[component].post_build() + # Print job summary if output mode is ALL if not args.quiet: logging.info("Job Summary:") diff --git a/deploy/scripts/sem_dom_import.py b/deploy/scripts/sem_dom_import.py index 17e8b3cb90..5c099e13b7 100755 --- a/deploy/scripts/sem_dom_import.py +++ b/deploy/scripts/sem_dom_import.py @@ -210,7 +210,7 @@ def get_sem_doms(node: ElementTree.Element, parent: SemDomTreeMap, prev: SemDomM elif field.tag == "Abbreviation": for abbrev_node in field: lang, id_text = get_auni_text(abbrev_node) - logging.info(f"id[{lang}]='{id_text}'") + logging.debug(f"id[{lang}]='{id_text}'") domain_set[lang].id = id_text elif field.tag == "Description": for descr_node in field: @@ -265,17 +265,8 @@ def write_json(output_dir: Path) -> None: file.write(f"{domain_tree[lang][id].to_json()}\n") -def main() -> None: - args = parse_args() - # setup logging levels - if args.debug: - log_level = logging.DEBUG - elif args.verbose: - log_level = logging.INFO - else: - log_level = logging.WARNING - logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) - for xmlfile in args.input_files: +def generate_semantic_domains(input_files: List[Path], output_dir: Path) -> None: + for xmlfile in input_files: logging.info(f"Parsing {xmlfile}") tree = ElementTree.parse(xmlfile) root = tree.getroot() @@ -299,7 +290,20 @@ def main() -> None: logging.info(f"Number of {lang} Domains: {len(domain_nodes[lang])}") for lang in domain_tree: logging.info(f"Number of {lang} Tree Nodes: {len(domain_tree[lang])}") - write_json(args.output_dir) + write_json(output_dir) + + +def main() -> None: + args = parse_args() + # setup logging levels + if args.debug: + log_level = logging.DEBUG + elif args.verbose: + log_level = logging.INFO + else: + log_level = logging.WARNING + logging.basicConfig(format="%(levelname)s:%(message)s", level=log_level) + generate_semantic_domains(args.input_files, args.output_dir) if __name__ == "__main__":