support for centralized DB / Solr Cloud, cca-operator stability impro…

…vements (#19) * added support for centralized DB * support for centralized DB and Solr Cloud * add solrcloud to travis * fixed for cca-operator ssh keys and centralized infra. * cca-operator create db scripts can only run once * solrcloud fixes * cca-operator: db init fixes * cca-operator: permission fixes to centralized datastore creation script * cca-operator: fix polling of traefik status, support central infra for create-instance, fix server keys * cca-operator fixes: delete-instance status polling, update-instance admin user creation
datopian · Dec 6, 2018 · d9cfea3 · d9cfea3
1 parent 8936ad6
commit d9cfea3
Show file tree

Hide file tree

Showing 20 changed files with 521 additions and 78 deletions.
diff --git a/.docker-compose-centralized.yaml b/.docker-compose-centralized.yaml
@@ -0,0 +1,27 @@
+version: '3.2'
+
+services:
+  ckan:
+    environment:
+    - INSTANCE_ID
+    - CKAN_K8S_SECRETS=/etc/ckan-conf/secrets/secrets-centralized.sh
+    volumes:
+    - ./docker-compose/ckan-secrets-centralized.sh:/etc/ckan-conf/secrets/secrets-centralized.sh
+
+  jobs:
+    environment:
+    - INSTANCE_ID
+    - CKAN_K8S_SECRETS=/etc/ckan-conf/secrets/secrets-centralized.sh
+    volumes:
+    - ./docker-compose/ckan-secrets-centralized.sh:/etc/ckan-conf/secrets/secrets-centralized.sh
+
+  solr:
+    image: viderum/ckan-cloud-docker:solrcloud-latest
+    build:
+      context: solr
+      dockerfile: solrcloud.Dockerfile
+      args:
+        SCHEMA_XML: ${SCHEMA_XML:-schema.xml}
+    entrypoint: [docker-entrypoint.sh, solr, start, -c, -f]
+    ports:
+     - "8983:8983"
diff --git a/.travis.sh b/.travis.sh
@@ -15,9 +15,12 @@ elif [ "${1}" == "deploy" ]; then
     ! tag_images "${TAG}" && exit 1
     if [ "${TRAVIS_BRANCH}" == "master" ]; then
         ! push_latest_images && exit 1
+        PUSHED_LATEST=1
+    else
+        PUSHED_LATEST=0
     fi
     ! push_tag_images "${TAG}" && exit 1
-    print_summary "${TAG}"
+    print_summary "${TAG}" "${PUSHED_LATEST}"
     if [ "${TRAVIS_TAG}" != "" ]; then
         if ! [ -z "${SLACK_TAG_NOTIFICATION_CHANNEL}" ] && ! [ -z "${SLACK_TAG_NOTIFICATION_WEBHOOK_URL}" ]; then
             ! curl -X POST \

diff --git a/README.md b/README.md
@@ -38,7 +38,7 @@ docker-compose pull
 Start the Docker compose environment
 
 ```
-docker-compose up -d ckan
+docker-compose up -d nginx
 ```
 
 Add a hosts entry mapping domain `nginx` to `127.0.0.1`:
@@ -62,6 +62,12 @@ docker-compose exec ckan ckan-paster --plugin=ckan \
 
 Login to CKAN at http://nginx:8080 with username `admin` and password `12345678`
 
+To start the jobs server for uploading to the datastore DB:
+
+```
+docker-compose up -d jobs
+```
+
 
 ## Making modifications to the docker images / configuration
 
@@ -82,7 +88,7 @@ docker-compose build | grep "Successfully tagged"
 Start the environment
 
 ```
-docker-compose up -d ckan
+docker-compose up -d nginx
 ```
 
 
@@ -142,15 +148,15 @@ services:
 Start the docker-compose environment with the modified config:
 
 ```
-docker-compose -f docker-compose.yaml -f .docker-compose.my-ckan.yaml up -d --build ckan
+docker-compose -f docker-compose.yaml -f .docker-compose.my-ckan.yaml up -d --build nginx
 ```
 
 You can persist the modified configurations in Git for reference and documentation.
 
 For example, to start the datagov-theme configuration:
 
 ```
-docker-compose -f docker-compose.yaml -f .docker-compose.datagov-theme.yaml up -d --build ckan
+docker-compose -f docker-compose.yaml -f .docker-compose.datagov-theme.yaml up -d --build nginx
 ```
 
 ## Running cca-operator
@@ -180,3 +186,51 @@ export PUBLIC_KEY="$(cat docker-compose/provisioning-api/public.pem | while read
 
 docker-compose up -d --build provisioning-api
 ```
+
+## Testing the centralized DB
+
+Create a bash alias to run docker-compose with the centralized configuration
+
+```
+alias docker-compose="`which docker-compose` -f docker-compose.yaml -f .docker-compose-centralized.yaml"
+```
+
+Start a clean environment with only the db and solr cloud -
+
+```
+docker-compose down -v
+docker-compose up -d db solr
+```
+
+Set the instance id which is used for database names and the solr core name
+
+```
+INSTANCE_ID=test1
+```
+
+Create the dbs
+
+```
+docker-compose run --rm cca-operator -c "source functions.sh; PGPASSWORD=123456 create_db db postgres ${INSTANCE_ID} 654321" &&\
+docker-compose run --rm cca-operator -c "source functions.sh; PGPASSWORD=123456 create_datastore_db db postgres ${INSTANCE_ID} ${INSTANCE_ID}-datastore 654321 ${INSTANCE_ID}-datastore-readonly 654321"
+```
+
+Create the solrcloud collection
+
+```
+docker-compose exec solr bin/solr create_collection -c ${INSTANCE_ID} -d ckan_default -n ckan_default
+```
+
+Start ckan
+
+```
+docker-compose up -d --force-recreate jobs
+```
+
+by default it uses `test1` as the INSTANCE_ID, to modify, override the ckan secrets.sh
+
+You might need to reload the solr collection after recreate:
+
+```
+curl "http://localhost:8983/solr/admin/collections?action=RELOAD&name=${INSTANCE_ID}&wt=json"
+```
diff --git a/cca-operator/Dockerfile b/cca-operator/Dockerfile
@@ -1,22 +1,18 @@
-FROM alpine
-
-RUN wget -qO kubectl https://storage.googleapis.com/kubernetes-release/release/$(wget -qO - https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl &&\
-    chmod +x kubectl && mv ./kubectl /usr/local/bin/kubectl
-
-RUN while ! ( apk update && apk add --no-cache bash python grep jq python3 libcurl git docker curl openssl ); do sleep 1; done &&\
-    python3 -m pip install pyyaml
-
-RUN curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > get_helm.sh &&\
+FROM alpine:3.6
+
+RUN while ! ( apk update && apk add --no-cache \
+        bash python grep jq python3 libcurl git docker openssl curl ca-certificates wget \
+        openssh-server openssh-sftp-server postgresql-client \
+    ); do sleep 1; done &&\
+    wget -qO kubectl https://storage.googleapis.com/kubernetes-release/release/$(wget -qO - https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl &&\
+    chmod +x kubectl && mv ./kubectl /usr/local/bin/kubectl &&\
+    python3 -m pip install pyyaml &&\
+    curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get > get_helm.sh &&\
     chmod 700 get_helm.sh && ./get_helm.sh --version v2.11.0 && helm version --client && rm ./get_helm.sh
 
-RUN apk update && apk add openssh-server openssh-sftp-server &&\
-    ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -N "" &&\
-    ssh-keygen -t dsa -f /etc/ssh/ssh_host_dsa_key -N "" &&\
-    ssh-keygen -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key -N "" &&\
-    ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -N ""
-
 COPY *.sh /cca-operator/
 COPY *.py /cca-operator/
+COPY *.template /cca-operator/
 
 RUN chmod +x /cca-operator/*.sh /cca-operator/*.py
 

diff --git a/cca-operator/README.md b/cca-operator/README.md
@@ -7,7 +7,7 @@ cca-operator manages, provisions and configures Ckan Cloud components inside a [
 Build and run using docker-compose:
 
 ```
-docker-compose build cca-operator && docker-compose run --rm cca-operator
+docker-compose build cca-operator && docker-compose run --rm cca-operator --help
 ```
 
 Cca-operator mounts /etc/ckan-cloud directory from the host into the container
@@ -92,11 +92,18 @@ cca-operator() {
 }
 ```
 
+Delete secrets to re-create
+
+```
+kubectl --context minikube -n $CKAN_NAMESPACE delete secret ckan-env-vars ckan-secrets
+```
+
 Run the cca-operator CKAN commands:
 
-* Create the ckan env vars secret: `cca-operator initialize-ckan-env-vars ckan-env-vars`
+* create the ckan env vars secret: `cca-operator initialize-ckan-env-vars ckan-env-vars`
+  * If you use the centralized infra, set the env vars: `--env CKAN_CLOUD_INSTANCE_ID=$CKAN_NAMESPACE --env CKAN_CLOUD_POSTGRES_HOST=db.ckan-cloud --env CKAN_CLOUD_POSTGRES_USER=postgres --env PGPASSWORD=123456 --env CKAN_CLOUD_SOLR_HOST=solr.ckan-cloud --env CKAN_CLOUD_SOLR_PORT=8983`
 * Initialize the CKAN secrets.sh: `cca-operator initialize-ckan-secrets ckan-env-vars ckan-secrets`
-* Write the CKAN secrets to secrets.sh: `cca-operator --command -- bash -c "./cca-operator.sh get-ckan-secrets ckan-secrets secrets.sh && cat secrets.sh"
+* Write the CKAN secrets to secrets.sh: `cca-operator --command -- bash -c "./cca-operator.sh get-ckan-secrets ckan-secrets secrets.sh && cat secrets.sh"`
 
 
 ## cca-operator server

diff --git a/cca-operator/cca-operator.sh b/cca-operator/cca-operator.sh
@@ -6,16 +6,61 @@ if [ "${1}" == "initialize-ckan-env-vars" ]; then
     ENV_VARS_SECRET="${2}"
     [ -z "${ENV_VARS_SECRET}" ] && echo usage: cca-operator initialize-ckan-env-vars '<ENV_VARS_SECRET_NAME>' && exit 1
     if ! kubectl $KUBECTL_GLOBAL_ARGS get secret $ENV_VARS_SECRET; then
+        POSTGRES_PASSWORD=`python -c "import binascii,os;print(binascii.hexlify(os.urandom(12)))"`
+        DATASTORE_POSTGRES_PASSWORD=`python -c "import binascii,os;print(binascii.hexlify(os.urandom(12)))"`
+        DATASTORE_RO_PASSWORD=`python -c "import binascii,os;print(binascii.hexlify(os.urandom(12)))"`
+        if [ -z "${CKAN_CLOUD_POSTGRES_HOST}" ]; then
+            echo Using self-hosted DB
+            POSTGRES_USER=ckan
+            POSTGRES_DB_NAME="${POSTGRES_USER}"
+            POSTGRES_HOST=db
+            DATASTORE_RO_USER=readonly
+            DATASTORE_POSTGRES_USER=postgres
+        else
+            echo Using centralized DB
+            if [ -z "${CKAN_CLOUD_INSTANCE_ID}" ]; then
+                POSTGRES_USER=`python -c "import binascii,os;print(binascii.hexlify(os.urandom(8)))"`
+                POSTGRES_USER="ckan-${POSTGRES_USER}"
+            else
+                POSTGRES_USER="${CKAN_CLOUD_INSTANCE_ID}"
+            fi
+            ! create_db "${CKAN_CLOUD_POSTGRES_HOST}" "${CKAN_CLOUD_POSTGRES_USER:-postgres}" "${POSTGRES_USER}" "${POSTGRES_PASSWORD}" \
+                && exit 1
+            POSTGRES_DB_NAME="${POSTGRES_USER}"
+            POSTGRES_HOST="${CKAN_CLOUD_POSTGRES_HOST}"
+            DATASTORE_RO_USER="${POSTGRES_DB_NAME}-datastore-readonly"
+            DATASTORE_POSTGRES_USER="${POSTGRES_DB_NAME}-datastore"
+            ! create_datastore_db "${POSTGRES_HOST}" "${CKAN_CLOUD_POSTGRES_USER:-postgres}" "${POSTGRES_DB_NAME}" \
+                                  "${DATASTORE_POSTGRES_USER}" "${DATASTORE_POSTGRES_PASSWORD}" \
+                                  "${DATASTORE_RO_USER}" "${DATASTORE_RO_PASSWORD}" \
+                && exit 1
+        fi
+        if [ -z "${CKAN_CLOUD_SOLR_HOST}" ]; then
+            echo using self-hosted solr
+            SOLR_URL="http://solr:8983/solr/ckan"
+        else
+            echo using centralized solr cloud
+            if [ -z "${CKAN_CLOUD_INSTANCE_ID}" ]; then
+                SOLRCLOUD_COLLECTION=`python -c "import binascii,os;print(binascii.hexlify(os.urandom(8)))"`
+                SOLRCLOUD_COLLECTION="ckan-${SOLRCLOUD_COLLECTION}"
+            else
+                SOLRCLOUD_COLLECTION="${CKAN_CLOUD_INSTANCE_ID}"
+            fi
+            SOLR_URL="http://${CKAN_CLOUD_SOLR_HOST}:${CKAN_CLOUD_SOLR_PORT:-8983}/solr/${SOLRCLOUD_COLLECTION}"
+        fi
         echo "Creating ckan env vars secret ${ENV_VARS_SECRET}"
         ! kubectl $KUBECTL_GLOBAL_ARGS create secret generic $ENV_VARS_SECRET \
                   --from-literal=CKAN_APP_INSTANCE_UUID=`python -c "import uuid;print(uuid.uuid1())"` \
                   --from-literal=CKAN_BEAKER_SESSION_SECRET=`python -c "import binascii,os;print(binascii.hexlify(os.urandom(25)))"` \
-                  --from-literal=POSTGRES_PASSWORD=`python -c "import binascii,os;print(binascii.hexlify(os.urandom(12)))"` \
-                  --from-literal=POSTGRES_USER=ckan \
-                  --from-literal=DATASTORE_POSTGRES_PASSWORD=`python -c "import binascii,os;print(binascii.hexlify(os.urandom(12)))"` \
-                  --from-literal=DATASTORE_POSTGRES_USER=postgres \
-                  --from-literal=DATASTORE_RO_USER=readonly \
-                  --from-literal=DATASTORE_RO_PASSWORD=`python -c "import binascii,os;print(binascii.hexlify(os.urandom(12)))"` \
+                  --from-literal=POSTGRES_PASSWORD=${POSTGRES_PASSWORD} \
+                  --from-literal=POSTGRES_USER=${POSTGRES_USER} \
+                  --from-literal=POSTGRES_HOST=${POSTGRES_HOST} \
+                  --from-literal=POSTGRES_DB_NAME=${POSTGRES_DB_NAME} \
+                  --from-literal=DATASTORE_POSTGRES_PASSWORD=${DATASTORE_POSTGRES_PASSWORD} \
+                  --from-literal=DATASTORE_POSTGRES_USER=${DATASTORE_POSTGRES_USER} \
+                  --from-literal=DATASTORE_RO_USER=${DATASTORE_RO_USER} \
+                  --from-literal=DATASTORE_RO_PASSWORD=${DATASTORE_RO_PASSWORD} \
+                  --from-literal=SOLR_URL=${SOLR_URL} \
             && echo Failed to create ckan env vars secret && exit 1
         echo Created ckan env vars secret && exit 0
     else
@@ -33,18 +78,19 @@ elif [ "${1}" == "initialize-ckan-secrets" ]; then
         ! export_ckan_env_vars $ENV_VARS_SECRET && exit 1
         TEMPFILE=`mktemp`
         echo "export BEAKER_SESSION_SECRET=${CKAN_BEAKER_SESSION_SECRET}
-        export APP_INSTANCE_UUID=${CKAN_APP_INSTANCE_UUID}
-        export SQLALCHEMY_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db/ckan
-        export CKAN_DATASTORE_WRITE_URL=postgresql://${DATASTORE_POSTGRES_USER}:${DATASTORE_POSTGRES_PASSWORD}@datastore-db/datastore
-        export CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_RO_USER}:${DATASTORE_RO_PASSWORD}@datastore-db/datastore
-        export SOLR_URL=http://solr:8983/solr/ckan
-        export CKAN_REDIS_URL=redis://redis:6379/1
-        export CKAN_DATAPUSHER_URL=
-        export SMTP_SERVER=
-        export SMTP_STARTTLS=
-        export SMTP_USER=
-        export SMTP_PASSWORD=
-        export SMTP_MAIL_FROM=" > $TEMPFILE
+export APP_INSTANCE_UUID=${CKAN_APP_INSTANCE_UUID}
+export SQLALCHEMY_URL=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST:-db}/${POSTGRES_DB_NAME:-ckan}
+export CKAN_DATASTORE_WRITE_URL=postgresql://${DATASTORE_POSTGRES_USER}:${DATASTORE_POSTGRES_PASSWORD}@${POSTGRES_HOST:-datastore-db}/${DATASTORE_POSTGRES_USER:-datastore}
+export CKAN_DATASTORE_READ_URL=postgresql://${DATASTORE_RO_USER}:${DATASTORE_RO_PASSWORD}@${POSTGRES_HOST:-datastore-db}/${DATASTORE_POSTGRES_USER:-datastore}
+export SOLR_URL=${SOLR_URL}
+export CKAN_REDIS_URL=redis://redis:6379/1
+export CKAN_DATAPUSHER_URL=
+export SMTP_SERVER=
+export SMTP_STARTTLS=
+export SMTP_USER=
+export SMTP_PASSWORD=
+export SMTP_MAIL_FROM=" > $TEMPFILE
+        cat $TEMPFILE
         kubectl $KUBECTL_GLOBAL_ARGS create secret generic "${CKAN_SECRETS_SECRET}" --from-file=secrets.sh=$TEMPFILE
         CKAN_SECRET_RES="$?"
         rm $TEMPFILE

diff --git a/cca-operator/datastore-permissions.sql.template b/cca-operator/datastore-permissions.sql.template
@@ -0,0 +1,53 @@
+CREATE OR REPLACE VIEW "_table_metadata" AS
+    SELECT DISTINCT
+        substr(md5(dependee.relname || COALESCE(dependent.relname, '')), 0, 17) AS "_id",
+        dependee.relname AS name,
+        dependee.oid AS oid,
+        dependent.relname AS alias_of
+    FROM
+        pg_class AS dependee
+        LEFT OUTER JOIN pg_rewrite AS r ON r.ev_class = dependee.oid
+        LEFT OUTER JOIN pg_depend AS d ON d.objid = r.oid
+        LEFT OUTER JOIN pg_class AS dependent ON d.refobjid = dependent.oid
+    WHERE
+        (dependee.oid != dependent.oid OR dependent.oid IS NULL) AND
+        -- is a table (from pg_tables view definition)
+        -- or is a view (from pg_views view definition)
+        (dependee.relkind = 'r'::"char" OR dependee.relkind = 'v'::"char")
+        AND dependee.relnamespace = (
+            SELECT oid FROM pg_namespace WHERE nspname='public')
+    ORDER BY dependee.oid DESC;
+ALTER VIEW "_table_metadata" OWNER TO "{{SITE_USER}}";
+GRANT SELECT ON "_table_metadata" TO "{{DS_RO_USER}}";
+
+CREATE OR REPLACE FUNCTION populate_full_text_trigger() RETURNS trigger
+AS $body$
+    BEGIN
+        IF NEW._full_text IS NOT NULL THEN
+            RETURN NEW;
+        END IF;
+        NEW._full_text := (
+            SELECT to_tsvector(string_agg(value, ' '))
+            FROM json_each_text(row_to_json(NEW.*))
+            WHERE key NOT LIKE '\_%');
+        RETURN NEW;
+    END;
+$body$ LANGUAGE plpgsql;
+ALTER FUNCTION populate_full_text_trigger() OWNER TO "{{SITE_USER}}";
+
+DO $body$
+    BEGIN
+        EXECUTE coalesce(
+            (SELECT string_agg(
+                'CREATE TRIGGER zfulltext BEFORE INSERT OR UPDATE ON ' ||
+                quote_ident(relname) || ' FOR EACH ROW EXECUTE PROCEDURE ' ||
+                'populate_full_text_trigger();', ' ')
+            FROM pg_class
+            LEFT OUTER JOIN pg_trigger AS t
+                ON t.tgrelid = relname::regclass AND t.tgname = 'zfulltext'
+            WHERE relkind = 'r'::"char" AND t.tgname IS NULL
+                AND relnamespace = (
+                    SELECT oid FROM pg_namespace WHERE nspname='public')),
+            'SELECT 1;');
+    END;
+$body$;
diff --git a/cca-operator/delete-instance.sh b/cca-operator/delete-instance.sh
@@ -8,12 +8,20 @@ source functions.sh
 if kubectl $KUBECTL_GLOBAL_ARGS get ns "${INSTANCE_NAMESPACE}"; then
     echo Deleting instance namespace: ${INSTANCE_NAMESPACE}
 
-    ! kubectl $KUBECTL_GLOBAL_ARGS -n ${INSTANCE_NAMESPACE} delete deployment ckan jobs && echo WARNING: failed to delete ckan pods
-    ! kubectl $KUBECTL_GLOBAL_ARGS delete ns "${INSTANCE_NAMESPACE}" && echo WARNING: failed to delete instance namespace
+    ! kubectl $KUBECTL_GLOBAL_ARGS -n ${INSTANCE_NAMESPACE} delete deployment ckan jobs --wait=false && echo WARNING: failed to delete ckan pods
+    echo waiting 60 seconds to let ckan pods stop
+    sleep 60
+    ! kubectl $KUBECTL_GLOBAL_ARGS delete ns "${INSTANCE_NAMESPACE}" --wait=false && echo WARNING: failed to delete instance namespace
+    echo waiting 60 seconds to let namespace terminate
+    echo waiting for all pods to be removed from namespace
+    while [ "$(kubectl get pods -n jenkins23 --no-headers | tee /dev/stderr | wc -l)" != "0" ]; do
+        sleep 5
+        echo .
+    done
 
     echo WARNING! instance was not removed from the load balancer
 
-    echo Instance namespace ${INSTANCE_NAMESPACE} deleted
+    echo Instance namespace ${INSTANCE_NAMESPACE} terminated successfully
 else
     echo Instance namespace does not exist: ${INSTANCE_NAMESPACE}
 fi