Add datagov theme (plus some infrastructure fixes) (#20)

datopian · Dec 6, 2018 · b7835ca · b7835ca
1 parent d9cfea3
commit b7835ca
Show file tree

Hide file tree

Showing 6 changed files with 86 additions and 15 deletions.
diff --git a/.docker-compose.datagov-theme.yaml b/.docker-compose.datagov-theme.yaml
@@ -7,10 +7,32 @@ services:
       args:
         POST_INSTALL: |
           install_standard_ckan_extension_github ckan/ckanext-spatial ckanext-spatial &&\
+          install_standard_ckan_extension_github ckan/ckanext-xloader ckanext-xloader &&\
+          install_standard_ckan_extension_github datagovuk/ckanext-report ckanext-report &&\
+          install_standard_ckan_extension_github ckan/ckanext-archiver ckanext-archiver &&\
           install_standard_ckan_extension_github ckan/ckanext-harvest ckanext-harvest &&\
-          install_standard_ckan_extension_github GSA/ckanext-geodatagov ckanext-geodatagov &&\
-          install_standard_ckan_extension_github GSA/ckanext-datagovtheme ckanext-datagovtheme
+          install_standard_ckan_extension_github ckan/ckanext-dcat ckanext-dcat &&\
+          install_standard_ckan_extension_github GSA/ckanext-datajson ckanext-datajson &&\
+          install_standard_ckan_extension_github GSA/ckanext-geodatagov ckanext-geodatagov
         POST_DOCKER_BUILD: |
           mkdir -p /var/tmp/ckan/dynamic_menu
+        CKAN_INIT: |
+          ckan-paster --plugin=ckanext-archiver archiver init  -c "CKAN_CONFIG/production.ini"
+          ckan-paster --plugin=ckanext-report report initdb  -c "CKAN_CONFIG/production.ini"
+          ckan-paster --plugin=ckanext-harvest harvester initdb  -c "CKAN_CONFIG/production.ini"
     environment:
     - CKAN_CONFIG_TEMPLATE_PREFIX=datagov-theme-
+  jobs:
+    image: viderum/ckan-cloud-docker:ckan-latest-datagov-theme
+    environment:
+    - CKAN_CONFIG_TEMPLATE_PREFIX=datagov-theme-
+
+  db:
+    image: mdillon/postgis
+    build:
+      args:
+        DB_INIT: |
+          psql --dbname="ckan" -c "CREATE EXTENSION IF NOT EXISTS postgis;                  \
+                                   CREATE EXTENSION IF NOT EXISTS postgis_topology;         \
+                                   CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;            \
+                                   CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder;"
diff --git a/ckan/Dockerfile b/ckan/Dockerfile
@@ -75,6 +75,9 @@ COPY templater.sh /templater.sh
 ARG POST_DOCKER_BUILD
 RUN . /post_install_functions.sh && eval "${POST_DOCKER_BUILD}"
 
+ARG CKAN_INIT
+RUN echo "${CKAN_INIT}" | sed s@CKAN_CONFIG@${CKAN_CONFIG}@g > ${CKAN_CONFIG}/ckan_extra_init.sh
+
 ENTRYPOINT ["/ckan-entrypoint.sh"]
 
 EXPOSE 5000

diff --git a/ckan/entrypoint.sh b/ckan/entrypoint.sh
@@ -5,17 +5,22 @@ rm -f $CKAN_CONFIG/*.ini &&\
 cp -f $CKAN_K8S_TEMPLATES/${CKAN_WHO_TEMPLATE_PREFIX}who.ini $CKAN_CONFIG/who.ini &&\
 bash /templater.sh $CKAN_K8S_TEMPLATES/${CKAN_CONFIG_TEMPLATE_PREFIX}production.ini.template > $CKAN_CONFIG/production.ini &&\
 bash /templater.sh $CKAN_K8S_TEMPLATES/${CKAN_INIT_TEMPLATE_PREFIX}ckan_init.sh.template > $CKAN_CONFIG/ckan_init.sh &&\
-bash $CKAN_CONFIG/ckan_init.sh &&\
-echo running ckan-paster db init &&\
-ckan-paster --plugin=ckan db init --verbose -c "${CKAN_CONFIG}/production.ini" &&\
-echo db initialization complete
-INIT_RES=$?
-
-[ "${INIT_RES}" != "0" ] && echo ERROR: Initialization failed
+bash $CKAN_CONFIG/ckan_init.sh
+[ "$?" != "0" ] && echo ERROR: CKAN Initialization failed && exit 1
 
 if [ "$*" == "" ]; then
-    [ "${INIT_RES}" != "0" ] && exit 1
+    echo running ckan-paster db init &&\
+    ckan-paster --plugin=ckan db init -c "${CKAN_CONFIG}/production.ini" &&\
+    echo db initialization complete
+    [ "$?" != "0" ] && echo ERROR: DB Initialization failed && exit 1
+
+    echo running ckan_extra_init &&\
+    . $CKAN_CONFIG/ckan_extra_init.sh &&\
+    echo ckan_extra_init complete
+    [ "$?" != "0" ] && echo ERROR: CKAN extra initialization failed && exit 1
+
     exec ${CKAN_VENV}/bin/gunicorn --paste ${CKAN_CONFIG}/production.ini --workers ${GUNICORN_WORKERS}
 else
+    sleep 180
     exec "$@"
 fi
diff --git a/db/Dockerfile b/db/Dockerfile
@@ -1,11 +1,14 @@
-FROM postgres:9.6.3-alpine
+FROM mdillon/postgis:9.6-alpine
 
 RUN apk --update add supervisor
 
-COPY init_db.sh /docker-entrypoint-initdb.d/
+COPY init_ckan_db.sh /docker-entrypoint-initdb.d/
 COPY *.sh /db-scripts/
 COPY datastore-permissions.sql.template /db-scripts/
 COPY datastore-public-ro-supervisord.conf /db-scripts/
 
+ARG DB_INIT
+RUN echo "${DB_INIT}" >> /docker-entrypoint-initdb.d/init_ckan_db.sh
+
 ENTRYPOINT ["/db-scripts/entrypoint.sh"]
 CMD ["postgres"]
diff --git a/db/init_db.sh → db/init_ckan_db.sh b/db/init_db.sh → db/init_ckan_db.sh
@@ -8,7 +8,7 @@ if [ -z "${DATASTORE_RO_USER}" ]; then
     echo Initializing CKAN DB &&\
     echo Creating db &&\
     createdb ckan -E utf-8 &&\
-    echo Creating role &&\
+    echo Creating role "create role ckan with login password '${POSTGRES_PASSWORD}'" &&\
     psql -c "create role ckan with login password '${POSTGRES_PASSWORD}';" &&\
     echo Granting privileges &&\
     psql -c 'GRANT ALL PRIVILEGES ON DATABASE "ckan" to ckan;' &&\

diff --git a/docker-compose/ckan-conf-templates/datagov-theme-production.ini.template b/docker-compose/ckan-conf-templates/datagov-theme-production.ini.template
@@ -77,8 +77,36 @@ ckan.redis.url = {{CKAN_REDIS_URL}}
 
 #		Add ``resource_proxy`` to enable resorce proxying and get around the
 #		same origin policy
-ckan.plugins = stats text_view image_view recline_view datastore xloader geodatagov datagovtheme
-
+ckan.plugins = stats
+   text_view
+   image_view
+   recline_view
+   datastore
+   xloader
+   harvest
+   ckan_harvester
+   datajson_harvest
+   datajson
+   archiver
+   report
+   dcat
+   dcat_rdf_harvester
+   dcat_json_harvester
+   dcat_json_interface
+   structured_data
+   geodatagov
+   geodatagov_miscs
+   z3950_harvester
+   arcgis_harvester
+   geodatagov_geoportal_harvester
+   waf_harvester_collection
+   geodatagov_csw_harvester
+   geodatagov_doc_harvester
+   geodatagov_waf_harvester
+   spatial_metadata
+   spatial_query
+
+
 # Define which views should be created by default
 # (plugins must be loaded in ckan.plugins)
 ckan.views.default_views = image_view text_view recline_view
@@ -163,13 +191,23 @@ smtp.user = {{SMTP_USER}}
 smtp.password = {{SMTP_PASSWORD}}
 smtp.mail_from = {{SMTP_MAIL_FROM}}
 
+###
+
+## datajson settings
+ckanext.datajson.inventory_links_enabled = True
+ckanext.datajson.url_enabled = False
+
 
 ## Datagov settings
 ckanext.geodatagov.dynamic_menu.url_default = http://www.data.gov/app/plugins/datagov-custom/wp_download_links.php
 ckanext.geodatagov.dynamic_menu.url =
 ckanext.geodatagov.bureau_csv.url_default = https://project-open-data.cio.gov/data/omb_bureau_codes.csv
 ckanext.geodatagov.bureau_csv.url = https://project-open-data.cio.gov/data/omb_bureau_codes.csv
 
+## Harvest settings
+ckan.harvest.mq.type = redis
+ckan.harvest.mq.hostname = redis
+ckan.harvest.mq.redis_db = 9
 
 ## Logging configuration
 [loggers]