diff --git a/.docker-compose.datagov-theme.yaml b/.docker-compose.datagov-theme.yaml index 2aed65c..dd466a0 100644 --- a/.docker-compose.datagov-theme.yaml +++ b/.docker-compose.datagov-theme.yaml @@ -7,10 +7,32 @@ services: args: POST_INSTALL: | install_standard_ckan_extension_github ckan/ckanext-spatial ckanext-spatial &&\ + install_standard_ckan_extension_github ckan/ckanext-xloader ckanext-xloader &&\ + install_standard_ckan_extension_github datagovuk/ckanext-report ckanext-report &&\ + install_standard_ckan_extension_github ckan/ckanext-archiver ckanext-archiver &&\ install_standard_ckan_extension_github ckan/ckanext-harvest ckanext-harvest &&\ - install_standard_ckan_extension_github GSA/ckanext-geodatagov ckanext-geodatagov &&\ - install_standard_ckan_extension_github GSA/ckanext-datagovtheme ckanext-datagovtheme + install_standard_ckan_extension_github ckan/ckanext-dcat ckanext-dcat &&\ + install_standard_ckan_extension_github GSA/ckanext-datajson ckanext-datajson &&\ + install_standard_ckan_extension_github GSA/ckanext-geodatagov ckanext-geodatagov POST_DOCKER_BUILD: | mkdir -p /var/tmp/ckan/dynamic_menu + CKAN_INIT: | + ckan-paster --plugin=ckanext-archiver archiver init -c "CKAN_CONFIG/production.ini" + ckan-paster --plugin=ckanext-report report initdb -c "CKAN_CONFIG/production.ini" + ckan-paster --plugin=ckanext-harvest harvester initdb -c "CKAN_CONFIG/production.ini" environment: - CKAN_CONFIG_TEMPLATE_PREFIX=datagov-theme- + jobs: + image: viderum/ckan-cloud-docker:ckan-latest-datagov-theme + environment: + - CKAN_CONFIG_TEMPLATE_PREFIX=datagov-theme- + + db: + image: mdillon/postgis + build: + args: + DB_INIT: | + psql --dbname="ckan" -c "CREATE EXTENSION IF NOT EXISTS postgis; \ + CREATE EXTENSION IF NOT EXISTS postgis_topology; \ + CREATE EXTENSION IF NOT EXISTS fuzzystrmatch; \ + CREATE EXTENSION IF NOT EXISTS postgis_tiger_geocoder;" diff --git a/ckan/Dockerfile b/ckan/Dockerfile index 7bf5e0c..1dc2733 100644 --- a/ckan/Dockerfile +++ b/ckan/Dockerfile @@ -75,6 +75,9 @@ COPY templater.sh /templater.sh ARG POST_DOCKER_BUILD RUN . /post_install_functions.sh && eval "${POST_DOCKER_BUILD}" +ARG CKAN_INIT +RUN echo "${CKAN_INIT}" | sed s@CKAN_CONFIG@${CKAN_CONFIG}@g > ${CKAN_CONFIG}/ckan_extra_init.sh + ENTRYPOINT ["/ckan-entrypoint.sh"] EXPOSE 5000 diff --git a/ckan/entrypoint.sh b/ckan/entrypoint.sh index e367953..c1162e2 100755 --- a/ckan/entrypoint.sh +++ b/ckan/entrypoint.sh @@ -5,17 +5,22 @@ rm -f $CKAN_CONFIG/*.ini &&\ cp -f $CKAN_K8S_TEMPLATES/${CKAN_WHO_TEMPLATE_PREFIX}who.ini $CKAN_CONFIG/who.ini &&\ bash /templater.sh $CKAN_K8S_TEMPLATES/${CKAN_CONFIG_TEMPLATE_PREFIX}production.ini.template > $CKAN_CONFIG/production.ini &&\ bash /templater.sh $CKAN_K8S_TEMPLATES/${CKAN_INIT_TEMPLATE_PREFIX}ckan_init.sh.template > $CKAN_CONFIG/ckan_init.sh &&\ -bash $CKAN_CONFIG/ckan_init.sh &&\ -echo running ckan-paster db init &&\ -ckan-paster --plugin=ckan db init --verbose -c "${CKAN_CONFIG}/production.ini" &&\ -echo db initialization complete -INIT_RES=$? - -[ "${INIT_RES}" != "0" ] && echo ERROR: Initialization failed +bash $CKAN_CONFIG/ckan_init.sh +[ "$?" != "0" ] && echo ERROR: CKAN Initialization failed && exit 1 if [ "$*" == "" ]; then - [ "${INIT_RES}" != "0" ] && exit 1 + echo running ckan-paster db init &&\ + ckan-paster --plugin=ckan db init -c "${CKAN_CONFIG}/production.ini" &&\ + echo db initialization complete + [ "$?" != "0" ] && echo ERROR: DB Initialization failed && exit 1 + + echo running ckan_extra_init &&\ + . $CKAN_CONFIG/ckan_extra_init.sh &&\ + echo ckan_extra_init complete + [ "$?" != "0" ] && echo ERROR: CKAN extra initialization failed && exit 1 + exec ${CKAN_VENV}/bin/gunicorn --paste ${CKAN_CONFIG}/production.ini --workers ${GUNICORN_WORKERS} else + sleep 180 exec "$@" fi diff --git a/db/Dockerfile b/db/Dockerfile index 30dda30..935527d 100644 --- a/db/Dockerfile +++ b/db/Dockerfile @@ -1,11 +1,14 @@ -FROM postgres:9.6.3-alpine +FROM mdillon/postgis:9.6-alpine RUN apk --update add supervisor -COPY init_db.sh /docker-entrypoint-initdb.d/ +COPY init_ckan_db.sh /docker-entrypoint-initdb.d/ COPY *.sh /db-scripts/ COPY datastore-permissions.sql.template /db-scripts/ COPY datastore-public-ro-supervisord.conf /db-scripts/ +ARG DB_INIT +RUN echo "${DB_INIT}" >> /docker-entrypoint-initdb.d/init_ckan_db.sh + ENTRYPOINT ["/db-scripts/entrypoint.sh"] CMD ["postgres"] diff --git a/db/init_db.sh b/db/init_ckan_db.sh similarity index 91% rename from db/init_db.sh rename to db/init_ckan_db.sh index 9bf4635..3ea9678 100755 --- a/db/init_db.sh +++ b/db/init_ckan_db.sh @@ -8,7 +8,7 @@ if [ -z "${DATASTORE_RO_USER}" ]; then echo Initializing CKAN DB &&\ echo Creating db &&\ createdb ckan -E utf-8 &&\ - echo Creating role &&\ + echo Creating role "create role ckan with login password '${POSTGRES_PASSWORD}'" &&\ psql -c "create role ckan with login password '${POSTGRES_PASSWORD}';" &&\ echo Granting privileges &&\ psql -c 'GRANT ALL PRIVILEGES ON DATABASE "ckan" to ckan;' &&\ diff --git a/docker-compose/ckan-conf-templates/datagov-theme-production.ini.template b/docker-compose/ckan-conf-templates/datagov-theme-production.ini.template index b081620..ca638fc 100644 --- a/docker-compose/ckan-conf-templates/datagov-theme-production.ini.template +++ b/docker-compose/ckan-conf-templates/datagov-theme-production.ini.template @@ -77,8 +77,36 @@ ckan.redis.url = {{CKAN_REDIS_URL}} # Add ``resource_proxy`` to enable resorce proxying and get around the # same origin policy -ckan.plugins = stats text_view image_view recline_view datastore xloader geodatagov datagovtheme - +ckan.plugins = stats + text_view + image_view + recline_view + datastore + xloader + harvest + ckan_harvester + datajson_harvest + datajson + archiver + report + dcat + dcat_rdf_harvester + dcat_json_harvester + dcat_json_interface + structured_data + geodatagov + geodatagov_miscs + z3950_harvester + arcgis_harvester + geodatagov_geoportal_harvester + waf_harvester_collection + geodatagov_csw_harvester + geodatagov_doc_harvester + geodatagov_waf_harvester + spatial_metadata + spatial_query + + # Define which views should be created by default # (plugins must be loaded in ckan.plugins) ckan.views.default_views = image_view text_view recline_view @@ -163,6 +191,12 @@ smtp.user = {{SMTP_USER}} smtp.password = {{SMTP_PASSWORD}} smtp.mail_from = {{SMTP_MAIL_FROM}} +### + +## datajson settings +ckanext.datajson.inventory_links_enabled = True +ckanext.datajson.url_enabled = False + ## Datagov settings ckanext.geodatagov.dynamic_menu.url_default = http://www.data.gov/app/plugins/datagov-custom/wp_download_links.php @@ -170,6 +204,10 @@ ckanext.geodatagov.dynamic_menu.url = ckanext.geodatagov.bureau_csv.url_default = https://project-open-data.cio.gov/data/omb_bureau_codes.csv ckanext.geodatagov.bureau_csv.url = https://project-open-data.cio.gov/data/omb_bureau_codes.csv +## Harvest settings +ckan.harvest.mq.type = redis +ckan.harvest.mq.hostname = redis +ckan.harvest.mq.redis_db = 9 ## Logging configuration [loggers]