Merge pull request #13 from qld-gov-au/develop

Develop to master - sync with upstream

.ahoy.yml

@@ -0,0 +1,196 @@
+---
+ahoyapi: v2
+
+commands:
+
+  # Docker commands.
+  build:
+    usage: Build or rebuild project.
+    cmd: |
+      ahoy title "Building project"
+      ahoy pre-flight
+      ahoy clean
+      ahoy build-network
+      ahoy up -- --build --force-recreate
+      ahoy install-site
+      ahoy title "Build complete"
+      ahoy doctor
+      ahoy info 1
+
+  build-network:
+    usage: Ensure that the amazeeio network exists.
+    cmd: |
+      docker network prune -f > /dev/null
+      docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network
+
+  info:
+    usage: Print information about this project.
+    cmd: |
+      ahoy line "Project                  : " ${PROJECT}
+      ahoy line "Site local URL           : " ${LAGOON_LOCALDEV_URL}
+      ahoy line "DB port on host          : " $(docker port $(docker-compose ps -q postgres) 5432 | cut -d : -f 2)
+      ahoy line "Solr port on host        : " $(docker port $(docker-compose ps -q solr) 8983 | cut -d : -f 2)
+      ahoy line "Mailhog URL              : " http://mailhog.docker.amazee.io/
+
+  up:
+    usage: Build and start Docker containers.
+    cmd: |
+      docker-compose up -d "$@"
+      sleep 10
+      docker-compose logs
+      ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m"
+      if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi
+      if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi
+      docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_
+      export DOCTOR_CHECK_CLI=0
+
+  down:
+    usage: Stop Docker containers and remove container, images, volumes and networks.
+    cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi'
+
+  start:
+    usage: Start existing Docker containers.
+    cmd: docker-compose start "$@"
+
+  stop:
+    usage: Stop running Docker containers.
+    cmd: docker-compose stop "$@"
+
+  restart:
+    usage: Restart all stopped and running Docker containers.
+    cmd: docker-compose restart "$@"
+
+  logs:
+    usage: Show Docker logs.
+    cmd: docker-compose logs "$@"
+
+  pull:
+    usage: Pull latest docker images.
+    cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep amazeeio/ | grep -v none | xargs -n1 docker pull | cat; fi
+
+  cli:
+    usage: Start a shell inside CLI container or run a command.
+    cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'; fi
+
+  doctor:
+    usage: Find problems with current project setup.
+    cmd: bin/doctor.sh "$@"
+
+  install-site:
+    usage: Install a site.
+    cmd: |
+      ahoy title "Installing a fresh site"
+      ahoy cli '$APP_DIR/bin/init.sh'
+
+  clean:
+    usage: Remove containers and all build files.
+    cmd: |
+      ahoy down
+      # Remove other directories.
+      # @todo: Add destinations below.
+      rm -rf \
+        ./ckan
+
+  reset:
+    usage: "Reset environment: remove containers, all build, manually created and Drupal-Dev files."
+    cmd: |
+      ahoy clean
+      git ls-files --others -i --exclude-from=.git/info/exclude | xargs chmod 777
+      git ls-files --others -i --exclude-from=.git/info/exclude | xargs rm -Rf
+      find . -type d -not -path "./.git/*" -empty -delete
+
+  flush-redis:
+    usage: Flush Redis cache.
+    cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null
+
+  lint:
+    usage: Lint code.
+    cmd: |
+      ahoy cli "flake8 ${@:-ckanext}" || \
+      [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ]
+
+  copy-local-files:
+    usage: Update files from local repo.
+    cmd: |
+      docker cp . $(docker-compose ps -q ckan):/srv/app/
+      docker cp bin/ckan_cli $(docker-compose ps -q ckan):/usr/bin/
+      ahoy cli 'chmod -v u+x /usr/bin/ckan_cli; cp -v .docker/test.ini $CKAN_INI'
+
+  test-unit:
+    usage: Run unit tests.
+    cmd: |
+      ahoy cli 'pytest --ckan-ini=${CKAN_INI} $APP_DIR/ckanext' || \
+      [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
+
+  test-bdd:
+    usage: Run BDD tests.
+    cmd: |
+      ahoy start-ckan-job-worker &
+      ahoy start-mailmock &
+      sleep 5 &&
+      ahoy cli "behave -k ${*:-test/features} --tags=-format_autocomplete" || \
+      [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
+      ahoy stop-mailmock
+      ahoy stop-ckan-job-worker
+
+  start-mailmock:
+    usage: Starts email mock server used for email BDD tests
+    cmd: |
+      ahoy title 'Starting mailmock'
+      ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails' # for debugging mailmock email output remove --no-stdout
+
+  stop-mailmock:
+    usage: Stops email mock server used for email BDD tests
+    cmd: |
+      ahoy title 'Stopping mailmock'
+      ahoy cli "killall -2 mailmock"
+
+  start-ckan-job-worker:
+    usage: Starts CKAN background job worker
+    cmd: |
+      ahoy title 'Starting CKAN background job worker'
+      ahoy cli "ckan_cli jobs clear && \
+                ckan_cli jobs worker"
+
+  stop-ckan-job-worker:
+    usage: Stops CKAN background job worker
+    cmd: |
+      ahoy title 'Stopping CKAN background job worker'
+      ahoy cli "pkill -f 'jobs worker'"
+
+  # Utilities.
+  title:
+    cmd: printf "$(tput -Txterm setaf 4)==> ${1}$(tput -Txterm sgr0)\n"
+    hide: true
+
+  line:
+    cmd: printf "$(tput -Txterm setaf 2)${1}$(tput -Txterm sgr0)${2}\n"
+    hide: true
+
+  getvar:
+    cmd: eval echo "${@}"
+    hide: true
+
+  pre-flight:
+    cmd: |
+      export DOCTOR_CHECK_DB=${DOCTOR_CHECK_DB:-1}
+      export DOCTOR_CHECK_TOOLS=${DOCTOR_CHECK_TOOLS:-1}
+      export DOCTOR_CHECK_PORT=${DOCTOR_CHECK_PORT:-0}
+      export DOCTOR_CHECK_PYGMY=${DOCTOR_CHECK_PYGMY:-1}
+      export DOCTOR_CHECK_CLI=${DOCTOR_CHECK_CLI:-0}
+      export DOCTOR_CHECK_SSH=${DOCTOR_CHECK_SSH:-0}
+      export DOCTOR_CHECK_WEBSERVER=${DOCTOR_CHECK_WEBSERVER:-0}
+      export DOCTOR_CHECK_BOOTSTRAP=${DOCTOR_CHECK_BOOTSTRAP:-0}
+      ahoy doctor
+    hide: true
+
+entrypoint:
+  - bash
+  - "-c"
+  - "-e"
+  - |
+    export LAGOON_LOCALDEV_URL=http://$PROJECT.docker.amazee.io
+    [ -f .env ] && [ -s .env ] && export $(grep -v '^#' .env | xargs) && if [ -f .env.local ] && [ -s .env.local ]; then export $(grep -v '^#' .env.local | xargs); fi
+    bash -e -c "$0" "$@"
+  - "{{cmd}}"
+  - "{{name}}"

.docker/Dockerfile-template.ckan

@@ -0,0 +1,28 @@
+FROM openknowledge/ckan-dev:{CKAN_VERSION}
+
+ARG SITE_URL=http://ckan:5000/
+ENV PYTHON_VERSION={PYTHON_VERSION}
+ENV CKAN_SITE_URL="${SITE_URL}"
+ENV PYTHON={PYTHON}
+
+WORKDIR "${APP_DIR}"
+
+ENV DOCKERIZE_VERSION v0.6.1
+RUN apk add --no-cache build-base \
+    && curl -sL https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+        | tar -C /usr/local/bin -xzvf -
+
+# Install CKAN.
+
+COPY .docker/test.ini $CKAN_INI
+
+COPY . ${APP_DIR}/
+
+COPY bin/ckan_cli /usr/bin/
+
+RUN chmod +x ${APP_DIR}/bin/*.sh /usr/bin/ckan_cli
+
+# Init current extension.
+RUN ${APP_DIR}/bin/init-ext.sh
+
+CMD ["/srv/app/bin/serve.sh"]

.docker/test.ini

@@ -0,0 +1,103 @@
+[DEFAULT]
+debug = false
+smtp_server = localhost
+error_email_from = paste@localhost
+
+[server:main]
+use = egg:Paste#http
+host = 0.0.0.0
+port = 5000
+
+[app:main]
+ckan.devserver.host = 0.0.0.0
+ckan.devserver.port = 5000
+
+use = egg:ckan
+full_stack = true
+cache_dir = /tmp/%(ckan.site_id)s/
+beaker.session.key = ckan
+
+# This is the secret token that the beaker library uses to hash the cookie sent
+# to the client. `paster make-config` generates a unique value for this each
+# time it generates a config file.
+beaker.session.secret = bSmgPpaxg2M+ZRes3u1TXwIcE
+
+# `paster make-config` generates a unique value for this each time it generates
+# a config file.
+app_instance_uuid = 6e3daf8e-1c6b-443b-911f-c7ab4c5f9605
+
+# repoze.who config
+who.config_file = %(here)s/who.ini
+who.log_level = warning
+who.log_file = %(cache_dir)s/who_log.ini
+
+## Database Settings
+sqlalchemy.url = postgresql://ckan_default:pass@postgres/ckan_test
+
+## Site Settings.
+ckan.site_url = http://ckan:5000/
+
+## Search Settings
+
+ckan.site_id = default
+solr_url = http://solr:8983/solr/ckan
+
+
+## Redis Settings
+
+# URL to your Redis instance, including the database to be used.
+ckan.redis.url = redis://redis:6379
+
+
+## Plugins Settings
+
+ckan.plugins = report tagless_report
+
+## Email settings
+
+#email_to = [email protected]
+#error_email_from = [email protected]
+#smtp.server = localhost
+#smtp.starttls = False
+#smtp.user = [email protected]
+#smtp.password = your_password
+#smtp.mail_from =
+# If 'smtp.test_server' is configured we assume we're running tests,
+# and don't use the smtp.server, starttls, user, password etc. options.
+smtp.test_server = localhost:8025
+smtp.mail_from = [email protected]
+
+# Logging configuration
+[loggers]
+keys = root, ckan, ckanext
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+
+[logger_ckan]
+level = INFO
+handlers = console
+qualname = ckan
+propagate = 0
+
+[logger_ckanext]
+level = DEBUG
+handlers = console
+qualname = ckanext
+propagate = 0
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s

.env

@@ -0,0 +1,29 @@
+##
+# Project environment variables.
+#
+# It is used by Ahoy and other scripts to read default values.
+#
+# The values must be scalar (cannot be another variable).
+#
+# You may also create .env.local file to override any values locally (it is
+# excluded from git).
+#
+
+# Project name.
+PROJECT="ckanext-report"
+
+# Docker Compose project name. All containers will have this name.
+COMPOSE_PROJECT_NAME="$PROJECT"
+
+# Flag to allow code linting failures. 0=enforce, 1=ignore
+ALLOW_LINT_FAIL=0
+
+# Flag to allow unit tests failures. 0=enforce, 1=ignore
+ALLOW_UNIT_FAIL=0
+
+# Flag to allow BDD tests failures. 0=enforce, 1=ignore
+ALLOW_BDD_FAIL=0
+
+# Disable amazeeio based health checks
+DOCTOR_CHECK_WEBSERVER=0
+DOCTOR_CHECK_BOOTSTRAP=0

.flake8

@@ -3,13 +3,13 @@
 
 exclude =
     ckan
-    scripts
 
 # Extended output format.
 format = pylint
 
 # Show the source of errors.
 show_source = True
+statistics = True
 
 max-complexity = 10
 max-line-length = 127