diff --git a/.ahoy.yml b/.ahoy.yml
new file mode 100644
index 0000000..000bc15
--- /dev/null
+++ b/.ahoy.yml
@@ -0,0 +1,196 @@
+---
+ahoyapi: v2
+
+commands:
+
+ # Docker commands.
+ build:
+ usage: Build or rebuild project.
+ cmd: |
+ ahoy title "Building project"
+ ahoy pre-flight
+ ahoy clean
+ ahoy build-network
+ ahoy up -- --build --force-recreate
+ ahoy install-site
+ ahoy title "Build complete"
+ ahoy doctor
+ ahoy info 1
+
+ build-network:
+ usage: Ensure that the amazeeio network exists.
+ cmd: |
+ docker network prune -f > /dev/null
+ docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network
+
+ info:
+ usage: Print information about this project.
+ cmd: |
+ ahoy line "Project : " ${PROJECT}
+ ahoy line "Site local URL : " ${LAGOON_LOCALDEV_URL}
+ ahoy line "DB port on host : " $(docker port $(docker-compose ps -q postgres) 5432 | cut -d : -f 2)
+ ahoy line "Solr port on host : " $(docker port $(docker-compose ps -q solr) 8983 | cut -d : -f 2)
+ ahoy line "Mailhog URL : " http://mailhog.docker.amazee.io/
+
+ up:
+ usage: Build and start Docker containers.
+ cmd: |
+ docker-compose up -d "$@"
+ sleep 10
+ docker-compose logs
+ ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m"
+ if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi
+ if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi
+ docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_
+ export DOCTOR_CHECK_CLI=0
+
+ down:
+ usage: Stop Docker containers and remove container, images, volumes and networks.
+ cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi'
+
+ start:
+ usage: Start existing Docker containers.
+ cmd: docker-compose start "$@"
+
+ stop:
+ usage: Stop running Docker containers.
+ cmd: docker-compose stop "$@"
+
+ restart:
+ usage: Restart all stopped and running Docker containers.
+ cmd: docker-compose restart "$@"
+
+ logs:
+ usage: Show Docker logs.
+ cmd: docker-compose logs "$@"
+
+ pull:
+ usage: Pull latest docker images.
+ cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep amazeeio/ | grep -v none | xargs -n1 docker pull | cat; fi
+
+ cli:
+ usage: Start a shell inside CLI container or run a command.
+ cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'; fi
+
+ doctor:
+ usage: Find problems with current project setup.
+ cmd: bin/doctor.sh "$@"
+
+ install-site:
+ usage: Install a site.
+ cmd: |
+ ahoy title "Installing a fresh site"
+ ahoy cli '$APP_DIR/bin/init.sh'
+
+ clean:
+ usage: Remove containers and all build files.
+ cmd: |
+ ahoy down
+ # Remove other directories.
+ # @todo: Add destinations below.
+ rm -rf \
+ ./ckan
+
+ reset:
+ usage: "Reset environment: remove containers, all build, manually created and Drupal-Dev files."
+ cmd: |
+ ahoy clean
+ git ls-files --others -i --exclude-from=.git/info/exclude | xargs chmod 777
+ git ls-files --others -i --exclude-from=.git/info/exclude | xargs rm -Rf
+ find . -type d -not -path "./.git/*" -empty -delete
+
+ flush-redis:
+ usage: Flush Redis cache.
+ cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null
+
+ lint:
+ usage: Lint code.
+ cmd: |
+ ahoy cli "flake8 ${@:-ckanext}" || \
+ [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ]
+
+ copy-local-files:
+ usage: Update files from local repo.
+ cmd: |
+ docker cp . $(docker-compose ps -q ckan):/srv/app/
+ docker cp bin/ckan_cli $(docker-compose ps -q ckan):/usr/bin/
+ ahoy cli 'chmod -v u+x /usr/bin/ckan_cli; cp -v .docker/test.ini $CKAN_INI'
+
+ test-unit:
+ usage: Run unit tests.
+ cmd: |
+ ahoy cli 'pytest --ckan-ini=${CKAN_INI} $APP_DIR/ckanext' || \
+ [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ]
+
+ test-bdd:
+ usage: Run BDD tests.
+ cmd: |
+ ahoy start-ckan-job-worker &
+ ahoy start-mailmock &
+ sleep 5 &&
+ ahoy cli "behave -k ${*:-test/features} --tags=-format_autocomplete" || \
+ [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ]
+ ahoy stop-mailmock
+ ahoy stop-ckan-job-worker
+
+ start-mailmock:
+ usage: Starts email mock server used for email BDD tests
+ cmd: |
+ ahoy title 'Starting mailmock'
+ ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails' # for debugging mailmock email output remove --no-stdout
+
+ stop-mailmock:
+ usage: Stops email mock server used for email BDD tests
+ cmd: |
+ ahoy title 'Stopping mailmock'
+ ahoy cli "killall -2 mailmock"
+
+ start-ckan-job-worker:
+ usage: Starts CKAN background job worker
+ cmd: |
+ ahoy title 'Starting CKAN background job worker'
+ ahoy cli "ckan_cli jobs clear && \
+ ckan_cli jobs worker"
+
+ stop-ckan-job-worker:
+ usage: Stops CKAN background job worker
+ cmd: |
+ ahoy title 'Stopping CKAN background job worker'
+ ahoy cli "pkill -f 'jobs worker'"
+
+ # Utilities.
+ title:
+ cmd: printf "$(tput -Txterm setaf 4)==> ${1}$(tput -Txterm sgr0)\n"
+ hide: true
+
+ line:
+ cmd: printf "$(tput -Txterm setaf 2)${1}$(tput -Txterm sgr0)${2}\n"
+ hide: true
+
+ getvar:
+ cmd: eval echo "${@}"
+ hide: true
+
+ pre-flight:
+ cmd: |
+ export DOCTOR_CHECK_DB=${DOCTOR_CHECK_DB:-1}
+ export DOCTOR_CHECK_TOOLS=${DOCTOR_CHECK_TOOLS:-1}
+ export DOCTOR_CHECK_PORT=${DOCTOR_CHECK_PORT:-0}
+ export DOCTOR_CHECK_PYGMY=${DOCTOR_CHECK_PYGMY:-1}
+ export DOCTOR_CHECK_CLI=${DOCTOR_CHECK_CLI:-0}
+ export DOCTOR_CHECK_SSH=${DOCTOR_CHECK_SSH:-0}
+ export DOCTOR_CHECK_WEBSERVER=${DOCTOR_CHECK_WEBSERVER:-0}
+ export DOCTOR_CHECK_BOOTSTRAP=${DOCTOR_CHECK_BOOTSTRAP:-0}
+ ahoy doctor
+ hide: true
+
+entrypoint:
+ - bash
+ - "-c"
+ - "-e"
+ - |
+ export LAGOON_LOCALDEV_URL=http://$PROJECT.docker.amazee.io
+ [ -f .env ] && [ -s .env ] && export $(grep -v '^#' .env | xargs) && if [ -f .env.local ] && [ -s .env.local ]; then export $(grep -v '^#' .env.local | xargs); fi
+ bash -e -c "$0" "$@"
+ - "{{cmd}}"
+ - "{{name}}"
diff --git a/.docker/Dockerfile-template.ckan b/.docker/Dockerfile-template.ckan
new file mode 100644
index 0000000..eab5e4c
--- /dev/null
+++ b/.docker/Dockerfile-template.ckan
@@ -0,0 +1,28 @@
+FROM openknowledge/ckan-dev:{CKAN_VERSION}
+
+ARG SITE_URL=http://ckan:5000/
+ENV PYTHON_VERSION={PYTHON_VERSION}
+ENV CKAN_SITE_URL="${SITE_URL}"
+ENV PYTHON={PYTHON}
+
+WORKDIR "${APP_DIR}"
+
+ENV DOCKERIZE_VERSION v0.6.1
+RUN apk add --no-cache build-base \
+ && curl -sL https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \
+ | tar -C /usr/local/bin -xzvf -
+
+# Install CKAN.
+
+COPY .docker/test.ini $CKAN_INI
+
+COPY . ${APP_DIR}/
+
+COPY bin/ckan_cli /usr/bin/
+
+RUN chmod +x ${APP_DIR}/bin/*.sh /usr/bin/ckan_cli
+
+# Init current extension.
+RUN ${APP_DIR}/bin/init-ext.sh
+
+CMD ["/srv/app/bin/serve.sh"]
diff --git a/.docker/test.ini b/.docker/test.ini
new file mode 100644
index 0000000..71bfbf7
--- /dev/null
+++ b/.docker/test.ini
@@ -0,0 +1,103 @@
+[DEFAULT]
+debug = false
+smtp_server = localhost
+error_email_from = paste@localhost
+
+[server:main]
+use = egg:Paste#http
+host = 0.0.0.0
+port = 5000
+
+[app:main]
+ckan.devserver.host = 0.0.0.0
+ckan.devserver.port = 5000
+
+use = egg:ckan
+full_stack = true
+cache_dir = /tmp/%(ckan.site_id)s/
+beaker.session.key = ckan
+
+# This is the secret token that the beaker library uses to hash the cookie sent
+# to the client. `paster make-config` generates a unique value for this each
+# time it generates a config file.
+beaker.session.secret = bSmgPpaxg2M+ZRes3u1TXwIcE
+
+# `paster make-config` generates a unique value for this each time it generates
+# a config file.
+app_instance_uuid = 6e3daf8e-1c6b-443b-911f-c7ab4c5f9605
+
+# repoze.who config
+who.config_file = %(here)s/who.ini
+who.log_level = warning
+who.log_file = %(cache_dir)s/who_log.ini
+
+## Database Settings
+sqlalchemy.url = postgresql://ckan_default:pass@postgres/ckan_test
+
+## Site Settings.
+ckan.site_url = http://ckan:5000/
+
+## Search Settings
+
+ckan.site_id = default
+solr_url = http://solr:8983/solr/ckan
+
+
+## Redis Settings
+
+# URL to your Redis instance, including the database to be used.
+ckan.redis.url = redis://redis:6379
+
+
+## Plugins Settings
+
+ckan.plugins = report tagless_report
+
+## Email settings
+
+#email_to = errors@example.com
+#error_email_from = ckan-errors@example.com
+#smtp.server = localhost
+#smtp.starttls = False
+#smtp.user = username@example.com
+#smtp.password = your_password
+#smtp.mail_from =
+# If 'smtp.test_server' is configured we assume we're running tests,
+# and don't use the smtp.server, starttls, user, password etc. options.
+smtp.test_server = localhost:8025
+smtp.mail_from = info@test.ckan.net
+
+# Logging configuration
+[loggers]
+keys = root, ckan, ckanext
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+
+[logger_ckan]
+level = INFO
+handlers = console
+qualname = ckan
+propagate = 0
+
+[logger_ckanext]
+level = DEBUG
+handlers = console
+qualname = ckanext
+propagate = 0
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s
diff --git a/.env b/.env
new file mode 100644
index 0000000..6e8dfe2
--- /dev/null
+++ b/.env
@@ -0,0 +1,29 @@
+##
+# Project environment variables.
+#
+# It is used by Ahoy and other scripts to read default values.
+#
+# The values must be scalar (cannot be another variable).
+#
+# You may also create .env.local file to override any values locally (it is
+# excluded from git).
+#
+
+# Project name.
+PROJECT="ckanext-report"
+
+# Docker Compose project name. All containers will have this name.
+COMPOSE_PROJECT_NAME="$PROJECT"
+
+# Flag to allow code linting failures. 0=enforce, 1=ignore
+ALLOW_LINT_FAIL=0
+
+# Flag to allow unit tests failures. 0=enforce, 1=ignore
+ALLOW_UNIT_FAIL=0
+
+# Flag to allow BDD tests failures. 0=enforce, 1=ignore
+ALLOW_BDD_FAIL=0
+
+# Disable amazeeio based health checks
+DOCTOR_CHECK_WEBSERVER=0
+DOCTOR_CHECK_BOOTSTRAP=0
diff --git a/.flake8 b/.flake8
index d6b0f6b..ee7a7cc 100644
--- a/.flake8
+++ b/.flake8
@@ -3,13 +3,13 @@
exclude =
ckan
- scripts
# Extended output format.
format = pylint
# Show the source of errors.
show_source = True
+statistics = True
max-complexity = 10
max-line-length = 127
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 43efa8a..614aa66 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -1,92 +1,45 @@
---
name: Tests
-on: [push, pull_request]
-jobs:
- lint:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-python@v2
- with:
- python-version: '3.6'
- - name: Cache pip
- uses: actions/cache@v2
- with:
- # This path is specific to Ubuntu
- path: ~/.cache/pip
- # Look to see if there is a cache hit for the corresponding requirements file
- key: ${{ runner.os }}-pip-flake8-${{ hashFiles('requirements.txt') }}
- restore-keys: |
- ${{ runner.os }}-pip-flake8-
- ${{ runner.os }}-
- - name: Install requirements
- run: pip install flake8 pycodestyle
- - name: Check syntax
- run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan
+on:
+ push:
+ pull_request:
+ branches:
+ - master
+jobs:
test:
- needs: lint
strategy:
- matrix:
- ckan-version: [2.9, 2.9-py2, 2.8, 2.7]
fail-fast: false
+ matrix:
+ ckan-version: ["2.10", 2.9, 2.9-py2, 2.8, 2.7]
- name: CKAN ${{ matrix.ckan-version }}
+ name: Continuous Integration build on CKAN ${{ matrix.ckan-version }}
runs-on: ubuntu-latest
- container:
- image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
- services:
- solr:
- image: ckan/ckan-solr-dev:${{ matrix.ckan-version }}
- postgres:
- image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
- env:
- POSTGRES_USER: postgres
- POSTGRES_PASSWORD: postgres
- POSTGRES_DB: postgres
- options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
- redis:
- image: redis:3
+ container: integratedexperts/ci-builder
env:
- CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
- CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
- CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
- CKAN_SOLR_URL: http://solr:8983/solr/ckan
- CKAN_REDIS_URL: redis://redis:6379/1
+ CKAN_VERSION: ${{ matrix.ckan-version }}
steps:
- uses: actions/checkout@v2
+ timeout-minutes: 2
- - name: Cache pip
- uses: actions/cache@v2
- with:
- # This path is specific to Ubuntu
- path: ~/.cache/pip
- # Look to see if there is a cache hit for the corresponding requirements file
- key: ${{ runner.os }}-pip-${{ hashFiles('*requirements.txt') }}
- restore-keys: |
- ${{ runner.os }}-pip-
- ${{ runner.os }}-
+ - name: Build
+ run: bin/build.sh
+ timeout-minutes: 15
- - name: Install requirements
- run: |
- pip install -r requirements-dev.txt
- pip install -e .
- # Replace default path to CKAN core config file with the one on the container
- sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
+ - name: Test
+ run: bin/test.sh
+ timeout-minutes: 30
- - name: Setup extension (CKAN >= 2.9)
- if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }}
- run: |
- ckan -c test.ini db init
- ckan -c test.ini report initdb
- ckan -c test.ini report generate tagless-datasets
- - name: Setup extension (CKAN < 2.9)
- if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }}
- run: |
- paster --plugin=ckan db init -c test.ini
- paster --plugin=ckanext-report report initdb -c test.ini
- paster --plugin=ckanext-report report generate tagless-datasets -c test.ini
+ - name: Retrieve screenshots
+ if: failure()
+ run: bin/process-artifacts.sh
+ timeout-minutes: 1
- - name: Run tests
- run: pytest --ckan-ini=test.ini --cov=ckanext.report --disable-warnings ckanext/report/tests
+ - name: Upload screenshots
+ if: failure()
+ uses: actions/upload-artifact@v2
+ with:
+ name: CKAN ${{ matrix.ckan-version }} screenshots
+ path: /tmp/artifacts/behave/screenshots
+ timeout-minutes: 1
diff --git a/.gitignore b/.gitignore
index 860e403..6544185 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,4 +3,6 @@
*.swp
*.swo
-ckanext/report/i18n/_reused_translations.pot
\ No newline at end of file
+ckanext/report/i18n/_reused_translations.pot
+.docker/Dockerfile.ckan
+
diff --git a/README.md b/README.md
index cd935cb..73bbe53 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,5 @@
[![Tests](https://github.com/qld-gov-au/ckanext-report/actions/workflows/test.yml/badge.svg)](https://github.com/qld-gov-au/ckanext-report/actions/workflows/test.yml)
-ckanext-report
+# ckanext-report
====================
ckanext-report is a CKAN extension that provides a reporting infrastructure. Here are the features offered:
@@ -28,7 +28,9 @@ TODO:
| 2.6 and earlier | yes |
| 2.7 | yes |
| 2.8 | yes |
-| 2.9 | yes |
+| 2.9-py2 | yes |
+| 2.9 (py3) | yes |
+| 2.10 (py3) | yes |
Status: was in production at data.gov.uk around 2014-2016, but since that uses its own CSS rather than core CKAN's, for others to use it CSS needs adding. For an example, see this branch: see https://github.com/GSA/ckanext-report/tree/geoversion
@@ -43,7 +45,8 @@ Install ckanext-report into your CKAN virtual environment in the usual way:
Initialize the database tables needed by ckanext-report:
- (pyenv) $ paster --plugin=ckanext-report report initdb --config=mysite.ini
+ CKAN < 2.9 (pyenv) $ paster --plugin=ckanext-report report initdb --config=mysite.ini
+ CKAN >= 2.9 (pyenv) $ ckan -c mysite.ini report initdb
Enable the plugin. In your config (e.g. development.ini or production.ini) add ``report`` to your ckan.plugins. e.g.:
@@ -151,7 +154,7 @@ data - other data values, as a dict
Average tags per package: {{ data['average_tags_per_package'] }} tags
-
+
Dataset |
@@ -230,10 +233,7 @@ class TaglessReportPlugin(p.SingletonPlugin):
The last line refers to `tag_report_info` which is a dictionary with properties of the report. This is stored in `reports.py` together with the report code (see above). The info dict looks like this:
```python
-try:
- from collections import OrderedDict # from python 2.7
-except ImportError:
- from sqlalchemy.util import OrderedDict
+from collections import OrderedDict
tagless_report_info = {
'name': 'tagless-datasets',
'description': 'Datasets which have no tags.',
diff --git a/bin/activate b/bin/activate
new file mode 100644
index 0000000..982827c
--- /dev/null
+++ b/bin/activate
@@ -0,0 +1,3 @@
+if [ "$VENV_DIR" != "" ]; then
+ . ${VENV_DIR}/bin/activate
+fi
diff --git a/bin/build.sh b/bin/build.sh
new file mode 100755
index 0000000..d4088e2
--- /dev/null
+++ b/bin/build.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+##
+# Build site in CI.
+#
+set -e
+
+# Process Docker Compose configuration. This is used to avoid multiple
+# docker-compose.yml files.
+# Remove lines containing '###'.
+sed -i -e "/###/d" docker-compose.yml
+# Uncomment lines containing '##'.
+sed -i -e "s/##//" docker-compose.yml
+
+# Pull the latest images.
+ahoy pull
+
+PYTHON=python
+if [ "$CKAN_VERSION" = "2.7" ] || [ "$CKAN_VERSION" = "2.8" ] || [ "$CKAN_VERSION" = "2.9-py2" ]; then
+ PYTHON_VERSION=py2
+else
+ PYTHON_VERSION=py3
+ PYTHON="${PYTHON}3"
+fi
+
+sed "s|{CKAN_VERSION}|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \
+ | sed "s|{PYTHON}|$PYTHON|g" \
+ | sed "s|{PYTHON_VERSION}|$PYTHON_VERSION|g" > .docker/Dockerfile.ckan
+
+ahoy build || (ahoy logs; exit 1)
diff --git a/bin/ckan_cli b/bin/ckan_cli
new file mode 100644
index 0000000..7757dc8
--- /dev/null
+++ b/bin/ckan_cli
@@ -0,0 +1,75 @@
+#!/bin/sh
+
+# Call either 'ckan' (from CKAN >= 2.9) or 'paster' (from CKAN <= 2.8)
+# with appropriate syntax, depending on what is present on the system.
+# This is intended to smooth the upgrade process from 2.8 to 2.9.
+# Eg:
+# ckan_cli jobs list
+# could become either:
+# paster --plugin=ckan jobs list -c /etc/ckan/default/production.ini
+# or:
+# ckan -c /etc/ckan/default/production.ini jobs list
+
+# This script is aware of the VIRTUAL_ENV environment variable, and will
+# attempt to respect it with similar behaviour to commands like 'pip'.
+# Eg placing this script in a virtualenv 'bin' directory will cause it
+# to call the 'ckan' or 'paster' command in that directory, while
+# placing this script elsewhere will cause it to rely on the VIRTUAL_ENV
+# variable, or if that is not set, the system PATH.
+
+# Since the positioning of the CKAN configuration file is central to the
+# differences between 'paster' and 'ckan', this script needs to be aware
+# of the config file location. It will use the CKAN_INI environment
+# variable if it exists, or default to /etc/ckan/default/production.ini.
+
+# If 'paster' is being used, the default plugin is 'ckan'. A different
+# plugin can be specified by setting the PASTER_PLUGIN environment
+# variable. This variable is irrelevant if using the 'ckan' command.
+
+CKAN_INI="${CKAN_INI:-/etc/ckan/default/production.ini}"
+PASTER_PLUGIN="${PASTER_PLUGIN:-ckan}"
+# First, look for a command alongside this file
+ENV_DIR=$(dirname "$0")
+if [ -f "$ENV_DIR/ckan" ]; then
+ COMMAND=ckan
+elif [ -f "$ENV_DIR/paster" ]; then
+ COMMAND=paster
+elif [ "$VIRTUAL_ENV" != "" ]; then
+ # If command not found alongside this file, check the virtualenv
+ ENV_DIR="$VIRTUAL_ENV/bin"
+ if [ -f "$ENV_DIR/ckan" ]; then
+ COMMAND=ckan
+ elif [ -f "$ENV_DIR/paster" ]; then
+ COMMAND=paster
+ fi
+else
+ # if no virtualenv is active, try the system path
+ if (which ckan > /dev/null 2>&1); then
+ ENV_DIR=$(dirname $(which ckan))
+ COMMAND=ckan
+ elif (which paster > /dev/null 2>&1); then
+ ENV_DIR=$(dirname $(which paster))
+ COMMAND=paster
+ else
+ echo "Unable to locate 'ckan' or 'paster' command" >&2
+ exit 1
+ fi
+fi
+
+if [ "$COMMAND" = "ckan" ]; then
+ # adjust args to match ckan expectations
+ COMMAND=$(echo "$1" | sed -e 's/create-test-data/seed/')
+ echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2
+ shift
+ exec $ENV_DIR/ckan -c ${CKAN_INI} $COMMAND "$@" $CLICK_ARGS
+elif [ "$COMMAND" = "paster" ]; then
+ # adjust args to match paster expectations
+ COMMAND=$1
+ echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2
+ shift
+ if [ "$1" = "show" ]; then shift; fi
+ exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN $COMMAND "$@" -c ${CKAN_INI}
+else
+ echo "Unable to locate 'ckan' or 'paster' command in $ENV_DIR" >&2
+ exit 1
+fi
diff --git a/bin/create-test-data.sh b/bin/create-test-data.sh
new file mode 100644
index 0000000..79fbdfd
--- /dev/null
+++ b/bin/create-test-data.sh
@@ -0,0 +1,116 @@
+#!/usr/bin/env sh
+##
+# Create some example content for extension BDD tests.
+#
+set -e
+
+CKAN_ACTION_URL=${CKAN_SITE_URL}api/action
+CKAN_USER_NAME="${CKAN_USER_NAME:-admin}"
+CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}"
+CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}"
+
+. ${APP_DIR}/bin/activate
+
+add_user_if_needed () {
+ echo "Adding user '$2' ($1) with email address [$3]"
+ ckan_cli user show "$1" | grep "$1" || ckan_cli user add "$1"\
+ fullname="$2"\
+ email="$3"\
+ password="${4:-Password123!}"
+}
+
+add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL"
+ckan_cli sysadmin add "${CKAN_USER_NAME}"
+
+API_KEY=$(ckan_cli user show "${CKAN_USER_NAME}" | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/')
+if [ "$API_KEY" = "None" ]; then
+ echo "No API Key found on ${CKAN_USER_NAME}, generating API Token..."
+ API_KEY=$(ckan_cli user token add "${CKAN_USER_NAME}" test_setup |tail -1 | tr -d '[:space:]')
+fi
+
+##
+# BEGIN: Create a test organisation with test users for admin, editor and member
+#
+TEST_ORG_NAME=test-organisation
+TEST_ORG_TITLE="Test Organisation"
+
+echo "Creating test users for ${TEST_ORG_TITLE} Organisation:"
+
+add_user_if_needed ckan_user "CKAN User" ckan_user@localhost
+add_user_if_needed test_org_admin "Test Admin" test_org_admin@localhost
+add_user_if_needed test_org_editor "Test Editor" test_org_editor@localhost
+add_user_if_needed test_org_member "Test Member" test_org_member@localhost
+
+echo "Creating ${TEST_ORG_TITLE} organisation:"
+
+TEST_ORG=$( \
+ curl -LsH "Authorization: ${API_KEY}" \
+ --data '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'"}' \
+ ${CKAN_ACTION_URL}/organization_create
+)
+
+TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON ${APP_DIR}/bin/extract-id.py)
+
+echo "Assigning test users to '${TEST_ORG_TITLE}' organisation (${TEST_ORG_ID}):"
+
+curl -LsH "Authorization: ${API_KEY}" \
+ --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' \
+ ${CKAN_ACTION_URL}/member_create
+
+curl -LsH "Authorization: ${API_KEY}" \
+ --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_editor", "object_type": "user", "capacity": "editor"}' \
+ ${CKAN_ACTION_URL}/member_create
+
+curl -LsH "Authorization: ${API_KEY}" \
+ --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_member", "object_type": "user", "capacity": "member"}' \
+ ${CKAN_ACTION_URL}/member_create
+##
+# END.
+#
+
+##
+# BEGIN: Create a Reporting organisation with test users
+#
+
+REPORT_ORG_NAME=reporting
+REPORT_ORG_TITLE=Reporting
+
+echo "Creating test users for ${REPORT_ORG_TITLE} Organisation:"
+
+add_user_if_needed report_admin "Reporting Admin" report_admin@localhost
+add_user_if_needed report_editor "Reporting Editor" report_editor@localhost
+
+echo "Creating ${REPORT_ORG_TITLE} Organisation:"
+
+REPORT_ORG=$( \
+ curl -LsH "Authorization: ${API_KEY}" \
+ --data '{"name": "'"${REPORT_ORG_NAME}"'", "title": "'"${REPORT_ORG_TITLE}"'"}' \
+ ${CKAN_ACTION_URL}/organization_create
+)
+
+REPORT_ORG_ID=$(echo $REPORT_ORG | $PYTHON ${APP_DIR}/bin/extract-id.py)
+
+echo "Assigning test users to ${REPORT_ORG_TITLE} Organisation:"
+
+curl -LsH "Authorization: ${API_KEY}" \
+ --data '{"id": "'"${REPORT_ORG_ID}"'", "object": "report_admin", "object_type": "user", "capacity": "admin"}' \
+ ${CKAN_ACTION_URL}/member_create
+
+curl -LsH "Authorization: ${API_KEY}" \
+ --data '{"id": "'"${REPORT_ORG_ID}"'", "object": "report_editor", "object_type": "user", "capacity": "editor"}' \
+ ${CKAN_ACTION_URL}/member_create
+
+echo "Creating test dataset for reporting:"
+
+curl -LsH "Authorization: ${API_KEY}" \
+ --data '{"name": "reporting", "title": "Reporting dataset", "description": "Dataset for testing reports",
+"owner_org": "'"${REPORT_ORG_ID}"'", "update_frequency": "near-realtime", "author_email": "report_admin@localhost",
+"version": "1.0", "license_id": "cc-by-4", "data_driven_application": "NO", "security_classification": "PUBLIC",
+"notes": "test", "de_identified_data": "NO"}'\
+ ${CKAN_ACTION_URL}/package_create
+
+##
+# END.
+#
+
+. ${APP_DIR}/bin/deactivate
diff --git a/bin/deactivate b/bin/deactivate
new file mode 100644
index 0000000..7cd77b9
--- /dev/null
+++ b/bin/deactivate
@@ -0,0 +1,3 @@
+if [ "$VENV_DIR" != "" ]; then
+ deactivate
+fi
diff --git a/bin/doctor.sh b/bin/doctor.sh
new file mode 100755
index 0000000..9c7f455
--- /dev/null
+++ b/bin/doctor.sh
@@ -0,0 +1,202 @@
+#!/usr/bin/env bash
+#
+# Check Drupal-Dev project requirements.
+#
+set -e
+
+DOCTOR_CHECK_TOOLS="${DOCTOR_CHECK_TOOLS:-1}"
+DOCTOR_CHECK_PORT="${DOCTOR_CHECK_PORT:-0}"
+DOCTOR_CHECK_PYGMY="${DOCTOR_CHECK_PYGMY:-1}"
+DOCTOR_CHECK_CLI="${DOCTOR_CHECK_CLI:-1}"
+DOCTOR_CHECK_SSH="${DOCTOR_CHECK_SSH:-0}"
+DOCTOR_CHECK_WEBSERVER="${DOCTOR_CHECK_WEBSERVER:-1}"
+DOCTOR_CHECK_BOOTSTRAP="${DOCTOR_CHECK_BOOTSTRAP:-1}"
+
+APP_PORT="${APP_PORT:-5000}"
+CLI="${CLI:-cli}"
+LAGOON_LOCALDEV_URL="${LAGOON_LOCALDEV_URL:-http://your-site.docker.amazee.io/}"
+SSH_KEY_FILE="${SSH_KEY_FILE:-$HOME/.ssh/id_rsa}"
+DATAROOT="${DATAROOT:-.data}"
+
+#-------------------------------------------------------------------------------
+# DO NOT CHANGE ANYTHING BELOW THIS LINE
+#-------------------------------------------------------------------------------
+
+
+#
+# Main entry point.
+#
+main() {
+ status "Checking project requirements"
+
+ if [ "${DOCTOR_CHECK_TOOLS}" == "1" ]; then
+ [ "$(command_exists docker)" == "1" ] && error "Please install Docker (https://www.docker.com/get-started)" && exit 1
+ [ "$(command_exists docker-compose)" == "1" ] && error "Please install docker-compose (https://docs.docker.com/compose/install/)" && exit 1
+ [ "$(command_exists composer)" == "1" ] && error "Please install Composer (https://getcomposer.org/)" && exit 1
+ [ "$(command_exists pygmy)" == "1" ] && error "Please install Pygmy (https://pygmy.readthedocs.io/)" && exit 1
+ [ "$(command_exists ahoy)" == "1" ] && error "Please install Ahoy (https://ahoy-cli.readthedocs.io/)" && exit 1
+ success "All required tools are present"
+ fi
+
+ if [ "${DOCTOR_CHECK_PORT}" == "1" ]; then
+ if ! lsof -i :$APP_PORT | grep LISTEN | grep -q om.docke; then
+ error "Port $APP_PORT is occupied by a service other than Docker. Stop this service and run 'pygmy up'"
+ fi
+ success "Port $APP_PORT is available"
+ fi
+
+ if [ "${DOCTOR_CHECK_PYGMY}" == "1" ]; then
+ if ! pygmy status > /dev/null 2>&1; then
+ error "pygmy is not running. Run 'pygmy up' to start pygmy."
+ exit 1
+ fi
+ success "Pygmy is running"
+ fi
+
+ # Check that the stack is running.
+ if [ "${DOCTOR_CHECK_CLI}" == "1" ]; then
+ if ! docker ps -q --no-trunc | grep "$(docker-compose ps -q ckan)" > /dev/null 2>&1; then
+ error "CLI container is not running. Run 'ahoy up'."
+ exit 1
+ fi
+ success "CLI container is running"
+ fi
+
+ if [ "${DOCTOR_CHECK_SSH}" == "1" ]; then
+ # SSH key injection is required to access Lagoon services from within
+ # containers. For example, to connect to production environment to run
+ # drush script.
+ # Pygmy makes this possible in the following way:
+ # 1. Pygmy starts `amazeeio/ssh-agent` container with a volume `/tmp/amazeeio_ssh-agent`
+ # 2. Pygmy adds a default SSH key from the host into this volume.
+ # 3. `docker-compose.yml` should have volume inclusion specified for CLI container:
+ # ```
+ # volumes_from:
+ # - container:amazeeio-ssh-agent
+ # ```
+ # 4. When CLI container starts, the volume is mounted and an entrypoint script
+ # adds SSH key into agent.
+ # @see https://github.com/amazeeio/lagoon/blob/master/images/php/cli/10-ssh-agent.sh
+ #
+ # Running `ssh-add -L` within CLI container should show that the SSH key
+ # is correctly loaded.
+ #
+ # As rule of a thumb, one must restart the CLI container after restarting
+ # Pygmy ONLY if SSH key was not loaded in pygmy before the stack starts.
+ # No need to restart CLI container if key was added, but pygmy was
+ # restarted - the volume mount will retain and the key will still be
+ # available in CLI container.
+
+ # Check that the key is injected into pygmy ssh-agent container.
+ if ! pygmy status 2>&1 | grep -q "${SSH_KEY_FILE}"; then
+ error "SSH key is not added to pygmy. Run 'pygmy stop && pygmy start' and then 'ahoy up -- --build'."
+ exit 1
+ fi
+
+ # Check that the volume is mounted into CLI container.
+ if ! docker exec -i "$(docker-compose ps -q ckan)" sh -c "grep \"^/dev\" /etc/mtab|grep -q /tmp/amazeeio_ssh-agent"; then
+ error "SSH key is added to Pygmy, but the volume is not mounted into container. Make sure that your your \"docker-compose.yml\" has the following lines:"
+ error "volumes_from:"
+ error " - container:amazeeio-ssh-agent"
+ error "After adding these lines, run 'ahoy up -- --build'"
+ exit 1
+ fi
+
+ # Check that ssh key is available in the container.
+ if ! docker exec -i "$(docker-compose ps -q ckan)" bash -c "ssh-add -L | grep -q 'ssh-rsa'" ; then
+ error "SSH key was not added into container. Run 'ahoy up -- --build'."
+ exit 1
+ fi
+
+ success "SSH key is available within CLI container"
+ fi
+
+
+ if [ "${DOCTOR_CHECK_WEBSERVER}" == "1" ]; then
+ host_app_port="$(docker port $(docker-compose ps -q ckan) $APP_PORT | cut -d : -f 2)"
+ if ! curl -L -s -o /dev/null -w "%{http_code}" "${LAGOON_LOCALDEV_URL}:${host_app_port}" | grep -q 200; then
+ error "Web server is not accessible at ${LAGOON_LOCALDEV_URL}:${host_app_port}"
+ exit 1
+ fi
+ success "Web server is running and accessible at ${LAGOON_LOCALDEV_URL}:${host_app_port}"
+ fi
+
+ if [ "${DOCTOR_CHECK_BOOTSTRAP}" == "1" ]; then
+ host_app_port="$(docker port $(docker-compose ps -q ckan) $APP_PORT | cut -d : -f 2)"
+ if ! curl -L -s -N "${LAGOON_LOCALDEV_URL}:${host_app_port}" | grep -q -i "meta name=\"generator\" content=\"ckan"; then
+ error "Website is running, but cannot be bootstrapped. Try pulling latest container images with 'ahoy pull'"
+ exit 1
+ fi
+ success "Successfully bootstrapped website at ${LAGOON_LOCALDEV_URL}:${host_app_port}"
+ fi
+
+ status "All required checks have passed"
+}
+
+#
+# Check that command exists.
+#
+command_exists() {
+ local cmd=$1
+ command -v "${cmd}" | grep -ohq "${cmd}"
+ local res=$?
+
+ # Try homebrew lookup, if brew is available.
+ if command -v "brew" | grep -ohq "brew" && [ "$res" == "1" ] ; then
+ brew --prefix "${cmd}" > /dev/null
+ res=$?
+ fi
+
+ echo ${res}
+}
+
+#
+# Status echo.
+#
+status() {
+ cecho blue "✚ $1";
+}
+
+#
+# Success echo.
+#
+success() {
+ cecho green " ✓ $1";
+}
+
+#
+# Error echo.
+#
+error() {
+ cecho red " ✘ $1";
+ exit 1
+}
+
+#
+# Colored echo.
+#
+cecho() {
+ local prefix="\033["
+ local input_color=$1
+ local message="$2"
+
+ local color=""
+ case "$input_color" in
+ black | bk) color="${prefix}0;30m";;
+ red | r) color="${prefix}1;31m";;
+ green | g) color="${prefix}1;32m";;
+ yellow | y) color="${prefix}1;33m";;
+ blue | b) color="${prefix}1;34m";;
+ purple | p) color="${prefix}1;35m";;
+ cyan | c) color="${prefix}1;36m";;
+ gray | gr) color="${prefix}0;37m";;
+ *) message="$1"
+ esac
+
+ # Format message with color codes, but only if a correct color was provided.
+ [ -n "$color" ] && message="${color}${message}${prefix}0m"
+
+ echo -e "$message"
+}
+
+main "$@"
diff --git a/bin/extract-id.py b/bin/extract-id.py
new file mode 100644
index 0000000..ae9cf79
--- /dev/null
+++ b/bin/extract-id.py
@@ -0,0 +1,5 @@
+# encoding: utf-8
+import json
+import sys
+
+print(json.loads(sys.stdin.read())['result']['id'])
diff --git a/bin/init-ext.sh b/bin/init-ext.sh
new file mode 100755
index 0000000..4ae5b57
--- /dev/null
+++ b/bin/init-ext.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env sh
+##
+# Install current extension.
+#
+set -e
+
+install_requirements () {
+ PROJECT_DIR=$1
+ shift
+ # Identify the best match requirements file, ignore the others.
+ # If there is one specific to our Python version, use that.
+ for filename_pattern in "$@"; do
+ filename="$PROJECT_DIR/${filename_pattern}-$PYTHON_VERSION.txt"
+ if [ -f "$filename" ]; then
+ pip install -r "$filename"
+ return 0
+ fi
+ done
+ for filename_pattern in "$@"; do
+ filename="$PROJECT_DIR/$filename_pattern.txt"
+ if [ -f "$filename" ]; then
+ pip install -r "$filename"
+ return 0
+ fi
+ done
+}
+
+. ${APP_DIR}/bin/activate
+
+install_requirements . dev-requirements requirements-dev
+for extension in . `ls -d $SRC_DIR/ckanext-*`; do
+ install_requirements $extension requirements pip-requirements
+done
+pip install -e .
+installed_name=$(grep '^\s*name=' setup.py |sed "s|[^']*'\([-a-zA-Z0-9]*\)'.*|\1|")
+
+# Validate that the extension was installed correctly.
+if ! pip list | grep "$installed_name" > /dev/null; then echo "Unable to find the extension in the list"; exit 1; fi
+
+. ${APP_DIR}/bin/deactivate
diff --git a/bin/init.sh b/bin/init.sh
new file mode 100755
index 0000000..5610d25
--- /dev/null
+++ b/bin/init.sh
@@ -0,0 +1,17 @@
+#!/usr/bin/env sh
+##
+# Initialise CKAN data for testing.
+#
+set -e
+
+. ${APP_DIR}/bin/activate
+CLICK_ARGS="--yes" ckan_cli db clean
+ckan_cli db init
+ckan_cli db upgrade
+
+# Initialise report tables
+PASTER_PLUGIN=ckanext-report ckan_cli report initdb
+PASTER_PLUGIN=ckanext-report ckan_cli report generate tagless-datasets
+
+# Create some base test data
+. $APP_DIR/bin/create-test-data.sh
diff --git a/bin/process-artifacts.sh b/bin/process-artifacts.sh
new file mode 100755
index 0000000..6324f60
--- /dev/null
+++ b/bin/process-artifacts.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+##
+# Process test artifacts.
+#
+set -e
+
+# Create screenshots directory in case it was not created before. This is to
+# avoid this script to fail when copying artifacts.
+ahoy cli "mkdir -p test/screenshots"
+
+# Copy from the app container to the build host for storage.
+mkdir -p /tmp/artifacts/behave
+docker cp "$(docker-compose ps -q ckan)":/srv/app/test/screenshots /tmp/artifacts/behave/
diff --git a/bin/serve.sh b/bin/serve.sh
new file mode 100755
index 0000000..eafaa3e
--- /dev/null
+++ b/bin/serve.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env sh
+set -e
+
+dockerize -wait tcp://postgres:5432 -timeout 1m
+dockerize -wait tcp://solr:8983 -timeout 1m
+dockerize -wait tcp://redis:6379 -timeout 1m
+
+for i in {1..60}; do
+ if (PGPASSWORD=pass psql -h postgres -U ckan_default -d ckan_test -c "\q"); then
+ break
+ else
+ sleep 1
+ fi
+done
+
+. ${APP_DIR}/bin/activate
+if (which ckan > /dev/null); then
+ ckan -c ${CKAN_INI} run -r
+else
+ paster serve ${CKAN_INI}
+fi
diff --git a/bin/test.sh b/bin/test.sh
new file mode 100755
index 0000000..a2d91da
--- /dev/null
+++ b/bin/test.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+##
+# Run tests in CI.
+#
+set -e
+
+echo "==> Lint code"
+ahoy lint
+
+echo "==> Run Unit tests"
+ahoy test-unit
+
+echo "==> Run BDD tests"
+ahoy install-site
+ahoy test-bdd || (ahoy logs; exit 1)
diff --git a/ckanext/__init__.py b/ckanext/__init__.py
index 2e2033b..ed48ed0 100644
--- a/ckanext/__init__.py
+++ b/ckanext/__init__.py
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
# this is a namespace package
try:
import pkg_resources
diff --git a/ckanext/report/blueprint.py b/ckanext/report/blueprint.py
new file mode 100644
index 0000000..8695fbe
--- /dev/null
+++ b/ckanext/report/blueprint.py
@@ -0,0 +1,37 @@
+# encoding: utf-8
+
+import six
+
+from flask import Blueprint, make_response
+
+from ckan.plugins import toolkit
+
+from .utils import report_index as index, report_view
+
+
+report = Blueprint(u'report', __name__)
+
+
+def redirect_to_index():
+ return toolkit.redirect_to('/report')
+
+
+def view(report_name, organization=None, refresh=False):
+ body, headers = report_view(report_name, organization, refresh)
+ if headers:
+ response = make_response(body)
+ for key, value in six.iteritems(headers):
+ response.headers[key] = value
+ return response
+ else:
+ return body
+
+
+report.add_url_rule(u'/report', 'index', view_func=index)
+report.add_url_rule(u'/reports', 'reports', view_func=redirect_to_index)
+report.add_url_rule(u'/report/', view_func=view, methods=('GET', 'POST'))
+report.add_url_rule(u'/report//', 'org', view_func=view, methods=('GET', 'POST',))
+
+
+def get_blueprints():
+ return [report]
diff --git a/ckanext/report/cli.py b/ckanext/report/cli.py
new file mode 100644
index 0000000..08a0f64
--- /dev/null
+++ b/ckanext/report/cli.py
@@ -0,0 +1,57 @@
+# encoding: utf-8
+
+import click
+
+from . import utils
+
+
+def get_commands():
+ return [report]
+
+
+@click.group()
+def report():
+ """Generates reports"""
+ pass
+
+
+@report.command()
+def initdb():
+ """Creates necessary db tables"""
+ utils.initdb()
+ click.secho(u'Report table is setup', fg=u"green")
+
+
+@report.command()
+@click.argument(u'report_list', required=False)
+def generate(report_list):
+ """
+ Generate and cache reports - all of them unless you specify
+ a comma separated list of them.
+ """
+ if report_list:
+ report_list = [s.strip() for s in report_list.split(',')]
+ timings = utils.generate(report_list)
+
+ click.secho(u'Report generation complete %s' % timings, fg=u"green")
+
+
+@report.command()
+def list():
+ """ Lists the reports
+ """
+ utils.list_reports()
+
+
+@report.command()
+@click.argument(u'report_name')
+@click.argument(u'report_options', nargs=-1)
+def generate_for_options(report_name, report_options):
+ """
+ Generate and cache a report for one combination of option values.
+ You can leave it with the defaults or specify options
+ as more parameters: key1=value key2=value
+ """
+ message = utils.generate_for_options(report_name, report_options)
+ if message:
+ click.secho(message, fg=u"red")
diff --git a/ckanext/report/cli/click_cli.py b/ckanext/report/cli/click_cli.py
deleted file mode 100644
index 314b42a..0000000
--- a/ckanext/report/cli/click_cli.py
+++ /dev/null
@@ -1,70 +0,0 @@
-# encoding: utf-8
-
-import click
-
-from ckanext.report.cli.command import Reporting
-
-# Click commands for CKAN 2.9 and above
-
-
-@click.group()
-def report():
- """ XLoader commands
- """
- pass
-
-
-@report.command()
-def list():
- """ Lists the reports
- """
- cmd = Reporting()
- cmd.list()
-
-
-@report.command()
-def initdb():
- """ Initialize the database tables for this extension
- """
- cmd = Reporting()
- cmd.initdb()
-
-
-@report.command()
-@click.argument(u'report_names')
-def generate(report_names):
- """
- Generate and cache reports - all of them unless you specify
- a comma separated list of them.
- """
- cmd = Reporting()
- report_list = [s.strip() for s in report_names.split(',')]
- cmd.generate(report_list)
-
-
-@report.command()
-@click.argument(u'report_name')
-@click.argument(u'options', nargs=-1)
-def generate_for_options(report_name, options):
- """
- Generate and cache a report for one combination of option values.
- You can leave it with the defaults or specify options
- as more parameters: key1=value key2=value
- """
- cmd = Reporting()
- report_options = {}
- for option_arg in options:
- if '=' not in option_arg:
- raise click.BadParameter(
- 'Option needs an "=" sign in it',
- options)
- equal_pos = option_arg.find('=')
- key, value = option_arg[:equal_pos], option_arg[equal_pos + 1:]
- if value == '':
- value = None # this is what the web i/f does with params
- report_options[key] = value
- cmd.generate_for_options(report_name, report_options)
-
-
-def get_commands():
- return [report]
diff --git a/ckanext/report/cli/command.py b/ckanext/report/cli/command.py
deleted file mode 100644
index cbf5439..0000000
--- a/ckanext/report/cli/command.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# encoding: utf-8
-
-import logging
-import time
-
-
-class Reporting():
-
- def __init__(self):
- self.log = logging.getLogger("ckanext.report.cli")
-
- def initdb(self):
- from ckanext.report import model
- model.init_tables()
- self.log.info('Report table is setup')
-
- def list(self):
- from ckanext.report.report_registry import ReportRegistry
- registry = ReportRegistry.instance()
- for plugin, report_name, report_title in registry.get_names():
- report = registry.get_report(report_name)
- date = report.get_cached_date()
- print('%s: %s %s' % (plugin, report_name,
- date.strftime('%d/%m/%Y %H:%M') if date else '(not cached)'))
-
- def generate(self, report_list=None):
- from ckanext.report.report_registry import ReportRegistry
- timings = {}
-
- self.log.info("Running reports => %s", report_list)
- registry = ReportRegistry.instance()
- if report_list:
- for report_name in report_list:
- s = time.time()
- registry.get_report(report_name).refresh_cache_for_all_options()
- timings[report_name] = time.time() - s
- else:
- s = time.time()
- registry.refresh_cache_for_all_reports()
- timings["All Reports"] = time.time() - s
-
- self.log.info("Report generation complete %s", timings)
-
- def generate_for_options(self, report_name, options):
- from ckanext.report.report_registry import ReportRegistry
- self.log.info("Running report => %s", report_name)
- registry = ReportRegistry.instance()
- report = registry.get_report(report_name)
- all_options = report.add_defaults_to_options(options,
- report.option_defaults)
- report.refresh_cache(all_options)
diff --git a/ckanext/report/cli/paster_cli.py b/ckanext/report/command.py
similarity index 68%
rename from ckanext/report/cli/paster_cli.py
rename to ckanext/report/command.py
index f656408..83e0a24 100644
--- a/ckanext/report/cli/paster_cli.py
+++ b/ckanext/report/command.py
@@ -2,7 +2,7 @@
import ckan.plugins as p
-from ckanext.report.cli.command import Reporting
+from . import utils
class ReportCommand(p.toolkit.CkanCommand):
@@ -56,28 +56,31 @@ def command(self):
self.log = logging.getLogger("ckan.lib.cli")
cmd = self.args[0]
- reporter = Reporting()
if cmd == 'initdb':
- reporter.initdb()
+ self._initdb()
elif cmd == 'list':
- reporter.list()
+ self._list()
elif cmd == 'generate':
report_list = None
if len(self.args) == 2:
report_list = [s.strip() for s in self.args[1].split(',')]
- reporter.generate(report_list)
+ self.log.info("Running reports => %s", report_list)
+ self._generate(report_list)
elif cmd == 'generate-for-options':
- report_name = self.args[1]
- report_options = {}
- for option_arg in self.args[2:]:
- if '=' not in option_arg:
- self.parser.error('Option needs an "=" sign in it: "%s"'
- % option_arg)
- equal_pos = option_arg.find('=')
- key, value = option_arg[:equal_pos], option_arg[equal_pos + 1:]
- if value == '':
- value = None # this is what the web i/f does with params
- report_options[key] = value
- reporter.generate_for_options(report_name, report_options)
+ message = utils.generate_for_options(self.args[1], self.self.args[2:])
+ if message:
+ self.parser.error(message)
else:
self.parser.error('Command not recognized: %r' % cmd)
+
+ def _initdb(self):
+ utils.initdb()
+ self.log.info('Report table is setup')
+
+ def _list(self):
+ utils.list_reports()
+
+ def _generate(self, report_list=None):
+
+ timings = utils.generate(report_list)
+ self.log.info("Report generation complete %s", timings)
diff --git a/ckanext/report/controllers/pylons_controllers.py b/ckanext/report/controllers.py
similarity index 87%
rename from ckanext/report/controllers/pylons_controllers.py
rename to ckanext/report/controllers.py
index 82517a6..60f61d8 100644
--- a/ckanext/report/controllers/pylons_controllers.py
+++ b/ckanext/report/controllers.py
@@ -3,7 +3,7 @@
import six
import ckan.plugins.toolkit as t
-from ckanext.report.controllers import report_index, report_view
+from .utils import report_index, report_view
class ReportController(t.BaseController):
diff --git a/ckanext/report/controllers/blueprints.py b/ckanext/report/controllers/blueprints.py
deleted file mode 100644
index ba109a7..0000000
--- a/ckanext/report/controllers/blueprints.py
+++ /dev/null
@@ -1,47 +0,0 @@
-# encoding: utf-8
-
-import six
-
-from flask import Blueprint, Response
-
-from ckan.plugins import toolkit
-from . import report_index, report_view
-
-
-reporting = Blueprint(
- u'report',
- __name__
-)
-
-
-def redirect_to_index():
- return toolkit.redirect_to('/report')
-
-
-def view(report_name, organization=None, refresh=False):
- body, headers = report_view(report_name, organization, refresh)
- if headers:
- response = Response(body)
- for key, value in six.iteritems(headers):
- response.headers[key] = value
- return response
- else:
- return body
-
-
-reporting.add_url_rule(
- u'/report', 'index', view_func=report_index
-)
-reporting.add_url_rule(
- u'/reports', 'reports', view_func=redirect_to_index
-)
-reporting.add_url_rule(
- u'/report/', 'view', view_func=view, methods=('GET', 'POST',)
-)
-reporting.add_url_rule(
- u'/report//', 'org', view_func=view, methods=('GET', 'POST',)
-)
-
-
-def get_blueprints():
- return [reporting]
diff --git a/ckanext/report/helpers.py b/ckanext/report/helpers.py
index dca49f8..8762ffd 100644
--- a/ckanext/report/helpers.py
+++ b/ckanext/report/helpers.py
@@ -16,9 +16,8 @@ def relative_url_for(**kwargs):
# being an open redirect.
disallowed_params = set(('controller', 'action', 'anchor', 'host',
'protocol', 'qualified'))
- user_specified_params = [(k, v) for k, v in list(tk.request.params.items())
+ user_specified_params = [(k, v) for k, v in tk.request.params.items()
if k not in disallowed_params]
-
if tk.check_ckan_version(min_version="2.9.0"):
from flask import request
args = dict(list(request.args.items())
@@ -29,14 +28,15 @@ def relative_url_for(**kwargs):
for k, v in list(args.items()):
if not v:
del args[k]
- return tk.url_for(request.url_rule.rule, **args)
+ return tk.url_for(request.path, **args)
else:
args = dict(list(tk.request.environ['pylons.routes_dict'].items())
+ user_specified_params
+ list(kwargs.items()))
+
# remove blanks
- for k, v in list(args.items()):
+ for k, v in args.items():
if not v:
del args[k]
return tk.url_for(**args)
@@ -88,3 +88,11 @@ def explicit_default_options(report_name):
if options[key] is True:
explicit_defaults[key] = 1
return explicit_defaults
+
+
+def is_ckan_29():
+ """
+ Returns True if using CKAN 2.9+, with Flask and Webassets.
+ Returns False if those are not present.
+ """
+ return tk.check_ckan_version(min_version='2.9.0')
diff --git a/ckanext/report/lib.py b/ckanext/report/lib.py
index 1c8f818..217a2f9 100644
--- a/ckanext/report/lib.py
+++ b/ckanext/report/lib.py
@@ -1,15 +1,16 @@
'''
These functions are for use by other extensions for their reports.
'''
-import datetime
+from datetime import datetime
import six
-from six.moves import zip
+from six.moves import cStringIO as StringIO, zip
try:
from collections import OrderedDict # from python 2.7
except ImportError:
from sqlalchemy.util import OrderedDict
import ckan.plugins as p
+from ckan.plugins.toolkit import config
def all_organizations(include_none=False):
@@ -59,10 +60,6 @@ def filter_by_organizations(query, organization, include_sub_organizations):
def dataset_notes(pkg):
'''Returns a string with notes about the given package. It is
configurable.'''
- if p.toolkit.check_ckan_version(min_version="2.6.0"):
- from ckan.plugins.toolkit import config
- else:
- from pylons import config
expression = config.get('ckanext-report.notes.dataset')
if not expression:
return ''
@@ -72,13 +69,13 @@ def dataset_notes(pkg):
def percent(numerator, denominator):
if denominator == 0:
return 100 if numerator else 0
- return int((numerator * 100.0) // denominator)
+ return int((numerator * 100.0) / denominator)
def make_csv_from_dicts(rows):
import csv
- csvout = six.StringIO()
+ csvout = StringIO()
csvwriter = csv.writer(
csvout,
dialect='excel',
@@ -91,7 +88,7 @@ def make_csv_from_dicts(rows):
for row in rows:
new_headers = set(row.keys()) - headers_set
headers_set |= new_headers
- for header in list(row.keys()):
+ for header in row.keys():
if header in new_headers:
headers_ordered.append(header)
csvwriter.writerow(headers_ordered)
@@ -99,9 +96,9 @@ def make_csv_from_dicts(rows):
items = []
for header in headers_ordered:
item = row.get(header, 'no record')
- if isinstance(item, datetime.datetime):
+ if isinstance(item, datetime):
item = item.strftime('%Y-%m-%d %H:%M')
- elif isinstance(item, (int, int, float, list, tuple)):
+ elif isinstance(item, (int, float, list, tuple)):
item = six.text_type(item)
elif item is None:
item = ''
diff --git a/ckanext/report/model.py b/ckanext/report/model.py
index 986b7a1..6151770 100644
--- a/ckanext/report/model.py
+++ b/ckanext/report/model.py
@@ -1,4 +1,4 @@
-from __future__ import absolute_import
+# encoding: utf-8
import datetime
import logging
@@ -65,7 +65,7 @@ class DataCache(object):
"""
def __init__(self, **kwargs):
- for k, v in list(kwargs.items()):
+ for k, v in kwargs.items():
setattr(self, k, v)
@classmethod
@@ -79,7 +79,6 @@ def get(cls, object_id, key, convert_json=False, max_age=None):
.filter(cls.object_id == object_id)\
.first()
if not item:
- # log.debug('Does not exist in cache: %s/%s', object_id, key)
return (None, None)
if max_age:
@@ -91,7 +90,9 @@ def get(cls, object_id, key, convert_json=False, max_age=None):
value = item.value
if convert_json:
- # Preserve the order of the columns in the data
+ # Use OrderedDict instead of dict, so that the order of the columns
+ # in the data is preserved from the data when it was written (assuming
+ # it was written as an OrderedDict in the report's code).
try:
# Python 2.7's json library has object_pairs_hook
import json
@@ -100,7 +101,6 @@ def get(cls, object_id, key, convert_json=False, max_age=None):
# Python 2.4-2.6
import simplejson as json
value = json.loads(value, object_pairs_hook=OrderedDict)
- # log.debug('Cache load: %s/%s "%s"...', object_id, key, repr(value)[:40])
return value, item.created
@classmethod
diff --git a/ckanext/report/plugin/__init__.py b/ckanext/report/plugin.py
similarity index 76%
rename from ckanext/report/plugin/__init__.py
rename to ckanext/report/plugin.py
index ddbeb29..a521206 100644
--- a/ckanext/report/plugin/__init__.py
+++ b/ckanext/report/plugin.py
@@ -2,18 +2,16 @@
import ckan.plugins as p
from ckan.plugins import toolkit
-from ckanext.report.interfaces import IReport
-import ckanext.report.logic.action.get as action_get
-import ckanext.report.logic.action.update as action_update
-import ckanext.report.logic.auth.get as auth_get
-import ckanext.report.logic.auth.update as auth_update
+from . import helpers as h
+from .interfaces import IReport
+from .logic.action import get as action_get, update as action_update
+from .logic.auth import get as auth_get, update as auth_update
-
-if toolkit.check_ckan_version("2.9"):
- from ckanext.report.plugin.flask_plugin import MixinPlugin
+if h.is_ckan_29():
+ from .plugin_mixins.flask_plugin import MixinPlugin
else:
- from ckanext.report.plugin.pylons_plugin import MixinPlugin
+ from .plugin_mixins.pylons_plugin import MixinPlugin
class ReportPlugin(MixinPlugin, p.SingletonPlugin):
@@ -25,18 +23,18 @@ class ReportPlugin(MixinPlugin, p.SingletonPlugin):
# IConfigurer
def update_config(self, config):
- toolkit.add_template_directory(config, '../templates')
+ toolkit.add_template_directory(config, 'templates')
# ITemplateHelpers
def get_helpers(self):
- from ckanext.report import helpers as h
return {
'report__relative_url_for': h.relative_url_for,
'report__chunks': h.chunks,
'report__organization_list': h.organization_list,
'report__render_datetime': h.render_datetime,
'report__explicit_default_options': h.explicit_default_options,
+ 'is_ckan_29': h.is_ckan_29,
}
# IActions
diff --git a/ckanext/report/cli/__init__.py b/ckanext/report/plugin_mixins/__init__.py
similarity index 100%
rename from ckanext/report/cli/__init__.py
rename to ckanext/report/plugin_mixins/__init__.py
diff --git a/ckanext/report/plugin/flask_plugin.py b/ckanext/report/plugin_mixins/flask_plugin.py
similarity index 57%
rename from ckanext/report/plugin/flask_plugin.py
rename to ckanext/report/plugin_mixins/flask_plugin.py
index 0bfca3f..9702341 100644
--- a/ckanext/report/plugin/flask_plugin.py
+++ b/ckanext/report/plugin_mixins/flask_plugin.py
@@ -1,8 +1,7 @@
# encoding: utf-8
import ckan.plugins as p
-from ckanext.report.controllers import blueprints
-from ckanext.report.cli import click_cli
+from ckanext.report import blueprint, cli
class MixinPlugin(p.SingletonPlugin):
@@ -12,9 +11,9 @@ class MixinPlugin(p.SingletonPlugin):
# IBlueprint
def get_blueprint(self):
- return blueprints.get_blueprints()
+ return blueprint.get_blueprints()
# IClick
def get_commands(self):
- return click_cli.get_commands()
+ return cli.get_commands()
diff --git a/ckanext/report/plugin/pylons_plugin.py b/ckanext/report/plugin_mixins/pylons_plugin.py
similarity index 75%
rename from ckanext/report/plugin/pylons_plugin.py
rename to ckanext/report/plugin_mixins/pylons_plugin.py
index 44136ca..37f33a0 100644
--- a/ckanext/report/plugin/pylons_plugin.py
+++ b/ckanext/report/plugin_mixins/pylons_plugin.py
@@ -9,8 +9,9 @@ class MixinPlugin(p.SingletonPlugin):
# IRoutes
def before_map(self, map):
- report_ctlr = 'ckanext.report.controllers.pylons_controllers:ReportController'
- map.connect('report.index', '/report', controller=report_ctlr, action='index')
+ report_ctlr = 'ckanext.report.controllers:ReportController'
+ map.connect('report.index', '/report', controller=report_ctlr,
+ action='index')
map.connect('reports', '/report', controller=report_ctlr,
action='index')
map.redirect('/reports', '/report')
@@ -18,6 +19,8 @@ def before_map(self, map):
action='view')
map.connect('report', '/report/:report_name', controller=report_ctlr,
action='view')
+ map.connect('report-org', '/report/:report_name/:organization',
+ controller=report_ctlr, action='view')
map.connect('report.org', '/report/:report_name/:organization',
controller=report_ctlr, action='view')
return map
diff --git a/ckanext/report/report_registry.py b/ckanext/report/report_registry.py
index ed1d24b..048b7b2 100644
--- a/ckanext/report/report_registry.py
+++ b/ckanext/report/report_registry.py
@@ -7,12 +7,13 @@
from ckan import model
from ckan.plugins.toolkit import asbool
-from ckanext.report.interfaces import IReport
try:
from collections import OrderedDict # from python 2.7
except ImportError:
from sqlalchemy.util import OrderedDict
+from ckanext.report.interfaces import IReport
+
log = logging.getLogger(__name__)
REPORT_KEYS_REQUIRED = set(('name', 'generate', 'template', 'option_defaults',
@@ -29,8 +30,7 @@ def __init__(self, report_info_dict, plugin):
missing_required_keys = REPORT_KEYS_REQUIRED - set(report_info_dict.keys())
assert not missing_required_keys, 'Report info dict missing keys %r: '\
'%r' % (missing_required_keys, report_info_dict)
- unknown_keys = set(report_info_dict.keys()) - REPORT_KEYS_REQUIRED - \
- REPORT_KEYS_OPTIONAL
+ unknown_keys = set(report_info_dict.keys()) - REPORT_KEYS_REQUIRED - REPORT_KEYS_OPTIONAL
assert not unknown_keys, 'Report info dict has unrecognized keys %r: '\
'%r' % (unknown_keys, report_info_dict)
if not report_info_dict['option_defaults']:
diff --git a/ckanext/report/reports.py b/ckanext/report/reports.py
index cf11b3c..aa45db5 100644
--- a/ckanext/report/reports.py
+++ b/ckanext/report/reports.py
@@ -3,12 +3,14 @@
'''
from ckan import model
-from ckanext.report import lib
+
try:
from collections import OrderedDict # from python 2.7
except ImportError:
from sqlalchemy.util import OrderedDict
+from ckanext.report import lib
+
def tagless_report(organization, include_sub_organizations=False):
'''
@@ -29,7 +31,7 @@ def tagless_report(organization, include_sub_organizations=False):
# Find the packages without tags
q = model.Session.query(model.Package) \
.outerjoin(model.PackageTag) \
- .filter(model.PackageTag.id is None)
+ .filter(model.PackageTag.id == None) # noqa: E711
if organization:
q = lib.filter_by_organizations(q, organization,
include_sub_organizations)
diff --git a/ckanext/report/templates/report/index.html b/ckanext/report/templates/report/index.html
index 46b77e1..0433dc0 100644
--- a/ckanext/report/templates/report/index.html
+++ b/ckanext/report/templates/report/index.html
@@ -3,7 +3,7 @@
{% block title %}{{ _('Reports') }} - {{ super() }}{% endblock %}
{% block breadcrumb_content %}
- {{ h.nav_link(_('Reports'), named_route='report.index') }}
+ {{ h.nav_link(_('Reports'), named_route='report.index') }}
{% endblock %}
{% block primary_content_inner %}
diff --git a/ckanext/report/templates/report/tagless-datasets.html b/ckanext/report/templates/report/tagless-datasets.html
index 0a75c8a..5444ab6 100644
--- a/ckanext/report/templates/report/tagless-datasets.html
+++ b/ckanext/report/templates/report/tagless-datasets.html
@@ -4,12 +4,14 @@
table - main data, as a list of rows, each row is a dict
data - other data values, as a dict
#}
+
+{% set dataset_read_route = 'dataset.read' if h.is_ckan_29() else 'dataset_read' %}
- {% trans %}Datasets without tags{% endtrans %}: {{ table|length }} / {{ data['num_packages'] }} ({{ data['packages_without_tags_percent'] }})
- {% trans %}Average tags per package{% endtrans %}: {{ data['average_tags_per_package'] }} tags
-
+
{% trans %}Dataset{% endtrans %} |
@@ -22,7 +24,7 @@
{% for row in table %}
-
+
{{ row.title }}
|
diff --git a/ckanext/report/templates/report/view.html b/ckanext/report/templates/report/view.html
index f700100..ddc4833 100644
--- a/ckanext/report/templates/report/view.html
+++ b/ckanext/report/templates/report/view.html
@@ -3,72 +3,66 @@
{% block title %}{{ report.title }} - {{ _('Reports') }} - {{ super() }}{% endblock %}
{% block breadcrumb_content %}
- {{ h.nav_link(_('Reports'), named_route='report.index') }}
- {{ h.nav_link(report.title, named_route='report.org' if '/organization' in request.environ.get('PATH_INFO', '') else 'report.view', report_name=report_name) }}
+ {{ h.nav_link(_('Reports'), named_route='report.index') }}
+ {{ h.nav_link(report.title, named_route='report.org' if '/organization' in request.environ.get('PATH_INFO', '') else 'report.view', report_name=report_name) }}
{% endblock%}
{% block primary_content_inner %}
- {{ report.title }}
- {{ report.description }}
-
- {{ _('Generated') }}: {{ h.report__render_datetime(report_date, '%d/%m/%Y %H:%M') }}
-
- {% if c.userobj.sysadmin %}
-
-
{% trans %}Refresh report{% endtrans %}
-
-
-
{{ _('As a system administrator you are able to refresh this report on demand by clicking the \'Refresh\' button.') }}
-
+
{{ report.title }}
+
{{ report.description }}
+
+ {{ _('Generated') }}: {{ h.report__render_datetime(report_date, '%d/%m/%Y %H:%M') }}
+
+ {% if c.userobj.sysadmin %}
+
+
{% trans %}Refresh report{% endtrans %}
+
+
+
{{ _('As a system administrator you are able to refresh this report on demand by clicking the \'Refresh\' button.') }}
- {% endif %}
+
+ {% endif %}
- {% if options %}
-
{{ _('Options') }}
-
- {% endif %}
+ {% if options %}
+
{{ _('Options') }}
+
+ {% endif %}
- {% if are_some_results %}
-
- {{ _('Download') }}:
-
CSV
-
JSON
-
- {% endif %}
-
{{ _('Results') }}
- {% if not are_some_results %}
-
{{ _('No results found.') }}
- {% else %}
-
- {% snippet report_template, table=data['table'], data=data, report_name=report_name, options=options %}
-
- {% endif %}
-
+ {% if are_some_results %}
+
+ {{ _('Download') }}:
+
CSV
+
JSON
+
+ {% endif %}
+ {{ _('Results') }}
+ {% if not are_some_results %}
+ {{ _('No results found.') }}
+ {% else %}
+
+ {% snippet report_template, table=data['table'], data=data, report_name=report_name, options=options %}
+
+ {% endif %}
{% endblock%}
{% block scripts %}
{{ super() }}
-
-