diff --git a/.ahoy.yml b/.ahoy.yml new file mode 100644 index 0000000..000bc15 --- /dev/null +++ b/.ahoy.yml @@ -0,0 +1,196 @@ +--- +ahoyapi: v2 + +commands: + + # Docker commands. + build: + usage: Build or rebuild project. + cmd: | + ahoy title "Building project" + ahoy pre-flight + ahoy clean + ahoy build-network + ahoy up -- --build --force-recreate + ahoy install-site + ahoy title "Build complete" + ahoy doctor + ahoy info 1 + + build-network: + usage: Ensure that the amazeeio network exists. + cmd: | + docker network prune -f > /dev/null + docker network inspect amazeeio-network > /dev/null || docker network create amazeeio-network + + info: + usage: Print information about this project. + cmd: | + ahoy line "Project : " ${PROJECT} + ahoy line "Site local URL : " ${LAGOON_LOCALDEV_URL} + ahoy line "DB port on host : " $(docker port $(docker-compose ps -q postgres) 5432 | cut -d : -f 2) + ahoy line "Solr port on host : " $(docker port $(docker-compose ps -q solr) 8983 | cut -d : -f 2) + ahoy line "Mailhog URL : " http://mailhog.docker.amazee.io/ + + up: + usage: Build and start Docker containers. + cmd: | + docker-compose up -d "$@" + sleep 10 + docker-compose logs + ahoy cli "dockerize -wait tcp://ckan:5000 -timeout 1m" + if docker-compose logs | grep -q "\[Error\]"; then docker-compose logs; exit 1; fi + if docker-compose logs | grep -q "Exception"; then docker-compose logs; exit 1; fi + docker ps -a --filter name=^/${COMPOSE_PROJECT_NAME}_ + export DOCTOR_CHECK_CLI=0 + + down: + usage: Stop Docker containers and remove container, images, volumes and networks. + cmd: 'if [ -f "docker-compose.yml" ]; then docker-compose down --volumes; fi' + + start: + usage: Start existing Docker containers. + cmd: docker-compose start "$@" + + stop: + usage: Stop running Docker containers. + cmd: docker-compose stop "$@" + + restart: + usage: Restart all stopped and running Docker containers. + cmd: docker-compose restart "$@" + + logs: + usage: Show Docker logs. + cmd: docker-compose logs "$@" + + pull: + usage: Pull latest docker images. + cmd: if [ ! -z "$(docker image ls -q)" ]; then docker image ls --format \"{{.Repository}}:{{.Tag}}\" | grep amazeeio/ | grep -v none | xargs -n1 docker pull | cat; fi + + cli: + usage: Start a shell inside CLI container or run a command. + cmd: if \[ "${#}" -ne 0 \]; then docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate; cd $APP_DIR;'" $*"; else docker exec $(docker-compose ps -q ckan) sh -c '. ${APP_DIR}/bin/activate && cd $APP_DIR && sh'; fi + + doctor: + usage: Find problems with current project setup. + cmd: bin/doctor.sh "$@" + + install-site: + usage: Install a site. + cmd: | + ahoy title "Installing a fresh site" + ahoy cli '$APP_DIR/bin/init.sh' + + clean: + usage: Remove containers and all build files. + cmd: | + ahoy down + # Remove other directories. + # @todo: Add destinations below. + rm -rf \ + ./ckan + + reset: + usage: "Reset environment: remove containers, all build, manually created and Drupal-Dev files." + cmd: | + ahoy clean + git ls-files --others -i --exclude-from=.git/info/exclude | xargs chmod 777 + git ls-files --others -i --exclude-from=.git/info/exclude | xargs rm -Rf + find . -type d -not -path "./.git/*" -empty -delete + + flush-redis: + usage: Flush Redis cache. + cmd: docker exec -i $(docker-compose ps -q redis) redis-cli flushall > /dev/null + + lint: + usage: Lint code. + cmd: | + ahoy cli "flake8 ${@:-ckanext}" || \ + [ "${ALLOW_LINT_FAIL:-0}" -eq 1 ] + + copy-local-files: + usage: Update files from local repo. + cmd: | + docker cp . $(docker-compose ps -q ckan):/srv/app/ + docker cp bin/ckan_cli $(docker-compose ps -q ckan):/usr/bin/ + ahoy cli 'chmod -v u+x /usr/bin/ckan_cli; cp -v .docker/test.ini $CKAN_INI' + + test-unit: + usage: Run unit tests. + cmd: | + ahoy cli 'pytest --ckan-ini=${CKAN_INI} $APP_DIR/ckanext' || \ + [ "${ALLOW_UNIT_FAIL:-0}" -eq 1 ] + + test-bdd: + usage: Run BDD tests. + cmd: | + ahoy start-ckan-job-worker & + ahoy start-mailmock & + sleep 5 && + ahoy cli "behave -k ${*:-test/features} --tags=-format_autocomplete" || \ + [ "${ALLOW_BDD_FAIL:-0}" -eq 1 ] + ahoy stop-mailmock + ahoy stop-ckan-job-worker + + start-mailmock: + usage: Starts email mock server used for email BDD tests + cmd: | + ahoy title 'Starting mailmock' + ahoy cli 'mailmock -p 8025 -o ${APP_DIR}/test/emails' # for debugging mailmock email output remove --no-stdout + + stop-mailmock: + usage: Stops email mock server used for email BDD tests + cmd: | + ahoy title 'Stopping mailmock' + ahoy cli "killall -2 mailmock" + + start-ckan-job-worker: + usage: Starts CKAN background job worker + cmd: | + ahoy title 'Starting CKAN background job worker' + ahoy cli "ckan_cli jobs clear && \ + ckan_cli jobs worker" + + stop-ckan-job-worker: + usage: Stops CKAN background job worker + cmd: | + ahoy title 'Stopping CKAN background job worker' + ahoy cli "pkill -f 'jobs worker'" + + # Utilities. + title: + cmd: printf "$(tput -Txterm setaf 4)==> ${1}$(tput -Txterm sgr0)\n" + hide: true + + line: + cmd: printf "$(tput -Txterm setaf 2)${1}$(tput -Txterm sgr0)${2}\n" + hide: true + + getvar: + cmd: eval echo "${@}" + hide: true + + pre-flight: + cmd: | + export DOCTOR_CHECK_DB=${DOCTOR_CHECK_DB:-1} + export DOCTOR_CHECK_TOOLS=${DOCTOR_CHECK_TOOLS:-1} + export DOCTOR_CHECK_PORT=${DOCTOR_CHECK_PORT:-0} + export DOCTOR_CHECK_PYGMY=${DOCTOR_CHECK_PYGMY:-1} + export DOCTOR_CHECK_CLI=${DOCTOR_CHECK_CLI:-0} + export DOCTOR_CHECK_SSH=${DOCTOR_CHECK_SSH:-0} + export DOCTOR_CHECK_WEBSERVER=${DOCTOR_CHECK_WEBSERVER:-0} + export DOCTOR_CHECK_BOOTSTRAP=${DOCTOR_CHECK_BOOTSTRAP:-0} + ahoy doctor + hide: true + +entrypoint: + - bash + - "-c" + - "-e" + - | + export LAGOON_LOCALDEV_URL=http://$PROJECT.docker.amazee.io + [ -f .env ] && [ -s .env ] && export $(grep -v '^#' .env | xargs) && if [ -f .env.local ] && [ -s .env.local ]; then export $(grep -v '^#' .env.local | xargs); fi + bash -e -c "$0" "$@" + - "{{cmd}}" + - "{{name}}" diff --git a/.docker/Dockerfile-template.ckan b/.docker/Dockerfile-template.ckan new file mode 100644 index 0000000..eab5e4c --- /dev/null +++ b/.docker/Dockerfile-template.ckan @@ -0,0 +1,28 @@ +FROM openknowledge/ckan-dev:{CKAN_VERSION} + +ARG SITE_URL=http://ckan:5000/ +ENV PYTHON_VERSION={PYTHON_VERSION} +ENV CKAN_SITE_URL="${SITE_URL}" +ENV PYTHON={PYTHON} + +WORKDIR "${APP_DIR}" + +ENV DOCKERIZE_VERSION v0.6.1 +RUN apk add --no-cache build-base \ + && curl -sL https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-alpine-linux-amd64-${DOCKERIZE_VERSION}.tar.gz \ + | tar -C /usr/local/bin -xzvf - + +# Install CKAN. + +COPY .docker/test.ini $CKAN_INI + +COPY . ${APP_DIR}/ + +COPY bin/ckan_cli /usr/bin/ + +RUN chmod +x ${APP_DIR}/bin/*.sh /usr/bin/ckan_cli + +# Init current extension. +RUN ${APP_DIR}/bin/init-ext.sh + +CMD ["/srv/app/bin/serve.sh"] diff --git a/.docker/test.ini b/.docker/test.ini new file mode 100644 index 0000000..71bfbf7 --- /dev/null +++ b/.docker/test.ini @@ -0,0 +1,103 @@ +[DEFAULT] +debug = false +smtp_server = localhost +error_email_from = paste@localhost + +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + +[app:main] +ckan.devserver.host = 0.0.0.0 +ckan.devserver.port = 5000 + +use = egg:ckan +full_stack = true +cache_dir = /tmp/%(ckan.site_id)s/ +beaker.session.key = ckan + +# This is the secret token that the beaker library uses to hash the cookie sent +# to the client. `paster make-config` generates a unique value for this each +# time it generates a config file. +beaker.session.secret = bSmgPpaxg2M+ZRes3u1TXwIcE + +# `paster make-config` generates a unique value for this each time it generates +# a config file. +app_instance_uuid = 6e3daf8e-1c6b-443b-911f-c7ab4c5f9605 + +# repoze.who config +who.config_file = %(here)s/who.ini +who.log_level = warning +who.log_file = %(cache_dir)s/who_log.ini + +## Database Settings +sqlalchemy.url = postgresql://ckan_default:pass@postgres/ckan_test + +## Site Settings. +ckan.site_url = http://ckan:5000/ + +## Search Settings + +ckan.site_id = default +solr_url = http://solr:8983/solr/ckan + + +## Redis Settings + +# URL to your Redis instance, including the database to be used. +ckan.redis.url = redis://redis:6379 + + +## Plugins Settings + +ckan.plugins = report tagless_report + +## Email settings + +#email_to = errors@example.com +#error_email_from = ckan-errors@example.com +#smtp.server = localhost +#smtp.starttls = False +#smtp.user = username@example.com +#smtp.password = your_password +#smtp.mail_from = +# If 'smtp.test_server' is configured we assume we're running tests, +# and don't use the smtp.server, starttls, user, password etc. options. +smtp.test_server = localhost:8025 +smtp.mail_from = info@test.ckan.net + +# Logging configuration +[loggers] +keys = root, ckan, ckanext + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console + +[logger_ckan] +level = INFO +handlers = console +qualname = ckan +propagate = 0 + +[logger_ckanext] +level = DEBUG +handlers = console +qualname = ckanext +propagate = 0 + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s diff --git a/.env b/.env new file mode 100644 index 0000000..6e8dfe2 --- /dev/null +++ b/.env @@ -0,0 +1,29 @@ +## +# Project environment variables. +# +# It is used by Ahoy and other scripts to read default values. +# +# The values must be scalar (cannot be another variable). +# +# You may also create .env.local file to override any values locally (it is +# excluded from git). +# + +# Project name. +PROJECT="ckanext-report" + +# Docker Compose project name. All containers will have this name. +COMPOSE_PROJECT_NAME="$PROJECT" + +# Flag to allow code linting failures. 0=enforce, 1=ignore +ALLOW_LINT_FAIL=0 + +# Flag to allow unit tests failures. 0=enforce, 1=ignore +ALLOW_UNIT_FAIL=0 + +# Flag to allow BDD tests failures. 0=enforce, 1=ignore +ALLOW_BDD_FAIL=0 + +# Disable amazeeio based health checks +DOCTOR_CHECK_WEBSERVER=0 +DOCTOR_CHECK_BOOTSTRAP=0 diff --git a/.flake8 b/.flake8 index d6b0f6b..ee7a7cc 100644 --- a/.flake8 +++ b/.flake8 @@ -3,13 +3,13 @@ exclude = ckan - scripts # Extended output format. format = pylint # Show the source of errors. show_source = True +statistics = True max-complexity = 10 max-line-length = 127 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 43efa8a..614aa66 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,92 +1,45 @@ --- name: Tests -on: [push, pull_request] -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - with: - python-version: '3.6' - - name: Cache pip - uses: actions/cache@v2 - with: - # This path is specific to Ubuntu - path: ~/.cache/pip - # Look to see if there is a cache hit for the corresponding requirements file - key: ${{ runner.os }}-pip-flake8-${{ hashFiles('requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip-flake8- - ${{ runner.os }}- - - name: Install requirements - run: pip install flake8 pycodestyle - - name: Check syntax - run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan +on: + push: + pull_request: + branches: + - master +jobs: test: - needs: lint strategy: - matrix: - ckan-version: [2.9, 2.9-py2, 2.8, 2.7] fail-fast: false + matrix: + ckan-version: ["2.10", 2.9, 2.9-py2, 2.8, 2.7] - name: CKAN ${{ matrix.ckan-version }} + name: Continuous Integration build on CKAN ${{ matrix.ckan-version }} runs-on: ubuntu-latest - container: - image: openknowledge/ckan-dev:${{ matrix.ckan-version }} - services: - solr: - image: ckan/ckan-solr-dev:${{ matrix.ckan-version }} - postgres: - image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }} - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - redis: - image: redis:3 + container: integratedexperts/ci-builder env: - CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test - CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test - CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test - CKAN_SOLR_URL: http://solr:8983/solr/ckan - CKAN_REDIS_URL: redis://redis:6379/1 + CKAN_VERSION: ${{ matrix.ckan-version }} steps: - uses: actions/checkout@v2 + timeout-minutes: 2 - - name: Cache pip - uses: actions/cache@v2 - with: - # This path is specific to Ubuntu - path: ~/.cache/pip - # Look to see if there is a cache hit for the corresponding requirements file - key: ${{ runner.os }}-pip-${{ hashFiles('*requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- + - name: Build + run: bin/build.sh + timeout-minutes: 15 - - name: Install requirements - run: | - pip install -r requirements-dev.txt - pip install -e . - # Replace default path to CKAN core config file with the one on the container - sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini + - name: Test + run: bin/test.sh + timeout-minutes: 30 - - name: Setup extension (CKAN >= 2.9) - if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }} - run: | - ckan -c test.ini db init - ckan -c test.ini report initdb - ckan -c test.ini report generate tagless-datasets - - name: Setup extension (CKAN < 2.9) - if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }} - run: | - paster --plugin=ckan db init -c test.ini - paster --plugin=ckanext-report report initdb -c test.ini - paster --plugin=ckanext-report report generate tagless-datasets -c test.ini + - name: Retrieve screenshots + if: failure() + run: bin/process-artifacts.sh + timeout-minutes: 1 - - name: Run tests - run: pytest --ckan-ini=test.ini --cov=ckanext.report --disable-warnings ckanext/report/tests + - name: Upload screenshots + if: failure() + uses: actions/upload-artifact@v2 + with: + name: CKAN ${{ matrix.ckan-version }} screenshots + path: /tmp/artifacts/behave/screenshots + timeout-minutes: 1 diff --git a/.gitignore b/.gitignore index 860e403..6544185 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ *.swp *.swo -ckanext/report/i18n/_reused_translations.pot \ No newline at end of file +ckanext/report/i18n/_reused_translations.pot +.docker/Dockerfile.ckan + diff --git a/README.md b/README.md index cd935cb..73bbe53 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Tests](https://github.com/qld-gov-au/ckanext-report/actions/workflows/test.yml/badge.svg)](https://github.com/qld-gov-au/ckanext-report/actions/workflows/test.yml) -ckanext-report +# ckanext-report ==================== ckanext-report is a CKAN extension that provides a reporting infrastructure. Here are the features offered: @@ -28,7 +28,9 @@ TODO: | 2.6 and earlier | yes | | 2.7 | yes | | 2.8 | yes | -| 2.9 | yes | +| 2.9-py2 | yes | +| 2.9 (py3) | yes | +| 2.10 (py3) | yes | Status: was in production at data.gov.uk around 2014-2016, but since that uses its own CSS rather than core CKAN's, for others to use it CSS needs adding. For an example, see this branch: see https://github.com/GSA/ckanext-report/tree/geoversion @@ -43,7 +45,8 @@ Install ckanext-report into your CKAN virtual environment in the usual way: Initialize the database tables needed by ckanext-report: - (pyenv) $ paster --plugin=ckanext-report report initdb --config=mysite.ini + CKAN < 2.9 (pyenv) $ paster --plugin=ckanext-report report initdb --config=mysite.ini + CKAN >= 2.9 (pyenv) $ ckan -c mysite.ini report initdb Enable the plugin. In your config (e.g. development.ini or production.ini) add ``report`` to your ckan.plugins. e.g.: @@ -151,7 +154,7 @@ data - other data values, as a dict
  • Average tags per package: {{ data['average_tags_per_package'] }} tags
  • - +
    @@ -230,10 +233,7 @@ class TaglessReportPlugin(p.SingletonPlugin): The last line refers to `tag_report_info` which is a dictionary with properties of the report. This is stored in `reports.py` together with the report code (see above). The info dict looks like this: ```python -try: - from collections import OrderedDict # from python 2.7 -except ImportError: - from sqlalchemy.util import OrderedDict +from collections import OrderedDict tagless_report_info = { 'name': 'tagless-datasets', 'description': 'Datasets which have no tags.', diff --git a/bin/activate b/bin/activate new file mode 100644 index 0000000..982827c --- /dev/null +++ b/bin/activate @@ -0,0 +1,3 @@ +if [ "$VENV_DIR" != "" ]; then + . ${VENV_DIR}/bin/activate +fi diff --git a/bin/build.sh b/bin/build.sh new file mode 100755 index 0000000..d4088e2 --- /dev/null +++ b/bin/build.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +## +# Build site in CI. +# +set -e + +# Process Docker Compose configuration. This is used to avoid multiple +# docker-compose.yml files. +# Remove lines containing '###'. +sed -i -e "/###/d" docker-compose.yml +# Uncomment lines containing '##'. +sed -i -e "s/##//" docker-compose.yml + +# Pull the latest images. +ahoy pull + +PYTHON=python +if [ "$CKAN_VERSION" = "2.7" ] || [ "$CKAN_VERSION" = "2.8" ] || [ "$CKAN_VERSION" = "2.9-py2" ]; then + PYTHON_VERSION=py2 +else + PYTHON_VERSION=py3 + PYTHON="${PYTHON}3" +fi + +sed "s|{CKAN_VERSION}|$CKAN_VERSION|g" .docker/Dockerfile-template.ckan \ + | sed "s|{PYTHON}|$PYTHON|g" \ + | sed "s|{PYTHON_VERSION}|$PYTHON_VERSION|g" > .docker/Dockerfile.ckan + +ahoy build || (ahoy logs; exit 1) diff --git a/bin/ckan_cli b/bin/ckan_cli new file mode 100644 index 0000000..7757dc8 --- /dev/null +++ b/bin/ckan_cli @@ -0,0 +1,75 @@ +#!/bin/sh + +# Call either 'ckan' (from CKAN >= 2.9) or 'paster' (from CKAN <= 2.8) +# with appropriate syntax, depending on what is present on the system. +# This is intended to smooth the upgrade process from 2.8 to 2.9. +# Eg: +# ckan_cli jobs list +# could become either: +# paster --plugin=ckan jobs list -c /etc/ckan/default/production.ini +# or: +# ckan -c /etc/ckan/default/production.ini jobs list + +# This script is aware of the VIRTUAL_ENV environment variable, and will +# attempt to respect it with similar behaviour to commands like 'pip'. +# Eg placing this script in a virtualenv 'bin' directory will cause it +# to call the 'ckan' or 'paster' command in that directory, while +# placing this script elsewhere will cause it to rely on the VIRTUAL_ENV +# variable, or if that is not set, the system PATH. + +# Since the positioning of the CKAN configuration file is central to the +# differences between 'paster' and 'ckan', this script needs to be aware +# of the config file location. It will use the CKAN_INI environment +# variable if it exists, or default to /etc/ckan/default/production.ini. + +# If 'paster' is being used, the default plugin is 'ckan'. A different +# plugin can be specified by setting the PASTER_PLUGIN environment +# variable. This variable is irrelevant if using the 'ckan' command. + +CKAN_INI="${CKAN_INI:-/etc/ckan/default/production.ini}" +PASTER_PLUGIN="${PASTER_PLUGIN:-ckan}" +# First, look for a command alongside this file +ENV_DIR=$(dirname "$0") +if [ -f "$ENV_DIR/ckan" ]; then + COMMAND=ckan +elif [ -f "$ENV_DIR/paster" ]; then + COMMAND=paster +elif [ "$VIRTUAL_ENV" != "" ]; then + # If command not found alongside this file, check the virtualenv + ENV_DIR="$VIRTUAL_ENV/bin" + if [ -f "$ENV_DIR/ckan" ]; then + COMMAND=ckan + elif [ -f "$ENV_DIR/paster" ]; then + COMMAND=paster + fi +else + # if no virtualenv is active, try the system path + if (which ckan > /dev/null 2>&1); then + ENV_DIR=$(dirname $(which ckan)) + COMMAND=ckan + elif (which paster > /dev/null 2>&1); then + ENV_DIR=$(dirname $(which paster)) + COMMAND=paster + else + echo "Unable to locate 'ckan' or 'paster' command" >&2 + exit 1 + fi +fi + +if [ "$COMMAND" = "ckan" ]; then + # adjust args to match ckan expectations + COMMAND=$(echo "$1" | sed -e 's/create-test-data/seed/') + echo "Using 'ckan' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2 + shift + exec $ENV_DIR/ckan -c ${CKAN_INI} $COMMAND "$@" $CLICK_ARGS +elif [ "$COMMAND" = "paster" ]; then + # adjust args to match paster expectations + COMMAND=$1 + echo "Using 'paster' command from $ENV_DIR with config ${CKAN_INI} to run $COMMAND..." >&2 + shift + if [ "$1" = "show" ]; then shift; fi + exec $ENV_DIR/paster --plugin=$PASTER_PLUGIN $COMMAND "$@" -c ${CKAN_INI} +else + echo "Unable to locate 'ckan' or 'paster' command in $ENV_DIR" >&2 + exit 1 +fi diff --git a/bin/create-test-data.sh b/bin/create-test-data.sh new file mode 100644 index 0000000..79fbdfd --- /dev/null +++ b/bin/create-test-data.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env sh +## +# Create some example content for extension BDD tests. +# +set -e + +CKAN_ACTION_URL=${CKAN_SITE_URL}api/action +CKAN_USER_NAME="${CKAN_USER_NAME:-admin}" +CKAN_DISPLAY_NAME="${CKAN_DISPLAY_NAME:-Administrator}" +CKAN_USER_EMAIL="${CKAN_USER_EMAIL:-admin@localhost}" + +. ${APP_DIR}/bin/activate + +add_user_if_needed () { + echo "Adding user '$2' ($1) with email address [$3]" + ckan_cli user show "$1" | grep "$1" || ckan_cli user add "$1"\ + fullname="$2"\ + email="$3"\ + password="${4:-Password123!}" +} + +add_user_if_needed "$CKAN_USER_NAME" "$CKAN_DISPLAY_NAME" "$CKAN_USER_EMAIL" +ckan_cli sysadmin add "${CKAN_USER_NAME}" + +API_KEY=$(ckan_cli user show "${CKAN_USER_NAME}" | tr -d '\n' | sed -r 's/^(.*)apikey=(\S*)(.*)/\2/') +if [ "$API_KEY" = "None" ]; then + echo "No API Key found on ${CKAN_USER_NAME}, generating API Token..." + API_KEY=$(ckan_cli user token add "${CKAN_USER_NAME}" test_setup |tail -1 | tr -d '[:space:]') +fi + +## +# BEGIN: Create a test organisation with test users for admin, editor and member +# +TEST_ORG_NAME=test-organisation +TEST_ORG_TITLE="Test Organisation" + +echo "Creating test users for ${TEST_ORG_TITLE} Organisation:" + +add_user_if_needed ckan_user "CKAN User" ckan_user@localhost +add_user_if_needed test_org_admin "Test Admin" test_org_admin@localhost +add_user_if_needed test_org_editor "Test Editor" test_org_editor@localhost +add_user_if_needed test_org_member "Test Member" test_org_member@localhost + +echo "Creating ${TEST_ORG_TITLE} organisation:" + +TEST_ORG=$( \ + curl -LsH "Authorization: ${API_KEY}" \ + --data '{"name": "'"${TEST_ORG_NAME}"'", "title": "'"${TEST_ORG_TITLE}"'"}' \ + ${CKAN_ACTION_URL}/organization_create +) + +TEST_ORG_ID=$(echo $TEST_ORG | $PYTHON ${APP_DIR}/bin/extract-id.py) + +echo "Assigning test users to '${TEST_ORG_TITLE}' organisation (${TEST_ORG_ID}):" + +curl -LsH "Authorization: ${API_KEY}" \ + --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_admin", "object_type": "user", "capacity": "admin"}' \ + ${CKAN_ACTION_URL}/member_create + +curl -LsH "Authorization: ${API_KEY}" \ + --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_editor", "object_type": "user", "capacity": "editor"}' \ + ${CKAN_ACTION_URL}/member_create + +curl -LsH "Authorization: ${API_KEY}" \ + --data '{"id": "'"${TEST_ORG_ID}"'", "object": "test_org_member", "object_type": "user", "capacity": "member"}' \ + ${CKAN_ACTION_URL}/member_create +## +# END. +# + +## +# BEGIN: Create a Reporting organisation with test users +# + +REPORT_ORG_NAME=reporting +REPORT_ORG_TITLE=Reporting + +echo "Creating test users for ${REPORT_ORG_TITLE} Organisation:" + +add_user_if_needed report_admin "Reporting Admin" report_admin@localhost +add_user_if_needed report_editor "Reporting Editor" report_editor@localhost + +echo "Creating ${REPORT_ORG_TITLE} Organisation:" + +REPORT_ORG=$( \ + curl -LsH "Authorization: ${API_KEY}" \ + --data '{"name": "'"${REPORT_ORG_NAME}"'", "title": "'"${REPORT_ORG_TITLE}"'"}' \ + ${CKAN_ACTION_URL}/organization_create +) + +REPORT_ORG_ID=$(echo $REPORT_ORG | $PYTHON ${APP_DIR}/bin/extract-id.py) + +echo "Assigning test users to ${REPORT_ORG_TITLE} Organisation:" + +curl -LsH "Authorization: ${API_KEY}" \ + --data '{"id": "'"${REPORT_ORG_ID}"'", "object": "report_admin", "object_type": "user", "capacity": "admin"}' \ + ${CKAN_ACTION_URL}/member_create + +curl -LsH "Authorization: ${API_KEY}" \ + --data '{"id": "'"${REPORT_ORG_ID}"'", "object": "report_editor", "object_type": "user", "capacity": "editor"}' \ + ${CKAN_ACTION_URL}/member_create + +echo "Creating test dataset for reporting:" + +curl -LsH "Authorization: ${API_KEY}" \ + --data '{"name": "reporting", "title": "Reporting dataset", "description": "Dataset for testing reports", +"owner_org": "'"${REPORT_ORG_ID}"'", "update_frequency": "near-realtime", "author_email": "report_admin@localhost", +"version": "1.0", "license_id": "cc-by-4", "data_driven_application": "NO", "security_classification": "PUBLIC", +"notes": "test", "de_identified_data": "NO"}'\ + ${CKAN_ACTION_URL}/package_create + +## +# END. +# + +. ${APP_DIR}/bin/deactivate diff --git a/bin/deactivate b/bin/deactivate new file mode 100644 index 0000000..7cd77b9 --- /dev/null +++ b/bin/deactivate @@ -0,0 +1,3 @@ +if [ "$VENV_DIR" != "" ]; then + deactivate +fi diff --git a/bin/doctor.sh b/bin/doctor.sh new file mode 100755 index 0000000..9c7f455 --- /dev/null +++ b/bin/doctor.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# +# Check Drupal-Dev project requirements. +# +set -e + +DOCTOR_CHECK_TOOLS="${DOCTOR_CHECK_TOOLS:-1}" +DOCTOR_CHECK_PORT="${DOCTOR_CHECK_PORT:-0}" +DOCTOR_CHECK_PYGMY="${DOCTOR_CHECK_PYGMY:-1}" +DOCTOR_CHECK_CLI="${DOCTOR_CHECK_CLI:-1}" +DOCTOR_CHECK_SSH="${DOCTOR_CHECK_SSH:-0}" +DOCTOR_CHECK_WEBSERVER="${DOCTOR_CHECK_WEBSERVER:-1}" +DOCTOR_CHECK_BOOTSTRAP="${DOCTOR_CHECK_BOOTSTRAP:-1}" + +APP_PORT="${APP_PORT:-5000}" +CLI="${CLI:-cli}" +LAGOON_LOCALDEV_URL="${LAGOON_LOCALDEV_URL:-http://your-site.docker.amazee.io/}" +SSH_KEY_FILE="${SSH_KEY_FILE:-$HOME/.ssh/id_rsa}" +DATAROOT="${DATAROOT:-.data}" + +#------------------------------------------------------------------------------- +# DO NOT CHANGE ANYTHING BELOW THIS LINE +#------------------------------------------------------------------------------- + + +# +# Main entry point. +# +main() { + status "Checking project requirements" + + if [ "${DOCTOR_CHECK_TOOLS}" == "1" ]; then + [ "$(command_exists docker)" == "1" ] && error "Please install Docker (https://www.docker.com/get-started)" && exit 1 + [ "$(command_exists docker-compose)" == "1" ] && error "Please install docker-compose (https://docs.docker.com/compose/install/)" && exit 1 + [ "$(command_exists composer)" == "1" ] && error "Please install Composer (https://getcomposer.org/)" && exit 1 + [ "$(command_exists pygmy)" == "1" ] && error "Please install Pygmy (https://pygmy.readthedocs.io/)" && exit 1 + [ "$(command_exists ahoy)" == "1" ] && error "Please install Ahoy (https://ahoy-cli.readthedocs.io/)" && exit 1 + success "All required tools are present" + fi + + if [ "${DOCTOR_CHECK_PORT}" == "1" ]; then + if ! lsof -i :$APP_PORT | grep LISTEN | grep -q om.docke; then + error "Port $APP_PORT is occupied by a service other than Docker. Stop this service and run 'pygmy up'" + fi + success "Port $APP_PORT is available" + fi + + if [ "${DOCTOR_CHECK_PYGMY}" == "1" ]; then + if ! pygmy status > /dev/null 2>&1; then + error "pygmy is not running. Run 'pygmy up' to start pygmy." + exit 1 + fi + success "Pygmy is running" + fi + + # Check that the stack is running. + if [ "${DOCTOR_CHECK_CLI}" == "1" ]; then + if ! docker ps -q --no-trunc | grep "$(docker-compose ps -q ckan)" > /dev/null 2>&1; then + error "CLI container is not running. Run 'ahoy up'." + exit 1 + fi + success "CLI container is running" + fi + + if [ "${DOCTOR_CHECK_SSH}" == "1" ]; then + # SSH key injection is required to access Lagoon services from within + # containers. For example, to connect to production environment to run + # drush script. + # Pygmy makes this possible in the following way: + # 1. Pygmy starts `amazeeio/ssh-agent` container with a volume `/tmp/amazeeio_ssh-agent` + # 2. Pygmy adds a default SSH key from the host into this volume. + # 3. `docker-compose.yml` should have volume inclusion specified for CLI container: + # ``` + # volumes_from: + # - container:amazeeio-ssh-agent + # ``` + # 4. When CLI container starts, the volume is mounted and an entrypoint script + # adds SSH key into agent. + # @see https://github.com/amazeeio/lagoon/blob/master/images/php/cli/10-ssh-agent.sh + # + # Running `ssh-add -L` within CLI container should show that the SSH key + # is correctly loaded. + # + # As rule of a thumb, one must restart the CLI container after restarting + # Pygmy ONLY if SSH key was not loaded in pygmy before the stack starts. + # No need to restart CLI container if key was added, but pygmy was + # restarted - the volume mount will retain and the key will still be + # available in CLI container. + + # Check that the key is injected into pygmy ssh-agent container. + if ! pygmy status 2>&1 | grep -q "${SSH_KEY_FILE}"; then + error "SSH key is not added to pygmy. Run 'pygmy stop && pygmy start' and then 'ahoy up -- --build'." + exit 1 + fi + + # Check that the volume is mounted into CLI container. + if ! docker exec -i "$(docker-compose ps -q ckan)" sh -c "grep \"^/dev\" /etc/mtab|grep -q /tmp/amazeeio_ssh-agent"; then + error "SSH key is added to Pygmy, but the volume is not mounted into container. Make sure that your your \"docker-compose.yml\" has the following lines:" + error "volumes_from:" + error " - container:amazeeio-ssh-agent" + error "After adding these lines, run 'ahoy up -- --build'" + exit 1 + fi + + # Check that ssh key is available in the container. + if ! docker exec -i "$(docker-compose ps -q ckan)" bash -c "ssh-add -L | grep -q 'ssh-rsa'" ; then + error "SSH key was not added into container. Run 'ahoy up -- --build'." + exit 1 + fi + + success "SSH key is available within CLI container" + fi + + + if [ "${DOCTOR_CHECK_WEBSERVER}" == "1" ]; then + host_app_port="$(docker port $(docker-compose ps -q ckan) $APP_PORT | cut -d : -f 2)" + if ! curl -L -s -o /dev/null -w "%{http_code}" "${LAGOON_LOCALDEV_URL}:${host_app_port}" | grep -q 200; then + error "Web server is not accessible at ${LAGOON_LOCALDEV_URL}:${host_app_port}" + exit 1 + fi + success "Web server is running and accessible at ${LAGOON_LOCALDEV_URL}:${host_app_port}" + fi + + if [ "${DOCTOR_CHECK_BOOTSTRAP}" == "1" ]; then + host_app_port="$(docker port $(docker-compose ps -q ckan) $APP_PORT | cut -d : -f 2)" + if ! curl -L -s -N "${LAGOON_LOCALDEV_URL}:${host_app_port}" | grep -q -i "meta name=\"generator\" content=\"ckan"; then + error "Website is running, but cannot be bootstrapped. Try pulling latest container images with 'ahoy pull'" + exit 1 + fi + success "Successfully bootstrapped website at ${LAGOON_LOCALDEV_URL}:${host_app_port}" + fi + + status "All required checks have passed" +} + +# +# Check that command exists. +# +command_exists() { + local cmd=$1 + command -v "${cmd}" | grep -ohq "${cmd}" + local res=$? + + # Try homebrew lookup, if brew is available. + if command -v "brew" | grep -ohq "brew" && [ "$res" == "1" ] ; then + brew --prefix "${cmd}" > /dev/null + res=$? + fi + + echo ${res} +} + +# +# Status echo. +# +status() { + cecho blue "✚ $1"; +} + +# +# Success echo. +# +success() { + cecho green " ✓ $1"; +} + +# +# Error echo. +# +error() { + cecho red " ✘ $1"; + exit 1 +} + +# +# Colored echo. +# +cecho() { + local prefix="\033[" + local input_color=$1 + local message="$2" + + local color="" + case "$input_color" in + black | bk) color="${prefix}0;30m";; + red | r) color="${prefix}1;31m";; + green | g) color="${prefix}1;32m";; + yellow | y) color="${prefix}1;33m";; + blue | b) color="${prefix}1;34m";; + purple | p) color="${prefix}1;35m";; + cyan | c) color="${prefix}1;36m";; + gray | gr) color="${prefix}0;37m";; + *) message="$1" + esac + + # Format message with color codes, but only if a correct color was provided. + [ -n "$color" ] && message="${color}${message}${prefix}0m" + + echo -e "$message" +} + +main "$@" diff --git a/bin/extract-id.py b/bin/extract-id.py new file mode 100644 index 0000000..ae9cf79 --- /dev/null +++ b/bin/extract-id.py @@ -0,0 +1,5 @@ +# encoding: utf-8 +import json +import sys + +print(json.loads(sys.stdin.read())['result']['id']) diff --git a/bin/init-ext.sh b/bin/init-ext.sh new file mode 100755 index 0000000..4ae5b57 --- /dev/null +++ b/bin/init-ext.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env sh +## +# Install current extension. +# +set -e + +install_requirements () { + PROJECT_DIR=$1 + shift + # Identify the best match requirements file, ignore the others. + # If there is one specific to our Python version, use that. + for filename_pattern in "$@"; do + filename="$PROJECT_DIR/${filename_pattern}-$PYTHON_VERSION.txt" + if [ -f "$filename" ]; then + pip install -r "$filename" + return 0 + fi + done + for filename_pattern in "$@"; do + filename="$PROJECT_DIR/$filename_pattern.txt" + if [ -f "$filename" ]; then + pip install -r "$filename" + return 0 + fi + done +} + +. ${APP_DIR}/bin/activate + +install_requirements . dev-requirements requirements-dev +for extension in . `ls -d $SRC_DIR/ckanext-*`; do + install_requirements $extension requirements pip-requirements +done +pip install -e . +installed_name=$(grep '^\s*name=' setup.py |sed "s|[^']*'\([-a-zA-Z0-9]*\)'.*|\1|") + +# Validate that the extension was installed correctly. +if ! pip list | grep "$installed_name" > /dev/null; then echo "Unable to find the extension in the list"; exit 1; fi + +. ${APP_DIR}/bin/deactivate diff --git a/bin/init.sh b/bin/init.sh new file mode 100755 index 0000000..5610d25 --- /dev/null +++ b/bin/init.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env sh +## +# Initialise CKAN data for testing. +# +set -e + +. ${APP_DIR}/bin/activate +CLICK_ARGS="--yes" ckan_cli db clean +ckan_cli db init +ckan_cli db upgrade + +# Initialise report tables +PASTER_PLUGIN=ckanext-report ckan_cli report initdb +PASTER_PLUGIN=ckanext-report ckan_cli report generate tagless-datasets + +# Create some base test data +. $APP_DIR/bin/create-test-data.sh diff --git a/bin/process-artifacts.sh b/bin/process-artifacts.sh new file mode 100755 index 0000000..6324f60 --- /dev/null +++ b/bin/process-artifacts.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +## +# Process test artifacts. +# +set -e + +# Create screenshots directory in case it was not created before. This is to +# avoid this script to fail when copying artifacts. +ahoy cli "mkdir -p test/screenshots" + +# Copy from the app container to the build host for storage. +mkdir -p /tmp/artifacts/behave +docker cp "$(docker-compose ps -q ckan)":/srv/app/test/screenshots /tmp/artifacts/behave/ diff --git a/bin/serve.sh b/bin/serve.sh new file mode 100755 index 0000000..eafaa3e --- /dev/null +++ b/bin/serve.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env sh +set -e + +dockerize -wait tcp://postgres:5432 -timeout 1m +dockerize -wait tcp://solr:8983 -timeout 1m +dockerize -wait tcp://redis:6379 -timeout 1m + +for i in {1..60}; do + if (PGPASSWORD=pass psql -h postgres -U ckan_default -d ckan_test -c "\q"); then + break + else + sleep 1 + fi +done + +. ${APP_DIR}/bin/activate +if (which ckan > /dev/null); then + ckan -c ${CKAN_INI} run -r +else + paster serve ${CKAN_INI} +fi diff --git a/bin/test.sh b/bin/test.sh new file mode 100755 index 0000000..a2d91da --- /dev/null +++ b/bin/test.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +## +# Run tests in CI. +# +set -e + +echo "==> Lint code" +ahoy lint + +echo "==> Run Unit tests" +ahoy test-unit + +echo "==> Run BDD tests" +ahoy install-site +ahoy test-bdd || (ahoy logs; exit 1) diff --git a/ckanext/__init__.py b/ckanext/__init__.py index 2e2033b..ed48ed0 100644 --- a/ckanext/__init__.py +++ b/ckanext/__init__.py @@ -1,3 +1,5 @@ +# encoding: utf-8 + # this is a namespace package try: import pkg_resources diff --git a/ckanext/report/blueprint.py b/ckanext/report/blueprint.py new file mode 100644 index 0000000..8695fbe --- /dev/null +++ b/ckanext/report/blueprint.py @@ -0,0 +1,37 @@ +# encoding: utf-8 + +import six + +from flask import Blueprint, make_response + +from ckan.plugins import toolkit + +from .utils import report_index as index, report_view + + +report = Blueprint(u'report', __name__) + + +def redirect_to_index(): + return toolkit.redirect_to('/report') + + +def view(report_name, organization=None, refresh=False): + body, headers = report_view(report_name, organization, refresh) + if headers: + response = make_response(body) + for key, value in six.iteritems(headers): + response.headers[key] = value + return response + else: + return body + + +report.add_url_rule(u'/report', 'index', view_func=index) +report.add_url_rule(u'/reports', 'reports', view_func=redirect_to_index) +report.add_url_rule(u'/report/', view_func=view, methods=('GET', 'POST')) +report.add_url_rule(u'/report//', 'org', view_func=view, methods=('GET', 'POST',)) + + +def get_blueprints(): + return [report] diff --git a/ckanext/report/cli.py b/ckanext/report/cli.py new file mode 100644 index 0000000..08a0f64 --- /dev/null +++ b/ckanext/report/cli.py @@ -0,0 +1,57 @@ +# encoding: utf-8 + +import click + +from . import utils + + +def get_commands(): + return [report] + + +@click.group() +def report(): + """Generates reports""" + pass + + +@report.command() +def initdb(): + """Creates necessary db tables""" + utils.initdb() + click.secho(u'Report table is setup', fg=u"green") + + +@report.command() +@click.argument(u'report_list', required=False) +def generate(report_list): + """ + Generate and cache reports - all of them unless you specify + a comma separated list of them. + """ + if report_list: + report_list = [s.strip() for s in report_list.split(',')] + timings = utils.generate(report_list) + + click.secho(u'Report generation complete %s' % timings, fg=u"green") + + +@report.command() +def list(): + """ Lists the reports + """ + utils.list_reports() + + +@report.command() +@click.argument(u'report_name') +@click.argument(u'report_options', nargs=-1) +def generate_for_options(report_name, report_options): + """ + Generate and cache a report for one combination of option values. + You can leave it with the defaults or specify options + as more parameters: key1=value key2=value + """ + message = utils.generate_for_options(report_name, report_options) + if message: + click.secho(message, fg=u"red") diff --git a/ckanext/report/cli/click_cli.py b/ckanext/report/cli/click_cli.py deleted file mode 100644 index 314b42a..0000000 --- a/ckanext/report/cli/click_cli.py +++ /dev/null @@ -1,70 +0,0 @@ -# encoding: utf-8 - -import click - -from ckanext.report.cli.command import Reporting - -# Click commands for CKAN 2.9 and above - - -@click.group() -def report(): - """ XLoader commands - """ - pass - - -@report.command() -def list(): - """ Lists the reports - """ - cmd = Reporting() - cmd.list() - - -@report.command() -def initdb(): - """ Initialize the database tables for this extension - """ - cmd = Reporting() - cmd.initdb() - - -@report.command() -@click.argument(u'report_names') -def generate(report_names): - """ - Generate and cache reports - all of them unless you specify - a comma separated list of them. - """ - cmd = Reporting() - report_list = [s.strip() for s in report_names.split(',')] - cmd.generate(report_list) - - -@report.command() -@click.argument(u'report_name') -@click.argument(u'options', nargs=-1) -def generate_for_options(report_name, options): - """ - Generate and cache a report for one combination of option values. - You can leave it with the defaults or specify options - as more parameters: key1=value key2=value - """ - cmd = Reporting() - report_options = {} - for option_arg in options: - if '=' not in option_arg: - raise click.BadParameter( - 'Option needs an "=" sign in it', - options) - equal_pos = option_arg.find('=') - key, value = option_arg[:equal_pos], option_arg[equal_pos + 1:] - if value == '': - value = None # this is what the web i/f does with params - report_options[key] = value - cmd.generate_for_options(report_name, report_options) - - -def get_commands(): - return [report] diff --git a/ckanext/report/cli/command.py b/ckanext/report/cli/command.py deleted file mode 100644 index cbf5439..0000000 --- a/ckanext/report/cli/command.py +++ /dev/null @@ -1,51 +0,0 @@ -# encoding: utf-8 - -import logging -import time - - -class Reporting(): - - def __init__(self): - self.log = logging.getLogger("ckanext.report.cli") - - def initdb(self): - from ckanext.report import model - model.init_tables() - self.log.info('Report table is setup') - - def list(self): - from ckanext.report.report_registry import ReportRegistry - registry = ReportRegistry.instance() - for plugin, report_name, report_title in registry.get_names(): - report = registry.get_report(report_name) - date = report.get_cached_date() - print('%s: %s %s' % (plugin, report_name, - date.strftime('%d/%m/%Y %H:%M') if date else '(not cached)')) - - def generate(self, report_list=None): - from ckanext.report.report_registry import ReportRegistry - timings = {} - - self.log.info("Running reports => %s", report_list) - registry = ReportRegistry.instance() - if report_list: - for report_name in report_list: - s = time.time() - registry.get_report(report_name).refresh_cache_for_all_options() - timings[report_name] = time.time() - s - else: - s = time.time() - registry.refresh_cache_for_all_reports() - timings["All Reports"] = time.time() - s - - self.log.info("Report generation complete %s", timings) - - def generate_for_options(self, report_name, options): - from ckanext.report.report_registry import ReportRegistry - self.log.info("Running report => %s", report_name) - registry = ReportRegistry.instance() - report = registry.get_report(report_name) - all_options = report.add_defaults_to_options(options, - report.option_defaults) - report.refresh_cache(all_options) diff --git a/ckanext/report/cli/paster_cli.py b/ckanext/report/command.py similarity index 68% rename from ckanext/report/cli/paster_cli.py rename to ckanext/report/command.py index f656408..83e0a24 100644 --- a/ckanext/report/cli/paster_cli.py +++ b/ckanext/report/command.py @@ -2,7 +2,7 @@ import ckan.plugins as p -from ckanext.report.cli.command import Reporting +from . import utils class ReportCommand(p.toolkit.CkanCommand): @@ -56,28 +56,31 @@ def command(self): self.log = logging.getLogger("ckan.lib.cli") cmd = self.args[0] - reporter = Reporting() if cmd == 'initdb': - reporter.initdb() + self._initdb() elif cmd == 'list': - reporter.list() + self._list() elif cmd == 'generate': report_list = None if len(self.args) == 2: report_list = [s.strip() for s in self.args[1].split(',')] - reporter.generate(report_list) + self.log.info("Running reports => %s", report_list) + self._generate(report_list) elif cmd == 'generate-for-options': - report_name = self.args[1] - report_options = {} - for option_arg in self.args[2:]: - if '=' not in option_arg: - self.parser.error('Option needs an "=" sign in it: "%s"' - % option_arg) - equal_pos = option_arg.find('=') - key, value = option_arg[:equal_pos], option_arg[equal_pos + 1:] - if value == '': - value = None # this is what the web i/f does with params - report_options[key] = value - reporter.generate_for_options(report_name, report_options) + message = utils.generate_for_options(self.args[1], self.self.args[2:]) + if message: + self.parser.error(message) else: self.parser.error('Command not recognized: %r' % cmd) + + def _initdb(self): + utils.initdb() + self.log.info('Report table is setup') + + def _list(self): + utils.list_reports() + + def _generate(self, report_list=None): + + timings = utils.generate(report_list) + self.log.info("Report generation complete %s", timings) diff --git a/ckanext/report/controllers/pylons_controllers.py b/ckanext/report/controllers.py similarity index 87% rename from ckanext/report/controllers/pylons_controllers.py rename to ckanext/report/controllers.py index 82517a6..60f61d8 100644 --- a/ckanext/report/controllers/pylons_controllers.py +++ b/ckanext/report/controllers.py @@ -3,7 +3,7 @@ import six import ckan.plugins.toolkit as t -from ckanext.report.controllers import report_index, report_view +from .utils import report_index, report_view class ReportController(t.BaseController): diff --git a/ckanext/report/controllers/blueprints.py b/ckanext/report/controllers/blueprints.py deleted file mode 100644 index ba109a7..0000000 --- a/ckanext/report/controllers/blueprints.py +++ /dev/null @@ -1,47 +0,0 @@ -# encoding: utf-8 - -import six - -from flask import Blueprint, Response - -from ckan.plugins import toolkit -from . import report_index, report_view - - -reporting = Blueprint( - u'report', - __name__ -) - - -def redirect_to_index(): - return toolkit.redirect_to('/report') - - -def view(report_name, organization=None, refresh=False): - body, headers = report_view(report_name, organization, refresh) - if headers: - response = Response(body) - for key, value in six.iteritems(headers): - response.headers[key] = value - return response - else: - return body - - -reporting.add_url_rule( - u'/report', 'index', view_func=report_index -) -reporting.add_url_rule( - u'/reports', 'reports', view_func=redirect_to_index -) -reporting.add_url_rule( - u'/report/', 'view', view_func=view, methods=('GET', 'POST',) -) -reporting.add_url_rule( - u'/report//', 'org', view_func=view, methods=('GET', 'POST',) -) - - -def get_blueprints(): - return [reporting] diff --git a/ckanext/report/helpers.py b/ckanext/report/helpers.py index dca49f8..8762ffd 100644 --- a/ckanext/report/helpers.py +++ b/ckanext/report/helpers.py @@ -16,9 +16,8 @@ def relative_url_for(**kwargs): # being an open redirect. disallowed_params = set(('controller', 'action', 'anchor', 'host', 'protocol', 'qualified')) - user_specified_params = [(k, v) for k, v in list(tk.request.params.items()) + user_specified_params = [(k, v) for k, v in tk.request.params.items() if k not in disallowed_params] - if tk.check_ckan_version(min_version="2.9.0"): from flask import request args = dict(list(request.args.items()) @@ -29,14 +28,15 @@ def relative_url_for(**kwargs): for k, v in list(args.items()): if not v: del args[k] - return tk.url_for(request.url_rule.rule, **args) + return tk.url_for(request.path, **args) else: args = dict(list(tk.request.environ['pylons.routes_dict'].items()) + user_specified_params + list(kwargs.items())) + # remove blanks - for k, v in list(args.items()): + for k, v in args.items(): if not v: del args[k] return tk.url_for(**args) @@ -88,3 +88,11 @@ def explicit_default_options(report_name): if options[key] is True: explicit_defaults[key] = 1 return explicit_defaults + + +def is_ckan_29(): + """ + Returns True if using CKAN 2.9+, with Flask and Webassets. + Returns False if those are not present. + """ + return tk.check_ckan_version(min_version='2.9.0') diff --git a/ckanext/report/lib.py b/ckanext/report/lib.py index 1c8f818..217a2f9 100644 --- a/ckanext/report/lib.py +++ b/ckanext/report/lib.py @@ -1,15 +1,16 @@ ''' These functions are for use by other extensions for their reports. ''' -import datetime +from datetime import datetime import six -from six.moves import zip +from six.moves import cStringIO as StringIO, zip try: from collections import OrderedDict # from python 2.7 except ImportError: from sqlalchemy.util import OrderedDict import ckan.plugins as p +from ckan.plugins.toolkit import config def all_organizations(include_none=False): @@ -59,10 +60,6 @@ def filter_by_organizations(query, organization, include_sub_organizations): def dataset_notes(pkg): '''Returns a string with notes about the given package. It is configurable.''' - if p.toolkit.check_ckan_version(min_version="2.6.0"): - from ckan.plugins.toolkit import config - else: - from pylons import config expression = config.get('ckanext-report.notes.dataset') if not expression: return '' @@ -72,13 +69,13 @@ def dataset_notes(pkg): def percent(numerator, denominator): if denominator == 0: return 100 if numerator else 0 - return int((numerator * 100.0) // denominator) + return int((numerator * 100.0) / denominator) def make_csv_from_dicts(rows): import csv - csvout = six.StringIO() + csvout = StringIO() csvwriter = csv.writer( csvout, dialect='excel', @@ -91,7 +88,7 @@ def make_csv_from_dicts(rows): for row in rows: new_headers = set(row.keys()) - headers_set headers_set |= new_headers - for header in list(row.keys()): + for header in row.keys(): if header in new_headers: headers_ordered.append(header) csvwriter.writerow(headers_ordered) @@ -99,9 +96,9 @@ def make_csv_from_dicts(rows): items = [] for header in headers_ordered: item = row.get(header, 'no record') - if isinstance(item, datetime.datetime): + if isinstance(item, datetime): item = item.strftime('%Y-%m-%d %H:%M') - elif isinstance(item, (int, int, float, list, tuple)): + elif isinstance(item, (int, float, list, tuple)): item = six.text_type(item) elif item is None: item = '' diff --git a/ckanext/report/model.py b/ckanext/report/model.py index 986b7a1..6151770 100644 --- a/ckanext/report/model.py +++ b/ckanext/report/model.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +# encoding: utf-8 import datetime import logging @@ -65,7 +65,7 @@ class DataCache(object): """ def __init__(self, **kwargs): - for k, v in list(kwargs.items()): + for k, v in kwargs.items(): setattr(self, k, v) @classmethod @@ -79,7 +79,6 @@ def get(cls, object_id, key, convert_json=False, max_age=None): .filter(cls.object_id == object_id)\ .first() if not item: - # log.debug('Does not exist in cache: %s/%s', object_id, key) return (None, None) if max_age: @@ -91,7 +90,9 @@ def get(cls, object_id, key, convert_json=False, max_age=None): value = item.value if convert_json: - # Preserve the order of the columns in the data + # Use OrderedDict instead of dict, so that the order of the columns + # in the data is preserved from the data when it was written (assuming + # it was written as an OrderedDict in the report's code). try: # Python 2.7's json library has object_pairs_hook import json @@ -100,7 +101,6 @@ def get(cls, object_id, key, convert_json=False, max_age=None): # Python 2.4-2.6 import simplejson as json value = json.loads(value, object_pairs_hook=OrderedDict) - # log.debug('Cache load: %s/%s "%s"...', object_id, key, repr(value)[:40]) return value, item.created @classmethod diff --git a/ckanext/report/plugin/__init__.py b/ckanext/report/plugin.py similarity index 76% rename from ckanext/report/plugin/__init__.py rename to ckanext/report/plugin.py index ddbeb29..a521206 100644 --- a/ckanext/report/plugin/__init__.py +++ b/ckanext/report/plugin.py @@ -2,18 +2,16 @@ import ckan.plugins as p from ckan.plugins import toolkit -from ckanext.report.interfaces import IReport -import ckanext.report.logic.action.get as action_get -import ckanext.report.logic.action.update as action_update -import ckanext.report.logic.auth.get as auth_get -import ckanext.report.logic.auth.update as auth_update +from . import helpers as h +from .interfaces import IReport +from .logic.action import get as action_get, update as action_update +from .logic.auth import get as auth_get, update as auth_update - -if toolkit.check_ckan_version("2.9"): - from ckanext.report.plugin.flask_plugin import MixinPlugin +if h.is_ckan_29(): + from .plugin_mixins.flask_plugin import MixinPlugin else: - from ckanext.report.plugin.pylons_plugin import MixinPlugin + from .plugin_mixins.pylons_plugin import MixinPlugin class ReportPlugin(MixinPlugin, p.SingletonPlugin): @@ -25,18 +23,18 @@ class ReportPlugin(MixinPlugin, p.SingletonPlugin): # IConfigurer def update_config(self, config): - toolkit.add_template_directory(config, '../templates') + toolkit.add_template_directory(config, 'templates') # ITemplateHelpers def get_helpers(self): - from ckanext.report import helpers as h return { 'report__relative_url_for': h.relative_url_for, 'report__chunks': h.chunks, 'report__organization_list': h.organization_list, 'report__render_datetime': h.render_datetime, 'report__explicit_default_options': h.explicit_default_options, + 'is_ckan_29': h.is_ckan_29, } # IActions diff --git a/ckanext/report/cli/__init__.py b/ckanext/report/plugin_mixins/__init__.py similarity index 100% rename from ckanext/report/cli/__init__.py rename to ckanext/report/plugin_mixins/__init__.py diff --git a/ckanext/report/plugin/flask_plugin.py b/ckanext/report/plugin_mixins/flask_plugin.py similarity index 57% rename from ckanext/report/plugin/flask_plugin.py rename to ckanext/report/plugin_mixins/flask_plugin.py index 0bfca3f..9702341 100644 --- a/ckanext/report/plugin/flask_plugin.py +++ b/ckanext/report/plugin_mixins/flask_plugin.py @@ -1,8 +1,7 @@ # encoding: utf-8 import ckan.plugins as p -from ckanext.report.controllers import blueprints -from ckanext.report.cli import click_cli +from ckanext.report import blueprint, cli class MixinPlugin(p.SingletonPlugin): @@ -12,9 +11,9 @@ class MixinPlugin(p.SingletonPlugin): # IBlueprint def get_blueprint(self): - return blueprints.get_blueprints() + return blueprint.get_blueprints() # IClick def get_commands(self): - return click_cli.get_commands() + return cli.get_commands() diff --git a/ckanext/report/plugin/pylons_plugin.py b/ckanext/report/plugin_mixins/pylons_plugin.py similarity index 75% rename from ckanext/report/plugin/pylons_plugin.py rename to ckanext/report/plugin_mixins/pylons_plugin.py index 44136ca..37f33a0 100644 --- a/ckanext/report/plugin/pylons_plugin.py +++ b/ckanext/report/plugin_mixins/pylons_plugin.py @@ -9,8 +9,9 @@ class MixinPlugin(p.SingletonPlugin): # IRoutes def before_map(self, map): - report_ctlr = 'ckanext.report.controllers.pylons_controllers:ReportController' - map.connect('report.index', '/report', controller=report_ctlr, action='index') + report_ctlr = 'ckanext.report.controllers:ReportController' + map.connect('report.index', '/report', controller=report_ctlr, + action='index') map.connect('reports', '/report', controller=report_ctlr, action='index') map.redirect('/reports', '/report') @@ -18,6 +19,8 @@ def before_map(self, map): action='view') map.connect('report', '/report/:report_name', controller=report_ctlr, action='view') + map.connect('report-org', '/report/:report_name/:organization', + controller=report_ctlr, action='view') map.connect('report.org', '/report/:report_name/:organization', controller=report_ctlr, action='view') return map diff --git a/ckanext/report/report_registry.py b/ckanext/report/report_registry.py index ed1d24b..048b7b2 100644 --- a/ckanext/report/report_registry.py +++ b/ckanext/report/report_registry.py @@ -7,12 +7,13 @@ from ckan import model from ckan.plugins.toolkit import asbool -from ckanext.report.interfaces import IReport try: from collections import OrderedDict # from python 2.7 except ImportError: from sqlalchemy.util import OrderedDict +from ckanext.report.interfaces import IReport + log = logging.getLogger(__name__) REPORT_KEYS_REQUIRED = set(('name', 'generate', 'template', 'option_defaults', @@ -29,8 +30,7 @@ def __init__(self, report_info_dict, plugin): missing_required_keys = REPORT_KEYS_REQUIRED - set(report_info_dict.keys()) assert not missing_required_keys, 'Report info dict missing keys %r: '\ '%r' % (missing_required_keys, report_info_dict) - unknown_keys = set(report_info_dict.keys()) - REPORT_KEYS_REQUIRED - \ - REPORT_KEYS_OPTIONAL + unknown_keys = set(report_info_dict.keys()) - REPORT_KEYS_REQUIRED - REPORT_KEYS_OPTIONAL assert not unknown_keys, 'Report info dict has unrecognized keys %r: '\ '%r' % (unknown_keys, report_info_dict) if not report_info_dict['option_defaults']: diff --git a/ckanext/report/reports.py b/ckanext/report/reports.py index cf11b3c..aa45db5 100644 --- a/ckanext/report/reports.py +++ b/ckanext/report/reports.py @@ -3,12 +3,14 @@ ''' from ckan import model -from ckanext.report import lib + try: from collections import OrderedDict # from python 2.7 except ImportError: from sqlalchemy.util import OrderedDict +from ckanext.report import lib + def tagless_report(organization, include_sub_organizations=False): ''' @@ -29,7 +31,7 @@ def tagless_report(organization, include_sub_organizations=False): # Find the packages without tags q = model.Session.query(model.Package) \ .outerjoin(model.PackageTag) \ - .filter(model.PackageTag.id is None) + .filter(model.PackageTag.id == None) # noqa: E711 if organization: q = lib.filter_by_organizations(q, organization, include_sub_organizations) diff --git a/ckanext/report/templates/report/index.html b/ckanext/report/templates/report/index.html index 46b77e1..0433dc0 100644 --- a/ckanext/report/templates/report/index.html +++ b/ckanext/report/templates/report/index.html @@ -3,7 +3,7 @@ {% block title %}{{ _('Reports') }} - {{ super() }}{% endblock %} {% block breadcrumb_content %} - {{ h.nav_link(_('Reports'), named_route='report.index') }} +
  • {{ h.nav_link(_('Reports'), named_route='report.index') }}
  • {% endblock %} {% block primary_content_inner %} diff --git a/ckanext/report/templates/report/tagless-datasets.html b/ckanext/report/templates/report/tagless-datasets.html index 0a75c8a..5444ab6 100644 --- a/ckanext/report/templates/report/tagless-datasets.html +++ b/ckanext/report/templates/report/tagless-datasets.html @@ -4,12 +4,14 @@ table - main data, as a list of rows, each row is a dict data - other data values, as a dict #} + +{% set dataset_read_route = 'dataset.read' if h.is_ckan_29() else 'dataset_read' %}
    • {% trans %}Datasets without tags{% endtrans %}: {{ table|length }} / {{ data['num_packages'] }} ({{ data['packages_without_tags_percent'] }})
    • {% trans %}Average tags per package{% endtrans %}: {{ data['average_tags_per_package'] }} tags
    -
    Dataset
    +
    @@ -22,7 +24,7 @@ {% for row in table %} diff --git a/ckanext/report/templates/report/view.html b/ckanext/report/templates/report/view.html index f700100..ddc4833 100644 --- a/ckanext/report/templates/report/view.html +++ b/ckanext/report/templates/report/view.html @@ -3,72 +3,66 @@ {% block title %}{{ report.title }} - {{ _('Reports') }} - {{ super() }}{% endblock %} {% block breadcrumb_content %} - {{ h.nav_link(_('Reports'), named_route='report.index') }} - {{ h.nav_link(report.title, named_route='report.org' if '/organization' in request.environ.get('PATH_INFO', '') else 'report.view', report_name=report_name) }} +
  • {{ h.nav_link(_('Reports'), named_route='report.index') }}
  • +
  • {{ h.nav_link(report.title, named_route='report.org' if '/organization' in request.environ.get('PATH_INFO', '') else 'report.view', report_name=report_name) }}
  • {% endblock%} {% block primary_content_inner %} -

    {{ report.title }}

    -

    {{ report.description }}

    -

    - {{ _('Generated') }}: {{ h.report__render_datetime(report_date, '%d/%m/%Y %H:%M') }} -

    - {% if c.userobj.sysadmin %} -
    -
    {% trans %}Refresh report{% endtrans %}
    -
    -
    - - -

    {{ _('As a system administrator you are able to refresh this report on demand by clicking the \'Refresh\' button.') }}

    -
    +

    {{ report.title }}

    +

    {{ report.description }}

    +

    + {{ _('Generated') }}: {{ h.report__render_datetime(report_date, '%d/%m/%Y %H:%M') }} +

    + {% if c.userobj.sysadmin %} +
    +
    {% trans %}Refresh report{% endtrans %}
    +
    +
    + + +

    {{ _('As a system administrator you are able to refresh this report on demand by clicking the \'Refresh\' button.') }}

    - {% endif %} +
    + {% endif %} - {% if options %} -

    {{ _('Options') }}

    -
    - {% for key, value in options.items() %} - {% if key in options_html %} - {{ options_html[key]|safe }} - {% else %} - {{ key }}: {{ value }} - - {% endif %} -
    - {% endfor %} - - {% endif %} + {% if options %} +

    {{ _('Options') }}

    +
    + {% for key, value in options.items() %} + {% if key in options_html %} + {{ options_html[key]|safe }} + {% else %} + {{ key }}: {{ value }} + + {% endif %} +
    + {% endfor %} + + {% endif %} - {% if are_some_results %} -
    - {{ _('Download') }}: - CSV - JSON -
    - {% endif %} -

    {{ _('Results') }}

    - {% if not are_some_results %} -

    {{ _('No results found.') }}

    - {% else %} -
    - {% snippet report_template, table=data['table'], data=data, report_name=report_name, options=options %} -
    - {% endif %} -
    + {% if are_some_results %} +
    + {{ _('Download') }}: + CSV + JSON +
    + {% endif %} +

    {{ _('Results') }}

    + {% if not are_some_results %} +

    {{ _('No results found.') }}

    + {% else %} +
    + {% snippet report_template, table=data['table'], data=data, report_name=report_name, options=options %} +
    + {% endif %} {% endblock%} {% block scripts %} {{ super() }} - -
    {% trans %}Dataset{% endtrans %}
    - + {{ row.title }}