#820: create simple regression test suite

.gitignore

@@ -152,3 +152,4 @@ cython_debug/
 !/config/sample.config.yaml
 !/config/logging.yaml
 /tests/*/fixtures/approved/*.received.txt
+/regression-tests/report/

CHANGELOG.md

@@ -6,7 +6,16 @@
 
 - report endpoint: has been deleted ([#818])
 
+
+### Other Changes
+
+- a new regression test suite has been added to support safer deployments of new versions ([#820])
+
+
+
 [#818]: https://github.com/GIScience/ohsome-quality-api/pull/818
+[#820]: https://github.com/GIScience/ohsome-quality-api/issues/820
+
 
 ## Release 1.4.0
 

docs/development_setup.md

@@ -131,6 +131,21 @@ approval tests will show the difference to the previously approved version of
 the output using a diff tool. If it es as expected resolve the difference using
 the diff-tool save and rerun the tests.
 
+### Regression Tests
+
+Unlike the above-mentioned unit and integration tests,
+regression tests do not primarily serve to guide and validate the development,
+but to check if an upcoming version caused bugs in the existing system.
+
+The regression tests are implemented in `hurl`
+and can be run manually on the command line against any of the staging systems.
+Please consider adding new `hurl` tests if you add new endpoints.
+A `hurl` plugin for JetBrains IDEs (e.g. PyCharm, Intellij Idea etc.)
+is available for syntax highlighting and execution within the IDE.
+
+For details, check the
+ [regression test README](../regression-tests/README.md).
+
 ## Logging
 
 Logging is enabled by default.

regression-tests/README.md

@@ -0,0 +1,20 @@
+# Regression tests with hurl
+
+Hurl is an HTTP command line client and HTTP test tool:
+
+https://hurl.dev
+
+In order to run the HTTP tests in this directory, `hurl` must be installed.
+The tests can be run against the different stages using the following scripts:
+
+* [run_hurl_tests_DEV.sh](./run_hurl_tests_DEV.sh)
+* [run_hurl_tests_TEST.sh](./run_hurl_tests_TEST.sh)
+* [run_hurl_tests_PROD.sh](./run_hurl_tests_PROD.sh)
+
+Please note that you may get different results depending on the supported endpoints and connected spatial database of the system running on a given stage.
+
+An HTML-report with the results is generated here:
+
+[report/index.html](./report/index.html)
+
+

regression-tests/__run_hurl_tests_for_stage.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# requires the env variable $HURL_BASE_URL
+
+rm -rf report
+mkdir report
+
+hurl *.hurl --report-html report
+printf "\n\nhurl report: file://$PWD/report/index.html\n"
+
+if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+  xdg-open report/index.html
+elif [[ "$OSTYPE" == "darwin"* ]]; then
+  open report/index.html
+else
+  printf "\nOS could not be detected. Please open report manually!\n"
+fi

regression-tests/buildingarea_adminarea.json

@@ -0,0 +1,264 @@
+{
+  "topic": "building-area",
+  "bpolys": {
+    "type": "FeatureCollection",
+    "features": [
+      {
+        "type": "Feature",
+        "id": "admin_world_water.-1392136-_-Mechtersen",
+        "geometry": {
+          "type": "MultiPolygon",
+          "coordinates": [
+            [
+              [
+                [
+                  10.265479,
+                  53.292841
+                ],
+                [
+                  10.268466,
+                  53.289283
+                ],
+                [
+                  10.269601,
+                  53.287949
+                ],
+                [
+                  10.273601,
+                  53.285098
+                ],
+                [
+                  10.282639,
+                  53.278703
+                ],
+                [
+                  10.276988,
+                  53.275121
+                ],
+                [
+                  10.277516,
+                  53.273322
+                ],
+                [
+                  10.275928,
+                  53.273074
+                ],
+                [
+                  10.269857,
+                  53.273559
+                ],
+                [
+                  10.269889,
+                  53.272413
+                ],
+                [
+                  10.269892,
+                  53.272318
+                ],
+                [
+                  10.268551,
+                  53.26638
+                ],
+                [
+                  10.267856,
+                  53.2634
+                ],
+                [
+                  10.267252,
+                  53.260808
+                ],
+                [
+                  10.268023,
+                  53.259773
+                ],
+                [
+                  10.273413,
+                  53.25984
+                ],
+                [
+                  10.278369,
+                  53.259919
+                ],
+                [
+                  10.282533,
+                  53.264359
+                ],
+                [
+                  10.317224,
+                  53.263145
+                ],
+                [
+                  10.320234,
+                  53.262952
+                ],
+                [
+                  10.332321,
+                  53.273117
+                ],
+                [
+                  10.332535,
+                  53.273305
+                ],
+                [
+                  10.334421,
+                  53.279821
+                ],
+                [
+                  10.333494,
+                  53.279896
+                ],
+                [
+                  10.333618,
+                  53.281157
+                ],
+                [
+                  10.333542,
+                  53.281765
+                ],
+                [
+                  10.332881,
+                  53.283022
+                ],
+                [
+                  10.332489,
+                  53.284407
+                ],
+                [
+                  10.332515,
+                  53.284634
+                ],
+                [
+                  10.332286,
+                  53.285151
+                ],
+                [
+                  10.332067,
+                  53.28524
+                ],
+                [
+                  10.331666,
+                  53.285226
+                ],
+                [
+                  10.331364,
+                  53.285316
+                ],
+                [
+                  10.329092,
+                  53.285174
+                ],
+                [
+                  10.327606,
+                  53.2852
+                ],
+                [
+                  10.317079,
+                  53.298367
+                ],
+                [
+                  10.316994,
+                  53.298383
+                ],
+                [
+                  10.305875,
+                  53.30052
+                ],
+                [
+                  10.302164,
+                  53.301261
+                ],
+                [
+                  10.301776,
+                  53.301339
+                ],
+                [
+                  10.297672,
+                  53.29707
+                ],
+                [
+                  10.29316,
+                  53.298999
+                ],
+                [
+                  10.292739,
+                  53.298855
+                ],
+                [
+                  10.287022,
+                  53.298217
+                ],
+                [
+                  10.283959,
+                  53.296305
+                ],
+                [
+                  10.281243,
+                  53.29531
+                ],
+                [
+                  10.27852,
+                  53.294257
+                ],
+                [
+                  10.277945,
+                  53.29438
+                ],
+                [
+                  10.277808,
+                  53.294409
+                ],
+                [
+                  10.275742,
+                  53.294849
+                ],
+                [
+                  10.273617,
+                  53.295818
+                ],
+                [
+                  10.272054,
+                  53.297669
+                ],
+                [
+                  10.269823,
+                  53.294204
+                ],
+                [
+                  10.267451,
+                  53.295826
+                ],
+                [
+                  10.265479,
+                  53.292841
+                ]
+              ]
+            ]
+          ]
+        },
+        "geometry_name": "geom",
+        "properties": {
+          "id": -1392136,
+          "iso": null,
+          "name": "Mechtersen",
+          "name_en": null,
+          "local_name": "Mechtersen",
+          "long_name": null,
+          "boundary": "administrative",
+          "admin_level": 8,
+          "wikidata": "Q503766",
+          "wikipedia": "de:Mechtersen",
+          "parents": "[Ljava.lang.Integer;@4e286c37",
+          "parent": -1969559,
+          "min_scale": 1,
+          "max_scale": 500000
+        },
+        "bbox": [
+          10.265478700000036,
+          53.259772999999974,
+          10.334420899999964,
+          53.301338699999974
+        ]
+      }
+    ]
+  }
+}

regression-tests/buildingarea_adminarea_buildingcomparison.hurl

@@ -0,0 +1,23 @@
+POST {{BASE_URL}}/indicators/building-comparison
+
+
+accept: application/json
+[Options]
+verbose: true
+file,buildingarea_adminarea.json;
+
+HTTP *
+
+[Asserts]
+duration < 20000
+status == 200
+bytes count > 2400
+
+jsonpath "$.result[0].topic.name" == "Building Area"
+jsonpath "$.result[0].result.description" contains "The completeness of OSM buildings in your area-of-interest is high. The completeness in comparison to EUBUCCO is 104.27%. The completeness in comparison to Microsoft Building Footprints is 96.16%."
+jsonpath "$.result[0].result.label" == "green"
+
+
+
+
+