Add tests via worker (#498)

* cleanup tests

* prepare tests for worker

.dockerignore

@@ -4,8 +4,8 @@ docker
 !docker/actinia-core-alpine/actinia.cfg
 !docker/actinia-core-dev/actinia.cfg
 !docker/actinia-core-dev/endpoints.csv
-!docker/actinia-core-tests/actinia-test.cfg
-!docker/actinia-core-tests/actinia-test-noauth.cfg
+!docker/actinia-core-tests/*.cfg
+
 .github
 .travis
 .travis.yml

.gitignore

@@ -51,5 +51,4 @@ docker/redis_queue_data/dump.rdb
 !docker/actinia-core-alpine/actinia.cfg
 !docker/actinia-core-dev/actinia.cfg
 !docker/actinia-core-dev/endpoints.csv
-!docker/actinia-core-tests/actinia-test.cfg
-!docker/actinia-core-tests/actinia-test-noauth.cfg
+!docker/actinia-core-tests/*.cfg

Makefile

@@ -23,3 +23,6 @@ noauthtest:
 
 integrationtest:
 	sh ./tests_with_redis.sh integrationtest
+
+workertest:
+	sh ./tests_with_redis.sh worker

docker/actinia-core-tests/Dockerfile

@@ -39,11 +39,14 @@ RUN rmdir /actinia_core/workspace/temp_db
 RUN rmdir /actinia_core/workspace/tmp
 RUN rmdir /actinia_core/workspace
 RUN rmdir /actinia_core
+RUN rm /etc/default/actinia
 
 # copy needed files and configs for test
-COPY docker/actinia-core-alpine/actinia.cfg /etc/default/actinia
+# COPY docker/actinia-core-alpine/actinia.cfg /etc/default/actinia
 COPY docker/actinia-core-tests/actinia-test.cfg /etc/default/actinia_test
 COPY docker/actinia-core-tests/actinia-test-noauth.cfg /etc/default/actinia_test_noauth
+COPY docker/actinia-core-tests/actinia-test-worker-usedby-api.cfg /etc/default/actinia_test_worker_usedby_api
+COPY docker/actinia-core-tests/actinia-test-worker-usedby-worker.cfg /etc/default/actinia_test_worker_usedby_worker
 
 # TODO: Postgres for tests
 # using tests/data/poly.gpkg

docker/actinia-core-tests/actinia-test-worker-usedby-api.cfg

@@ -0,0 +1,31 @@
+[GRASS]
+grass_database = /tmp/actinia_core/grassdb
+grass_user_database = /tmp/actinia_core/userdata
+grass_tmp_database = /tmp/actinia_core/workspace/temp_db
+grass_resource_dir = /tmp/actinia_core/resources
+
+[API]
+force_https_urls = True
+
+[LIMITS]
+max_cell_limit = 22500000
+process_time_limt = 60
+process_num_limit = 20
+
+[REDIS]
+worker_logfile = /tmp/actinia_core/workspace/actinia_worker_test.log
+redis_server_url = localhost
+redis_server_port = 6379
+
+[QUEUE]
+queue_type = redis
+number_of_workers = 1
+redis_queue_server_url = localhost
+worker_queue_prefix = job_queue
+
+[LOGGING]
+log_level = 1
+
+[MISC]
+tmp_workdir = /tmp/actinia_core/workspace
+download_cache = /tmp/actinia_core/workspace/download_cache

docker/actinia-core-tests/actinia-test-worker-usedby-worker.cfg

@@ -0,0 +1,31 @@
+[GRASS]
+grass_database = /tmp/actinia_core/grassdb
+grass_user_database = /tmp/actinia_core/userdata
+grass_tmp_database = /tmp/actinia_core/workspace/temp_db
+grass_resource_dir = /tmp/actinia_core/resources
+
+[API]
+force_https_urls = True
+
+[LIMITS]
+max_cell_limit = 22500000
+process_time_limt = 60
+process_num_limit = 20
+
+[REDIS]
+worker_logfile = /tmp/actinia_core/workspace/actinia_worker_test.log
+redis_server_url = localhost
+redis_server_port = 6379
+
+[QUEUE]
+queue_type = redis
+number_of_workers = 1
+redis_queue_server_url = localhost
+worker_queue_prefix = job_queue
+
+[LOGGING]
+log_level = 1
+
+[MISC]
+tmp_workdir = /tmp/actinia_core/workspace
+download_cache = /tmp/actinia_core/workspace/download_cache

tests_with_redis.sh

@@ -1,13 +1,26 @@
 #!/usr/bin/env sh
 
 run_tests_noauth (){
-  # change config run tests and change config bag
+  # change config run tests and change config back
   mv ${DEFAULT_CONFIG_PATH} ${DEFAULT_CONFIG_PATH}_tmp
   cp /etc/default/actinia_test_noauth ${DEFAULT_CONFIG_PATH}
   pytest -m 'noauth'
   mv ${DEFAULT_CONFIG_PATH}_tmp ${DEFAULT_CONFIG_PATH}
 }
 
+run_tests_worker (){
+  # change config run tests and change config back
+  mv ${DEFAULT_CONFIG_PATH} ${DEFAULT_CONFIG_PATH}_tmp
+  cp /etc/default/actinia_test_worker_usedby_api ${DEFAULT_CONFIG_PATH}
+  echo "Starting worker..."
+  # TODO: make sure the worker is not overwriting config path from env var
+  actinia-worker job_queue_0 -c /etc/default/actinia_test_worker_usedby_worker &
+  WORKER_PID=`echo $!`
+  echo "Running tests"
+  pytest -m 'not unittest and not noauth'
+  kill $WORKER_PID
+  mv ${DEFAULT_CONFIG_PATH}_tmp ${DEFAULT_CONFIG_PATH}
+}
 
 # start redis server
 redis-server &
@@ -19,7 +32,7 @@ webhook-server --host "0.0.0.0" --port "5005" &
 sleep 10
 
 # run tests
-echo $DEFAULT_CONFIG_PATH
+# echo $DEFAULT_CONFIG_PATH
 
 TEST_RES=1
 if [ "$1" == "dev" ]
@@ -42,6 +55,10 @@ elif [ "$1" == "noauth" ]
 then
   run_tests_noauth
   TEST_RES=$?
+elif [ "$1" == "worker" ]
+then
+  run_tests_worker
+  TEST_RES=$?
 else
   pytest
   TEST_RES=$?