test: scaffold test setup for running as command

.github/workflows/docker.test.command.yml

@@ -0,0 +1,63 @@
+name: Docker test run as command
+
+on:
+  push:
+    branches:
+      - 'main'
+    tags:
+      - '*'
+  pull_request:
+
+jobs:
+  docker-test:
+    env:
+      COMPOSE_ARGS: -f docker-compose.yml -f docker-compose.ci.command.yml
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: docker-compose up
+        run: docker-compose ${{ env.COMPOSE_ARGS }} up -d
+
+      - name: Create entities
+        run: |
+          docker compose exec api /usr/src/app/create-entities.js 40
+
+      - name: Run updater
+        run: |
+          docker compose exec wdqs-updater /wdqsup/runUpdateWbStack.sh -- \
+            --wikibaseHost wikibase.svc \
+            --ids Q1,Q2,Q3,Q4,Q5,Q6,Q7,Q8,Q9,Q10,Q11,Q12,Q13,Q14,Q15 \
+            --entityNamespaces 120,122,146 \
+            --sparqlUrl http://wdqs.svc:9999/bigdata/namespace/wdq/sparql \
+            --wikibaseScheme http \
+            --conceptUri https://wikibase.svc
+
+            docker compose exec wdqs-updater /wdqsup/runUpdateWbStack.sh -- \
+            --wikibaseHost wikibase.svc \
+            --ids Q16,Q17,Q18,Q19,Q20,Q21,Q22,Q23,Q24,Q25,Q26,Q27,Q28,Q29,Q30 \
+            --entityNamespaces 120,122,146 \
+            --sparqlUrl http://wdqs.svc:9999/bigdata/namespace/wdq/sparql \
+            --wikibaseScheme http \
+            --conceptUri https://wikibase.svc
+
+      - name: Make a sparql request
+        run: |
+          NUM_BINDINGS=$(curl 'http://localhost:9999/bigdata/namespace/wdq/sparql' -H 'Accept: application/sparql-results+json' --data-raw 'query=SELECT+*+WHERE%7B+%3Fs+%3Fp+%3Fo+%7D' | jq '.results.bindings | length')
+          # should be 30
+          if [[ "$NUM_BINDINGS" -lt 30 ]]; then
+              exit 1
+          fi
+      - name: docker-compose logs > output.log
+        if: always()
+        run: docker-compose ${{ env.COMPOSE_ARGS }} logs --no-color > "output.log"
+
+      - name: Archive output.log
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: DockerTestLog
+          if-no-files-found: error
+          path: output.log
+

.github/workflows/docker.test.yml → .github/workflows/docker.test.service.yml

@@ -1,4 +1,4 @@
-name: Docker test
+name: Docker test run as service
 
 on:
   push:
@@ -11,7 +11,7 @@ on:
 jobs:
   docker-test:
     env:
-      COMPOSE_ARGS: -f docker-compose.yml -f docker-compose.ci.yml
+      COMPOSE_ARGS: -f docker-compose.yml -f docker-compose.ci.service.yml
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

docker-compose.ci.command.yml

@@ -0,0 +1,13 @@
+version: '2'
+services:
+  wdqs-updater:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    restart: unless-stopped
+    tty: true
+    entrypoint: /bin/bash
+    depends_on:
+      - wdqs
+    environment:
+      - HEAP_SIZE=32m

seeder/create-entities.js

@@ -0,0 +1,32 @@
+#!/usr/bin/env node
+
+const wbEdit = require('wikibase-edit')(require('./wikibase-edit.config'));
+
+const numEntities = parseInt(process.argv[2], 10)
+if (!Number.isFinite(numEntities)) {
+    throw new Error(`Expected numeric argument to be given, got ${process.argv[2]}`)
+}
+
+;(async function () {
+    for (let i = 1; i <= numEntities; i++) {
+        await wbEdit.entity.create({
+            type: 'item',
+            labels: {
+                'en': new Date().toISOString()
+            },
+            descriptions: {
+                'en': new Date().toDateString() + new Date().toISOString()
+            }
+        })
+    }
+
+    return `Sucessfully created ${numEntities} entities`
+})()
+    .then((result) => {
+        console.log(result)
+    })
+    .catch((err) => {
+        console.error("Failed to run command.")
+        console.error(err)
+        process.exit(1)
+    })

src/main/java/org/wikidata/query/rdf/tool/WbStackUpdate.java

@@ -95,7 +95,7 @@ private WbStackUpdate() {
         // utility class, should never be constructed
     }
 
-    private static void setValuesFromEnvOrDie() {
+    private static void setServiceValuesFromEnvOrDie() {
         if (System.getenv("WBSTACK_API_ENDPOINT_GET_BATCHES") == null
                 || System.getenv("WBSTACK_API_ENDPOINT_MARK_NOT_DONE") == null
                 || System.getenv("WBSTACK_API_ENDPOINT_MARK_DONE") == null
@@ -116,6 +116,10 @@ private static void setValuesFromEnvOrDie() {
         wbStackLoopLimit = Integer.parseInt(System.getenv("WBSTACK_LOOP_LIMIT"));
     }
 
+    public static void setCommandValuesFromEnvOrDie() {
+        wbStackUpdaterThreadCount = Integer.parseInt(System.getenv().getOrDefault("WBSTACK_THREAD_COUNT", "10"));
+    }
+
     private static void setSingleUseServicesAndObjects() {
         gson = new GsonBuilder().setPrettyPrinting().create();
         buildProps = loadBuildProperties();
@@ -128,8 +132,12 @@ private static void closeSingleUseServicesAndObjects() throws IOException {
     }
 
     public static void main(String[] args) throws InterruptedException, IOException {
+        setSingleUseServicesAndObjects();
+
         if (args.length != 0) {
+            setCommandValuesFromEnvOrDie();
             boolean success = runUpdaterWithArgs(args);
+            closeSingleUseServicesAndObjects();
             if (!success) {
                 System.err.println("Failed to run update command.");
                 System.exit(1);
@@ -138,8 +146,7 @@ public static void main(String[] args) throws InterruptedException, IOException
             System.exit(0);
         }
 
-        setValuesFromEnvOrDie();
-        setSingleUseServicesAndObjects();
+        setServiceValuesFromEnvOrDie();
 
         int loopCount = 0;
         long apiLastCalled;
@@ -290,7 +297,7 @@ private static boolean runUpdaterWithArgs(String[] args) {
             }
 
         } catch (Exception e) {
-            System.err.println("Failed batch!");
+            System.err.println("Failed batch: " + e.getMessage());
             e.printStackTrace();
             return false;
         }