feat: allow running updater as one-off command (#143)

* feat: allow running updater as one-off job

* test: add integration test for running image as command

* refactor: deduplicate entity seeding

* docs: add changelog

.github/workflows/docker.test.command.yml

@@ -0,0 +1,63 @@
+name: Docker test run as command
+
+on:
+  push:
+    branches:
+      - 'main'
+    tags:
+      - '*'
+  pull_request:
+
+jobs:
+  docker-test:
+    env:
+      COMPOSE_ARGS: -f docker-compose.yml -f docker-compose.ci.command.yml
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: docker-compose up
+        run: docker-compose ${{ env.COMPOSE_ARGS }} up -d
+
+      - name: Create entities
+        run: |
+          docker compose exec api /usr/src/app/seeder.js 40
+
+      - name: Run updater
+        run: |
+          docker compose exec wdqs-updater /wdqsup/runUpdateWbStack.sh -- \
+            --wikibaseHost wikibase.svc \
+            --ids Q1,Q2,Q3,Q4,Q5,Q6,Q7,Q8,Q9,Q10,Q11,Q12,Q13,Q14,Q15 \
+            --entityNamespaces 120,122,146 \
+            --sparqlUrl http://wdqs.svc:9999/bigdata/namespace/wdq/sparql \
+            --wikibaseScheme http \
+            --conceptUri https://wikibase.svc
+
+          docker compose exec wdqs-updater /wdqsup/runUpdateWbStack.sh -- \
+            --wikibaseHost wikibase.svc \
+            --ids Q16,Q17,Q18,Q19,Q20,Q21,Q22,Q23,Q24,Q25,Q26,Q27,Q28,Q29,Q30 \
+            --entityNamespaces 120,122,146 \
+            --sparqlUrl http://wdqs.svc:9999/bigdata/namespace/wdq/sparql \
+            --wikibaseScheme http \
+            --conceptUri https://wikibase.svc
+
+      - name: Make a sparql request
+        run: |
+          NUM_BINDINGS=$(curl 'http://localhost:9999/bigdata/namespace/wdq/sparql' -H 'Accept: application/sparql-results+json' --data-raw 'query=SELECT+*+WHERE%7B+%3Fs+%3Fp+%3Fo+%7D' | jq '.results.bindings | length')
+          # should be plenty more than 100
+          if [[ "$NUM_BINDINGS" -lt 100 ]]; then
+              exit 1
+          fi
+      - name: docker-compose logs > output.log
+        if: always()
+        run: docker-compose ${{ env.COMPOSE_ARGS }} logs --no-color > "output.log"
+
+      - name: Archive output.log
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: DockerTestLog
+          if-no-files-found: error
+          path: output.log
+

.github/workflows/docker.test.yml → .github/workflows/docker.test.service.yml

@@ -1,4 +1,4 @@
-name: Docker test
+name: Docker test run as service
 
 on:
   push:
@@ -11,7 +11,7 @@ on:
 jobs:
   docker-test:
     env:
-      COMPOSE_ARGS: -f docker-compose.yml -f docker-compose.ci.yml
+      COMPOSE_ARGS: -f docker-compose.yml -f docker-compose.ci.service.yml
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

CHANGELOG.md

@@ -1,5 +1,9 @@
 # queryservice-updater
 
+## 0.3.84_3.12 - December 2023
+
+- T352656: Support running updater as one off command with command line args
+
 ## 0.3.84_3.11 - November 2023
 
 - T352157: Support running loop forever and without forced exit

docker-compose.ci.command.yml

@@ -0,0 +1,13 @@
+version: '2'
+services:
+  wdqs-updater:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    restart: unless-stopped
+    tty: true
+    entrypoint: /bin/bash
+    depends_on:
+      - wdqs
+    environment:
+      - HEAP_SIZE=32m

runUpdate.sh

@@ -100,4 +100,4 @@ SPARQL_URL=$HOST/$CONTEXT/namespace/$NAMESPACE/sparql
 echo "Updating via $SPARQL_URL"
 exec java -cp ${CLASSPATH} ${GC_LOGS} ${LOG_OPTIONS} ${EXTRA_JVM_OPTS} \
      ${TIMEOUT_ARG} ${UPDATER_OPTS} \
-     ${MAIN} ${ARGS} --sparqlUrl ${SPARQL_URL} "$@"
+     ${MAIN} ${ARGS} "$@"

runUpdateWbStack.sh

@@ -7,15 +7,4 @@ set -e
 # Call runUpdate with all of the magic stuff that is in there for logging and JVM etc
 # But don't actually pass any real arguments into Java
 # Also pass in our custom Main class that will handle the "magic"
-# TODO also pass class path in...
-./runUpdate.sh \
- -m org.wikidata.query.rdf.tool.WbStackUpdate \
- -h IGNOREDHOST \
- -n IGNOREDNAMESPACE \
- -- \
- --wikibaseHost IGNOREDWIKIBASEHOST \
- --wikibaseScheme IGNOREDSCHEME \
- --conceptUri IGNOREDCONCEPTURI \
- --entityNamespaces IGNOREDENTITYNS \
- --ids IGNOREDIDS
-
+./runUpdate.sh -m org.wikidata.query.rdf.tool.WbStackUpdate "$@"

seeder/create-entities.js

@@ -0,0 +1,18 @@
+const wbEdit = require('wikibase-edit')(require('./wikibase-edit.config'));
+
+module.exports = async function (numEntities) {
+    const result = []
+    for (let i = 1; i <= numEntities; i++) {
+        const { entity } = await wbEdit.entity.create({
+            type: 'item',
+            labels: {
+                'en': new Date().toISOString()
+            },
+            descriptions: {
+                'en': new Date().toDateString() + new Date().toISOString()
+            }
+        })
+        result.push(entity)
+    }
+    return result
+}

seeder/seeder.js

@@ -0,0 +1,21 @@
+#!/usr/bin/env node
+
+const createEntities = require('./create-entities');
+
+const numEntities = parseInt(process.argv[2], 10)
+if (!Number.isFinite(numEntities)) {
+    throw new Error(`Expected numeric argument to be given, got ${process.argv[2]}`)
+}
+
+;(async function () {
+    const result = await createEntities(numEntities)
+    return `Sucessfully created ${result.length} entities`
+})()
+    .then((result) => {
+        console.log(result)
+    })
+    .catch((err) => {
+        console.error("Failed to run command.")
+        console.error(err)
+        process.exit(1)
+    })

seeder/server.js

@@ -1,6 +1,6 @@
 const http = require('http');
 const assert = require('assert');
-const wbEdit = require( 'wikibase-edit' )( require( './wikibase-edit.config' ) );
+const createEntities = require('./create-entities');
 
 let batchId = 1;
 
@@ -50,26 +50,11 @@ http.createServer(function (req, res) {
                 return Promise.reject(err);
             }
             const numEntities = 20;
-            const entities = [];
-
-            for (let i = 1; i <= numEntities; ++i) {
-                const { entity } = await wbEdit.entity.create({
-                    type: 'item',
-                    labels: {
-                        'en': new Date().toISOString()
-                    },
-                    descriptions: {
-                        'en': new Date().toDateString() + new Date().toISOString()
-                    }
-                });
-
-                console.log('created item id', entity.id)
-                entities.push(entity.id)
-            }
+            const entities = await createEntities(numEntities);
 
             responseObject = {
                 'id': batchId++,
-                'entityIds': entities.join(','),
+                'entityIds': entities.map(e => e.id).join(','),
                 'wiki': {
                     'domain': process.env.API_WIKIBASE_DOMAIN,
                     'wiki_queryservice_namespace': {

src/main/java/org/wikidata/query/rdf/tool/WbStackUpdate.java

@@ -95,7 +95,7 @@ private WbStackUpdate() {
         // utility class, should never be constructed
     }
 
-    private static void setValuesFromEnvOrDie() {
+    private static void setServiceValuesFromEnvOrDie() {
         if (System.getenv("WBSTACK_API_ENDPOINT_GET_BATCHES") == null
                 || System.getenv("WBSTACK_API_ENDPOINT_MARK_NOT_DONE") == null
                 || System.getenv("WBSTACK_API_ENDPOINT_MARK_DONE") == null
@@ -116,6 +116,10 @@ private static void setValuesFromEnvOrDie() {
         wbStackLoopLimit = Integer.parseInt(System.getenv("WBSTACK_LOOP_LIMIT"));
     }
 
+    public static void setCommandValuesFromEnvOrDie() {
+        wbStackUpdaterThreadCount = Integer.parseInt(System.getenv().getOrDefault("WBSTACK_THREAD_COUNT", "10"));
+    }
+
     private static void setSingleUseServicesAndObjects() {
         gson = new GsonBuilder().setPrettyPrinting().create();
         buildProps = loadBuildProperties();
@@ -128,9 +132,22 @@ private static void closeSingleUseServicesAndObjects() throws IOException {
     }
 
     public static void main(String[] args) throws InterruptedException, IOException {
-        setValuesFromEnvOrDie();
         setSingleUseServicesAndObjects();
 
+        if (args.length != 0) {
+            setCommandValuesFromEnvOrDie();
+            boolean success = runUpdaterWithArgs(args);
+            closeSingleUseServicesAndObjects();
+            if (!success) {
+                System.err.println("Failed to run update command.");
+                System.exit(1);
+            }
+            System.err.println("Successfully ran update command.");
+            System.exit(0);
+        }
+
+        setServiceValuesFromEnvOrDie();
+
         int loopCount = 0;
         long apiLastCalled;
 
@@ -280,7 +297,7 @@ private static boolean runUpdaterWithArgs(String[] args) {
             }
 
         } catch (Exception e) {
-            System.err.println("Failed batch!");
+            System.err.println("Failed batch: " + e.getMessage());
             e.printStackTrace();
             return false;
         }