From c77998ba2b286ba6fa4b4af6454a1aa9543e8f1f Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Tue, 5 Dec 2023 08:04:23 +0100 Subject: [PATCH] feat: allow running updater as one-off command (#143) * feat: allow running updater as one-off job * test: add integration test for running image as command * refactor: deduplicate entity seeding * docs: add changelog --- .github/workflows/docker.test.command.yml | 63 +++++++++++++++++++ ...ocker.test.yml => docker.test.service.yml} | 4 +- CHANGELOG.md | 4 ++ docker-compose.ci.command.yml | 13 ++++ ...se.ci.yml => docker-compose.ci.service.yml | 0 runUpdate.sh | 2 +- runUpdateWbStack.sh | 13 +--- seeder/create-entities.js | 18 ++++++ seeder/seeder.js | 21 +++++++ seeder/server.js | 21 +------ .../query/rdf/tool/WbStackUpdate.java | 23 ++++++- 11 files changed, 146 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/docker.test.command.yml rename .github/workflows/{docker.test.yml => docker.test.service.yml} (94%) create mode 100644 docker-compose.ci.command.yml rename docker-compose.ci.yml => docker-compose.ci.service.yml (100%) create mode 100644 seeder/create-entities.js create mode 100755 seeder/seeder.js diff --git a/.github/workflows/docker.test.command.yml b/.github/workflows/docker.test.command.yml new file mode 100644 index 0000000..91599aa --- /dev/null +++ b/.github/workflows/docker.test.command.yml @@ -0,0 +1,63 @@ +name: Docker test run as command + +on: + push: + branches: + - 'main' + tags: + - '*' + pull_request: + +jobs: + docker-test: + env: + COMPOSE_ARGS: -f docker-compose.yml -f docker-compose.ci.command.yml + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: docker-compose up + run: docker-compose ${{ env.COMPOSE_ARGS }} up -d + + - name: Create entities + run: | + docker compose exec api /usr/src/app/seeder.js 40 + + - name: Run updater + run: | + docker compose exec wdqs-updater /wdqsup/runUpdateWbStack.sh -- \ + --wikibaseHost wikibase.svc \ + --ids Q1,Q2,Q3,Q4,Q5,Q6,Q7,Q8,Q9,Q10,Q11,Q12,Q13,Q14,Q15 \ + --entityNamespaces 120,122,146 \ + --sparqlUrl http://wdqs.svc:9999/bigdata/namespace/wdq/sparql \ + --wikibaseScheme http \ + --conceptUri https://wikibase.svc + + docker compose exec wdqs-updater /wdqsup/runUpdateWbStack.sh -- \ + --wikibaseHost wikibase.svc \ + --ids Q16,Q17,Q18,Q19,Q20,Q21,Q22,Q23,Q24,Q25,Q26,Q27,Q28,Q29,Q30 \ + --entityNamespaces 120,122,146 \ + --sparqlUrl http://wdqs.svc:9999/bigdata/namespace/wdq/sparql \ + --wikibaseScheme http \ + --conceptUri https://wikibase.svc + + - name: Make a sparql request + run: | + NUM_BINDINGS=$(curl 'http://localhost:9999/bigdata/namespace/wdq/sparql' -H 'Accept: application/sparql-results+json' --data-raw 'query=SELECT+*+WHERE%7B+%3Fs+%3Fp+%3Fo+%7D' | jq '.results.bindings | length') + # should be plenty more than 100 + if [[ "$NUM_BINDINGS" -lt 100 ]]; then + exit 1 + fi + - name: docker-compose logs > output.log + if: always() + run: docker-compose ${{ env.COMPOSE_ARGS }} logs --no-color > "output.log" + + - name: Archive output.log + uses: actions/upload-artifact@v2 + if: always() + with: + name: DockerTestLog + if-no-files-found: error + path: output.log + diff --git a/.github/workflows/docker.test.yml b/.github/workflows/docker.test.service.yml similarity index 94% rename from .github/workflows/docker.test.yml rename to .github/workflows/docker.test.service.yml index 9ac01a5..9fff6ed 100644 --- a/.github/workflows/docker.test.yml +++ b/.github/workflows/docker.test.service.yml @@ -1,4 +1,4 @@ -name: Docker test +name: Docker test run as service on: push: @@ -11,7 +11,7 @@ on: jobs: docker-test: env: - COMPOSE_ARGS: -f docker-compose.yml -f docker-compose.ci.yml + COMPOSE_ARGS: -f docker-compose.yml -f docker-compose.ci.service.yml runs-on: ubuntu-latest steps: - name: Checkout diff --git a/CHANGELOG.md b/CHANGELOG.md index 841975f..a50cf07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # queryservice-updater +## 0.3.84_3.12 - December 2023 + +- T352656: Support running updater as one off command with command line args + ## 0.3.84_3.11 - November 2023 - T352157: Support running loop forever and without forced exit diff --git a/docker-compose.ci.command.yml b/docker-compose.ci.command.yml new file mode 100644 index 0000000..38d0454 --- /dev/null +++ b/docker-compose.ci.command.yml @@ -0,0 +1,13 @@ +version: '2' +services: + wdqs-updater: + build: + context: . + dockerfile: Dockerfile + restart: unless-stopped + tty: true + entrypoint: /bin/bash + depends_on: + - wdqs + environment: + - HEAP_SIZE=32m diff --git a/docker-compose.ci.yml b/docker-compose.ci.service.yml similarity index 100% rename from docker-compose.ci.yml rename to docker-compose.ci.service.yml diff --git a/runUpdate.sh b/runUpdate.sh index c3e71c5..0ec2bf5 100755 --- a/runUpdate.sh +++ b/runUpdate.sh @@ -100,4 +100,4 @@ SPARQL_URL=$HOST/$CONTEXT/namespace/$NAMESPACE/sparql echo "Updating via $SPARQL_URL" exec java -cp ${CLASSPATH} ${GC_LOGS} ${LOG_OPTIONS} ${EXTRA_JVM_OPTS} \ ${TIMEOUT_ARG} ${UPDATER_OPTS} \ - ${MAIN} ${ARGS} --sparqlUrl ${SPARQL_URL} "$@" + ${MAIN} ${ARGS} "$@" diff --git a/runUpdateWbStack.sh b/runUpdateWbStack.sh index d43c061..33a0248 100755 --- a/runUpdateWbStack.sh +++ b/runUpdateWbStack.sh @@ -7,15 +7,4 @@ set -e # Call runUpdate with all of the magic stuff that is in there for logging and JVM etc # But don't actually pass any real arguments into Java # Also pass in our custom Main class that will handle the "magic" -# TODO also pass class path in... -./runUpdate.sh \ - -m org.wikidata.query.rdf.tool.WbStackUpdate \ - -h IGNOREDHOST \ - -n IGNOREDNAMESPACE \ - -- \ - --wikibaseHost IGNOREDWIKIBASEHOST \ - --wikibaseScheme IGNOREDSCHEME \ - --conceptUri IGNOREDCONCEPTURI \ - --entityNamespaces IGNOREDENTITYNS \ - --ids IGNOREDIDS - +./runUpdate.sh -m org.wikidata.query.rdf.tool.WbStackUpdate "$@" diff --git a/seeder/create-entities.js b/seeder/create-entities.js new file mode 100644 index 0000000..cf8f4ec --- /dev/null +++ b/seeder/create-entities.js @@ -0,0 +1,18 @@ +const wbEdit = require('wikibase-edit')(require('./wikibase-edit.config')); + +module.exports = async function (numEntities) { + const result = [] + for (let i = 1; i <= numEntities; i++) { + const { entity } = await wbEdit.entity.create({ + type: 'item', + labels: { + 'en': new Date().toISOString() + }, + descriptions: { + 'en': new Date().toDateString() + new Date().toISOString() + } + }) + result.push(entity) + } + return result +} diff --git a/seeder/seeder.js b/seeder/seeder.js new file mode 100755 index 0000000..b41962c --- /dev/null +++ b/seeder/seeder.js @@ -0,0 +1,21 @@ +#!/usr/bin/env node + +const createEntities = require('./create-entities'); + +const numEntities = parseInt(process.argv[2], 10) +if (!Number.isFinite(numEntities)) { + throw new Error(`Expected numeric argument to be given, got ${process.argv[2]}`) +} + +;(async function () { + const result = await createEntities(numEntities) + return `Sucessfully created ${result.length} entities` +})() + .then((result) => { + console.log(result) + }) + .catch((err) => { + console.error("Failed to run command.") + console.error(err) + process.exit(1) + }) diff --git a/seeder/server.js b/seeder/server.js index 7e24d77..eb26210 100644 --- a/seeder/server.js +++ b/seeder/server.js @@ -1,6 +1,6 @@ const http = require('http'); const assert = require('assert'); -const wbEdit = require( 'wikibase-edit' )( require( './wikibase-edit.config' ) ); +const createEntities = require('./create-entities'); let batchId = 1; @@ -50,26 +50,11 @@ http.createServer(function (req, res) { return Promise.reject(err); } const numEntities = 20; - const entities = []; - - for (let i = 1; i <= numEntities; ++i) { - const { entity } = await wbEdit.entity.create({ - type: 'item', - labels: { - 'en': new Date().toISOString() - }, - descriptions: { - 'en': new Date().toDateString() + new Date().toISOString() - } - }); - - console.log('created item id', entity.id) - entities.push(entity.id) - } + const entities = await createEntities(numEntities); responseObject = { 'id': batchId++, - 'entityIds': entities.join(','), + 'entityIds': entities.map(e => e.id).join(','), 'wiki': { 'domain': process.env.API_WIKIBASE_DOMAIN, 'wiki_queryservice_namespace': { diff --git a/src/main/java/org/wikidata/query/rdf/tool/WbStackUpdate.java b/src/main/java/org/wikidata/query/rdf/tool/WbStackUpdate.java index eef22d2..994321d 100644 --- a/src/main/java/org/wikidata/query/rdf/tool/WbStackUpdate.java +++ b/src/main/java/org/wikidata/query/rdf/tool/WbStackUpdate.java @@ -95,7 +95,7 @@ private WbStackUpdate() { // utility class, should never be constructed } - private static void setValuesFromEnvOrDie() { + private static void setServiceValuesFromEnvOrDie() { if (System.getenv("WBSTACK_API_ENDPOINT_GET_BATCHES") == null || System.getenv("WBSTACK_API_ENDPOINT_MARK_NOT_DONE") == null || System.getenv("WBSTACK_API_ENDPOINT_MARK_DONE") == null @@ -116,6 +116,10 @@ private static void setValuesFromEnvOrDie() { wbStackLoopLimit = Integer.parseInt(System.getenv("WBSTACK_LOOP_LIMIT")); } + public static void setCommandValuesFromEnvOrDie() { + wbStackUpdaterThreadCount = Integer.parseInt(System.getenv().getOrDefault("WBSTACK_THREAD_COUNT", "10")); + } + private static void setSingleUseServicesAndObjects() { gson = new GsonBuilder().setPrettyPrinting().create(); buildProps = loadBuildProperties(); @@ -128,9 +132,22 @@ private static void closeSingleUseServicesAndObjects() throws IOException { } public static void main(String[] args) throws InterruptedException, IOException { - setValuesFromEnvOrDie(); setSingleUseServicesAndObjects(); + if (args.length != 0) { + setCommandValuesFromEnvOrDie(); + boolean success = runUpdaterWithArgs(args); + closeSingleUseServicesAndObjects(); + if (!success) { + System.err.println("Failed to run update command."); + System.exit(1); + } + System.err.println("Successfully ran update command."); + System.exit(0); + } + + setServiceValuesFromEnvOrDie(); + int loopCount = 0; long apiLastCalled; @@ -280,7 +297,7 @@ private static boolean runUpdaterWithArgs(String[] args) { } } catch (Exception e) { - System.err.println("Failed batch!"); + System.err.println("Failed batch: " + e.getMessage()); e.printStackTrace(); return false; }