From ea4cd80ae9fe693fba91a9d001d247af27fd4ebb Mon Sep 17 00:00:00 2001 From: Paul Beaudoin Date: Thu, 17 Oct 2024 15:31:32 -0400 Subject: [PATCH] Improvements to assess-query-targets script --- lib/elasticsearch/utils.js | 11 +++- scripts/assess-query-targets.js | 105 ++++++++++++++++++++++++-------- 2 files changed, 91 insertions(+), 25 deletions(-) diff --git a/lib/elasticsearch/utils.js b/lib/elasticsearch/utils.js index d6a512cc..1a1abfbf 100644 --- a/lib/elasticsearch/utils.js +++ b/lib/elasticsearch/utils.js @@ -76,7 +76,10 @@ const buildEsMatchClause = (matcher, field, value, boost) => { /** * Given a field name and a value, returns a plainobject representing a ES -* `prefix` clause that can be inserted into a ES query +* `prefix` clause that can be inserted into a ES query. +* +* For example: +* const query = { prefixMatch('idIsbn', '1234' } */ const prefixMatch = (field, value, boost = 1) => { return buildEsMatchClause('prefix', field, value, boost) @@ -85,6 +88,9 @@ const prefixMatch = (field, value, boost = 1) => { /** * Given a field name and a value, returns a plainobject representing a ES * `term` clause that can be inserted into a ES query +* +* For example: +* const query = { termMatch('idBarcode', '1234567891011' } */ const termMatch = (field, value, boost = 1) => { return buildEsMatchClause('term', field, value, boost) @@ -93,6 +99,9 @@ const termMatch = (field, value, boost = 1) => { /** * Given a field name and a value, returns a plainobject representing a ES * `match_phrase` clause that can be inserted into a ES query +* +* For example: +* const query = { phraseMatch('title', 'Importance of Being' } */ const phraseMatch = (field, value, boost = 1) => { return { diff --git a/scripts/assess-query-targets.js b/scripts/assess-query-targets.js index a96f0b2b..4a3e741a 100644 --- a/scripts/assess-query-targets.js +++ b/scripts/assess-query-targets.js @@ -1,12 +1,16 @@ /** -* Assess query targets performance of the discovery-api +* Assess relevancy performance of discovery-api by running several sample +* searches and reporting on how many of the named bibs appear among the top +* results. * * This script loads "query targets" from a CSV, representing scoped keyword queries with 1 or more "good hits" * * Query targets CSV should be placed in /data/query-targets.csv, downloaded from * https://docs.google.com/spreadsheets/d/1cLYqK8MqJ8XtR4cVCCdYRc3hMPZNwoxirko93q8CAts/edit#gid=0 * -* For each query target, the script runs the query against the app, reporting on the successes and failures. +* For each query target, the script runs the query against the app, reporting on +* the successes and failures. Success is determined by how many of the named +* bibs appear close to the top of teh results. * * Usage: * ENV=qa node scripts/assess-query-targets.js [--rows 2,3] [--offset O] [--limit L] @@ -38,6 +42,16 @@ setKmsCredentials(fromIni({ profile: argv.profile })) const PORT = 3333 +// Define ratio threshold for passing each query +const OVERALL_PASS_RATIO = { + // If 100% pass, mark test as "pass" + pass: 1, + // If 50+% pass, mark test as "mixed" + mixed: 0.5, + // Otherwise ( < 50% pass), mark test as "fail" + fail: 0 +} + /** * Start the app server */ @@ -93,29 +107,46 @@ const runQuery = async (query) => { * returns two arrays of objects representing passes and fails */ const analyzeResults = (results, goodHits) => { - return Object.keys(goodHits) - .reduce((h, expectedIndex) => { - const bibid = goodHits[expectedIndex].bibid - if (!bibid) { - console.warn('No "good hits" defined?') - return h - } - const actualIndex = results.findIndex((result) => result.result.uri === bibid) - // Consider it a pass if within 5 of expected index (adjusting for number of good hits) - const pass = actualIndex >= 0 && Math.abs(actualIndex - expectedIndex) <= (5 + goodHits.length) - - const document = actualIndex >= 0 ? results[actualIndex].result : null - const report = { - bibid, - expectedIndex, - actualIndex, - document - } - h[pass ? 'pass' : 'fail'].push(report) + return goodHits + .map((goodHit) => analyzeResultsForTargetBibId(results, goodHit, goodHits.length)) + .reduce((h, analysis) => { + h[analysis.pass ? 'pass' : 'fail'].push(analysis) return h }, { pass: [], fail: [] }) } +/** +* Given an array of search results and a "good hit" (bibid and rank), returns +* an object representing the pass/fail for the expected bib +*/ +const analyzeResultsForTargetBibId = (results, goodHit, numberOfTargetHits) => { + // Where does the target bib occur in the results? + const actualIndex = results.findIndex((result) => result.result.uri === goodHit.bibid) + + // Assess whether and how well the expected hit appeared in the results + let pass = false + if (actualIndex >= 0) { + const distanceFromExpected = actualIndex >= 0 && Math.abs(actualIndex - goodHit.rank) + + // Consider it a pass if found within 5 of expected index (adjusting for + // number of good hits) + pass = distanceFromExpected <= (5 + numberOfTargetHits) + } + + // Store document in report, if found: + const document = actualIndex >= 0 ? results[actualIndex].result : null + + const report = { + bibid: goodHit.bibid, + expectedIndex: goodHit.rank, + actualIndex, + document, + pass + } + + return report +} + /** * Given an Express instance, an array of queries, and a index, executes the * next test query against the server, reports on the result, and calls itself @@ -141,6 +172,9 @@ const testNextQuery = async (queries, index = 0) => { console.info(tableFormat.table(table)) } + if (!query.orderedHits) { + console.warn('No "good hits" defined?') + } // Analyze results: const { pass, fail } = analyzeResults(results, query.orderedHits) const ratioPass = pass.length / query.orderedHits.length @@ -164,7 +198,7 @@ const testNextQuery = async (queries, index = 0) => { } // Summarize results: - reportOn(`${pass.length} out of ${query.orderedHits.length} PASS`, ratioPass === 1 ? 'success' : (ratioPass < 0.5 ? 'failure' : 'mixed')) + reportOn(`${pass.length} out of ${query.orderedHits.length} PASS`, overallPassFailLabel(ratioPass)) // More queries to run? if (queries[index + 1]) { @@ -176,6 +210,28 @@ const testNextQuery = async (queries, index = 0) => { } } +/** +* Given a ratio (between 0-1), returns "pass", "fail", or "mixed" +* +* E.g. overallPassFailLabel(0.3) => 'fail' +* overallPassFailLabel(0.7) => 'mixed' +* overallPassFailLabel(1.0) => 'pass' +*/ +const overallPassFailLabel = (ratio) => { + return Object.entries(OVERALL_PASS_RATIO) + // Sort by thresholds ascending: + .sort((pair1, pair2) => { + return pair1[1] - pair2[1] + }) + // Return highest pass/fail label where `ratio` exceeds configured thresshold + .reduce((result, [label, thresshold]) => { + if (ratio >= thresshold) { + result = label + } + return result + }, '') +} + // Await-able setTimeout: const delay = async (time) => new Promise((resolve) => setTimeout(resolve, time)) @@ -185,10 +241,11 @@ const delay = async (time) => new Promise((resolve) => setTimeout(resolve, time) const parseTargetQueryRow = (row, index) => { row.orderedHits = row['good hits'] .split('\n') - .map((hit) => { + .map((hit, index) => { return { url: hit, - bibid: hit.split('/').pop() + bibid: hit.split('/').pop(), + rank: index } }) delete row['good hits']