Updates following searchathon testing

Updates: - Add search_scope=journal_title (to ease testing and because it really ought to be implemented in the discovery-api, not the front-end - Match on shelfmark lowercased - Allow smart-quotes in fully-quoted searches - Remove boost=1 on quoted creator/contrib phrase matching - Match on new idIsbn.clean and idIssn.clean (added 2024-10-17) - Updates qa index to 2024-10-16 (added shelfMark.keywordLowercased)
NYPL · Oct 17, 2024 · ca37f6a · ca37f6a
1 parent ee44d51
commit ca37f6a
Show file tree

Hide file tree

Showing 41 changed files with 5,877 additions and 6,373 deletions.
diff --git a/config/qa.env b/config/qa.env
@@ -1,6 +1,6 @@
 # Greg built self-hosted qa domain:
 ENCRYPTED_ELASTICSEARCH_URI=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAJYwgZMGCSqGSIb3DQEHBqCBhTCBggIBADB9BgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDMIkDoQ9C/cCDCAq1wIBEIBQ+L3OgUGeOW9rs1CWkhpBjwM4LbbVRFIWedqew4UXIeSNMJ8cO9SNe4YGCUIoKwCDYt7W7ip3VtDRRRMVvz6QJw+Eg8ugTMVs2pbNFGNvaAQ=
-ENCRYPTED_RESOURCES_INDEX=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHIwcAYJKoZIhvcNAQcGoGMwYQIBADBcBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDNxOM6EK7KwwZE56SgIBEIAvjEuTWDQzzrtq8VX+gNIpc/LYi3Z1H6CDjlwAjmR0IvkPQT2DladPXJbaYiyD3/E=
+ENCRYPTED_RESOURCES_INDEX=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAHIwcAYJKoZIhvcNAQcGoGMwYQIBADBcBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDKS+EEsLB1iiqsC/YwIBEIAvBJpaq8IQcbTPYdgywjBfeUcMaWENUOftxSiVsdG01Rh56dDv/ITrYTHAqqXK/DU=
 ENCRYPTED_ELASTICSEARCH_API_KEY=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAJ4wgZsGCSqGSIb3DQEHBqCBjTCBigIBADCBhAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAx+kryf2KUmGdBYD9sCARCAV3ygz3eXIdq8JX/wpG9JRWlTNMRcpNE1qT0zNlN4t+ZvXEoedLQa/3p1YjgHw06GIAdA9xtkMV4eH9a1K8uCvjP8XxxNKekcMj59TlResnu9QF3r7pGXuQ==
 
 ENCRYPTED_SCSB_URL=AQECAHh7ea2tyZ6phZgT4B9BDKwguhlFtRC6hgt+7HbmeFsrsgAAAH8wfQYJKoZIhvcNAQcGoHAwbgIBADBpBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBKllElmWYLxGOGopQIBEIA8JJyKde/8m8iCJGKR5D8HoTJhXHeyvw9eIDeuUNKiXLfJwoVz+PDAZSxkCQtM9O91zGhXbe3l6Bk1RlYJ

diff --git a/lib/api-request.js b/lib/api-request.js
@@ -2,6 +2,10 @@
 * This class wraps request params to ease interpretting the query
 **/
 
+// Build regex pattern for matching a phrase fully enclosed in quotes (or smart quotes):
+const QUOTE_CHARS = '"\u201C\u201D\u201E\u201F\u2033\u2036'
+const IN_QUOTES_PATTERN = new RegExp(`^[${QUOTE_CHARS}][^${QUOTE_CHARS}]+[${QUOTE_CHARS}]$`)
+
 class ApiRequest {
   static ADVANCED_SEARCH_PARAMS = ['title', 'subject', 'contributor']
   static IDENTIFIER_NUMBER_PARAMS = ['isbn', 'issn', 'lccn', 'oclc']
@@ -43,10 +47,10 @@ class ApiRequest {
   }
 
   /**
-  * Returns true if query is fully wrapped in quotes
+  * Returns true if query is fully wrapped in quotes (or smart quotes)
   **/
   queryIsFullyQuoted () {
-    return /^"[^"]+"$/.test(this.params.q)
+    return IN_QUOTES_PATTERN.test(this.params.q)
   }
 
   /**

diff --git a/lib/elasticsearch/config.js b/lib/elasticsearch/config.js
@@ -62,6 +62,9 @@ const SEARCH_SCOPES = {
   series: {
     fields: ['seriesStatement.folded']
   },
+  journal_title: {
+    fields: null
+  },
   callnumber: {
     fields: null
   },

diff --git a/lib/elasticsearch/elastic-query-builder.js b/lib/elasticsearch/elastic-query-builder.js
@@ -22,6 +22,10 @@ class ElasticQueryBuilder {
       case 'title':
         this.buildTitleQuery()
         break
+      case 'journal_title':
+        this.buildTitleQuery()
+        this.requireIssuance('urn:biblevel:s')
+        break
       case 'subject':
         this.buildSubjectQuery()
         break
@@ -187,6 +191,18 @@ class ElasticQueryBuilder {
     this.boostOnItemShelfmarkPrefix()
   }
 
+  /**
+  * Require a specific issuance (e.g. require 'urn:biblevel:s' for journal-
+  * title searches)
+  */
+  requireIssuance (issuance) {
+    this.query.addFilter({
+      term: {
+        'issuance.id': issuance
+      }
+    })
+  }
+
   /**
   * Require exact phrase-match on named fields
   */
@@ -213,8 +229,9 @@ class ElasticQueryBuilder {
           prefixMatch('identifierV2.value', q),
           termMatch('uri', q),
           termMatch('items.idBarcode', q),
-          termMatch('idIsbn_clean', q),
-          prefixMatch('items.shelfMark.raw', q)
+          termMatch('idIsbn.clean', q),
+          termMatch('idIssn.clean', q),
+          prefixMatch('items.shelfMark.keywordLowercased', q)
         ]
       }
     })
@@ -233,17 +250,9 @@ class ElasticQueryBuilder {
         }
       })
     } else if (this.request.params.issn) {
-      this.query.addMust({ term: { idIssn: this.request.params.issn } })
+      this.query.addMust({ term: { 'idIssn.clean': this.request.params.issn } })
     } else if (this.request.params.isbn) {
-      this.query.addMust({
-        bool: {
-          should: [
-            { term: { idIsbn: this.request.params.isbn } },
-            { term: { idIsbn_clean: this.request.params.isbn } }
-          ],
-          minimum_should_match: 1
-        }
-      })
+      this.query.addMust({ term: { 'idIsbn.clean': this.request.params.isbn } })
     } else if (this.request.params.oclc) {
       this.query.addMust({ term: { idOclc: this.request.params.oclc } })
     }
@@ -258,8 +267,8 @@ class ElasticQueryBuilder {
     this.query.addMust({
       bool: {
         should: [
-          prefixMatch('shelfMark.raw', q),
-          prefixMatch('items.shelfMark.raw', q)
+          prefixMatch('shelfMark.keywordLowercased', q),
+          prefixMatch('items.shelfMark.keywordLowercased', q)
         ]
       }
     })
@@ -350,14 +359,18 @@ class ElasticQueryBuilder {
   * Boost bibs with items having exact callnumber match
   **/
   boostOnItemShelfmarkExact (boost = 50) {
-    this.query.addShould(termMatch('items.shelfMark.raw', this.request.querySansQuotes(), 50))
+    this.query.addShould(termMatch('items.shelfMark.keywordLowercased', this.request.querySansQuotes(), boost))
+    // Specially boost case-sensitive match:
+    this.query.addShould(termMatch('items.shelfMark.raw', this.request.querySansQuotes(), boost * 2))
   }
 
   /**
   * Boost bibs with items having prefix callnumber match
   **/
   boostOnItemShelfmarkPrefix (boost = 10) {
-    this.query.addShould(prefixMatch('items.shelfMark.raw', boost))
+    this.query.addShould(prefixMatch('items.shelfMark.keywordLowercased', this.request.querySansQuotes(), boost))
+    // Specially boost case-sensitive match:
+    this.query.addShould(prefixMatch('items.shelfMark.raw', this.request.querySansQuotes(), boost * 2))
   }
 
   /**
@@ -367,20 +380,27 @@ class ElasticQueryBuilder {
     const q = this.request.querySansQuotes()
 
     this.query.addShoulds([
-      prefixMatch('identifierV2.value', q, 10),
-      termMatch('uri', q, 10),
-      prefixMatch('idIsbn', q, 10),
+      prefixMatch('identifierV2.value', q, 5),
+      termMatch('uri', q, 5),
+
+      // Tiered matches on isbn:
+      prefixMatch('idIsbn.clean', q, 10),
+      termMatch('idIsbn.clean', q, 20),
+      termMatch('idIsbn', q, 20),
+
+      // Tiered matches on issn:
+      prefixMatch('idIssn.clean', q, 10),
+      termMatch('idIssn.clean', q, 20),
+      termMatch('idIssn', q, 20),
+
       prefixMatch('idLccn', q, 10),
       prefixMatch('idOclc', q, 10),
-      prefixMatch('shelfMark.raw', q, 10),
+      prefixMatch('shelfMark.keywordLowercased', q, 10),
+      prefixMatch('items.shelfMark.keywordLowercased', q, 10),
+
       // I would like this to be a prefix, but doesn't seem to work:
-      termMatch('items.identifierV2.value', q),
-      prefixMatch('items.shelfMark.raw', q)
+      termMatch('items.identifierV2.value', q)
     ])
-
-    if (/\d{6,}/.test(this.request.querySansQuotes())) {
-      this.query.addShould(prefixMatch('idIsbn_clean', q))
-    }
   }
 
   /**

diff --git a/lib/elasticsearch/utils.js b/lib/elasticsearch/utils.js
@@ -94,7 +94,7 @@ const termMatch = (field, value, boost = 1) => {
 * Given a field name and a value, returns a plainobject representing a ES
 * `match_phrase` clause that can be inserted into a ES query
 */
-const phraseMatch = (field, value, boost = 1) => {
+const phraseMatch = (field, value, boost = 0) => {
   return {
     match_phrase: {
       [field]: namedQuery({

diff --git a/swagger.v1.1.x.json b/swagger.v1.1.x.json
@@ -85,6 +85,7 @@
             "enum": [
               "all",
               "title",
+              "journal_title",
               "contributor",
               "subject",
               "series",
@@ -287,6 +288,7 @@
             "enum": [
               "all",
               "title",
+              "journal_title",
               "contributor",
               "subject",
               "series",

diff --git a/test/api-request.test.js b/test/api-request.test.js
@@ -31,6 +31,10 @@ describe('ApiRequest', function () {
     request = ApiRequest.fromParams({ q: '"toast"' })
     expect(request.queryIsFullyQuoted()).to.eq(true)
 
+    // Test smart-quotes, the great scorge
+    request = ApiRequest.fromParams({ q: '“toast“' })
+    expect(request.queryIsFullyQuoted()).to.eq(true)
+
     request = ApiRequest.fromParams({ q: '"toast" and jam' })
     expect(request.queryIsFullyQuoted()).to.eq(false)
   })

diff --git a/test/elastic-query-builder.test.js b/test/elastic-query-builder.test.js
@@ -107,7 +107,7 @@ describe('ElasticQueryBuilder', () => {
       // Expect boosting on several identifier fields:
       expect(inst.query.toJson().bool.should)
         .to.be.a('array')
-        .have.lengthOf.at.least(9).at.most(11)
+        .have.lengthOf.at.least(15).at.most(16)
     })
   })
 
@@ -118,13 +118,13 @@ describe('ElasticQueryBuilder', () => {
 
       // Expect multiple term/prefix matches on identifier fields:
       expect(inst.query.toJson()).to.nested
-        .include({ 'bool.must[0].bool.should[0].prefix.shelfMark\\.raw.value': 'toast' })
-        .include({ 'bool.must[0].bool.should[1].nested.query.prefix.items\\.shelfMark\\.raw.value': 'toast' })
+        .include({ 'bool.must[0].bool.should[0].prefix.shelfMark\\.keywordLowercased.value': 'toast' })
+        .include({ 'bool.must[0].bool.should[1].nested.query.prefix.items\\.shelfMark\\.keywordLowercased.value': 'toast' })
 
       // Expect boosting on several identifier fields:
       expect(inst.query.toJson().bool.should)
         .to.be.a('array')
-        .have.lengthOf.at.least(2).at.most(3)
+        .have.lengthOf.at.least(4).at.most(5)
     })
   })
 

diff --git a/...ery-3ab62cb3884964644f8c9019bd4afc7c.json → ...ery-0b94a3d48612b7def067c254f3ae08e2.json b/...ery-3ab62cb3884964644f8c9019bd4afc7c.json → ...ery-0b94a3d48612b7def067c254f3ae08e2.json
@@ -18,7 +18,7 @@
         {
           "_index": "resources-2024-07-11",
           "_id": "b22144813",
-          "_score": 11,
+          "_score": 16,
           "_source": {
             "extent": [
               "1234, [1] pages, x leaves : illustrations ;",
@@ -737,7 +737,7 @@
             ]
           },
           "sort": [
-            11,
+            16,
             "b22144813"
           ],
           "inner_hits": {

diff --git a/...ery-44e74368d4e871ba3ddfae0f5fb8dedc.json → ...ery-107fe3cf00cb821f9de8f70ebc8512ad.json b/...ery-44e74368d4e871ba3ddfae0f5fb8dedc.json → ...ery-107fe3cf00cb821f9de8f70ebc8512ad.json
@@ -1,6 +1,6 @@
 {
   "body": {
-    "took": 7,
+    "took": 13,
     "timed_out": false,
     "_shards": {
       "total": 2,

diff --git a/...ery-d5152a626e1519e3375a4d4e9cb0d226.json → ...ery-1731cd5247baa2a5b7065391a66eca0e.json b/...ery-d5152a626e1519e3375a4d4e9cb0d226.json → ...ery-1731cd5247baa2a5b7065391a66eca0e.json
@@ -18,7 +18,7 @@
         {
           "_index": "resources-2024-07-11",
           "_id": "b22144813",
-          "_score": 11,
+          "_score": 6,
           "_source": {
             "extent": [
               "1234, [1] pages, x leaves : illustrations ;",
@@ -737,7 +737,7 @@
             ]
           },
           "sort": [
-            11,
+            6,
             "b22144813"
           ],
           "inner_hits": {

diff --git a/...ery-2f8758c972bad5766e0b75f50e655159.json → ...ery-21b667ae1bee574cc4c2aae7f3e9c2b0.json b/...ery-2f8758c972bad5766e0b75f50e655159.json → ...ery-21b667ae1bee574cc4c2aae7f3e9c2b0.json
@@ -18,7 +18,7 @@
         {
           "_index": "resources-2024-07-11",
           "_id": "b22144813",
-          "_score": 11,
+          "_score": 6,
           "_source": {
             "extent": [
               "1234, [1] pages, x leaves : illustrations ;",
@@ -737,7 +737,7 @@
             ]
           },
           "sort": [
-            11,
+            6,
             "b22144813"
           ],
           "inner_hits": {

diff --git a/...ery-de2a8ddf12655ef8419998a6b00823f2.json → ...ery-240b5ecb8e3680a5141faf669f2a656b.json b/...ery-de2a8ddf12655ef8419998a6b00823f2.json → ...ery-240b5ecb8e3680a5141faf669f2a656b.json
@@ -1,6 +1,6 @@
 {
   "body": {
-    "took": 6,
+    "took": 36,
     "timed_out": false,
     "_shards": {
       "total": 2,

diff --git a/...ery-26a11afdafa6aea99ed004732d137f84.json → ...ery-24f6b3be52db457067c232fb97110254.json b/...ery-26a11afdafa6aea99ed004732d137f84.json → ...ery-24f6b3be52db457067c232fb97110254.json
@@ -1,6 +1,6 @@
 {
   "body": {
-    "took": 18,
+    "took": 117,
     "timed_out": false,
     "_shards": {
       "total": 2,

diff --git a/...ery-9fd51c5f2b1b58b560bd6406586ccabb.json → ...ery-2efeba804a2c39785fce03b93ece4558.json b/...ery-9fd51c5f2b1b58b560bd6406586ccabb.json → ...ery-2efeba804a2c39785fce03b93ece4558.json
@@ -1,6 +1,6 @@
 {
   "body": {
-    "took": 9,
+    "took": 11,
     "timed_out": false,
     "_shards": {
       "total": 2,
@@ -18,7 +18,7 @@
         {
           "_index": "resources-2024-07-11",
           "_id": "b13627363",
-          "_score": 23,
+          "_score": 6,
           "_source": {
             "extent": [
               "128 p. illus."
@@ -164,7 +164,7 @@
             ]
           },
           "sort": [
-            23,
+            6,
             "b13627363"
           ],
           "inner_hits": {

diff --git a/...ery-a36ab4b7594662598ecd3b1649e3c43c.json → ...ery-436ecd09cce9179142f59ee34413eb87.json b/...ery-a36ab4b7594662598ecd3b1649e3c43c.json → ...ery-436ecd09cce9179142f59ee34413eb87.json
@@ -1,6 +1,6 @@
 {
   "body": {
-    "took": 33,
+    "took": 4,
     "timed_out": false,
     "_shards": {
       "total": 2,
@@ -18,7 +18,7 @@
         {
           "_index": "resources-2024-07-11",
           "_id": "b11826883",
-          "_score": 677.532,
+          "_score": 1299.1257,
           "_source": {
             "extent": [
               "v. : ill. ;"
@@ -532,7 +532,7 @@
             ]
           },
           "sort": [
-            677.532,
+            1299.1257,
             "b11826883"
           ],
           "inner_hits": {

diff --git a/...ery-eb9e6b92cd06dc992a7801e4a7f74b8d.json → ...ery-4cce6dbd13eed818785235f84f251bfa.json b/...ery-eb9e6b92cd06dc992a7801e4a7f74b8d.json → ...ery-4cce6dbd13eed818785235f84f251bfa.json
@@ -1,6 +1,6 @@
 {
   "body": {
-    "took": 18,
+    "took": 148,
     "timed_out": false,
     "_shards": {
       "total": 2,

diff --git a/...ery-519aeb5f85865c28ba15cd0a5029e84a.json → ...ery-4f60ceb24a20299f5f126ff480a59961.json b/...ery-519aeb5f85865c28ba15cd0a5029e84a.json → ...ery-4f60ceb24a20299f5f126ff480a59961.json
@@ -1,6 +1,6 @@
 {
   "body": {
-    "took": 6,
+    "took": 4,
     "timed_out": false,
     "_shards": {
       "total": 2,
@@ -18,7 +18,7 @@
         {
           "_index": "resources-2024-07-11",
           "_id": "b22144813",
-          "_score": 21,
+          "_score": 6,
           "_source": {
             "extent": [
               "1234, [1] pages, x leaves : illustrations ;",
@@ -737,7 +737,7 @@
             ]
           },
           "sort": [
-            21,
+            6,
             "b22144813"
           ],
           "inner_hits": {