Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Package updates, use RCI source-mapper #415

Merged
merged 2 commits into from
Oct 31, 2024
Merged

Package updates, use RCI source-mapper #415

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
920798e
Package updates, use RCI source-mapper
nonword Oct 25, 2024
a5e6160
Fix small bug with identifying nypl checkin items
nonword Oct 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions lib/availability_resolver.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
const scsbClient = require('./scsb-client')
const logger = require('./logger')
const ResourceSerializer = require('./jsonld_serializers').ResourceSerializer
const NyplSourceMapper = require('discovery-store-models/lib/nypl-source-mapper')
const NyplSourceMapper = require('research-catalog-indexer/lib/utils/nypl-source-mapper')
const { nonRecapItemStatusAggregation } = require('./elasticsearch/client')
const { deepValue, isInRecap } = require('./util')

Expand Down Expand Up @@ -82,7 +82,7 @@ class AvailabilityResolver {
* If response contains an item status aggregation and appears to cover some
* ReCAP items, updates response to accurately reflect ReCAP statuses
*/
_fixItemStatusAggregation (options) {
async _fixItemStatusAggregation (options) {
const resp = this.elasticSearchResponse

// Return early if there are no item aggregations to fix:
Expand All @@ -97,7 +97,8 @@ class AvailabilityResolver {
const bnum = resp.hits.hits[0]?._id
if (!bnum) return Promise.resolve()

const { nyplSource } = NyplSourceMapper.instance().splitIdentifier(bnum)
const nyplSourceMapper = await NyplSourceMapper.instance()
const { nyplSource } = nyplSourceMapper.splitIdentifier(bnum)

// Get total number of items:
const numItems = (resp.hits.hits[0]._source.numItemsTotal || resp.hits.hits[0]._source.numItems)[0]
Expand Down
154 changes: 42 additions & 112 deletions lib/jsonld_serializers.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
'use strict'

const locations = require('@nypl/nypl-core-objects')('by-sierra-location')
const NyplSourceMapper = require('discovery-store-models/lib/nypl-source-mapper')
const NyplSourceMapper = require('research-catalog-indexer/lib/utils/nypl-source-mapper')

const util = require('./util.js')
const logger = require('./logger')
Expand All @@ -14,15 +14,16 @@ class JsonLdSerializer {
this.options = options || {}
}

format () {
async format () {
const base = {}
if (this.options.root) {
// If expandContext, dump full context doc into result, otherwise just give URL
if (this.options.expandContext) base['@context'] = util.buildJsonLdContext({})
else base['@context'] = `${this.options.baseUrl}/context_all.jsonld`
}
base['@type'] = this.type
Object.assign(base, this.statements())
const statements = await this.statements()
Object.assign(base, statements)
return base
}
}
Expand All @@ -45,7 +46,7 @@ class JsonLdListSerializer extends JsonLdSerializer {
this.type = 'itemList'
}

statements () {
async statements () {
return Object.assign({}, {
itemListElement: this.items.map(this.itemSerializer.bind(this))
}, (this.options.extraRootProperties || {}))
Expand All @@ -67,7 +68,7 @@ class JsonLdItemSerializer extends JsonLdSerializer {
return `res:${this.body.uri}`
}

statements () {
async statements () {
const stmts = { '@id': this.resultId() }

// Takes any kind of value (array, object, string) and ensures .id props are formatted as '@id'
Expand Down Expand Up @@ -221,8 +222,8 @@ class ResourceSerializer extends JsonLdItemSerializer {
this.type = [...new Set([item.type].concat('nypl:Resource').flat())]
}

statements () {
const stmts = JsonLdItemSerializer.prototype.statements.call(this)
async statements () {
const stmts = await JsonLdItemSerializer.prototype.statements.call(this)

if (this.body.parentUri) stmts.memberOf = [util.eachValue(this.body.parentUri, (id) => ({ '@type': 'nypl:Resource', '@id': `res:${id}` }))].flat()

Expand All @@ -234,12 +235,16 @@ class ResourceSerializer extends JsonLdItemSerializer {
}

if (this.body.items) {
stmts.items = this.body.items
// Amend items to include source identifier (e.g. urn:SierraNypl:1234, urn:RecapCul:4567)
.map(ItemResourceSerializer.addSourceIdentifier)
.map((item) => {
return (new ItemResourceSerializer(item)).statements()
})
stmts.items = await Promise.all(
this.body.items
// Amend items to include source identifier (e.g. urn:SierraNypl:1234, urn:RecapCul:4567)
.map(ItemResourceSerializer.addSourceIdentifier)
)
stmts.items = await Promise.all(
stmts.items
.map((item) => new ItemResourceSerializer(item))
.map((itemSerializer) => itemSerializer.statements())
)
}

if (this.body.itemAggregations) {
Expand Down Expand Up @@ -321,8 +326,8 @@ class ItemResourceSerializer extends JsonLdItemSerializer {
this.type = 'bf:Item'
}

statements () {
const stmts = JsonLdItemSerializer.prototype.statements.call(this)
async statements () {
const stmts = await JsonLdItemSerializer.prototype.statements.call(this)

if (stmts.identifier) {
// Add idNyplSourceId convenience property by parsing identifiers that match urn:[source]:[id]
Expand Down Expand Up @@ -354,11 +359,11 @@ class ItemResourceSerializer extends JsonLdItemSerializer {
// urn:RecapCul:4567
// urn:RecapPul:6789
// urn:RecapHl:87654321
static addSourceIdentifier (item) {
static async addSourceIdentifier (item) {
// Ensure identifiers array exists:
item.identifier = item.identifier || []

const { id, nyplSource, type } = NyplSourceMapper.instance().splitIdentifier(item.uri)
const nyplSourceMapper = await NyplSourceMapper.instance()
const { id, nyplSource, type } = nyplSourceMapper.splitIdentifier(item.uri)
if (type === 'item') {
// Build prefix nyplSource as camel case
const sourceIdentifierPrefix = ItemResourceSerializer.sourceIdentifierPrefixByNyplSource(nyplSource)
Expand Down Expand Up @@ -410,8 +415,15 @@ class ResourceResultsSerializer extends SearchResultsSerializer {
return `resources:${result.uri}`
}

static serialize (resp, opts) {
const results = resp.hits.hits.map((h) => ({ score: h._score, record: ResourceSerializer.serialize(h._source), matched_queries: h.matched_queries }))
static async serialize (resp, opts) {
const results = await Promise.all(
resp.hits.hits.map((h) => {
// Serialize the bib record:
return ResourceSerializer.serialize(h._source)
// Serialize the "result" record wrapping the bib record:
.then((record) => ({ score: h._score, record, matched_queries: h.matched_queries }))
})
)
const totalResults = typeof resp.hits.total?.value === 'number' ? resp.hits.total.value : resp.hits.total
opts = Object.assign({ extraRootProperties: { totalResults } }, opts)
return (new ResourceResultsSerializer(results, opts)).format()
Expand All @@ -432,27 +444,31 @@ class AggregationsSerializer extends JsonLdListSerializer {
return `field:${result.field}`
}

static serialize (resp, options) {
static async serialize (resp, options) {
if ((typeof options) === 'undefined') options = {}

const items = Object.keys(resp.aggregations).map((field) => AggregationSerializer.serialize(Object.assign({ id: field }, resp.aggregations[field]), options))
const items = await Promise.all(
Object.entries(resp.aggregations)
// Add id property to body of aggregation:
.map(([id, agg]) => Object.assign({ id }, agg))
.map((agg) => AggregationSerializer.serialize(agg, options))
)
return (new AggregationsSerializer(items, Object.assign({ extraRootProperties: { totalResults: resp.hits.total } }, options))).format()
}
}

class AggregationSerializer extends JsonLdItemSerializer {
constructor (item, options) {
super(item, options)
// Serialize both the most general type (Resource) as well as any resource-specific type (Collection, Component, Capture, etc)
this.type = 'nypl:Aggregation'
}

resultId () {
return `res:${this.body.id}`
}

statements () {
const stmts = JsonLdItemSerializer.prototype.statements.call(this)
async statements () {
const stmts = await JsonLdItemSerializer.prototype.statements.call(this)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can this line become this.statements() instead?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think one could use super.statements() at this point, but really I think this whole file needs to be rethought with no/flatter inheritance. That's a bigger effort beyond the scope of this PR, which is mainly about resolving vulnerabilities, so I'd like to punt on that kind of work if that sounds alright.


stmts.field = this.body.id
const field = this.body.id
Expand Down Expand Up @@ -486,90 +502,4 @@ class AggregationSerializer extends JsonLdItemSerializer {
}
}

/*
* Search Results: Agents
*/

class AgentResultsSerializer extends SearchResultsSerializer {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this no longer used?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It hasn't been used for about 5 years and will likely take another form entirely with Contributor Explorer, so taking the opportunity to remove it.

constructor (items, opts) {
super(items, opts)
this.resultType = 'nypl:Person'
}

resultId (result) {
return `agents:${result.uri}`
}

static serialize (resp) {
const results = []
const hits = resp.hits.hits
hits.forEach(function (h) {
const serialized = AgentSerializer.serialize(h._source)
results.push(serialized)
})
return (new AgentResultsSerializer(results, { extraRootProperties: { totalResults: resp.hits.total } })).format()
}
}
/*
* Agents
*/

class AgentSerializer extends JsonLdItemSerializer {
constructor (item, options) {
super(item, options)

this.type = ['edm:Agent']
let foafType = 'foaf:Person'
if (item.type === 'Meeting') foafType = 'foaf:Group'
if (item.type === 'Coporation') foafType = 'foaf:Organization'
if (item.type === 'Organization') foafType = 'foaf:Organization'
this.type.push(foafType)
}

resultId () {
return `agents:${this.body.uri}`
}

statements () {
const h = JsonLdItemSerializer.parsePackedStatements(this.body)

const stmts = JsonLdItemSerializer.prototype.statements.call(this)

if (h.label) stmts.prefLabel = h.label

if (h.dobString) stmts.birthDate = h.dobString
if (h.dobYear) stmts.birthYear = h.dobYear
if (h.dobDecade) stmts.birthDecade = h.dobDecade
if (h.dodString) stmts.deathDate = h.dodString
if (h.dodYear) stmts.deathYear = h.dodYear
if (h.dodDecade) stmts.deathDecade = h.dodDecade

stmts.topFiveTermsString = h.topFiveTerms
stmts.topFiveRolesString = h.topFiveRoles

if (h.description) stmts.description = h.description
if (h.viaf) stmts.uriViaf = 'viaf:' + h.viaf
if (h.wikidata) stmts.uriWikidata = 'wikidata:' + h.wikidata
if (h.lc) stmts.uriLc = 'lc:' + h.lc
if (h.dbpedia) stmts.uriDbpedia = 'dbpedia:' + h.dbpedia
if (h.depiction) stmts.depiction = h.depiction
if (h.wikipedia) stmts.wikipedia = 'https://wikipedia.org/wiki/' + h.wikipedia
if (h.label) stmts.prefLabel = h.label
if (h.useCount) stmts.useCount = h.useCount
if (h.score) stmts.searchResultScore = h.score

return stmts
}

static serialize (resp, options) {
if (resp.identifier) {
let bnum = null
if ((bnum = resp.identifier.filter((i) => i.match(/^urn:bnum:/))) && (bnum = bnum[0]) && (bnum = bnum.split(':')) && (bnum = bnum[bnum.length - 1])) {
resp.depiction = `https://s3.amazonaws.com/data.nypl.org/bookcovers/${bnum}_ol.jpg`
}
}
return (new AgentSerializer(resp, options)).format()
}
}

module.exports = { JsonLdSerializer, ResourceSerializer, ItemResourceSerializer, ItemResultsSerializer, ResourceResultsSerializer, AggregationsSerializer, AgentResultsSerializer, AgentSerializer, AggregationSerializer }
module.exports = { JsonLdSerializer, ResourceSerializer, ItemResourceSerializer, ItemResultsSerializer, ResourceResultsSerializer, AggregationsSerializer, AggregationSerializer }
8 changes: 3 additions & 5 deletions lib/ownership_determination.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
const NyplSourceMapper = require('discovery-store-models/lib/nypl-source-mapper')
// NYPL item ids start with an 'i'
const NYPL_ITEM_ID_PATTERN = /^i/

// This is based on uri now but should be changed to use the 'item.owner' field once it's more reliably serialized.
// This just talks about ownership, not recap vs non-recap
const isItemNyplOwned = (item) => {
const { nyplSource } = NyplSourceMapper.instance().splitIdentifier(item.uri)
return nyplSource === 'sierra-nypl'
return NYPL_ITEM_ID_PATTERN.test(item?.uri)
}

module.exports = { isItemNyplOwned }
21 changes: 7 additions & 14 deletions lib/resources.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
const NyplSourceMapper = require('discovery-store-models/lib/nypl-source-mapper')
const NyplSourceMapper = require('research-catalog-indexer/lib/utils/nypl-source-mapper')
const scsbClient = require('./scsb-client')

const ResourceResultsSerializer = require('./jsonld_serializers.js').ResourceResultsSerializer
Expand Down Expand Up @@ -104,7 +104,7 @@ module.exports = function (app, _private = null) {
app.resources = {}

// Get a single resource:
app.resources.findByUri = function (params, opts = {}, request) {
app.resources.findByUri = async function (params, opts = {}, request) {
// Parse all params we support:
params = parseParams(params, {
all_items: { type: 'boolean', default: false },
Expand All @@ -122,7 +122,8 @@ module.exports = function (app, _private = null) {
})

// Validate uri:
const { id, nyplSource } = NyplSourceMapper.instance().splitIdentifier(params.uri)
const nyplSourceMapper = await NyplSourceMapper.instance()
const { id, nyplSource } = nyplSourceMapper.splitIdentifier(params.uri)
if (!id || !nyplSource) {
throw new errors.InvalidParameterError(`Invalid bnum: ${params.uri}`)
}
Expand Down Expand Up @@ -194,8 +195,6 @@ module.exports = function (app, _private = null) {
app.logger.debug('Resources#findByUri', body)
return app.esClient.search(body)
.then((resp) => {
resp = resp.body

// Mindfully throw errors for known issues:
if (!resp || !resp.hits) {
throw new Error('Error connecting to index')
Expand All @@ -218,9 +217,10 @@ module.exports = function (app, _private = null) {
}

// Get a single raw annotated-marc resource:
app.resources.annotatedMarc = function (params, opts) {
app.resources.annotatedMarc = async function (params, opts) {
// Convert discovery id to nyplSource and un-prefixed id:
const { id, nyplSource } = NyplSourceMapper.instance().splitIdentifier(params.uri)
const nyplSourceMapper = await NyplSourceMapper.instance()
const { id, nyplSource } = nyplSourceMapper.splitIdentifier(params.uri)

app.logger.debug('Resources#annotatedMarc', { id, nyplSource })
return makeNyplDataApiClient().get(`bibs/${nyplSource}/${id}`)
Expand Down Expand Up @@ -259,8 +259,6 @@ module.exports = function (app, _private = null) {
app.logger.debug('Resources#itemsByFilter', body)
return app.esClient.search(body)
.then((resp) => {
resp = resp.body

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what made this change possible?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Subtle API change in the move from Elasticsearch client v7 to v8

if (!resp || !resp.hits || resp.hits.total === 0) return Promise.reject(new Error('No matching items'))
resp = new LocationLabelUpdater(resp).responseWithUpdatedLabels()
// Convert this ES bibs response into an array of flattened items:
Expand Down Expand Up @@ -634,7 +632,6 @@ module.exports = function (app, _private = null) {

return app.esClient.search(body)
.then((resp) => {
resp = resp.body
const massagedResponse = new ResponseMassager(resp)
return massagedResponse.massagedResponse(request)
.catch((e) => {
Expand Down Expand Up @@ -689,8 +686,6 @@ module.exports = function (app, _private = null) {
app.logger.debug('Resources#aggregations:', body)
return app.esClient.search(body)
.then((resp) => {
resp = resp.body

// Transform response slightly before serialization:
resp.aggregations = Object.keys(resp.aggregations)
.reduce((aggs, field) => {
Expand Down Expand Up @@ -735,8 +730,6 @@ module.exports = function (app, _private = null) {
app.logger.debug('Resources#aggregation:', body)
return app.esClient.search(body)
.then((resp) => {
resp = resp.body

// If it's nested, it will be in our special '_nested' prop:
resp = resp.aggregations[params.field]._nested || resp.aggregations[params.field]
resp.id = params.field
Expand Down
7 changes: 4 additions & 3 deletions lib/scsb-client.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
const scsbRestClient = require('@nypl/scsb-rest-client')
const NyplSourceMapper = require('discovery-store-models/lib/nypl-source-mapper')
const NyplSourceMapper = require('research-catalog-indexer/lib/utils/nypl-source-mapper')
const logger = require('./logger')
const { bNumberWithCheckDigit } = require('./util')

Expand Down Expand Up @@ -72,9 +72,10 @@ clientWrapper.recapCustomerCodeByBarcode = (barcode) => {
clientWrapper.getItemsAvailabilityForBarcodes = (barcodes) => scsbClient().getItemsAvailabilityForBarcodes(barcodes)

// bnum is a plain bnum without padding such as we use in the DiscoveryAPI
clientWrapper.getItemsAvailabilityForBnum = (bnum) => {
clientWrapper.getItemsAvailabilityForBnum = async (bnum) => {
// Identify nypl-source and unprefixed id:
const { nyplSource, id } = NyplSourceMapper.instance().splitIdentifier(bnum)
const nyplSourceMapper = await NyplSourceMapper.instance()
const { nyplSource, id } = nyplSourceMapper.splitIdentifier(bnum)
// Determine SCSB "institutionId" (e.g. NYPL, HL, CUL, PUL):
const institutionId = nyplSource.split('-').pop().toUpperCase()
// The "bibliographicId" in SCSB for our items is padded and prefixed;
Expand Down
Loading
Loading