Skip to content

Commit

Permalink
ODL with multiple license blocks (#39)
Browse files Browse the repository at this point in the history 
* Add additional tests for ODL
- Test a feed with multiple license blocks.
- Test a single license block with different numbers of
  licenses available.

* Update the ODL licensing model and add additional tests
* Fix ODL 2.x tests
* Update ODL_TEMPLATE_DIR to an absolute path

Co-authored-by: Viacheslav Bessonov <[email protected]>
  • Loading branch information
@jonathangreen @vbessonov
jonathangreen and vbessonov authored Oct 15, 2021
1 parent 50e3994 commit 8ec48c7
Show file tree
Hide file tree
Showing 7 changed files with 746 additions and 322 deletions.
212 changes: 142 additions & 70 deletions api/odl.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import datetime
import json
import logging
import uuid
from io import StringIO
from typing import Callable, Optional

import dateutil
import feedparser
Expand All @@ -14,12 +16,7 @@

from core import util
from core.analytics import Analytics
from core.metadata_layer import (
CirculationData,
FormatData,
LicenseData,
TimestampData,
)
from core.metadata_layer import CirculationData, FormatData, LicenseData, TimestampData
from core.model import (
Collection,
ConfigurationSetting,
Expand All @@ -32,38 +29,20 @@
LicensePool,
Loan,
MediaTypes,
Representation,
RightsStatus,
Session,
get_one,
get_one_or_create,
Representation)
from core.monitor import (
CollectionMonitor,
IdentifierSweepMonitor)
from core.opds_import import (
OPDSXMLParser,
OPDSImporter,
OPDSImportMonitor,
)
from core.testing import (
DatabaseTest,
MockRequestsResponse,
)
from core.util.datetime_helpers import (
utc_now,
)
from core.util.http import (
HTTP,
BadResponseException,
RemoteIntegrationException,
)
from core.monitor import CollectionMonitor, IdentifierSweepMonitor
from core.opds_import import OPDSImporter, OPDSImportMonitor, OPDSXMLParser
from core.testing import DatabaseTest, MockRequestsResponse
from core.util.datetime_helpers import utc_now
from core.util.http import HTTP, BadResponseException, RemoteIntegrationException
from core.util.string_helpers import base64
from .circulation import (
BaseCirculationAPI,
LoanInfo,
FulfillmentInfo,
HoldInfo,
)

from .circulation import BaseCirculationAPI, FulfillmentInfo, HoldInfo, LoanInfo
from .circulation_exceptions import *
from .shared_collection import BaseSharedCollectionAPI

Expand Down Expand Up @@ -786,6 +765,105 @@ class ODLImporter(OPDSImporter):
# about the license.
LICENSE_INFO_DOCUMENT_MEDIA_TYPE = 'application/vnd.odl.info+json'

@classmethod
def parse_license(
cls,
identifier: str,
total_checkouts: Optional[int],
concurrent_checkouts: Optional[int],
expires: Optional[datetime.datetime],
checkout_link: Optional[str],
odl_status_link: Optional[str],
do_get: Callable
) -> Optional[LicenseData]:
"""Check the license's attributes passed as parameters:
- if they're correct, turn them into a LicenseData object
- otherwise, return a None
:param identifier: License's identifier
:param total_checkouts: Total number of checkouts before the license expires
:param concurrent_checkouts: Number of concurrent checkouts allowed for this license
:param expires: Date & time until the license is valid
:param checkout_link: License's checkout link
:param odl_status_link: License Info Document's link
:param do_get: Callback performing HTTP GET method
:return: LicenseData if all the license's attributes are correct, None, otherwise
"""
remaining_checkouts = None
available_concurrent_checkouts = None

# This cycle ends in two different cases:
# - when at least one of the parameters is invalid; in this case, the method returns None.
# - when all the parameters are valid; in this case, the method returns a LicenseData object.
while True:
if total_checkouts is not None:
total_checkouts = int(total_checkouts)

if total_checkouts <= 0:
logging.info(
f"License # {identifier} expired since "
f"the total number of checkouts is {total_checkouts}"
)
break

if expires:
if not isinstance(expires, datetime.datetime):
expires = dateutil.parser.parse(expires)

expires = util.datetime_helpers.to_utc(expires)
now = util.datetime_helpers.utc_now()

if expires <= now:
logging.info(
f"License # {identifier} expired at {expires} (now is {now})"
)
break

if odl_status_link:
status_code, _, response = do_get(
odl_status_link, headers={}
)

if status_code in (200, 201):
status = json.loads(response)
checkouts = status.get("checkouts", {})
remaining_checkouts = checkouts.get("left")
available_concurrent_checkouts = checkouts.get("available")
else:
logging.warning(
f"License # {identifier}'s Info Document is not available. "
f"Status link failed with {status_code} code"
)
break

if remaining_checkouts is None:
remaining_checkouts = total_checkouts

if remaining_checkouts is not None:
remaining_checkouts = int(remaining_checkouts)

if remaining_checkouts <= 0:
logging.info(
f"License # {identifier} expired since "
f"the remaining number of checkouts is {remaining_checkouts}"
)
break

if available_concurrent_checkouts is None:
available_concurrent_checkouts = concurrent_checkouts

return LicenseData(
identifier=identifier,
checkout_url=checkout_link,
status_url=odl_status_link,
expires=expires,
remaining_checkouts=remaining_checkouts,
concurrent_checkouts=available_concurrent_checkouts,
)

return None

@classmethod
def _detail_for_elementtree_entry(cls, parser, entry_tag, feed_url=None, do_get=None):
do_get = do_get or Representation.cautious_http_get
Expand Down Expand Up @@ -843,11 +921,6 @@ def _detail_for_elementtree_entry(cls, parser, entry_tag, feed_url=None, do_get=

data['medium'] = medium

expires = None
remaining_checkouts = None
available_checkouts = None
concurrent_checkouts = None

checkout_link = None
for link_tag in parser._xpath(odl_license_tag, 'odl:tlink') or []:
rel = link_tag.attrib.get("rel")
Expand All @@ -866,37 +939,33 @@ def _detail_for_elementtree_entry(cls, parser, entry_tag, feed_url=None, do_get=
odl_status_link = attrib.get("href")
break

# If we found one, retrieve it and get licensing information about this book.
if odl_status_link:
ignore, ignore, response = do_get(odl_status_link, headers={})
status = json.loads(response)
checkouts = status.get("checkouts", {})
remaining_checkouts = checkouts.get("left")
available_checkouts = checkouts.get("available")
expires = None
total_checkouts = None
concurrent_checkouts = None

terms = parser._xpath(odl_license_tag, "odl:terms")
if terms:
total_checkouts = subtag(terms[0], "odl:total_checkouts")
concurrent_checkouts = subtag(terms[0], "odl:concurrent_checkouts")
expires = subtag(terms[0], "odl:expires")

if expires:
expires = util.datetime_helpers.to_utc(dateutil.parser.parse(expires))
now = util.datetime_helpers.utc_now()
license = cls.parse_license(
identifier,
total_checkouts,
concurrent_checkouts,
expires,
checkout_link,
odl_status_link,
do_get
)

if expires <= now:
continue
if not license:
continue

licenses_owned += int(concurrent_checkouts or 0)
licenses_available += int(available_checkouts or 0)
licenses_owned += int(license.remaining_checkouts or 0)
licenses_available += int(license.concurrent_checkouts or 0)

licenses.append(LicenseData(
identifier=identifier,
checkout_url=checkout_link,
status_url=odl_status_link,
expires=expires,
remaining_checkouts=remaining_checkouts,
concurrent_checkouts=concurrent_checkouts,
))
licenses.append(license)

if not data.get('circulation'):
data['circulation'] = dict()
Expand Down Expand Up @@ -1552,17 +1621,20 @@ def __init__(self, _db, collection):

def process_item(self, identifier):
for licensepool in identifier.licensed_through:
licenses_owned = licensepool.licenses_owned
licenses_available = licensepool.licenses_available

for license in licensepool.licenses:
if license.is_expired:
licenses_owned -= 1
licenses_available -= 1

if licenses_owned != licensepool.licenses_owned or licenses_available != licensepool.licenses_available:
licenses_owned = max(licenses_owned, 0)
licenses_available = max(licenses_available, 0)
remaining_checkouts = 0 # total number of checkouts across all the licenses in the pool
concurrent_checkouts = 0 # number of concurrent checkouts allowed across all the licenses in the pool

# 0 is a starting point,
# we're going through all the valid licenses in the pool and count up available checkouts.
for license_pool_license in licensepool.licenses:
if not license_pool_license.is_expired:
remaining_checkouts += license_pool_license.remaining_checkouts
concurrent_checkouts += license_pool_license.concurrent_checkouts

if remaining_checkouts != licensepool.licenses_owned or \
concurrent_checkouts != licensepool.licenses_available:
licenses_owned = max(remaining_checkouts, 0)
licenses_available = max(concurrent_checkouts, 0)

circulation_data = CirculationData(
data_source=licensepool.data_source,
Expand Down
57 changes: 21 additions & 36 deletions api/odl2.py
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
import json
import logging

from api.odl import ODLAPI, ODLExpiredItemsReaper, ODLImporter
from contextlib2 import contextmanager
from flask_babel import lazy_gettext as _
from webpub_manifest_parser.odl import ODLFeedParserFactory
from webpub_manifest_parser.opds2.registry import OPDS2LinkRelationsRegistry

from api.odl import ODLAPI, ODLExpiredItemsReaper
from core import util
from core.metadata_layer import FormatData, LicenseData
from core.metadata_layer import FormatData
from core.model import DeliveryMechanism, Edition, MediaTypes, RightsStatus
from core.model.configuration import (
ConfigurationAttributeType,
Expand Down Expand Up @@ -209,11 +208,6 @@ def _extract_publication_metadata(self, feed, publication, data_source_name):
)
)

expires = None
remaining_checkouts = None
available_checkouts = None
concurrent_checkouts = None

checkout_link = first_or_default(
license.links.get_by_rel(OPDS2LinkRelationsRegistry.BORROW.key)
)
Expand All @@ -226,41 +220,32 @@ def _extract_publication_metadata(self, feed, publication, data_source_name):
if odl_status_link:
odl_status_link = odl_status_link.href

if odl_status_link:
status_code, _, response = self.http_get(
odl_status_link, headers={}
)

if status_code < 400:
status = json.loads(response)
checkouts = status.get("checkouts", {})
remaining_checkouts = checkouts.get("left")
available_checkouts = checkouts.get("available")
expires = None
total_checkouts = None
concurrent_checkouts = None

if license.metadata.terms:
expires = license.metadata.terms.expires
total_checkouts = license.metadata.terms.checkouts
concurrent_checkouts = license.metadata.terms.concurrency
expires = license.metadata.terms.expires

if expires:
expires = util.datetime_helpers.to_utc(expires)
now = util.datetime_helpers.utc_now()
license = ODLImporter.parse_license(
identifier,
total_checkouts,
concurrent_checkouts,
expires,
checkout_link,
odl_status_link,
self.http_get
)

if expires <= now:
continue
if not license:
continue

licenses_owned += int(concurrent_checkouts or 0)
licenses_available += int(available_checkouts or 0)
licenses_owned += int(license.remaining_checkouts or 0)
licenses_available += int(license.concurrent_checkouts or 0)

licenses.append(
LicenseData(
identifier=identifier,
checkout_url=checkout_link,
status_url=odl_status_link,
expires=expires,
remaining_checkouts=remaining_checkouts,
concurrent_checkouts=concurrent_checkouts,
)
)
licenses.append(license)

metadata.circulation.licenses_owned = licenses_owned
metadata.circulation.licenses_available = licenses_available
Expand Down
Loading

0 comments on commit 8ec48c7

Please sign in to comment.