Remove expired ODL items

api/odl.py

@@ -1,83 +1,73 @@
 import datetime
-import dateutil
 import json
 import uuid
-from flask_babel import lazy_gettext as _
-import urllib.parse
-from collections import defaultdict
-import flask
-from flask import Response
-import feedparser
-from lxml import etree
-from .problem_details import NO_LICENSES
 from io import StringIO
-import re
-from uritemplate import URITemplate
 
+import dateutil
+import feedparser
+import flask
+from flask import url_for
+from flask_babel import lazy_gettext as _
+from lxml import etree
 from sqlalchemy.sql.expression import or_
+from uritemplate import URITemplate
 
-from core.opds_import import (
-    OPDSXMLParser,
-    OPDSImporter,
-    OPDSImportMonitor,
-)
-from core.monitor import (
-    CollectionMonitor,
-    TimelineMonitor,
+from core import util
+from core.analytics import Analytics
+from core.metadata_layer import (
+    CirculationData,
+    FormatData,
+    LicenseData,
+    TimestampData,
 )
 from core.model import (
     Collection,
     ConfigurationSetting,
-    Credential,
     DataSource,
     DeliveryMechanism,
     Edition,
     ExternalIntegration,
     Hold,
     Hyperlink,
-    Identifier,
-    IntegrationClient,
     LicensePool,
     Loan,
     MediaTypes,
     RightsStatus,
     Session,
-    create,
     get_one,
     get_one_or_create,
+    Representation)
+from core.monitor import (
+    CollectionMonitor,
+    IdentifierSweepMonitor)
+from core.opds_import import (
+    OPDSXMLParser,
+    OPDSImporter,
+    OPDSImportMonitor,
 )
-from core.metadata_layer import (
-    CirculationData,
-    FormatData,
-    IdentifierData,
-    LicenseData,
-    TimestampData,
-)
-from .circulation import (
-    BaseCirculationAPI,
-    LoanInfo,
-    FulfillmentInfo,
-    HoldInfo,
+from core.testing import (
+    DatabaseTest,
+    MockRequestsResponse,
 )
-from core.analytics import Analytics
 from core.util.datetime_helpers import (
     utc_now,
-    strptime_utc,
 )
 from core.util.http import (
     HTTP,
     BadResponseException,
     RemoteIntegrationException,
 )
 from core.util.string_helpers import base64
-from flask import url_for
-from core.testing import (
-    DatabaseTest,
-    MockRequestsResponse,
+from .circulation import (
+    BaseCirculationAPI,
+    LoanInfo,
+    FulfillmentInfo,
+    HoldInfo,
 )
 from .circulation_exceptions import *
 from .shared_collection import BaseSharedCollectionAPI
 
+
 class ODLAPI(BaseCirculationAPI, BaseSharedCollectionAPI):
     """ODL (Open Distribution to Libraries) is a specification that allows
     libraries to manage their own loans and holds. It offers a deeper level
@@ -782,6 +772,7 @@ class ODLXMLParser(OPDSXMLParser):
     NAMESPACES = dict(OPDSXMLParser.NAMESPACES,
                       odl="http://opds-spec.org/odl")
 
+
 class ODLImporter(OPDSImporter):
     """Import information and formats from an ODL feed.
 
@@ -795,6 +786,35 @@ class ODLImporter(OPDSImporter):
     # about the license.
     LICENSE_INFO_DOCUMENT_MEDIA_TYPE = 'application/vnd.odl.info+json'
 
+    _skip_expired_items = True
+
+    def __init__(
+            self,
+            _db,
+            collection,
+            data_source_name=None,
+            identifier_mapping=None,
+            http_get=None,
+            metadata_client=None,
+            content_modifier=None,
+            map_from_collection=None,
+            mirrors=None,
+            skip_expired_items=True
+    ):
+        super(ODLImporter, self).__init__(
+            _db,
+            collection,
+            data_source_name,
+            identifier_mapping,
+            http_get,
+            metadata_client,
+            content_modifier,
+            map_from_collection,
+            mirrors
+        )
+
+        ODLImporter._skip_expired_items = skip_expired_items
+
     @classmethod
     def _detail_for_elementtree_entry(cls, parser, entry_tag, feed_url=None, do_get=None):
         do_get = do_get or Representation.cautious_http_get
@@ -887,8 +907,15 @@ def _detail_for_elementtree_entry(cls, parser, entry_tag, feed_url=None, do_get=
             if terms:
                 concurrent_checkouts = subtag(terms[0], "odl:concurrent_checkouts")
                 expires = subtag(terms[0], "odl:expires")
+
                 if expires:
-                    expires = dateutil.parser.parse(expires)
+                    expires = util.datetime_helpers.to_utc(dateutil.parser.parse(expires))
+
+                    if ODLImporter._skip_expired_items:
+                        now = util.datetime_helpers.utc_now()
+
+                        if expires <= now:
+                            continue
 
             licenses_owned += int(concurrent_checkouts or 0)
             licenses_available += int(available_checkouts or 0)
@@ -914,6 +941,7 @@ def _detail_for_elementtree_entry(cls, parser, entry_tag, feed_url=None, do_get=
         data['circulation']['licenses_available'] = licenses_available
         return data
 
+
 class ODLImportMonitor(OPDSImportMonitor):
     """Import information from an ODL feed."""
     PROTOCOL = ODLImporter.NAME
@@ -1040,6 +1068,27 @@ def __init__(self, _db, collection):
 
         self.base_url = collection.external_account_id
 
+    @staticmethod
+    def _parse_feed_from_response(response):
+        """Parse ODL (Atom) feed from the HTTP response.
+
+        :param response: HTTP response
+        :type response: requests.Response
+
+        :return: Parsed ODL (Atom) feed
+        :rtype: dict
+        """
+        response_content = response.content
+
+        if isinstance(response_content, bytes):
+            response_content = response_content.decode("utf-8")
+        elif not isinstance(response_content, str):
+            raise ValueError("Response content must be a string or byte-encoded value")
+
+        feed = feedparser.parse(response_content)
+
+        return feed
+
     def internal_format(self, delivery_mechanism):
         """Each consolidated copy is only available in one format, so we don't need
         a mapping to internal formats.
@@ -1091,7 +1140,8 @@ def checkout(self, patron, pin, licensepool, internal_format):
                 hold_info_response = self._get(hold.external_identifier)
             except RemoteIntegrationException as e:
                 raise CannotLoan()
-            feed = feedparser.parse(str(hold_info_response.content))
+
+            feed = self._parse_feed_from_response(hold_info_response)
             entries = feed.get("entries")
             if len(entries) < 1:
                 raise CannotLoan()
@@ -1117,7 +1167,8 @@ def checkout(self, patron, pin, licensepool, internal_format):
         elif response.status_code == 404:
             if hasattr(response, 'json') and response.json().get('type', '') == NO_LICENSES.uri:
                 raise NoLicenses()
-        feed = feedparser.parse(str(response.content))
+
+        feed = self._parse_feed_from_response(response)
         entries = feed.get("entries")
         if len(entries) < 1:
             raise CannotLoan()
@@ -1181,7 +1232,8 @@ def checkin(self, patron, pin, licensepool):
             raise CannotReturn()
         if response.status_code == 404:
             raise NotCheckedOut()
-        feed = feedparser.parse(str(response.content))
+
+        feed = self._parse_feed_from_response(response)
         entries = feed.get("entries")
         if len(entries) < 1:
             raise CannotReturn()
@@ -1286,7 +1338,8 @@ def release_hold(self, patron, pin, licensepool):
             raise CannotReleaseHold()
         if response.status_code == 404:
             raise NotOnHold()
-        feed = feedparser.parse(str(response.content))
+
+        feed = self._parse_feed_from_response(response)
         entries = feed.get("entries")
         if len(entries) < 1:
             raise CannotReleaseHold()
@@ -1325,7 +1378,7 @@ def patron_activity(self, patron, pin):
             if response.status_code == 404:
                 # 404 is returned when the loan has been deleted. Leave this loan out of the result.
                 continue
-            feed = feedparser.parse(str(response.content))
+            feed = self._parse_feed_from_response(response)
             entries = feed.get("entries")
             if len(entries) < 1:
                 raise CirculationException()
@@ -1354,7 +1407,7 @@ def patron_activity(self, patron, pin):
             if response.status_code == 404:
                 # 404 is returned when the hold has been deleted. Leave this hold out of the result.
                 continue
-            feed = feedparser.parse(str(response.content))
+            feed = self._parse_feed_from_response(response)
             entries = feed.get("entries")
             if len(entries) < 1:
                 raise CirculationException()
@@ -1518,3 +1571,39 @@ def _get(self, url, patron=None, headers=None, allowed_response_codes=None):
         self.request_args.append((patron, headers, allowed_response_codes))
         response = self.responses.pop()
         return HTTP._process_response(url, response, allowed_response_codes=allowed_response_codes)
+
+
+class ODLExpiredItemsReaper(IdentifierSweepMonitor):
+    """Check for books that are in the local collection but have left our
+    Overdrive collection.
+    """
+    SERVICE_NAME = "ODL Expired Items Reaper"
+    PROTOCOL = ODLAPI.NAME
+
+    def __init__(self, _db, collection):
+        super(ODLExpiredItemsReaper, self).__init__(_db, collection)
+
+    def process_item(self, identifier):
+        for licensepool in identifier.licensed_through:
+            licenses_owned = licensepool.licenses_owned
+            licenses_available = licensepool.licenses_available
+
+            for license in licensepool.licenses:
+                if license.is_expired:
+                    licenses_owned -= 1
+                    licenses_available -= 1
+
+            if licenses_owned != licensepool.licenses_owned or licenses_available != licensepool.licenses_available:
+                licenses_owned = max(licenses_owned, 0)
+                licenses_available = max(licenses_available, 0)
+
+                circulation_data = CirculationData(
+                    data_source=licensepool.data_source,
+                    primary_identifier=identifier,
+                    licenses_owned=licenses_owned,
+                    licenses_available=licenses_available,
+                    licenses_reserved=0,
+                    patrons_in_hold_queue=0,
+                )
+
+                circulation_data.apply(self._db, self.collection)

api/odl2.py

@@ -1,11 +1,12 @@
 import json
 import logging
 
+from core import util
 from contextlib2 import contextmanager
 from flask_babel import lazy_gettext as _
 from webpub_manifest_parser.opds2.registry import OPDS2LinkRelationsRegistry
 
-from api.odl import ODLAPI
+from api.odl import ODLAPI, ODLExpiredItemsReaper
 from core.metadata_layer import FormatData, LicenseData
 from core.model import DeliveryMechanism, Edition, MediaTypes, RightsStatus
 from core.model.configuration import (
@@ -239,6 +240,13 @@ def _extract_publication_metadata(self, feed, publication, data_source_name):
                     expires = license.metadata.terms.expires
                     concurrent_checkouts = license.metadata.terms.concurrency
 
+                    if expires:
+                        expires = util.datetime_helpers.to_utc(expires)
+                        now = util.datetime_helpers.utc_now()
+
+                        if expires <= now:
+                            continue
+
                 licenses_owned += int(concurrent_checkouts or 0)
                 licenses_available += int(available_checkouts or 0)
 
@@ -270,3 +278,11 @@ class ODL2ImportMonitor(OPDS2ImportMonitor):
 
     PROTOCOL = ODL2Importer.NAME
     SERVICE_NAME = "ODL 2.x Import Monitor"
+
+
+class ODL2ExpiredItemsReaper(ODLExpiredItemsReaper):
+    """Check for books that are in the local collection but have left our
+    Overdrive collection.
+    """
+    SERVICE_NAME = "ODL 2 Expired Items Reaper"
+    PROTOCOL = ODL2Importer.NAME

bin/odl2_reaper

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+"""Monitor the Overdrive collections by looking for books with lost licenses."""
+import os
+import sys
+bin_dir = os.path.split(__file__)[0]
+package_dir = os.path.join(bin_dir, "..")
+sys.path.append(os.path.abspath(package_dir))
+from core.scripts import RunCollectionMonitorScript
+from api.odl2 import ODL2ExpiredItemsReaper
+RunCollectionMonitorScript(ODL2ExpiredItemsReaper).run()

bin/odl_reaper

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+"""Monitor the Overdrive collections by looking for books with lost licenses."""
+import os
+import sys
+bin_dir = os.path.split(__file__)[0]
+package_dir = os.path.join(bin_dir, "..")
+sys.path.append(os.path.abspath(package_dir))
+from core.scripts import RunCollectionMonitorScript
+from api.odl import ODLExpiredItemsReaper
+RunCollectionMonitorScript(ODLExpiredItemsReaper).run()