feat: import books from Wikisource

openlibrary/catalog/add_book/load_book.py

@@ -142,8 +142,7 @@ def walk_redirects(obj, seen):
             seen.add(obj['key'])
         return obj
 
-    # Try for open library ID, then other external identifiers (wikidata, bookbrainz, etc).
-    # If not found, try for an 'exact' (case-insensitive) name match, but fall back to alternate_names,
+    # Try for an 'exact' (case-insensitive) name match, but fall back to alternate_names,
     # then last name with identical birth and death dates (that are not themselves `None` or '').
     name = author["name"].replace("*", r"\*")
     queries = [
@@ -156,13 +155,6 @@ def walk_redirects(obj, seen):
             "death_date~": f"*{extract_year(author.get('death_date', '')) or -1}*",
         },  # Use `-1` to ensure an empty string from extract_year doesn't match empty dates.
     ]
-    if identifiers := author.get("identifiers"):
-        for id in identifiers:
-            queries.insert(
-                0, {"type": "/type/author", f"identifiers.{id}~": identifiers[id]}
-            )
-    if key := author.get("key"):
-        queries.insert(0, {"type": "/type/author", "key~": key})
     for query in queries:
         if reply := list(web.ctx.site.things(query)):
             break
@@ -254,20 +246,9 @@ def import_author(author: dict[str, Any], eastern=False) -> "Author | dict[str,
             new['death_date'] = author['death_date']
         return new
     a = {'type': {'key': '/type/author'}}
-    for f in (
-        'name',
-        'title',
-        'personal_name',
-        'birth_date',
-        'death_date',
-        'date',
-        'remote_ids',
-    ):
+    for f in 'name', 'title', 'personal_name', 'birth_date', 'death_date', 'date':
         if f in author:
             a[f] = author[f]
-    # Import record hitting endpoing should list external IDs under "identifiers", but needs to be "remote_ids" when going into the DB.
-    if "identifiers" in author:
-        a["remote_ids"] = author["identifiers"]
     return a
 
 
@@ -314,4 +295,4 @@ def build_query(rec: dict[str, Any]) -> dict[str, Any]:
                 book[k] = {'type': t, 'value': v}
         else:
             book[k] = v
-    return book
+    return book

openlibrary/catalog/add_book/tests/test_load_book.py

@@ -136,77 +136,9 @@ def test_author_wildcard_match_with_no_matches_creates_author_with_wildcard(
         new_author_name = import_author(author)
         assert author["name"] == new_author_name["name"]
 
-    def test_first_match_ol_key(self, mock_site):
+    def test_first_match_priority_name_and_dates(self, mock_site):
         """
-        Highest priority match is OL key.
-        """
-        self.add_three_existing_authors(mock_site)
-
-        # Author with VIAF
-        author = {
-            "name": "William H. Brewer",
-            "key": "/authors/OL3A",
-            "type": {"key": "/type/author"},
-            "identifiers": {"viaf": "12345678"},
-        }
-
-        # Another author with VIAF
-        author_different_key = {
-            "name": "William Brewer",
-            "key": "/authors/OL4A",
-            "type": {"key": "/type/author"},
-            "identifiers": {"viaf": "87654321"},
-        }
-
-        mock_site.save(author)
-        mock_site.save(author_different_key)
-
-        # Look for exact match on OL ID, regardless of other fields.
-        # We ideally shouldn't ever have a case where different authors have the same VIAF, but this demonstrates priority.
-        searched_author = {
-            "name": "William H. Brewer",
-            "key": "/authors/OL4A",
-            "identifiers": {"viaf": "12345678"},
-        }
-        found = import_author(searched_author)
-        assert found.key == author_different_key["key"]
-
-    def test_second_match_strong_identifier(self, mock_site):
-        """
-        Next highest priority match is any other strong identifier, such as VIAF, Goodreads ID, Amazon ID, etc.
-        """
-        self.add_three_existing_authors(mock_site)
-
-        # Author with VIAF
-        author = {
-            "name": "William H. Brewer",
-            "key": "/authors/OL3A",
-            "type": {"key": "/type/author"},
-            "identifiers": {"viaf": "12345678"},
-        }
-
-        # Another author with VIAF
-        author_different_viaf = {
-            "name": "William Brewer",
-            "key": "/authors/OL4A",
-            "type": {"key": "/type/author"},
-            "identifiers": {"viaf": "87654321"},
-        }
-
-        mock_site.save(author)
-        mock_site.save(author_different_viaf)
-
-        # Look for exact match on VIAF, regardless of name field.
-        searched_author = {
-            "name": "William Brewer",
-            "identifiers": {"viaf": "12345678"},
-        }
-        found = import_author(searched_author)
-        assert found.key == author["key"]
-
-    def test_third_match_priority_name_and_dates(self, mock_site):
-        """
-        Next highest priority match is name, birth date, and death date.
+        Highest priority match is name, birth date, and death date.
         """
         self.add_three_existing_authors(mock_site)
 
@@ -269,7 +201,7 @@ def test_non_matching_birth_death_creates_new_author(self, mock_site):
         assert isinstance(found, dict)
         assert found["death_date"] == searched_and_not_found_author["death_date"]
 
-    def test_match_priority_alternate_names_and_dates(self, mock_site):
+    def test_second_match_priority_alternate_names_and_dates(self, mock_site):
         """
         Matching, as a unit, alternate name, birth date, and death date, get
         second match priority.
@@ -392,4 +324,4 @@ def test_birth_and_death_date_match_is_on_year_strings(self, mock_site):
             "death_date": "November 1910",
         }
         found = import_author(searched_author)
-        assert found.key == author["key"]
+        assert found.key == author["key"]

openlibrary/components/AuthorIdentifiers.vue

@@ -35,8 +35,6 @@ const identifierPatterns  = {
     lc_naf: /^n[bors]?[0-9]+$/,
     amazon: /^B[0-9A-Za-z]{9}$/,
     youtube: /^@[A-Za-z0-9_\-.]{3,30}/,
-    imdb: /^\w{2}\d+$/,
-    opac_sbn: /^\D{2}[A-Z0-3]V\d{6}$/,
 }
 export default {
     // Props are for external options; if a subelement of this is modified,

openlibrary/plugins/importapi/import_edition_builder.py

@@ -100,15 +100,10 @@ def add_list(self, key, val):
             self.edition_dict[key] = [val]
 
     def add_author(self, key, val):
-        if isinstance(val, dict):
-            author_dict = val
-            if "name" in author_dict:
-                author_dict['personal_name'] = author_dict['name']
-            self.add_list('authors', author_dict)
-        else:
-            self.add_list(
-                'authors', {'personal_name': val, 'name': val, 'entity_type': 'person'}
-            )
+        # We don't know birth_date or death_date.
+        # Should name and personal_name be the same value?
+        author_dict = {'personal_name': val, 'name': val, 'entity_type': 'person'}
+        self.add_list('authors', author_dict)
 
     def add_illustrator(self, key, val):
         self.add_list('contributions', val + ' (Illustrator)')

requirements.txt

@@ -17,8 +17,6 @@ isbnlib==3.10.14
 luqum==0.11.0
 lxml==4.9.4
 multipart==0.2.4
-mwparserfromhell==0.6.6
-nameparser==1.1.3
 Pillow==10.4.0
 psycopg2==2.9.6
 pydantic==2.4.0
@@ -32,4 +30,3 @@ sentry-sdk==1.28.1
 simplejson==3.19.1
 statsd==4.0.1
 validate_email==1.3
-wikitextparser==0.56.1