From 89a4ce883ed68e3de626e717bd1067ec6960fc16 Mon Sep 17 00:00:00 2001 From: Samuele Kaplun Date: Wed, 28 Sep 2016 22:55:01 +0200 Subject: [PATCH] general: upstream invenio-pidstore * No longer uses INSPIRE fork of invenio-pidstore. * Introduces new get_pid_type_for() and get_endpoint_from_pid_type() helpers to map between endpoints and the corresponding pid_types, now that pid_types are constrained in length. (closes #1579) Signed-off-by: Samuele Kaplun --- inspirehep/config.py | 42 +++++++++---------- inspirehep/modules/pidstore/providers.py | 23 ++++++++-- inspirehep/modules/theme/views.py | 11 +++-- inspirehep/utils/record_getter.py | 10 +++-- requirements.txt | 3 -- setup.py | 1 + .../integration/disambiguation/test_daemon.py | 7 ++-- .../integration/disambiguation/test_logic.py | 3 +- .../disambiguation/test_receivers.py | 5 ++- .../disambiguation/test_records.py | 5 ++- .../disambiguation/test_workflows.py | 17 ++++---- tests/integration/test_record.py | 3 +- 12 files changed, 77 insertions(+), 53 deletions(-) diff --git a/inspirehep/config.py b/inspirehep/config.py index 357389d01e..7461fd4ba5 100644 --- a/inspirehep/config.py +++ b/inspirehep/config.py @@ -258,7 +258,7 @@ def _(x): # ======= RECORDS_REST_ENDPOINTS = dict( literature=dict( - pid_type='literature', + pid_type='lit', pid_minter='inspire_recid_minter', pid_fetcher='inspire_recid_fetcher', search_class='inspirehep.modules.search:LiteratureSearch', @@ -322,7 +322,7 @@ def _(x): record_class='inspirehep.modules.records.es_record:ESRecord' ), literature_db=dict( - pid_type='literature', + pid_type='lit', search_class='inspirehep.modules.search:LiteratureSearch', record_serializers={ 'application/json': ('invenio_records_rest.serializers' @@ -339,7 +339,7 @@ def _(x): ), authors=dict( - pid_type='authors', + pid_type='aut', pid_minter='inspire_recid_minter', pid_fetcher='inspire_recid_fetcher', search_class='inspirehep.modules.search:AuthorsSearch', @@ -362,7 +362,7 @@ def _(x): search_factory_imp='inspirehep.modules.search.query:inspire_search_factory', ), authors_citations=dict( - pid_type='authors', + pid_type='aut', pid_minter='inspire_recid_minter', pid_fetcher='inspire_recid_fetcher', search_class='inspirehep.modules.search:AuthorsSearch', @@ -386,7 +386,7 @@ def _(x): ':inspire_search_factory'), ), authors_coauthors=dict( - pid_type='authors', + pid_type='aut', pid_minter='inspire_recid_minter', pid_fetcher='inspire_recid_fetcher', search_class='inspirehep.modules.search:AuthorsSearch', @@ -410,7 +410,7 @@ def _(x): ':inspire_search_factory'), ), authors_publications=dict( - pid_type='authors', + pid_type='aut', pid_minter='inspire_recid_minter', pid_fetcher='inspire_recid_fetcher', search_class='inspirehep.modules.search:AuthorsSearch', @@ -434,7 +434,7 @@ def _(x): ':inspire_search_factory'), ), authors_stats=dict( - pid_type='authors', + pid_type='aut', pid_minter='inspire_recid_minter', pid_fetcher='inspire_recid_fetcher', search_class='inspirehep.modules.search:AuthorsSearch', @@ -458,7 +458,7 @@ def _(x): ':inspire_search_factory'), ), data=dict( - pid_type='data', + pid_type='dat', pid_minter='inspire_recid_minter', pid_fetcher='inspire_recid_fetcher', search_class='inspirehep.modules.search:DataSearch', @@ -481,7 +481,7 @@ def _(x): search_factory_imp='inspirehep.modules.search.query:inspire_search_factory', ), conferences=dict( - pid_type='conferences', + pid_type='con', pid_minter='inspire_recid_minter', pid_fetcher='inspire_recid_fetcher', search_class='inspirehep.modules.search:ConferencesSearch', @@ -504,7 +504,7 @@ def _(x): search_factory_imp='inspirehep.modules.search.query:inspire_search_factory', ), jobs=dict( - pid_type='jobs', + pid_type='job', pid_minter='inspire_recid_minter', pid_fetcher='inspire_recid_fetcher', search_class='inspirehep.modules.search:JobsSearch', @@ -527,7 +527,7 @@ def _(x): search_factory_imp='inspirehep.modules.search.query:inspire_search_factory', ), institutions=dict( - pid_type='institutions', + pid_type='ins', pid_minter='inspire_recid_minter', pid_fetcher='inspire_recid_fetcher', search_class='inspirehep.modules.search:InstitutionsSearch', @@ -550,7 +550,7 @@ def _(x): search_factory_imp='inspirehep.modules.search.query:inspire_search_factory', ), experiments=dict( - pid_type='experiments', + pid_type='exp', pid_minter='inspire_recid_minter', pid_fetcher='inspire_recid_fetcher', search_class='inspirehep.modules.search:ExperimentsSearch', @@ -573,7 +573,7 @@ def _(x): search_factory_imp='inspirehep.modules.search.query:inspire_search_factory', ), journals=dict( - pid_type='journals', + pid_type='jou', pid_minter='inspire_recid_minter', pid_fetcher='inspire_recid_fetcher', search_class='inspirehep.modules.search:JournalsSearch', @@ -599,7 +599,7 @@ def _(x): RECORDS_UI_ENDPOINTS = dict( literature=dict( - pid_type='literature', + pid_type='lit', route='/literature/', template='inspirehep_theme/format/record/' 'Inspire_Default_HTML_detailed.tpl', @@ -607,7 +607,7 @@ def _(x): permission_factory_imp='invenio_records_rest.utils:allow_all', ), authors=dict( - pid_type='authors', + pid_type='aut', route='/authors/', template='inspirehep_theme/format/record/' 'authors/Author_HTML_detailed.html', @@ -615,41 +615,41 @@ def _(x): permission_factory_imp='invenio_records_rest.utils:allow_all', ), data=dict( - pid_type='data', + pid_type='dat', route='/data/', template='inspirehep_theme/format/record/Data_HTML_detailed.tpl', permission_factory_imp='invenio_records_rest.utils:allow_all', ), conferences=dict( - pid_type='conferences', + pid_type='con', route='/conferences/', template='inspirehep_theme/format/record/Conference_HTML_detailed.tpl', record_class='inspirehep.modules.records.wrappers:ConferencesRecord', permission_factory_imp='invenio_records_rest.utils:allow_all', ), jobs=dict( - pid_type='jobs', + pid_type='job', route='/jobs/', template='inspirehep_theme/format/record/Job_HTML_detailed.tpl', record_class='inspirehep.modules.records.wrappers:JobsRecord', permission_factory_imp='invenio_records_rest.utils:allow_all', ), institutions=dict( - pid_type='institutions', + pid_type='ins', route='/institutions/', template='inspirehep_theme/format/record/Institution_HTML_detailed.tpl', record_class='inspirehep.modules.records.wrappers:InstitutionsRecord', permission_factory_imp='invenio_records_rest.utils:allow_all', ), experiments=dict( - pid_type='experiments', + pid_type='exp', route='/experiments/', template='inspirehep_theme/format/record/Experiment_HTML_detailed.tpl', record_class='inspirehep.modules.records.wrappers:ExperimentsRecord', permission_factory_imp='invenio_records_rest.utils:allow_all', ), journals=dict( - pid_type='journals', + pid_type='jou', route='/journals/', template='inspirehep_theme/format/record/Journal_HTML_detailed.tpl', record_class='inspirehep.modules.records.wrappers:JournalsRecord', diff --git a/inspirehep/modules/pidstore/providers.py b/inspirehep/modules/pidstore/providers.py index 8a4c694c72..666fa56366 100644 --- a/inspirehep/modules/pidstore/providers.py +++ b/inspirehep/modules/pidstore/providers.py @@ -35,6 +35,11 @@ from invenio_pidstore.providers.base import BaseProvider +from inspirehep.config import RECORDS_REST_ENDPOINTS + + +_PID_TYPE_TO_ENDPOINT = {value['pid_type']: key for key, value in RECORDS_REST_ENDPOINTS.items()} + def _get_next_pid_from_legacy(): """Reserve the next pid on legacy. @@ -84,8 +89,18 @@ def create(cls, object_type=None, object_uuid=None, **kwargs): @staticmethod def schema_to_pid_type(schema): - pid_type = os.path.splitext(os.path.basename(schema))[0] - if pid_type == 'hep': + endpoint = os.path.splitext(os.path.basename(schema))[0] + if endpoint == 'hep': # FIXME: temp hack until we rename hep.json to literature.json - return 'literature' - return pid_type + endpoint = 'literature' + return get_pid_type_for(endpoint) + + +def get_pid_type_for(endpoint): + """Get the pid_type code corresponding to the recid of the given endpoint.""" + return RECORDS_REST_ENDPOINTS[endpoint]['pid_type'] + + +def get_endpoint_from_pid_type(pid_type): + """Return the endpoint corresponding to the given pid_type.""" + return _PID_TYPE_TO_ENDPOINT[pid_type] diff --git a/inspirehep/modules/theme/views.py b/inspirehep/modules/theme/views.py index 0674416379..41ea12f00b 100644 --- a/inspirehep/modules/theme/views.py +++ b/inspirehep/modules/theme/views.py @@ -45,6 +45,7 @@ from invenio_pidstore.models import PersistentIdentifier from invenio_search import current_search_client +from inspirehep.modules.pidstore.providers import get_pid_type_for, get_endpoint_from_pid_type from inspirehep.modules.records.conference_series import ( CONFERENCE_CATEGORIES_TO_SERIES, ) @@ -226,7 +227,7 @@ def ajax_references(): recid = request.args.get('recid', '') collection = request.args.get('collection', '') - pid = PersistentIdentifier.get(collection, recid) + pid = PersistentIdentifier.get(get_pid_type_for(collection), recid) record = LiteratureSearch().get_source(pid.object_uuid) @@ -244,7 +245,7 @@ def ajax_citations(): recid = request.args.get('recid', '') collection = request.args.get('collection', '') - pid = PersistentIdentifier.get(collection, recid) + pid = PersistentIdentifier.get(get_pid_type_for(collection), recid) record = LiteratureSearch().get_source(pid.object_uuid) @@ -471,7 +472,7 @@ def ajax_institutions_experiments(): """Datatable handler to get experiments in an institution.""" recid = request.args.get('recid', '') - pid = PersistentIdentifier.get('institutions', recid) + pid = PersistentIdentifier.get(get_pid_type_for('institutions'), recid) record = InstitutionsSearch().get_source(pid.object_uuid) try: @@ -570,7 +571,9 @@ def record(control_number): abort(404) return redirect('/{collection}/{control_number}'.format( - collection=pid.pid_type, control_number=control_number)), 301 + collection=get_endpoint_from_pid_type(pid.pid_type), + control_number=control_number + )), 301 # diff --git a/inspirehep/utils/record_getter.py b/inspirehep/utils/record_getter.py index 73c1e750ce..e8a04bd805 100644 --- a/inspirehep/utils/record_getter.py +++ b/inspirehep/utils/record_getter.py @@ -32,6 +32,8 @@ from invenio_pidstore.models import PersistentIdentifier from invenio_records.api import Record +from inspirehep.modules.pidstore.providers import get_pid_type_for, get_endpoint_from_pid_type + class RecordGetterError(Exception): @@ -63,7 +65,7 @@ def wrapper(*args, **kwargs): @raise_record_getter_error_and_log def get_es_record(record_type, recid, **kwargs): - pid = PersistentIdentifier.get(record_type, recid) + pid = PersistentIdentifier.get(get_pid_type_for(record_type), recid) search_conf = current_app.config['RECORDS_REST_ENDPOINTS'][record_type] search_class = import_string(search_conf['search_class'])() return search_class.get_source(pid.object_uuid, **kwargs) @@ -80,7 +82,7 @@ def get_es_records(record_type, recids, **kwargs): """ uuids = PersistentIdentifier.query.filter( PersistentIdentifier.pid_value.in_(recids), - PersistentIdentifier.pid_type == record_type + PersistentIdentifier.pid_type == get_pid_type_for(record_type) ).all() uuids = [str(uuid.object_uuid) for uuid in uuids] search_conf = current_app.config['RECORDS_REST_ENDPOINTS'][record_type] @@ -91,12 +93,12 @@ def get_es_records(record_type, recids, **kwargs): @raise_record_getter_error_and_log def get_es_record_by_uuid(uuid): pid = PersistentIdentifier.query.filter_by(object_uuid=uuid).one() - search_conf = current_app.config['RECORDS_REST_ENDPOINTS'][pid.pid_type] + search_conf = current_app.config['RECORDS_REST_ENDPOINTS'][get_endpoint_from_pid_type(pid.pid_type)] search_class = import_string(search_conf['search_class'])() return search_class.get_source(uuid) @raise_record_getter_error_and_log def get_db_record(record_type, recid): - pid = PersistentIdentifier.get(record_type, recid) + pid = PersistentIdentifier.get(get_pid_type_for(record_type), recid) return Record.get_record(pid.object_uuid) diff --git a/requirements.txt b/requirements.txt index ca93a17a01..f5e6a880cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,9 +23,6 @@ # FIXME temporary branch for testing -e git+https://github.com/inspirehep/invenio-query-parser.git@invenio3-inspire#egg=invenio-query-parser==0.6.0 -# INSPIRE forks of Invenio packages --e git+https://github.com/inspirehep/invenio-pidstore.git#egg=invenio-pidstore - # Workflows and Holding Pen related dependencies -e git+https://github.com/inveniosoftware/invenio-classifier.git#egg=invenio-classifier -e git+https://github.com/inspirehep/invenio-matcher.git#egg=invenio-matcher diff --git a/setup.py b/setup.py index 6b96ffc9a0..6069e60f1a 100644 --- a/setup.py +++ b/setup.py @@ -58,6 +58,7 @@ 'invenio-logging>=1.0.0a3', 'invenio-mail>=1.0.0a4', 'invenio-oauthclient>=1.0.0a8', + 'invenio-pidstore>=1.0.0a9', 'invenio-records>=1.0.0a16', # Add [versioning] in the future 'invenio-rest[cors]>=1.0.0a7', 'invenio-search>=1.0.0a7', diff --git a/tests/integration/disambiguation/test_daemon.py b/tests/integration/disambiguation/test_daemon.py index 72aed38a10..3658275840 100644 --- a/tests/integration/disambiguation/test_daemon.py +++ b/tests/integration/disambiguation/test_daemon.py @@ -29,6 +29,7 @@ from invenio_search import current_search_client as es from inspirehep.modules.disambiguation.models import DisambiguationRecord +from inspirehep.modules.pidstore.providers import get_pid_type_for from inspirehep.utils.record_getter import get_es_record_by_uuid @@ -42,7 +43,7 @@ def test_count_phonetic_block_dispatched(small_app): # Signature #1. glashow_record_id = str(PersistentIdentifier.get( - "literature", 4328).object_uuid) + get_pid_type_for("literature"), 4328).object_uuid) glashow_record = get_es_record_by_uuid(glashow_record_id) # Add phonetic block to the record. @@ -53,7 +54,7 @@ def test_count_phonetic_block_dispatched(small_app): # Signature #2. higgs_record_id_first = str(PersistentIdentifier.get( - "literature", 1358492).object_uuid) + get_pid_type_for("literature"), 1358492).object_uuid) higgs_record_first = get_es_record_by_uuid(higgs_record_id_first) # Add phonetic block to the record. @@ -64,7 +65,7 @@ def test_count_phonetic_block_dispatched(small_app): # Signature #3. higgs_record_id_second = str(PersistentIdentifier.get( - "literature", 11883).object_uuid) + get_pid_type_for("literature"), 11883).object_uuid) higgs_record_second = get_es_record_by_uuid(higgs_record_id_second) # Add phonetic block to the record. diff --git a/tests/integration/disambiguation/test_logic.py b/tests/integration/disambiguation/test_logic.py index d05a0c841a..fcddb8560e 100644 --- a/tests/integration/disambiguation/test_logic.py +++ b/tests/integration/disambiguation/test_logic.py @@ -25,11 +25,12 @@ from invenio_pidstore.models import PersistentIdentifier from inspirehep.modules.disambiguation.logic import _create_distance_signature +from inspirehep.modules.pidstore.providers import get_pid_type_for def test_create_distance_signature_method(small_app): """Test the method responsible for creating data in Beard format.""" - pid = PersistentIdentifier.get("literature", 4328) + pid = PersistentIdentifier.get(get_pid_type_for("literature"), 4328) publication_id = str(pid.object_uuid) signatures_map = { diff --git a/tests/integration/disambiguation/test_receivers.py b/tests/integration/disambiguation/test_receivers.py index ceb674ad4f..2c67d44a0b 100644 --- a/tests/integration/disambiguation/test_receivers.py +++ b/tests/integration/disambiguation/test_receivers.py @@ -35,6 +35,7 @@ append_new_record_to_queue, append_updated_record_to_queue, ) +from inspirehep.modules.pidstore.providers import get_pid_type_for class _IdDict(dict): @@ -92,7 +93,7 @@ def test_append_new_record_to_queue_method_not_hep_record(small_app): def test_append_updated_record_to_queue(small_app): """Test the receiver responsible for queuing updated HEP records.""" - pid = PersistentIdentifier.get("literature", 4328) + pid = PersistentIdentifier.get(get_pid_type_for("literature"), 4328) publication_id = str(pid.object_uuid) record = Record.get_record(publication_id) @@ -159,7 +160,7 @@ def test_append_updated_record_to_queue_not_hep_record(small_app): def test_append_updated_record_to_queue_same_data(small_app): """Check if for the same record, the receiver will skip the publication.""" - pid = PersistentIdentifier.get("literature", 11883) + pid = PersistentIdentifier.get(get_pid_type_for("literature"), 11883) publication_id = str(pid.object_uuid) record = Record.get_record(publication_id) diff --git a/tests/integration/disambiguation/test_records.py b/tests/integration/disambiguation/test_records.py index bf1cb1b39f..03577e1183 100644 --- a/tests/integration/disambiguation/test_records.py +++ b/tests/integration/disambiguation/test_records.py @@ -29,6 +29,7 @@ _get_author_schema, create_author, ) +from inspirehep.modules.pidstore.providers import get_pid_type_for def test_get_author_schema_method(small_app): @@ -48,7 +49,7 @@ def test_create_author_method(small_app): } recid = create_author(signature) - pid = PersistentIdentifier.get("authors", recid) + pid = PersistentIdentifier.get(get_pid_type_for("authors"), recid) record = Record.get_record(pid.object_uuid) assert record['collections'] == [{'primary': 'HEPNAMES'}] @@ -61,7 +62,7 @@ def test_update_authors_recid_method(small_app): """Test the method responsible for updating author's recid.""" from inspirehep.modules.disambiguation.tasks import update_authors_recid - pid = PersistentIdentifier.get("literature", 4328) + pid = PersistentIdentifier.get(get_pid_type_for("literature"), 4328) publication_id = str(pid.object_uuid) signature = Record.get_record(publication_id)['authors'][0]['uuid'] diff --git a/tests/integration/disambiguation/test_workflows.py b/tests/integration/disambiguation/test_workflows.py index 768689d7b5..e5df758a65 100644 --- a/tests/integration/disambiguation/test_workflows.py +++ b/tests/integration/disambiguation/test_workflows.py @@ -32,6 +32,7 @@ from invenio_records.api import Record from invenio_search import current_search_client as es +from inspirehep.modules.pidstore.providers import get_pid_type_for from inspirehep.utils.record_getter import get_es_record_by_uuid @@ -54,7 +55,7 @@ def test_single_signature_with_no_profile(small_app): update_authors_recid ) - record_id = str(PersistentIdentifier.get("literature", 11883).object_uuid) + record_id = str(PersistentIdentifier.get(get_pid_type_for("literature"), 11883).object_uuid) record = get_es_record_by_uuid(record_id) author_uuid = record['authors'][0]['uuid'] @@ -81,7 +82,7 @@ def test_match_signature_with_existing_profile(small_app): ) old_record_id = str(PersistentIdentifier.get( - "literature", 11883).object_uuid) + get_pid_type_for("literature"), 11883).object_uuid) old_record = get_es_record_by_uuid(old_record_id) old_author_uuid = old_record['authors'][0]['uuid'] @@ -92,7 +93,7 @@ def test_match_signature_with_existing_profile(small_app): es.indices.refresh('records-hep') record_id = str(PersistentIdentifier.get( - "literature", 1358492).object_uuid) + get_pid_type_for("literature"), 1358492).object_uuid) record = get_es_record_by_uuid(record_id) author_uuid = record['authors'][0]['uuid'] @@ -123,7 +124,7 @@ def test_appoint_profile_from_claimed_signature(small_app): ) old_record_id = str(PersistentIdentifier.get( - "literature", 11883).object_uuid) + get_pid_type_for("literature"), 11883).object_uuid) old_record = get_es_record_by_uuid(old_record_id) old_author_uuid = old_record['authors'][0]['uuid'] @@ -135,7 +136,7 @@ def test_appoint_profile_from_claimed_signature(small_app): es.indices.refresh('records-hep') record_id = str(PersistentIdentifier.get( - "literature", 1358492).object_uuid) + get_pid_type_for("literature"), 1358492).object_uuid) record = get_es_record_by_uuid(record_id) author_uuid = record['authors'][0]['uuid'] @@ -171,7 +172,7 @@ def test_solve_claim_conflicts(small_app): # Claimed signature #1. glashow_record_id_claimed = str(PersistentIdentifier.get( - "literature", 4328).object_uuid) + get_pid_type_for("literature"), 4328).object_uuid) glashow_record_claimed = get_es_record_by_uuid( glashow_record_id_claimed) glashow_record_uuid_claimed = glashow_record_claimed[ @@ -187,7 +188,7 @@ def test_solve_claim_conflicts(small_app): # Claimed signature #2. higgs_record_id_claimed = str(PersistentIdentifier.get( - "literature", 1358492).object_uuid) + get_pid_type_for("literature"), 1358492).object_uuid) higgs_record_claimed = get_es_record_by_uuid( higgs_record_id_claimed) higgs_record_uuid_claimed = higgs_record_claimed[ @@ -203,7 +204,7 @@ def test_solve_claim_conflicts(small_app): # Not claimed signature. higgs_record_id_not_claimed = str(PersistentIdentifier.get( - "literature", 11883).object_uuid) + get_pid_type_for("literature"), 11883).object_uuid) higgs_record_not_claimed = get_es_record_by_uuid( higgs_record_id_not_claimed) higgs_record_uuid_not_claimed = higgs_record_not_claimed[ diff --git a/tests/integration/test_record.py b/tests/integration/test_record.py index d069e0f2e8..b3fda76928 100644 --- a/tests/integration/test_record.py +++ b/tests/integration/test_record.py @@ -32,6 +32,7 @@ from inspirehep.dojson.hep import hep from inspirehep.modules.migrator.tasks.records import record_upsert +from inspirehep.modules.pidstore.providers import get_pid_type_for from inspirehep.utils.record_getter import get_db_record @@ -105,7 +106,7 @@ def _delete_record_from_everywhere(collection, record_control_number): ri.delete(record) record.delete(force=True) - pid = PersistentIdentifier.get(collection, record_control_number) + pid = PersistentIdentifier.get(get_pid_type_for(collection), record_control_number) PersistentIdentifier.delete(pid) object_uuid = pid.object_uuid