From 1ad31fbc72c633a6538e30efc3d8793bee283c0f Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Fri, 18 Aug 2023 14:42:24 -0600 Subject: [PATCH 01/48] Started annotation for Germ Accession import --- .../GermplasmAccessionImporter.php | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php new file mode 100644 index 0000000..5a1effc --- /dev/null +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -0,0 +1,43 @@ + + *
  • Germplasm Name: Name of this germplasm accession.
    • + *
  • External Database: The institution who assigned the following accession.
    • + *
  • Accession Number: A unique identifier for the accession.
    • + *
  • Germplasm Genus: The genus of the accession.
    • + *
  • Germplasm Species: The species of the accession.
    • + *
  • Germplasm Subtaxa: Subtaxon can be used to store any additional taxonomic identifier.
    • + *
  • Institute Code: The code for the Institute that has bred the material.
    • + *
  • Institute Name: The name of the Institute that has bred the material.
    • + *
  • Country of Origin Code: 3-letter ISO 3166-1 code of the country in which the sample was originally.
    • + *
  • Biological Status of Accession: The 3 digit code representing the biological status of the accession.
    • + *
  • Breeding Method: The unique identifier for the breeding method used to create this germplasm.
    • + *
  • Pedigree: The cross name and optional selection history.
    • + *
  • Synonyms: The synonyms of the accession.
    • + * ", + * file_upload = True, + * file_load = True, + * file_remote = True, + * file_required = True, + * cardinality = 1, + * + * ) + */ +class GermplasmAccessionImporter extends ChadoImporterBase { + +} From 70976fb8b63089fcd37c0bdd14343a4fad2e3d40 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 21 Aug 2023 14:44:50 -0600 Subject: [PATCH 02/48] Completed form function --- .../GermplasmAccessionImporter.php | 93 +++++++++++++++---- 1 file changed, 77 insertions(+), 16 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 5a1effc..26f068b 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -15,29 +15,90 @@ * file_types = {"tsv", "txt"}, * use_analysis = FALSE, * require_analysis = FALSE, + * upload_title = "Germplasm Accession Import", * upload_description = "Germplasm file should be a tab separated file with the following colums:
      - *
    1. Germplasm Name: Name of this germplasm accession.
      2. - *
    2. External Database: The institution who assigned the following accession.
      3. - *
    3. Accession Number: A unique identifier for the accession.
      4. - *
    4. Germplasm Genus: The genus of the accession.
      5. - *
    5. Germplasm Species: The species of the accession.
      6. - *
    6. Germplasm Subtaxa: Subtaxon can be used to store any additional taxonomic identifier.
      7. - *
    7. Institute Code: The code for the Institute that has bred the material.
      8. - *
    8. Institute Name: The name of the Institute that has bred the material.
      9. - *
    9. Country of Origin Code: 3-letter ISO 3166-1 code of the country in which the sample was originally.
      10. - *
    10. Biological Status of Accession: The 3 digit code representing the biological status of the accession.
      11. - *
    11. Breeding Method: The unique identifier for the breeding method used to create this germplasm.
      12. - *
    12. Pedigree: The cross name and optional selection history.
      13. - *
    13. Synonyms: The synonyms of the accession.
      14. - *
    ", +
  • Germplasm Name: Name of this germplasm accession.
    • +
  • External Database: The institution who assigned the following accession.
    • +
  • Accession Number: A unique identifier for the accession.
    • +
  • Germplasm Species: The species of the accession.
    • +
  • Germplasm Subtaxa: Subtaxon can be used to store any additional taxonomic identifier.
    • +
  • Institute Code: The code for the Institute that has bred the material.
    • +
  • Institute Name: The name of the Institute that has bred the material.
    • +
  • Country of Origin Code: 3-letter ISO 3166-1 code of the country in which the sample was originally.
    • +
  • Biological Status of Accession: The 3 digit code representing the biological status of the accession.
    • +
  • Breeding Method: The unique identifier for the breeding method used to create this germplasm.
    • +
  • Pedigree: The cross name and optional selection history.
    • +
  • Synonyms: The synonyms of the accession.
    • + ", + * button_text = "Import Germplasm Accessions", * file_upload = True, * file_load = True, * file_remote = True, * file_required = True, - * cardinality = 1, - * + * cardinality = 1 * ) */ class GermplasmAccessionImporter extends ChadoImporterBase { + /** + * {@inheritDoc} + */ + public function form($form, &$form_state) { + + // Select the entire genus field and make sure it is sorted and distinct + $connection = \Drupal::service('tripal_chado.database'); + $genus_query = $connection->select('1:organism', 'o') + ->fields('o',['genus']) + ->orderBy('genus') + ->distinct(); + $genus = $genus_query->execute()->fetchAllKeyed(0,0); + + $form['instructions'] = [ + '#weight' => -99, + '#markup' => ' +

    Load germplasm into database

    +

    Upload file must be in TSV (tab-separated values) or TXT format. Please confirm the file format and column order before upload.

    + ', + ]; + + $form['genus_name'] = [ + '#weight' => -80, + '#type' => 'select', + '#title' => t('Genus'), + '#options' => $genus, + '#required' => TRUE, + '#description' => t('Select the genus of the germplasm accessions in your file. If your file consists of multiple genus, it is best practice to separate it into one file per genus and upload each one individually.'), + ]; + + return $form; + } + + /** + * @see TripalImporter::formValidate() + */ + public function formValidate($form, &$form_state){ + + } + + /** + * @see TripalImporter::run() + */ + public function run(){ + + } + + /* + * {@inheritdoc} + */ + public function postRun() { + + } + + /** + * @see TripalImporter::formSubmit() + */ + public function formSubmit($form, &$form_state) { + + } + } From 843bb97231a034559ac469072102c75ad284c303 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 21 Aug 2023 17:10:23 -0600 Subject: [PATCH 03/48] Created configuration for cvterms used by the germplasm accession importer --- .../trpcultivate_germplasm.settings.yml | 13 ++++++ .../schema/trpcultivate_germplasm.schema.yml | 44 +++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 trpcultivate_germplasm/config/install/trpcultivate_germplasm.settings.yml create mode 100644 trpcultivate_germplasm/config/schema/trpcultivate_germplasm.schema.yml diff --git a/trpcultivate_germplasm/config/install/trpcultivate_germplasm.settings.yml b/trpcultivate_germplasm/config/install/trpcultivate_germplasm.settings.yml new file mode 100644 index 0000000..558ff03 --- /dev/null +++ b/trpcultivate_germplasm/config/install/trpcultivate_germplasm.settings.yml @@ -0,0 +1,13 @@ +terms: + accession: 0 + genus: 0 + species: 0 + name: 0 + institute_code: 0 + institute_name: 0 + pedigree: 0 + country_of_origin_code: 0 + synonym: 0 + biological_status_of_accession_code: 0 + breeding_method_DbId: 0 + subtaxa: 0 diff --git a/trpcultivate_germplasm/config/schema/trpcultivate_germplasm.schema.yml b/trpcultivate_germplasm/config/schema/trpcultivate_germplasm.schema.yml new file mode 100644 index 0000000..ff6ea36 --- /dev/null +++ b/trpcultivate_germplasm/config/schema/trpcultivate_germplasm.schema.yml @@ -0,0 +1,44 @@ +trpcultivate_germplasm.settings: + type: config_object + label: 'Germplasm Package Configuration' + mapping: + terms: + type: mapping + label: 'Package default terms' + mapping: + accession: + type: integer + label: 'The cvterm ID of the accession for a germplasm' + genus: + type: integer + label: 'The cvterm ID of the genus of a germplasm' + species: + type: integer + label: 'The cvterm ID of the species of a germplasm' + name: + type: integer + label: 'The cvterm ID of the name of a germplasm' + institute_code: + type: integer + label: 'The cvterm ID of the institute code where the germplasm was bred' + institute_name: + type: integer + label: 'The cvterm ID of the institute name where the germplasm was bred' + pedigree: + type: integer + label: 'The cvterm ID of the pedigree information for a germplasm' + country_of_origin_code: + type: integer + label: 'The cvterm ID of a germplasm's country of origin' + synonym: + type: integer + label: 'The cvterm ID of a synonym of a germplasm name' + biological_status_of_accession_code: + type: integer + label: 'The cvterm ID of the biological status of a germplasm accession' + breeding_method_DbId: + type: integer + label: 'The cvterm ID of the germplasm breeding method' + subtaxa: + type: integer + label: 'The cvterm ID of the subtaxa of a germplasm' From ed299a2f6581242ae486271d6aa52c63f494f5b0 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 21 Aug 2023 17:18:55 -0600 Subject: [PATCH 04/48] Fixed syntax error in config --- .../config/schema/trpcultivate_germplasm.schema.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trpcultivate_germplasm/config/schema/trpcultivate_germplasm.schema.yml b/trpcultivate_germplasm/config/schema/trpcultivate_germplasm.schema.yml index ff6ea36..9d09f64 100644 --- a/trpcultivate_germplasm/config/schema/trpcultivate_germplasm.schema.yml +++ b/trpcultivate_germplasm/config/schema/trpcultivate_germplasm.schema.yml @@ -29,7 +29,7 @@ trpcultivate_germplasm.settings: label: 'The cvterm ID of the pedigree information for a germplasm' country_of_origin_code: type: integer - label: 'The cvterm ID of a germplasm's country of origin' + label: 'The cvterm ID of the country of origin of a germplasm' synonym: type: integer label: 'The cvterm ID of a synonym of a germplasm name' From 2712d469bdc133e608fe4e7908cb34e25781506a Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Fri, 8 Sep 2023 17:12:49 -0600 Subject: [PATCH 05/48] Started implementing the run method --- .../GermplasmAccessionImporter.php | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 26f068b..6dc688a 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -56,7 +56,7 @@ public function form($form, &$form_state) { '#weight' => -99, '#markup' => '

    Load germplasm into database

    -

    Upload file must be in TSV (tab-separated values) or TXT format. Please confirm the file format and column order before upload.

    +

    Please confirm the file format and column order before upload.

    ', ]; @@ -83,7 +83,28 @@ public function formValidate($form, &$form_state){ * @see TripalImporter::run() */ public function run(){ - + $arguments = $this->arguments['run_args']; + + // Grab the file path + $file_path = $arguments['files'][0]['file_path']; + $genus_name = $arguments['genus_name']; + + // Set up the ability to track progress so we can report it to the user + $filesize = filesize($file_path); + $this->setTotalItems($filesize); + $this->setItemsHandled(0); + $bytes_read = 0; + + // Open the file and start iterating through each line + $GERMPLASM_FILE = fopen($file_path, 'r'); + while (!feof($GERMPLASM_FILE)){ + $current_line = fgetcsv($GERMPLASM_FILE, 0, "\t"); + + // Check for empty lines, comment lines and a header line + if (empty($current_line)) continue; + if (preg_match('/^#/', $current_line)) continue; + if (preg_match('/^Germplasm/', $current_line)) continue; + } } /* From 87a9ed02dbbcc5744f54759abb7dc7fd06eb2da3 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 11 Sep 2023 17:21:43 -0600 Subject: [PATCH 06/48] Setup kernel testing of the form for the Germplasm Accession Importer --- .../GermplasmAccessionImporterTest.php | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php new file mode 100644 index 0000000..cfb376d --- /dev/null +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -0,0 +1,88 @@ +set('is_a_test_environment', TRUE); + + // Open connection to Chado + $this->connection = $this->getTestSchema(ChadoTestKernelBase::PREPARE_TEST_CHADO); + + // Ensure we can access file_managed related functionality from Drupal. + // ... users need access to system.action config? + $this->installConfig('system'); + // ... managed files are associated with a user. + $this->installEntitySchema('user'); + // ... Finally the file module + tables itself. + $this->installEntitySchema('file'); + $this->installSchema('file', ['file_usage']); + + } + + /** + * Tests focusing on the Germplasm Accession Importer form. + * + * @group tripal_importer + */ + public function testGermplasmAccessionImporterForm() { + + $plugin_id = 'trpcultivate-germplasm-accession'; + + // Build the form using the Drupal form builder. + $form = \Drupal::formBuilder()->getForm( + 'Drupal\tripal\Form\TripalImporterForm', + $plugin_id + ); + // Ensure we are able to build the form. + $this->assertIsArray($form, + 'We expect the form builder to return a form but it did not.'); + $this->assertEquals('tripal_admin_form_tripalimporter', $form['#form_id'], + 'We did not get the form id we expected.'); + + // Now that we have provided a plugin_id, we expect it to have... + // title matching our importer label. + $this->assertArrayHasKey('#title', $form, + "The form should have a title set."); + $this->assertEquals('Tripal Cultivate: Germplasm Accessions', $form['#title'], + "The title should match the label annotated for our plugin."); + // the plugin_id stored in a value form element. + $this->assertArrayHasKey('importer_plugin_id', $form, + "The form should have an element to save the plugin_id."); + $this->assertEquals($plugin_id, $form['importer_plugin_id']['#value'], + "The importer_plugin_id[#value] should be set to our plugin_id."); + // a submit button. + $this->assertArrayHasKey('button', $form, + "The form should not have a submit button since we indicated a specific importer."); + + // We should also have our importer-specific form elements added to the form! + $this->assertArrayHasKey('instructions', $form, + "The form should include an instructions form element."); + $this->assertArrayHasKey('genus_name', $form, + "The form should include a genus_name form element."); + $this->assertArrayHasKey('file', $form, + "The form should include a file form element."); + + // Our default annotation indicates there should be no analysis element. + $this->assertArrayNotHasKey('analysis_id', $form, + "The from should not include analysis element, yet one exists."); + } + +} \ No newline at end of file From 4fccd9e8a3e124b4a2e927d3aa74a8ef11a2fc3b Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 18 Sep 2023 17:05:37 -0600 Subject: [PATCH 07/48] Finished designing the run function --- .../GermplasmAccessionImporter.php | 92 ++++++++++++++++--- 1 file changed, 80 insertions(+), 12 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 6dc688a..ec37c3b 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -16,19 +16,19 @@ * use_analysis = FALSE, * require_analysis = FALSE, * upload_title = "Germplasm Accession Import", - * upload_description = "Germplasm file should be a tab separated file with the following colums:
      + * upload_description = "Germplasm file should be a tab separated file with the following columns:
      1. Germplasm Name: Name of this germplasm accession.
        2. -
      2. External Database: The institution who assigned the following accession.
        3. +
      3. External Database: The institution who assigned the accession.
      4. Accession Number: A unique identifier for the accession.
      5. Germplasm Species: The species of the accession.
        6. -
      6. Germplasm Subtaxa: Subtaxon can be used to store any additional taxonomic identifier.
        7. -
      7. Institute Code: The code for the Institute that has bred the material.
        8. -
      8. Institute Name: The name of the Institute that has bred the material.
        9. -
      9. Country of Origin Code: 3-letter ISO 3166-1 code of the country in which the sample was originally.
        10. +
      10. Germplasm Subtaxa: Subtaxon can be specified here or any additional taxonomic identifier.
        11. +
      11. Institute Code: The code for the Institute that bred the material.
        12. +
      12. Institute Name: The name of the Institute that bred the material.
        13. +
      13. Country of Origin Code: 3-letter ISO 3166-1 code of the country in which the sample was originally sourced.
      14. Biological Status of Accession: The 3 digit code representing the biological status of the accession.
      15. Breeding Method: The unique identifier for the breeding method used to create this germplasm.
      16. Pedigree: The cross name and optional selection history.
        17. -
      17. Synonyms: The synonyms of the accession.
        18. +
      18. Synonyms: Any synonyms of the accession.
      ", * button_text = "Import Germplasm Accessions", * file_upload = True, @@ -76,17 +76,24 @@ public function form($form, &$form_state) { * @see TripalImporter::formValidate() */ public function formValidate($form, &$form_state){ - + // Nothing to validate since the genus field is set to "required". } /** * @see TripalImporter::run() */ public function run(){ + // All values provided by the user in the Importer's form widgets are + // made available to us here by the Class' arguments member variable. $arguments = $this->arguments['run_args']; - // Grab the file path + // The path to the uploaded file is always made available using the + // 'files' argument. The importer can support multiple files, therefore + // this is an array of files, where each has a 'file_path' key specifying + // where the file is located on the server. $file_path = $arguments['files'][0]['file_path']; + + // Grab the genus name $genus_name = $arguments['genus_name']; // Set up the ability to track progress so we can report it to the user @@ -100,18 +107,81 @@ public function run(){ while (!feof($GERMPLASM_FILE)){ $current_line = fgetcsv($GERMPLASM_FILE, 0, "\t"); + // Calculate how many bytes we have read from the file and let the + // importer know how many have been processed so it can provide a + // progress indicator. + $bytes_read += drupal_strlen($current_line); + $this->setItemsHandled($bytes_read); + // Check for empty lines, comment lines and a header line if (empty($current_line)) continue; if (preg_match('/^#/', $current_line)) continue; if (preg_match('/^Germplasm/', $current_line)) continue; + + // Trim the current line for trailing whitespace and split columns + // into an array + $current_line = trim($current_line); + $germplasm_columns = explode("\t", $current_line); + + // Collect our values from our current line into variables + $germplasm_name = $germplasm_columns[0]; + $external_database = $germplasm_columns[1]; + $accession_number = $germplasm_columns[2]; + $germplasm_species = $germplasm_columns[3]; + $germplasm_subtaxa = $germplasm_columns[4]; + $institute_code = $germplasm_columns[5]; + $institute_name = $germplasm_columns[6]; + $country_of_origin_code = $germplasm_columns[7]; + $biological_status_of_accession_code = $germplasm_columns[8]; + $breeding_method_DbId = $germplasm_columns[9]; + $pedigree = $germplasm_columns[10]; + $synonyms = $germplasm_columns[11]; + + // STEP 1: Pull out the organism ID for the current germplasm + $organism_ID = $this->getOrganismID($genus_name, $germplasm_species, $germplasm_subtaxa); + + // STEP 2: Check/Insert this germplasm into the Chado stock table + $stock_id = $this->getStockID($germplasm_name, $accession_number, $organism_ID); + + // STEP 3: If $external_database is provided, then + // Load the external database info into Chado dbxref table + + // STEP 4: Load stock properties (if provided) for: + // $institute_code + // $institute_name + // $country_of_origin_code + // $biological_status_of_accession_code + // $breeding_method_DbId + // $pedigree + + // STEP 5: Load synonyms + } } + /** + * Checks if an organism exists in Chado and returns the primary key, + * otherwise throws an error if the organism does not exist + * + */ + public function getOrganismID() { + return 0; + } + + /** + * Checks if an stock exists in Chado, inserts it and returns the primary + * key in the stock table + * + */ + public function getStockID() { + return 0; + } + /* * {@inheritdoc} */ public function postRun() { - + // Nothing to clean up. } /** @@ -120,6 +190,4 @@ public function postRun() { public function formSubmit($form, &$form_state) { } - - } From 5704bf2ee727bcdcabda2360295f53d920bf1594 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Tue, 19 Sep 2023 17:11:10 -0600 Subject: [PATCH 08/48] Implemented the getOrganismID function --- .../GermplasmAccessionImporter.php | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index ec37c3b..481ea3a 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -154,18 +154,43 @@ public function run(){ // $breeding_method_DbId // $pedigree - // STEP 5: Load synonyms + // STEP 5: Load synonyms } } /** - * Checks if an organism exists in Chado and returns the primary key, - * otherwise throws an error if the organism does not exist - * + * Checks if an organism exists in Chado and returns the primary key, + * otherwise throws an error if the organism does not exist or there + * are multiple matches + * + * @param string $genus_name + * @param string $germplasm_species + * @param string $germplasm_subtaxa + * @return int|false + * The value of the primary key for the organism record in Chado. + * If no single primary key can be retrieved, then FALSE is returned. */ - public function getOrganismID() { - return 0; + public function getOrganismID($genus_name, $germplasm_species, $germplasm_subtaxa) { + + $organism_name = $genus_name . ' ' . $germplasm_species; + if ($germplasm_subtaxa) { + $organism_name = $organism_name . ' ' . $germplasm_subtaxa; + } + $organism_array = chado_get_organism_id_from_scientific_name($organism_name); + + if (!$organism_array) { + $this->logMessage("ERROR: Could not find an organism \"@organism_name\" in the database.", ['@organism_name' => $organism_name], TRIPAL_ERROR); + return false; + } + // We also want to check if we were given only one value back, as there is + // potential to retrieve multiple organism IDs + if (is_array($organism_array) && (count($organism_array) > 1)) { + $this->logMessage("ERROR: Found more than one organism ID for \"@organism_name\" when only 1 was expected.", ['@organism_name' => $organism_name], TRIPAL_ERROR); + return false; + } + + return $organism_array[0]; } /** From c5534c70b2ece9d217d636890a899b21e00151f4 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Wed, 20 Sep 2023 17:13:52 -0600 Subject: [PATCH 09/48] Mocked the logger for testing, and created first assert for the getOrganismID function --- .../GermplasmAccessionImporter.php | 8 ++- .../GermplasmAccessionImporterTest.php | 51 ++++++++++++++++++- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 481ea3a..1876877 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -165,8 +165,12 @@ public function run(){ * are multiple matches * * @param string $genus_name + * The genus of the organism. * @param string $germplasm_species + * The species of the organism. * @param string $germplasm_subtaxa + * Optional. Must consist of two strings, one of the subtaxon type + * followed by the name. For example: "subspecies chadoii". * @return int|false * The value of the primary key for the organism record in Chado. * If no single primary key can be retrieved, then FALSE is returned. @@ -180,13 +184,13 @@ public function getOrganismID($genus_name, $germplasm_species, $germplasm_subtax $organism_array = chado_get_organism_id_from_scientific_name($organism_name); if (!$organism_array) { - $this->logMessage("ERROR: Could not find an organism \"@organism_name\" in the database.", ['@organism_name' => $organism_name], TRIPAL_ERROR); + $this->logger->error("Could not find an organism \"@organism_name\" in the database.", ['@organism_name' => $organism_name]); return false; } // We also want to check if we were given only one value back, as there is // potential to retrieve multiple organism IDs if (is_array($organism_array) && (count($organism_array) > 1)) { - $this->logMessage("ERROR: Found more than one organism ID for \"@organism_name\" when only 1 was expected.", ['@organism_name' => $organism_name], TRIPAL_ERROR); + $this->logger->error("Found more than one organism ID for \"@organism_name\" when only 1 was expected.", ['@organism_name' => $organism_name]); return false; } diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index cfb376d..515bce3 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -14,6 +14,8 @@ class GermplasmAccessionImporterTest extends ChadoTestKernelBase { protected static $modules = ['system', 'user', 'file', 'tripal', 'tripal_chado', 'trpcultivate_germplasm']; + protected $importer; + /** * {@inheritdoc} */ @@ -35,12 +37,34 @@ protected function setUp(): void { $this->installEntitySchema('file'); $this->installSchema('file', ['file_usage']); + // We need to mock the logger to test the progress reporting. + $container = \Drupal::getContainer(); + $mock_logger = $this->getMockBuilder(\Drupal\tripal\Services\TripalLogger::class) + ->onlyMethods(['notice','error']) + ->getMock(); + // $mock_logger->method('notice') + // ->willReturnCallback(function($message, $context, $options) { + // print str_replace(array_keys($context), $context, $message); + // return NULL; + // }); + $mock_logger->method('error') + ->willReturnCallback(function($message, $context, $options) { + print str_replace(array_keys($context), $context, $message); + return NULL; + }); + $container->set('tripal.logger', $mock_logger); + + $this->importer = new \Drupal\trpcultivate_germplasm\Plugin\TripalImporter\GermplasmAccessionImporter( + [], + 'trpcultivate-germplasm-accession', + [] // Note: May need to add annotation in here. Ask Lacey if annotation things fail + ); } /** * Tests focusing on the Germplasm Accession Importer form. * - * @group tripal_importer + * @group germ_accession_importer */ public function testGermplasmAccessionImporterForm() { @@ -85,4 +109,27 @@ public function testGermplasmAccessionImporterForm() { "The from should not include analysis element, yet one exists."); } -} \ No newline at end of file + /** + * Tests focusing on the Germplasm Accession Importer getOrganismID() function + * + * @group germ_accession_importer + */ + public function testGermplasmAccessionImporterGetOrganismID() { + + // Insert an organism + $subtaxa_cvterm_id = $this->getCVtermID('TAXRANK', '0000023'); + + $organism_id = $this->connection->insert('1:organism') + ->fields([ + 'genus' => 'Tripalus', + 'species' => 'databasica', + 'infraspecific_name' => 'chadoii', + 'type_id' => $subtaxa_cvterm_id, + ]) + ->execute(); + + $grabbed_organism_id = $this->importer->getOrganismID('Tripalus', 'databasica', 'subspecies chadoii'); + $this->assertEquals($grabbed_organism_id, $organism_id, "The organism ID grabbed by the importer does not match the one that was inserted into the database."); + + } +} From 4a50d29972358d6e17a94f0d6b1ae89a9f27fa1c Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 25 Sep 2023 18:02:23 -0600 Subject: [PATCH 10/48] Started implementing getStockID function --- .../GermplasmAccessionImporter.php | 61 +++++++++++++++++-- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 1876877..7e8b199 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -198,12 +198,63 @@ public function getOrganismID($genus_name, $germplasm_species, $germplasm_subtax } /** - * Checks if an stock exists in Chado, inserts it and returns the primary - * key in the stock table - * + * Checks if a stock exists in Chado and if not, inserts it and returns the primary + * key in the stock table. If the stock already exists, logs an error + * + * @param string $germplasm_name + * The name of the germplasm. + * @param string $accession_number + * A unique identifier for the germplasm accession. + * @param int $organism_ID + * The primary key of the stock's organism in the organism table + * @return int|false + * The value of the primary key for the stock record in Chado. If the stock already + * exists or cannot be inserted, then FALSE is returned. */ - public function getStockID() { - return 0; + public function getStockID($germplasm_name, $accession_number, $organism_ID) { + + // First query the stock table just using the germplasm name and organism ID + $query = $this->connection->select('1:stock', 's') + ->fields('s', 'stock_id'); + $query->condition('s.name', $germplasm_name, '=') + ->condition('s.organism_id', $organism_ID, '='); + $record = $query->execute()->fetchAll(); + + if (sizeof($record) >= 2) { + $this->logger->error("Found more than one stock ID for \"@germplasm_name\".", ['@germplasm_name' => $germplasm_name]); + return false; + } + if (sizeof($record) == 1) { + // Handle the situation where a stock record exists + } + // Confirmed that a stock record doesn't yet exist, so now we create one + else { + $values = array( + 'organism_id' => $organism_ID, + 'name' => $germplasm_name, + 'uniquename' => $accession_number, + //'type_id' => get_CV_term(); + // Ask Lacey: Do we need to insert NULL values? + 'dbxref_id' => NULL, + 'description' => NULL, + $is_obsolete = 'f' + ); + + $this->logger->notice("Inserting \"@germplasm_name\".", ['@germplasm_name' => $germplasm_name]); + + $result = $this->connection->insert('1:stock') + ->fields($values) + ->execute(); + + // If the primary key is available, then the insert worked and we can return it + if ($result) { + return $result; + } + else { + $this->logger->error("Insertion of \"@germplasm_name\" failed.", ['@germplasm_name' => $germplasm_name]); + return false; + } + } } /* From f570b21853e424c0abbdd4085b1599d73c580a11 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Wed, 27 Sep 2023 12:02:02 -0600 Subject: [PATCH 11/48] Added a global error tracking variable to the Accession importer class --- .../GermplasmAccessionImporter.php | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 7e8b199..3100223 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -39,6 +39,13 @@ * ) */ class GermplasmAccessionImporter extends ChadoImporterBase { + + /** + * Used to track whether an error is logged during the import process. If it + * is set to TRUE, then the db transaction will not be committed. + */ + protected $error_tracker = FALSE; + /** * {@inheritDoc} */ @@ -141,7 +148,9 @@ public function run(){ $organism_ID = $this->getOrganismID($genus_name, $germplasm_species, $germplasm_subtaxa); // STEP 2: Check/Insert this germplasm into the Chado stock table - $stock_id = $this->getStockID($germplasm_name, $accession_number, $organism_ID); + if ($organism_ID) { + $stock_id = $this->getStockID($germplasm_name, $accession_number, $organism_ID); + } // STEP 3: If $external_database is provided, then // Load the external database info into Chado dbxref table @@ -185,12 +194,14 @@ public function getOrganismID($genus_name, $germplasm_species, $germplasm_subtax if (!$organism_array) { $this->logger->error("Could not find an organism \"@organism_name\" in the database.", ['@organism_name' => $organism_name]); + $this->error_tracker = TRUE; return false; } // We also want to check if we were given only one value back, as there is // potential to retrieve multiple organism IDs if (is_array($organism_array) && (count($organism_array) > 1)) { $this->logger->error("Found more than one organism ID for \"@organism_name\" when only 1 was expected.", ['@organism_name' => $organism_name]); + $this->error_tracker = TRUE; return false; } @@ -200,7 +211,7 @@ public function getOrganismID($genus_name, $germplasm_species, $germplasm_subtax /** * Checks if a stock exists in Chado and if not, inserts it and returns the primary * key in the stock table. If the stock already exists, logs an error - * + * * @param string $germplasm_name * The name of the germplasm. * @param string $accession_number @@ -212,7 +223,7 @@ public function getOrganismID($genus_name, $germplasm_species, $germplasm_subtax * exists or cannot be inserted, then FALSE is returned. */ public function getStockID($germplasm_name, $accession_number, $organism_ID) { - + // First query the stock table just using the germplasm name and organism ID $query = $this->connection->select('1:stock', 's') ->fields('s', 'stock_id'); @@ -222,11 +233,23 @@ public function getStockID($germplasm_name, $accession_number, $organism_ID) { if (sizeof($record) >= 2) { $this->logger->error("Found more than one stock ID for \"@germplasm_name\".", ['@germplasm_name' => $germplasm_name]); + $this->error_tracker = TRUE; return false; } if (sizeof($record) == 1) { // Handle the situation where a stock record exists - } + // Check the uniquename matches the accession_number column in the file + if ($accession_number != $record[0]->{uniquename}) { + $this->logger->error("A stock already exists for \"@germplasm_name\" but with an accession of \"@accession\" which does not match the input file.", ['@germplasm_name' => $germplasm_name, '@accession' => $record[0]->uniquename]); + $this->error_tracker = TRUE; + return false; + } + // Check the type_id is of type accession + // if ($record[0]->{type_id}) + + // Confirmed that the selected record matches what's in the upload file, so return the stock_id + return $record[0]->{stock_id}; + } // Confirmed that a stock record doesn't yet exist, so now we create one else { $values = array( @@ -234,10 +257,6 @@ public function getStockID($germplasm_name, $accession_number, $organism_ID) { 'name' => $germplasm_name, 'uniquename' => $accession_number, //'type_id' => get_CV_term(); - // Ask Lacey: Do we need to insert NULL values? - 'dbxref_id' => NULL, - 'description' => NULL, - $is_obsolete = 'f' ); $this->logger->notice("Inserting \"@germplasm_name\".", ['@germplasm_name' => $germplasm_name]); @@ -252,6 +271,7 @@ public function getStockID($germplasm_name, $accession_number, $organism_ID) { } else { $this->logger->error("Insertion of \"@germplasm_name\" failed.", ['@germplasm_name' => $germplasm_name]); + $this->error_tracker = TRUE; return false; } } From b86f3b3cb19984c64220f57dcedfd3c1f6fac395 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Thu, 28 Sep 2023 17:26:47 -0600 Subject: [PATCH 12/48] Updated method for specifying upload file format description --- .../GermplasmAccessionImporter.php | 54 ++++++++++++++----- .../GermplasmAccessionImporterTest.php | 25 ++++++++- 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 3100223..f60f824 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -16,20 +16,6 @@ * use_analysis = FALSE, * require_analysis = FALSE, * upload_title = "Germplasm Accession Import", - * upload_description = "Germplasm file should be a tab separated file with the following columns:
        -
      1. Germplasm Name: Name of this germplasm accession.
        2. -
      2. External Database: The institution who assigned the accession.
        3. -
      3. Accession Number: A unique identifier for the accession.
        4. -
      4. Germplasm Species: The species of the accession.
        5. -
      5. Germplasm Subtaxa: Subtaxon can be specified here or any additional taxonomic identifier.
        6. -
      6. Institute Code: The code for the Institute that bred the material.
        7. -
      7. Institute Name: The name of the Institute that bred the material.
        8. -
      8. Country of Origin Code: 3-letter ISO 3166-1 code of the country in which the sample was originally sourced.
        9. -
      9. Biological Status of Accession: The 3 digit code representing the biological status of the accession.
        10. -
      10. Breeding Method: The unique identifier for the breeding method used to create this germplasm.
        11. -
      11. Pedigree: The cross name and optional selection history.
        12. -
      12. Synonyms: Any synonyms of the accession.
        13. -
      ", * button_text = "Import Germplasm Accessions", * file_upload = True, * file_load = True, @@ -46,6 +32,46 @@ class GermplasmAccessionImporter extends ChadoImporterBase { */ protected $error_tracker = FALSE; + /** + * @{inheritdoc} + */ + public function describeUploadFileFormat() { + + $file_types = $this->plugin_definition['file_types']; + + $output = "Germplasm file should be a tab separated file (" . implode(', ', $file_types) . ") with the following columns:"; + + $columns = [ + 'Germplasm Name' => 'Name of this germplasm accession (e.g. CDC Redberry)', + 'External Database' => 'The institution who assigned the accession. (e.g. KnowPulse Germplasm)', + 'Accession Number' => 'A unique identifier for the accession (e.g. KP:GERM58)', + 'Germplasm Species' => 'The species of the accession (e.g. culinaris)', + 'Germplasm Subtaxa' => 'The rank below species is specified first, followed by the name (e.g. var. medullare)', + 'Institute Code' => 'The code for the Institute that bred the material (e.g. CUAC)', + 'Institute Name' => 'The name of the Institute that bred the material (e.g. "Crop Development Center, University of Saskatchewan")', + 'Country of Origin Code' => '3-letter ISO 3166-1 code of the country in which the sample was originally sourced (e.g. 124)', + 'Biological Status of Accession' => 'The 3 digit code representing the biological status of the accession (e.g. 410)', + 'Breeding Method' => 'The unique identifier for the breeding method used to create this germplasm (e.g. "Recurrent selection")', + 'Pedigree' => 'The cross name and optional selection history (e.g. 1049F^3/819-5R)', + 'Synonyms' => 'Any synonyms of the accession. (e.g. Redberry)', + ]; + + $required_col = ['Germplasm Name', 'Accession Number', 'Germplasm Species']; + + $output .= '
        '; + foreach ($columns as $title => $definition) { + if (in_array($title, $required_col)) { + $output .= '
      1. ' . $title . '*: ' . $definition . '
        2. '; + } + else { + $output .= '
      2. ' . $title . ': ' . $definition . '
        3. '; + } + } + $output .= '
      '; + + return $output; + } + /** * {@inheritDoc} */ diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index 515bce3..31c3eaa 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -16,6 +16,25 @@ class GermplasmAccessionImporterTest extends ChadoTestKernelBase { protected $importer; + protected $definitions = [ + 'test-germplasm-accession' => [ + 'id' => 'trpcultivate-germplasm-accession', + 'label' => 'Tripal Cultivate: Germplasm Accessions', + 'description' => 'Imports germplasm accessions into Chado with metadata meeting BrAPI standards.', + 'file_types' => ["tsv", "txt"], + 'use_analysis' => FALSE, + 'require_analysis' => FALSE, + 'upload_title' => 'Germplasm Accession Import', + 'upload_description' => 'This should not be visible!', + 'button_text' => 'Import Germplasm Accessions', + 'file_upload' => True, + 'file_load' => True, + 'file_remote' => True, + 'file_required' => True, + 'cardinality' => 1, + ], + ]; + /** * {@inheritdoc} */ @@ -57,7 +76,7 @@ protected function setUp(): void { $this->importer = new \Drupal\trpcultivate_germplasm\Plugin\TripalImporter\GermplasmAccessionImporter( [], 'trpcultivate-germplasm-accession', - [] // Note: May need to add annotation in here. Ask Lacey if annotation things fail + $this->definitions ); } @@ -131,5 +150,9 @@ public function testGermplasmAccessionImporterGetOrganismID() { $grabbed_organism_id = $this->importer->getOrganismID('Tripalus', 'databasica', 'subspecies chadoii'); $this->assertEquals($grabbed_organism_id, $organism_id, "The organism ID grabbed by the importer does not match the one that was inserted into the database."); + // Try an organism that does not currently exist + //$non_existent_organism_id = $this->importer->getOrganismID('Nullus', 'organismus'); + //assertTrue($this->importer->error_tracker); + } } From 2a5e8a4c9f25aece71e41cc03be22b9f06149484 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Thu, 5 Oct 2023 21:50:12 -0600 Subject: [PATCH 13/48] Fixed formatting for required columns in input file --- .../Plugin/TripalImporter/GermplasmAccessionImporter.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index f60f824..a000825 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -56,12 +56,14 @@ public function describeUploadFileFormat() { 'Synonyms' => 'Any synonyms of the accession. (e.g. Redberry)', ]; - $required_col = ['Germplasm Name', 'Accession Number', 'Germplasm Species']; + $required_col = ['Germplasm Name', 'External Database', 'Accession Number', 'Germplasm Species']; + // @TODO: Make this red, but Drupal makes it difficult :) + $required_markup = '*'; $output .= '
        '; foreach ($columns as $title => $definition) { if (in_array($title, $required_col)) { - $output .= '
      1. ' . $title . '*: ' . $definition . '
        2. '; + $output .= '
      2. ' . $title . $required_markup . ': ' . $definition . '
        3. '; } else { $output .= '
      3. ' . $title . ': ' . $definition . '
        4. '; From 64640f4791148e0eb3215b8b0c65336a78f762eb Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Wed, 11 Oct 2023 17:09:52 -0600 Subject: [PATCH 14/48] Added getStockId() tests and prepared for db dependency injection --- .../GermplasmAccessionImporter.php | 18 +++---- .../GermplasmAccessionImporterTest.php | 51 ++++++++++++++++++- 2 files changed, 59 insertions(+), 10 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index a000825..649a848 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -58,13 +58,13 @@ public function describeUploadFileFormat() { $required_col = ['Germplasm Name', 'External Database', 'Accession Number', 'Germplasm Species']; // @TODO: Make this red, but Drupal makes it difficult :) - $required_markup = '*'; + $required_markup = '*'; $output .= '
          '; foreach ($columns as $title => $definition) { if (in_array($title, $required_col)) { - $output .= '
        1. ' . $title . $required_markup . ': ' . $definition . '
          2. '; - } + $output .= '
        2. ' . $title . $required_markup . ': ' . $definition . '
          3. '; + } else { $output .= '
        3. ' . $title . ': ' . $definition . '
          4. '; } @@ -78,6 +78,7 @@ public function describeUploadFileFormat() { * {@inheritDoc} */ public function form($form, &$form_state) { + $form = parent::form($form, $form_state); // Select the entire genus field and make sure it is sorted and distinct $connection = \Drupal::service('tripal_chado.database'); @@ -180,8 +181,7 @@ public function run(){ $stock_id = $this->getStockID($germplasm_name, $accession_number, $organism_ID); } - // STEP 3: If $external_database is provided, then - // Load the external database info into Chado dbxref table + // STEP 3: Load the external database info into Chado dbxref table // STEP 4: Load stock properties (if provided) for: // $institute_code @@ -254,7 +254,7 @@ public function getStockID($germplasm_name, $accession_number, $organism_ID) { // First query the stock table just using the germplasm name and organism ID $query = $this->connection->select('1:stock', 's') - ->fields('s', 'stock_id'); + ->fields('s', ['stock_id', 'uniquename', 'type_id']); $query->condition('s.name', $germplasm_name, '=') ->condition('s.organism_id', $organism_ID, '='); $record = $query->execute()->fetchAll(); @@ -267,16 +267,16 @@ public function getStockID($germplasm_name, $accession_number, $organism_ID) { if (sizeof($record) == 1) { // Handle the situation where a stock record exists // Check the uniquename matches the accession_number column in the file - if ($accession_number != $record[0]->{uniquename}) { + if ($accession_number != $record[0]->uniquename) { $this->logger->error("A stock already exists for \"@germplasm_name\" but with an accession of \"@accession\" which does not match the input file.", ['@germplasm_name' => $germplasm_name, '@accession' => $record[0]->uniquename]); $this->error_tracker = TRUE; return false; } // Check the type_id is of type accession - // if ($record[0]->{type_id}) + // if ($record[0]->type_id) // Confirmed that the selected record matches what's in the upload file, so return the stock_id - return $record[0]->{stock_id}; + return $record[0]->stock_id; } // Confirmed that a stock record doesn't yet exist, so now we create one else { diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index 31c3eaa..7958a7d 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -78,6 +78,15 @@ protected function setUp(): void { 'trpcultivate-germplasm-accession', $this->definitions ); + + $run_args = [ + 'schema_name' => $this->testSchemaName, + 'genus' => 'Tripalus', + ]; + $file_details = [ + 'file_local' => '/pretend/we/have/a/file.txt', + ]; + $this->importer->create($run_args, $file_details); } /** @@ -151,8 +160,48 @@ public function testGermplasmAccessionImporterGetOrganismID() { $this->assertEquals($grabbed_organism_id, $organism_id, "The organism ID grabbed by the importer does not match the one that was inserted into the database."); // Try an organism that does not currently exist - //$non_existent_organism_id = $this->importer->getOrganismID('Nullus', 'organismus'); + //$non_existent_organism_id = $this->importer->getOrganismID('Nullus', 'organismus', ''); //assertTrue($this->importer->error_tracker); + // Check for multiple organism IDs? @AskLacey how we want to go about this considering Chado SHOULD restrict inserting multiple + + } + + /** + * Tests focusing on the Germplasm Accession Importer getStockID() function + * + * @group germ_accession_importer + */ + public function testGermplasmAccessionImporterGetStockID() { + + // Insert an organism + $subtaxa_cvterm_id = $this->getCVtermID('TAXRANK', '0000023'); + + $organism_id = $this->connection->insert('1:organism') + ->fields([ + 'genus' => 'Tripalus', + 'species' => 'databasica', + 'infraspecific_name' => 'chadoii', + 'type_id' => $subtaxa_cvterm_id, + ]) + ->execute(); + + // Insert a stock + // @TODO: FIND THE APPROPRIATE CVTERM ID FOR ACCESSION + $accession_cvterm_id = $this->getCVtermID('TAXRANK', '0000023'); + + $stock_id = $this->connection->insert('1:stock') + ->fields([ + 'organism_id' => $organism_id, + 'name' => 'stock1', + 'uniquename' => 'TEST:1', + 'type_id' => $accession_cvterm_id, + ]) + ->execute(); + + $grabbed_stock_id = $this->importer->getStockID('stock1', 'TEST:1', $organism_id); + $this->assertEquals($grabbed_stock_id, $stock_id, "The stock ID grabbed by the importer does not match the one that was inserted into the database."); + } + } From 33fe60017db10151150b432f60d31777b078ad73 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 16 Oct 2023 11:08:45 -0600 Subject: [PATCH 15/48] Fixed tests to work with dependency injection newly added to ChadoImporterBase --- .../GermplasmAccessionImporterTest.php | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index 7958a7d..fdca567 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -76,17 +76,9 @@ protected function setUp(): void { $this->importer = new \Drupal\trpcultivate_germplasm\Plugin\TripalImporter\GermplasmAccessionImporter( [], 'trpcultivate-germplasm-accession', - $this->definitions + $this->definitions, + $this->connection ); - - $run_args = [ - 'schema_name' => $this->testSchemaName, - 'genus' => 'Tripalus', - ]; - $file_details = [ - 'file_local' => '/pretend/we/have/a/file.txt', - ]; - $this->importer->create($run_args, $file_details); } /** From 92ab7c699138dd94b3d6ac138d9d702c2454abcb Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 16 Oct 2023 14:46:44 -0600 Subject: [PATCH 16/48] Added tests to getOrganismID and getStockID --- .../GermplasmAccessionImporterTest.php | 50 +++++++++++++++---- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index fdca567..3456c76 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -61,11 +61,11 @@ protected function setUp(): void { $mock_logger = $this->getMockBuilder(\Drupal\tripal\Services\TripalLogger::class) ->onlyMethods(['notice','error']) ->getMock(); - // $mock_logger->method('notice') - // ->willReturnCallback(function($message, $context, $options) { - // print str_replace(array_keys($context), $context, $message); - // return NULL; - // }); + $mock_logger->method('notice') + ->willReturnCallback(function($message, $context, $options) { + print str_replace(array_keys($context), $context, $message); + return NULL; + }); $mock_logger->method('error') ->willReturnCallback(function($message, $context, $options) { print str_replace(array_keys($context), $context, $message); @@ -152,11 +152,12 @@ public function testGermplasmAccessionImporterGetOrganismID() { $this->assertEquals($grabbed_organism_id, $organism_id, "The organism ID grabbed by the importer does not match the one that was inserted into the database."); // Try an organism that does not currently exist - //$non_existent_organism_id = $this->importer->getOrganismID('Nullus', 'organismus', ''); - //assertTrue($this->importer->error_tracker); - - // Check for multiple organism IDs? @AskLacey how we want to go about this considering Chado SHOULD restrict inserting multiple + ob_start(); + $non_existent_organism_id = $this->importer->getOrganismID('Nullus', 'organismus', ''); + $printed_output = ob_get_clean(); + $this->assertTrue($printed_output == 'Could not find an organism "Nullus organismus" in the database.', "Did not get the expected error message when testing for a non-existant organism."); + // Not testing if multiple organisms are retrieved, since Chado should be preventing such a situation } /** @@ -180,7 +181,7 @@ public function testGermplasmAccessionImporterGetStockID() { // Insert a stock // @TODO: FIND THE APPROPRIATE CVTERM ID FOR ACCESSION - $accession_cvterm_id = $this->getCVtermID('TAXRANK', '0000023'); + $accession_cvterm_id = $this->getCVtermID('TAXRANK', '0000024'); $stock_id = $this->connection->insert('1:stock') ->fields([ @@ -191,9 +192,38 @@ public function testGermplasmAccessionImporterGetStockID() { ]) ->execute(); + // Test that the stock just inserted gets selected $grabbed_stock_id = $this->importer->getStockID('stock1', 'TEST:1', $organism_id); $this->assertEquals($grabbed_stock_id, $stock_id, "The stock ID grabbed by the importer does not match the one that was inserted into the database."); + // Test that a stock not in the database successfully gets inserted + //ob_start(); + //$created_stock_id = $this->importer->getStockID('stock2', 'TEST:2', $organism_id); + //$printed_output = ob_get_clean(); + //$this->assertTrue($printed_output == 'Inserting "stock2".', "Did not get the expected notice message when inserting a new stock."); + + // Test for a stock name + organism that already exists but has a different accession + + ob_start(); + $grabbed_dup_stock_name = $this->importer->getStockID('stock1', 'TEST:1000', $organism_id); + $printed_output = ob_get_clean(); + $this->assertTrue($printed_output == 'A stock already exists for "stock1" but with an accession of "TEST:1" which does not match the input file.', "Did not get the expected error message when testing for duplicate stock names."); + + // Now test for multiple stocks with the same name and accession, but different type_id + $stock_id = $this->connection->insert('1:stock') + ->fields([ + 'organism_id' => $organism_id, + 'name' => 'stock1', + 'uniquename' => 'TEST:1', + 'type_id' => $subtaxa_cvterm_id, + ]) + ->execute(); + + ob_start(); + $grabbed_dup_stock_id = $this->importer->getStockID('stock1', 'TEST:1', $organism_id); + $printed_output = ob_get_clean(); + $this->assertTrue($printed_output == 'Found more than one stock ID for "stock1".', "Did not get the expected error message when testing for duplicate stock IDs."); + } } From 5b06962273044be7d2330900dc5e6b3d91a8e9eb Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 16 Oct 2023 17:19:05 -0600 Subject: [PATCH 17/48] Implemented the config factory and started setting and pulling config terms --- .../GermplasmAccessionImporter.php | 77 +++++++++++++++++-- .../GermplasmAccessionImporterTest.php | 39 +++++++--- 2 files changed, 99 insertions(+), 17 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 649a848..5ebf958 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -3,6 +3,9 @@ namespace Drupal\trpcultivate_germplasm\Plugin\TripalImporter; use Drupal\tripal_chado\TripalImporter\ChadoImporterBase; +use Symfony\Component\DependencyInjection\ContainerInterface; +use Drupal\tripal_chado\Database\ChadoConnection; +use Drupal\Core\Config\ConfigFactoryInterface; /** * Provides an importer for loading germplasm accessions from a tab-delimited @@ -32,6 +35,63 @@ class GermplasmAccessionImporter extends ChadoImporterBase { */ protected $error_tracker = FALSE; + /** + * The service for retreiving configuration values. + * + * @var \Drupal\Core\Config\ConfigFactoryInterface + */ + protected $config_factory; + + /** + * Implements ContainerFactoryPluginInterface->create(). + * + * OVERRIDES create() from the parent, ChadoImporterBase.php, in order to introduce the + * config factory + * + * Since we have implemented the ContainerFactoryPluginInterface this static function + * will be called behind the scenes when a Plugin Manager uses createInstance(). Specifically + * this method is used to determine the parameters to pass to the contructor. + * + * @param \Symfony\Component\DependencyInjection\ContainerInterface $container + * @param array $configuration + * @param string $plugin_id + * @param mixed $plugin_definition + * + * @return static + */ + public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition) { + return new static( + $configuration, + $plugin_id, + $plugin_definition, + $container->get('tripal_chado.database'), + $container->get('config.factory') + ); + } + + /** + * Implements __contruct(). + * + * OVERRIDES __construct() from the parent, ChadoImporterBase.php, in order to introduce + * the config factory + * + * Since we have implemented the ContainerFactoryPluginInterface, the constructor + * will be passed additional parameters added by the create() function. This allows + * our plugin to use dependency injection without our plugin manager service needing + * to worry about it. + * + * @param array $configuration + * @param string $plugin_id + * @param mixed $plugin_definition + * @param Drupal\tripal_chado\Database\ChadoConnection $connection + * @param + */ + public function __construct(array $configuration, $plugin_id, $plugin_definition, ChadoConnection $connection, ConfigFactoryInterface $config_factory) { + parent::__construct($configuration, $plugin_id, $plugin_definition, $connection); + + $this->config_factory = $config_factory; + } + /** * @{inheritdoc} */ @@ -177,12 +237,11 @@ public function run(){ $organism_ID = $this->getOrganismID($genus_name, $germplasm_species, $germplasm_subtaxa); // STEP 2: Check/Insert this germplasm into the Chado stock table - if ($organism_ID) { - $stock_id = $this->getStockID($germplasm_name, $accession_number, $organism_ID); - } + $stock_id = $this->getStockID($germplasm_name, $accession_number, $organism_ID); // STEP 3: Load the external database info into Chado dbxref table + // STEP 4: Load stock properties (if provided) for: // $institute_code // $institute_name @@ -252,6 +311,9 @@ public function getOrganismID($genus_name, $germplasm_species, $germplasm_subtax */ public function getStockID($germplasm_name, $accession_number, $organism_ID) { + $germplasm_config = $this->config_factory->get('trpcultivate_germplasm.settings'); + $accession_type_id = $germplasm_config->get('terms.accession'); + // First query the stock table just using the germplasm name and organism ID $query = $this->connection->select('1:stock', 's') ->fields('s', ['stock_id', 'uniquename', 'type_id']); @@ -264,6 +326,7 @@ public function getStockID($germplasm_name, $accession_number, $organism_ID) { $this->error_tracker = TRUE; return false; } + if (sizeof($record) == 1) { // Handle the situation where a stock record exists // Check the uniquename matches the accession_number column in the file @@ -273,7 +336,11 @@ public function getStockID($germplasm_name, $accession_number, $organism_ID) { return false; } // Check the type_id is of type accession - // if ($record[0]->type_id) + if ($accession_type_id != $record[0]->type_id) { + $this->logger->error("A stock already exists for \"@germplasm_name\" but with a type ID of \"@type\" which is not of type \"accession\".", ['@germplasm_name' => $germplasm_name, '@type' => $accession_type_id]); + $this->error_tracker = TRUE; + return false; + } // Confirmed that the selected record matches what's in the upload file, so return the stock_id return $record[0]->stock_id; @@ -284,7 +351,7 @@ public function getStockID($germplasm_name, $accession_number, $organism_ID) { 'organism_id' => $organism_ID, 'name' => $germplasm_name, 'uniquename' => $accession_number, - //'type_id' => get_CV_term(); + 'type_id' => $accession_type_id ); $this->logger->notice("Inserting \"@germplasm_name\".", ['@germplasm_name' => $germplasm_name]); diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index 3456c76..f3171fe 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -49,7 +49,7 @@ protected function setUp(): void { // Ensure we can access file_managed related functionality from Drupal. // ... users need access to system.action config? - $this->installConfig('system'); + $this->installConfig(['system', 'trpcultivate_germplasm']); // ... managed files are associated with a user. $this->installEntitySchema('user'); // ... Finally the file module + tables itself. @@ -73,11 +73,19 @@ protected function setUp(): void { }); $container->set('tripal.logger', $mock_logger); + $this->config_factory = \Drupal::configFactory(); + $germplasm_config = $this->config_factory->getEditable('trpcultivate_germplasm.settings'); + $subtaxa_cvterm_id = $this->getCVtermID('TAXRANK', '0000023'); + $germplasm_config->set('terms.subtaxa', $subtaxa_cvterm_id); + $germplasm_config->set('terms.accession', 9); + $germplasm_config->save(); + $this->importer = new \Drupal\trpcultivate_germplasm\Plugin\TripalImporter\GermplasmAccessionImporter( [], 'trpcultivate-germplasm-accession', $this->definitions, - $this->connection + $this->connection, + $this->config_factory ); } @@ -180,15 +188,12 @@ public function testGermplasmAccessionImporterGetStockID() { ->execute(); // Insert a stock - // @TODO: FIND THE APPROPRIATE CVTERM ID FOR ACCESSION - $accession_cvterm_id = $this->getCVtermID('TAXRANK', '0000024'); - $stock_id = $this->connection->insert('1:stock') ->fields([ 'organism_id' => $organism_id, 'name' => 'stock1', 'uniquename' => 'TEST:1', - 'type_id' => $accession_cvterm_id, + 'type_id' => 9, ]) ->execute(); @@ -197,13 +202,23 @@ public function testGermplasmAccessionImporterGetStockID() { $this->assertEquals($grabbed_stock_id, $stock_id, "The stock ID grabbed by the importer does not match the one that was inserted into the database."); // Test that a stock not in the database successfully gets inserted - //ob_start(); - //$created_stock_id = $this->importer->getStockID('stock2', 'TEST:2', $organism_id); - //$printed_output = ob_get_clean(); - //$this->assertTrue($printed_output == 'Inserting "stock2".', "Did not get the expected notice message when inserting a new stock."); + ob_start(); + $created_stock_id = $this->importer->getStockID('stock2', 'TEST:2', $organism_id); + $printed_output = ob_get_clean(); + $this->assertTrue($printed_output == 'Inserting "stock2".', "Did not get the expected notice message when inserting a new stock."); + + $stock2_query = $this->connection->select('1:stock', 's') + ->fields('s', ['stock_id']) + ->condition('organism_id', $organism_id, '=') + ->condition('name', 'stock2', '=') + ->condition('uniquename', 'TEST:2', '=') + ->condition('type_id', 9, '='); + $stock2_record = $stock2_query->execute()->fetchAll(); + $this->assertEquals($created_stock_id, $stock2_record[0]->stock_id, "The stock ID inserted for \"stock2\" does not match the stock ID returned by getStockID()."); + + // No test for if the insert fails, since most likely will get a complaint from Chado // Test for a stock name + organism that already exists but has a different accession - ob_start(); $grabbed_dup_stock_name = $this->importer->getStockID('stock1', 'TEST:1000', $organism_id); $printed_output = ob_get_clean(); @@ -215,7 +230,7 @@ public function testGermplasmAccessionImporterGetStockID() { 'organism_id' => $organism_id, 'name' => 'stock1', 'uniquename' => 'TEST:1', - 'type_id' => $subtaxa_cvterm_id, + 'type_id' => 10, ]) ->execute(); From 3012a5325f9fcd7ced421e4c1d1623dcf5f4008f Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Tue, 17 Oct 2023 17:19:08 -0600 Subject: [PATCH 18/48] Started implementing getDbxrefID --- .../GermplasmAccessionImporter.php | 109 +++++++++++++++--- 1 file changed, 93 insertions(+), 16 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 5ebf958..1b6437b 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -37,15 +37,15 @@ class GermplasmAccessionImporter extends ChadoImporterBase { /** * The service for retreiving configuration values. - * + * * @var \Drupal\Core\Config\ConfigFactoryInterface */ protected $config_factory; /** * Implements ContainerFactoryPluginInterface->create(). - * - * OVERRIDES create() from the parent, ChadoImporterBase.php, in order to introduce the + * + * OVERRIDES create() from the parent, ChadoImporterBase.php, in order to introduce the * config factory * * Since we have implemented the ContainerFactoryPluginInterface this static function @@ -71,7 +71,7 @@ public static function create(ContainerInterface $container, array $configuratio /** * Implements __contruct(). - * + * * OVERRIDES __construct() from the parent, ChadoImporterBase.php, in order to introduce * the config factory * @@ -84,7 +84,7 @@ public static function create(ContainerInterface $container, array $configuratio * @param string $plugin_id * @param mixed $plugin_definition * @param Drupal\tripal_chado\Database\ChadoConnection $connection - * @param + * @param */ public function __construct(array $configuration, $plugin_id, $plugin_definition, ChadoConnection $connection, ConfigFactoryInterface $config_factory) { parent::__construct($configuration, $plugin_id, $plugin_definition, $connection); @@ -141,8 +141,7 @@ public function form($form, &$form_state) { $form = parent::form($form, $form_state); // Select the entire genus field and make sure it is sorted and distinct - $connection = \Drupal::service('tripal_chado.database'); - $genus_query = $connection->select('1:organism', 'o') + $genus_query = $this->connection->select('1:organism', 'o') ->fields('o',['genus']) ->orderBy('genus') ->distinct(); @@ -234,13 +233,17 @@ public function run(){ $synonyms = $germplasm_columns[11]; // STEP 1: Pull out the organism ID for the current germplasm - $organism_ID = $this->getOrganismID($genus_name, $germplasm_species, $germplasm_subtaxa); + $organism_id = $this->getOrganismID($genus_name, $germplasm_species, $germplasm_subtaxa); // STEP 2: Check/Insert this germplasm into the Chado stock table - $stock_id = $this->getStockID($germplasm_name, $accession_number, $organism_ID); + if ($organism_id) { + $stock_id = $this->getStockID($germplasm_name, $accession_number, $organism_id); + } // STEP 3: Load the external database info into Chado dbxref table - + if ($stock_id) { + $dbxref_id = $this->getDbxrefID($external_database, $stock_id, $accession_number); + } // STEP 4: Load stock properties (if provided) for: // $institute_code @@ -303,13 +306,14 @@ public function getOrganismID($genus_name, $germplasm_species, $germplasm_subtax * The name of the germplasm. * @param string $accession_number * A unique identifier for the germplasm accession. - * @param int $organism_ID + * @param int $organism_id * The primary key of the stock's organism in the organism table * @return int|false * The value of the primary key for the stock record in Chado. If the stock already - * exists or cannot be inserted, then FALSE is returned. + * exists and does not match the accession number or type, or it cannot be inserted, + * then FALSE is returned. */ - public function getStockID($germplasm_name, $accession_number, $organism_ID) { + public function getStockID($germplasm_name, $accession_number, $organism_id) { $germplasm_config = $this->config_factory->get('trpcultivate_germplasm.settings'); $accession_type_id = $germplasm_config->get('terms.accession'); @@ -318,7 +322,7 @@ public function getStockID($germplasm_name, $accession_number, $organism_ID) { $query = $this->connection->select('1:stock', 's') ->fields('s', ['stock_id', 'uniquename', 'type_id']); $query->condition('s.name', $germplasm_name, '=') - ->condition('s.organism_id', $organism_ID, '='); + ->condition('s.organism_id', $organism_id, '='); $record = $query->execute()->fetchAll(); if (sizeof($record) >= 2) { @@ -327,7 +331,7 @@ public function getStockID($germplasm_name, $accession_number, $organism_ID) { return false; } - if (sizeof($record) == 1) { + elseif (sizeof($record) == 1) { // Handle the situation where a stock record exists // Check the uniquename matches the accession_number column in the file if ($accession_number != $record[0]->uniquename) { @@ -348,7 +352,7 @@ public function getStockID($germplasm_name, $accession_number, $organism_ID) { // Confirmed that a stock record doesn't yet exist, so now we create one else { $values = array( - 'organism_id' => $organism_ID, + 'organism_id' => $organism_id, 'name' => $germplasm_name, 'uniquename' => $accession_number, 'type_id' => $accession_type_id @@ -372,6 +376,79 @@ public function getStockID($germplasm_name, $accession_number, $organism_ID) { } } + /** + * Checks if a dbxref exists in Chado and if not, inserts it. Then, updates + * the stock table to include the dbxref_id. Returns the primary + * key in the dbxref table. + * + * @param string $external_database + * The name of the institution who assigned the accession. + * @param int $stock_id + * The value of the primary key for the stock record in Chado. + * @param string $accession_number + * A unique identifier for the germplasm accession. + * @return int|false + * The value of the primary key for the dbxref record in Chado. + */ + public function getDbxrefID($external_database, $stock_id, $accession_number) { + // Check if the external database exists in chado.db + $db_query = $this->connection->select('1:db', 'db') + ->fields('db', ['db_id']); + $db_query->condition('db.name', $external_database, '='); + $db_record = $db_query->execute()->fetchAll(); + + if (sizeof($db_record) >= 2) { + $this->logger->error("Found more than one db ID for \"@external_db\".", ['@external_db' => $external_database]); + $this->error_tracker = TRUE; + return false; + } + + elseif (sizeof($db_record) == 0) { + $this->logger->error("Couldn't find \"@external_db\" in chado.db.", ['@external_db' => $external_database]); + $this->error_tracker = TRUE; + return false; + } + + // Confirmed that a single record of this external database exists + $db_id = $db_record[0]->db_id; + + // Now to check for the dbxref record + $dbx_query = $this->connection->select('1:dbxref', 'dbx') + ->fields('dbx', ['dbxref_id']); + $dbx_query->condition('dbx.accession', $accession_number, '=') + ->condition('dbxdb_id', $db_id, '='); + $dbx_record = $dbx_query->execute()->fetchAll(); + + if (sizeof($dbx_record) >= 2) { + $this->logger->error("Found more than one dbxref ID for \"@accession\".", ['@accession' => $accession_number]); + $this->error_tracker = TRUE; + return false; + } + elseif (sizeof($dbx_record) == 1) { + $dbxref_id = $dbx_record[0]->dbxref_id; + } + // Couldn't find the dbxref_id for this accession, so insert it + else { + $values = [ + 'db_id' => $db_id, + 'accession' => $accession_number + ]; + $result = $this->connection->insert('1:dbxref') + ->fields($values) + ->execute(); + + // If the primary key is not available, then the insert failed + if (!$result) { + $this->logger->error("Insertion of \"@accession\" into chado.dbxref failed.", ['@accession' => $accession_number]); + $this->error_tracker = TRUE; + return false; + } + } + + // Now update the stock table to include the dbxref_id + + } + /* * {@inheritdoc} */ From 70cb9dce2f6337d097f5bee63ad2495d7cff2deb Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Wed, 18 Oct 2023 17:03:15 -0600 Subject: [PATCH 19/48] Finished the getDbxrefID function, and caught up with tests --- .../GermplasmAccessionImporter.php | 51 +++++++- .../GermplasmAccessionImporterTest.php | 121 +++++++++++++++++- 2 files changed, 165 insertions(+), 7 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 1b6437b..2636ccc 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -391,7 +391,10 @@ public function getStockID($germplasm_name, $accession_number, $organism_id) { * The value of the primary key for the dbxref record in Chado. */ public function getDbxrefID($external_database, $stock_id, $accession_number) { + // ------------------------------------------------- // Check if the external database exists in chado.db + // If not, report an error + // ------------------------------------------------- $db_query = $this->connection->select('1:db', 'db') ->fields('db', ['db_id']); $db_query->condition('db.name', $external_database, '='); @@ -404,7 +407,7 @@ public function getDbxrefID($external_database, $stock_id, $accession_number) { } elseif (sizeof($db_record) == 0) { - $this->logger->error("Couldn't find \"@external_db\" in chado.db.", ['@external_db' => $external_database]); + $this->logger->error("Unable to find \"@external_db\" in chado.db.", ['@external_db' => $external_database]); $this->error_tracker = TRUE; return false; } @@ -412,11 +415,14 @@ public function getDbxrefID($external_database, $stock_id, $accession_number) { // Confirmed that a single record of this external database exists $db_id = $db_record[0]->db_id; - // Now to check for the dbxref record + // ------------------------------------------------- + // Check if the dbxref for this stock already exists + // If not, insert it + // ------------------------------------------------- $dbx_query = $this->connection->select('1:dbxref', 'dbx') ->fields('dbx', ['dbxref_id']); $dbx_query->condition('dbx.accession', $accession_number, '=') - ->condition('dbxdb_id', $db_id, '='); + ->condition('dbx.db_id', $db_id, '='); $dbx_record = $dbx_query->execute()->fetchAll(); if (sizeof($dbx_record) >= 2) { @@ -443,9 +449,46 @@ public function getDbxrefID($external_database, $stock_id, $accession_number) { $this->error_tracker = TRUE; return false; } + else { + $dbxref_id = $result; + } } - // Now update the stock table to include the dbxref_id + // ------------------------------------------------------------------------ + // Update the stock table to include the dbxref_id + // After making sure a different one doesn't already exist (throw an error) + // ------------------------------------------------------------------------ + $stock_query = $this->connection->select('1:stock', 's') + ->fields('s', ['dbxref_id']); + $stock_query->condition('s.stock_id', $stock_id, '='); + $stock_record = $stock_query->execute()->fetchAll(); + + if ($stock_record[0]->dbxref_id == "") { + $update_stock = $this->connection->update('1:stock') + ->fields(['dbxref_id' => $dbxref_id]) + ->condition('stock_id', $stock_id, '=') + ->execute(); + + // Since update queries return the number of rows affected, check that only one row was changed + if ($update_stock != 1) { + $this->logger->error("An attempt to update the dbxref_id of \"@stock\" reported that \"@number\" rows were affected.", ['@stock' => $accession_number, '@number' => $update_stock]); + $this->error_tracker = TRUE; + return false; + } + else { + return $dbxref_id; + } + } + // Otherwise, the correct dbxref_id might already be set so we're good to go + elseif ($stock_record[0]->dbxref_id == $dbxref_id) { + return $dbxref_id; + } + // OR, it is something entirely different - so report an error + else { + $this->logger->error("There is already a primary dbxref_id for stock ID \"@stock\" that does not match the external database and accession provided in the file (@external_db:@accession).", ['@stock' => $stock_id, '@external_db' => $external_database, '@accession' => $accession_number]); + $this->error_tracker = TRUE; + return false; + } } diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index f3171fe..b937e63 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -206,7 +206,7 @@ public function testGermplasmAccessionImporterGetStockID() { $created_stock_id = $this->importer->getStockID('stock2', 'TEST:2', $organism_id); $printed_output = ob_get_clean(); $this->assertTrue($printed_output == 'Inserting "stock2".', "Did not get the expected notice message when inserting a new stock."); - + $stock2_query = $this->connection->select('1:stock', 's') ->fields('s', ['stock_id']) ->condition('organism_id', $organism_id, '=') @@ -215,7 +215,7 @@ public function testGermplasmAccessionImporterGetStockID() { ->condition('type_id', 9, '='); $stock2_record = $stock2_query->execute()->fetchAll(); $this->assertEquals($created_stock_id, $stock2_record[0]->stock_id, "The stock ID inserted for \"stock2\" does not match the stock ID returned by getStockID()."); - + // No test for if the insert fails, since most likely will get a complaint from Chado // Test for a stock name + organism that already exists but has a different accession @@ -238,7 +238,122 @@ public function testGermplasmAccessionImporterGetStockID() { $grabbed_dup_stock_id = $this->importer->getStockID('stock1', 'TEST:1', $organism_id); $printed_output = ob_get_clean(); $this->assertTrue($printed_output == 'Found more than one stock ID for "stock1".', "Did not get the expected error message when testing for duplicate stock IDs."); - } + /** + * Tests focusing on the Germplasm Accession Importer getDbxrefID() function + * + * @group germ_accession_importer + */ + public function testGermplasmAccessionImporterGetDbxrefID() { + + // Insert an organism + $subtaxa_cvterm_id = $this->getCVtermID('TAXRANK', '0000023'); + + $organism_id = $this->connection->insert('1:organism') + ->fields([ + 'genus' => 'Tripalus', + 'species' => 'databasica', + 'infraspecific_name' => 'chadoii', + 'type_id' => $subtaxa_cvterm_id, + ]) + ->execute(); + + // Insert a stock + $accession = 'TEST:1'; + + $stock_id = $this->connection->insert('1:stock') + ->fields([ + 'organism_id' => $organism_id, + 'name' => 'stock1', + 'uniquename' => $accession, + 'type_id' => 9, + ]) + ->execute(); + + // Attempt to call the function before inserting an external database + ob_start(); + $non_existing_external_db = $this->importer->getDbxrefID('PRETEND', $stock_id, $accession); + $printed_output = ob_get_clean(); + $this->assertTrue($printed_output == 'Unable to find "PRETEND" in chado.db.', "Did not get the expected error message when looking up an external database that does not yet exist."); + + // Verify that the stock has an empty dbxref_id + $empty_stock_query = $this->connection->select('1:stock', 's') + ->fields('s', ['dbxref_id']); + $empty_stock_query->condition('s.stock_id', $stock_id, '='); + $empty_stock_record = $empty_stock_query->execute()->fetchAll(); + + $this->assertEmpty($empty_stock_record[0]->dbxref_id, "The stock just inserted (stock1) already has a dbxref_id."); + + // Now add an external db + $db_id = $this->connection->insert('1:db') + ->fields([ + 'name' => 'Test DB', + ]) + ->execute(); + + // ----------------------------- ROUND 1 ------------------------------- + // Call the function and check that dbxref is inserted and stock updated + $round_one_dbxref = $this->importer->getDbxrefID('Test DB', $stock_id, $accession); + + // Check that the dbxref was inserted successfully + $r1_dbx_query = $this->connection->select('1:dbxref', 'dbx') + ->fields('dbx', ['dbxref_id']); + $r1_dbx_query->condition('dbx.accession', $accession, '=') + ->condition('dbx.db_id', $db_id, '='); + $r1_dbx_record = $r1_dbx_query->execute()->fetchAll(); + + $this->assertEquals($round_one_dbxref, $r1_dbx_record[0]->dbxref_id, "The dbxref_id that was inserted does not match what was queried."); + + // Check that the stock was updated successfully + $updated_stock_query = $this->connection->select('1:stock', 's') + ->fields('s', ['dbxref_id']); + $updated_stock_query->condition('s.stock_id', $stock_id, '='); + $updated_stock_record = $updated_stock_query->execute()->fetchAll(); + + $this->assertEquals($round_one_dbxref, $updated_stock_record[0]->dbxref_id, "The stock just inserted (stock1) was not successfully updated with a dbxref_id."); + + // ----------------------------- ROUND 2 ------------------------------- + // Call the function again and check that the results are still the same + // The purpose of this test is to trigger the elseif statements for both + // the dbxref check and stock.dbxref_id + $round_two_dbxref = $this->importer->getDbxrefID('Test DB', $stock_id, $accession); + + // Check that the dbxref was selected successfully + $r2_dbx_query = $this->connection->select('1:dbxref', 'dbx') + ->fields('dbx', ['dbxref_id']); + $r2_dbx_query->condition('dbx.accession', $accession, '=') + ->condition('dbx.db_id', $db_id, '='); + $r2_dbx_record = $r2_dbx_query->execute()->fetchAll(); + + $this->assertEquals($round_two_dbxref, $r2_dbx_record[0]->dbxref_id, "The dbxref_id has changed unexpectedly in round 2."); + + // Check that the stock was updated successfully + $r2_updated_stock_query = $this->connection->select('1:stock', 's') + ->fields('s', ['dbxref_id']); + $r2_updated_stock_query->condition('s.stock_id', $stock_id, '='); + $r2_updated_stock_record = $r2_updated_stock_query->execute()->fetchAll(); + + $this->assertEquals($round_two_dbxref, $r2_updated_stock_record[0]->dbxref_id, "The dbxref_id of stock1 was unexpectedly changed in round 2 from round 1."); + // --------------------------------------------------------------------- + // Manually insert a dbxref with the same accession but a different db_id + $second_db_name = 'Second Test DB'; + $second_db_id = $this->connection->insert('1:db') + ->fields([ + 'name' => $second_db_name, + ]) + ->execute(); + + $second_dbxref_id = $this->connection->insert('1:dbxref') + ->fields([ + 'db_id' => $second_db_id, + 'accession' => $accession + ]) + ->execute(); + + ob_start(); + $multiple_dbxref_accessions = $this->importer->getDbxrefID($second_db_name, $stock_id, $accession); + $printed_output = ob_get_clean(); + $this->assertTrue($printed_output == 'There is already a primary dbxref_id for stock ID "1" that does not match the external database and accession provided in the file (Second Test DB:TEST:1).', "Did not get the expected error message when inserting a dbxref with an existing accession with a different db."); + } } From 22628016ab919dc94131f305d0f00bcd818705da Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Thu, 19 Oct 2023 17:43:29 -0600 Subject: [PATCH 20/48] Implemented get and setCVterm methods and most of loadStockProperties --- .../GermplasmAccessionImporter.php | 163 ++++++++++++++++-- .../GermplasmAccessionImporterTest.php | 49 +++++- 2 files changed, 192 insertions(+), 20 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 2636ccc..e73f186 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -42,6 +42,11 @@ class GermplasmAccessionImporter extends ChadoImporterBase { */ protected $config_factory; + /** + * An associative array of cvterms as the key and the cvterm_id as the value. + */ + protected $cvterms = []; + /** * Implements ContainerFactoryPluginInterface->create(). * @@ -134,6 +139,31 @@ public function describeUploadFileFormat() { return $output; } + /** + * Set a cvterm with its cvterm_id + * + * @param string $key + * A key used in the config settings.yml + * @param int $cvterm_id + * @return TRUE + */ + public function setCVterm($key, $cvterm_id) { + $this->cvterms[$key] = $cvterm_id; + return TRUE; + } + + /** + * Get a cvterm ID, given a key that maps to the config settings.yml + * + * @param string $key + * The cvterm name + * @return int + * The cvterm ID + */ + public function getCVterm($key) { + return $this->cvterms[$key]; + } + /** * {@inheritDoc} */ @@ -191,6 +221,16 @@ public function run(){ // Grab the genus name $genus_name = $arguments['genus_name']; + // Set up our array of cv terms + $germplasm_config = $this->config_factory->get('trpcultivate_germplasm.settings'); + $this->setCVterm('accession', $germplasm_config->get('terms.accession')); + $this->setCVterm('institute_code', $germplasm_config->get('terms.institute_code')); + $this->setCVterm('institute_name', $germplasm_config->get('terms.institute_name')); + $this->setCVterm('country_of_origin_code', $germplasm_config->get('terms.country_of_origin_code')); + $this->setCVterm('biological_status_of_accession_code', $germplasm_config->get('terms.biological_status_of_accession_code')); + $this->setCVterm('breeding_method_DbId', $germplasm_config->get('terms.breeding_method_DbId')); + $this->setCVterm('pedigree', $germplasm_config->get('terms.pedigree')); + // Set up the ability to track progress so we can report it to the user $filesize = filesize($file_path); $this->setTotalItems($filesize); @@ -240,19 +280,26 @@ public function run(){ $stock_id = $this->getStockID($germplasm_name, $accession_number, $organism_id); } - // STEP 3: Load the external database info into Chado dbxref table if ($stock_id) { + // STEP 3: Load the external database info into Chado dbxref table $dbxref_id = $this->getDbxrefID($external_database, $stock_id, $accession_number); - } - - // STEP 4: Load stock properties (if provided) for: - // $institute_code - // $institute_name - // $country_of_origin_code - // $biological_status_of_accession_code - // $breeding_method_DbId - // $pedigree + // STEP 4: Load stock properties + // Since all stock properties are optional, check if they have an empty string + $stock_properties = [ + $institute_code, + $institute_name, + $country_of_origin_code, + $biological_status_of_accession_code, + $breeding_method_DbId, + $pedigree + ]; +// foreach ($stock_properties) { +// if (empty()) +// } + $this->loadStockProperties($stock_id, $stock_properties); + } + // STEP 5: Load synonyms } @@ -315,8 +362,7 @@ public function getOrganismID($genus_name, $germplasm_species, $germplasm_subtax */ public function getStockID($germplasm_name, $accession_number, $organism_id) { - $germplasm_config = $this->config_factory->get('trpcultivate_germplasm.settings'); - $accession_type_id = $germplasm_config->get('terms.accession'); + $accession_type_id = $this->getCVterm('accession'); // First query the stock table just using the germplasm name and organism ID $query = $this->connection->select('1:stock', 's') @@ -351,12 +397,12 @@ public function getStockID($germplasm_name, $accession_number, $organism_id) { } // Confirmed that a stock record doesn't yet exist, so now we create one else { - $values = array( + $values = [ 'organism_id' => $organism_id, 'name' => $germplasm_name, 'uniquename' => $accession_number, 'type_id' => $accession_type_id - ); + ]; $this->logger->notice("Inserting \"@germplasm_name\".", ['@germplasm_name' => $germplasm_name]); @@ -492,6 +538,95 @@ public function getDbxrefID($external_database, $stock_id, $accession_number) { } + /** + * Checks each property within an array and inserts them into chado.stockprop. + * Returns true if the insert was successful. + * + * @param int $stock_id + * The value of the primary key for the stock record in Chado. + * @param array $stock_properties + * An array of optional properties to be attached to a stock + * @return boolean + * Returns true if inserting all the properties was successful, + * including if there are no properties + */ + public function loadStockProperties($stock_id, $stock_properties) { + + // Log a notice if there's no properties to deal with, but since + // they are all optional, return true. + if (sizeof($stock_properties) == 0) { + $this->logger->notice("There are no stock properties to insert for stock ID \"@stock\".", ['@stock' => $stock_id]); + return true; + } + // Iterate through our properties + foreach ($stock_properties as $property) { + + // Lookup the CV term + $cvterm_id = getCVterm($property); + if ($cvterm_id) { + // Try to lookup the stockprop_id in Chado + $stockprop_query = $this->connection->select('1:stockprop', 'sp') + ->fields('sp', ['stockprop_id', 'value']) + ->condition('sp.stock_id', $stock_id, '=') + ->condition('sp.type_id', $cvterm_id, '='); + $stockprop_record = $stockprop_query->execute()->fetchAll(); + // If one or more record(s) exists for this stock, check if one is the same as in + // the file. If not, then add it but increase the rank by 1 + if (sizeof($stockprop_record) >= 1) { + $maxrank = 0; + foreach ($stockprop_record as $record) { + $found = false; + if ($record->value == $property) { + $found = true; + next; + } + else { + $rank = $record->rank; + if ($rank > $maxrank) { $maxrank = $rank; } + } + } + if ($found == false) { + // Insert this property into the stockprop table and increment the max rank by one + $values = [ + 'stock_id' => $stock_id, + 'type_id' => $cvterm_id, + 'value' => $property, + 'rank' => $maxrank++ + ]; + $result = $this->connection->insert('1:stockprop') + ->fields($values) + ->execute(); + + // If the primary key is not available, then the insert failed + if (!$result) { + $this->logger->error("Insertion of stock property \"@property\" into chado.stockprop failed.", ['@property' => $property]); + $this->error_tracker = TRUE; + return false; + } + } + } + + // If no records exist, then insert the property as normal + else { + $values = [ + 'stock_id' => $stock_id, + 'type_id' => $cvterm_id, + 'value' => $property + ]; + $result = $this->connection->insert('1:stockprop') + ->fields($values) + ->execute(); + + // If the primary key is not available, then the insert failed + if (!$result) { + $this->logger->error("Insertion of stock property \"@property\" into chado.stockprop failed.", ['@property' => $property]); + $this->error_tracker = TRUE; + return false; + } + } + } + } + } /* * {@inheritdoc} */ diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index b937e63..c87c27b 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -74,12 +74,6 @@ protected function setUp(): void { $container->set('tripal.logger', $mock_logger); $this->config_factory = \Drupal::configFactory(); - $germplasm_config = $this->config_factory->getEditable('trpcultivate_germplasm.settings'); - $subtaxa_cvterm_id = $this->getCVtermID('TAXRANK', '0000023'); - $germplasm_config->set('terms.subtaxa', $subtaxa_cvterm_id); - $germplasm_config->set('terms.accession', 9); - $germplasm_config->save(); - $this->importer = new \Drupal\trpcultivate_germplasm\Plugin\TripalImporter\GermplasmAccessionImporter( [], 'trpcultivate-germplasm-accession', @@ -87,6 +81,10 @@ protected function setUp(): void { $this->connection, $this->config_factory ); + + $subtaxa_cvterm_id = $this->getCVtermID('TAXRANK', '0000023'); + $this->importer->setCVterm('accession', 9); + $this->importer->setCVterm('subtaxa', $subtaxa_cvterm_id); } /** @@ -356,4 +354,43 @@ public function testGermplasmAccessionImporterGetDbxrefID() { $printed_output = ob_get_clean(); $this->assertTrue($printed_output == 'There is already a primary dbxref_id for stock ID "1" that does not match the external database and accession provided in the file (Second Test DB:TEST:1).', "Did not get the expected error message when inserting a dbxref with an existing accession with a different db."); } + + /** + * Tests focusing on the Germplasm Accession Importer loadStockProperties() function + * + * @group germ_accession_importer + */ + public function testGermplasmAccessionImporterloadStockProperties() { + // Insert an organism + $subtaxa_cvterm_id = $this->getCVtermID('TAXRANK', '0000023'); + + $organism_id = $this->connection->insert('1:organism') + ->fields([ + 'genus' => 'Tripalus', + 'species' => 'databasica', + 'infraspecific_name' => 'chadoii', + 'type_id' => $subtaxa_cvterm_id, + ]) + ->execute(); + + // Insert a stock + $stock_id = $this->connection->insert('1:stock') + ->fields([ + 'organism_id' => $organism_id, + 'name' => 'stock1', + 'uniquename' => 'TEST:1', + 'type_id' => 9, + ]) + ->execute(); + + // Try running the function with no properties. Expecting a notice but should + // return true + $stock_props = []; + ob_start(); + $this->importer->loadStockProperties($stock_id, $stock_props); + $printed_output = ob_get_clean(); + $this->assertTrue($printed_output == 'There are no stock properties to insert for stock ID "1".', "Expected a notice message when trying to insert 0 stock properties for a stock."); + + + } } From 40efcd7c3149eeafe7963fc4ffcfecebfffbadc6 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Fri, 20 Oct 2023 14:12:09 -0600 Subject: [PATCH 21/48] Finished tests for loadStockProperties --- .../GermplasmAccessionImporter.php | 60 +++++------- .../GermplasmAccessionImporterTest.php | 98 +++++++++++++++++-- 2 files changed, 115 insertions(+), 43 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index e73f186..ab58a84 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -264,12 +264,14 @@ public function run(){ $accession_number = $germplasm_columns[2]; $germplasm_species = $germplasm_columns[3]; $germplasm_subtaxa = $germplasm_columns[4]; - $institute_code = $germplasm_columns[5]; - $institute_name = $germplasm_columns[6]; - $country_of_origin_code = $germplasm_columns[7]; - $biological_status_of_accession_code = $germplasm_columns[8]; - $breeding_method_DbId = $germplasm_columns[9]; - $pedigree = $germplasm_columns[10]; + $stock_properties = [ + 'institute_code' => $germplasm_columns[5], + 'institute_name' => $germplasm_columns[6], + 'country_of_origin_code' => $germplasm_columns[7], + 'biological_status_of_accession_code' => $germplasm_columns[8], + 'breeding_method_DbId' => $germplasm_columns[9], + 'pedigree' => $germplasm_columns[10] + ]; $synonyms = $germplasm_columns[11]; // STEP 1: Pull out the organism ID for the current germplasm @@ -285,21 +287,9 @@ public function run(){ $dbxref_id = $this->getDbxrefID($external_database, $stock_id, $accession_number); // STEP 4: Load stock properties - // Since all stock properties are optional, check if they have an empty string - $stock_properties = [ - $institute_code, - $institute_name, - $country_of_origin_code, - $biological_status_of_accession_code, - $breeding_method_DbId, - $pedigree - ]; -// foreach ($stock_properties) { -// if (empty()) -// } - $this->loadStockProperties($stock_id, $stock_properties); + $load_props = $this->loadStockProperties($stock_id, $stock_properties); } - + // STEP 5: Load synonyms } @@ -552,21 +542,16 @@ public function getDbxrefID($external_database, $stock_id, $accession_number) { */ public function loadStockProperties($stock_id, $stock_properties) { - // Log a notice if there's no properties to deal with, but since - // they are all optional, return true. - if (sizeof($stock_properties) == 0) { - $this->logger->notice("There are no stock properties to insert for stock ID \"@stock\".", ['@stock' => $stock_id]); - return true; - } - // Iterate through our properties - foreach ($stock_properties as $property) { + foreach ($stock_properties as $property => $prop_value) { + // Skip if the value of this property is empty + if (($prop_value !== '0') && empty($prop_value)) { continue; } // Lookup the CV term - $cvterm_id = getCVterm($property); + $cvterm_id = $this->getCVterm($property); if ($cvterm_id) { // Try to lookup the stockprop_id in Chado $stockprop_query = $this->connection->select('1:stockprop', 'sp') - ->fields('sp', ['stockprop_id', 'value']) + ->fields('sp', ['stockprop_id', 'value', 'rank']) ->condition('sp.stock_id', $stock_id, '=') ->condition('sp.type_id', $cvterm_id, '='); $stockprop_record = $stockprop_query->execute()->fetchAll(); @@ -576,9 +561,9 @@ public function loadStockProperties($stock_id, $stock_properties) { $maxrank = 0; foreach ($stockprop_record as $record) { $found = false; - if ($record->value == $property) { + if ($record->value == $prop_value) { $found = true; - next; + break; } else { $rank = $record->rank; @@ -590,8 +575,8 @@ public function loadStockProperties($stock_id, $stock_properties) { $values = [ 'stock_id' => $stock_id, 'type_id' => $cvterm_id, - 'value' => $property, - 'rank' => $maxrank++ + 'value' => $prop_value, + 'rank' => ++$maxrank ]; $result = $this->connection->insert('1:stockprop') ->fields($values) @@ -611,7 +596,7 @@ public function loadStockProperties($stock_id, $stock_properties) { $values = [ 'stock_id' => $stock_id, 'type_id' => $cvterm_id, - 'value' => $property + 'value' => $prop_value ]; $result = $this->connection->insert('1:stockprop') ->fields($values) @@ -625,6 +610,11 @@ public function loadStockProperties($stock_id, $stock_properties) { } } } + else { + $this->logger->error("Unable to retrieve the cvterm_id of property \"@property\"", ['@property' => $property]); + $this->error_tracker = TRUE; + return false; + } } } /* diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index c87c27b..a257c07 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -85,6 +85,12 @@ protected function setUp(): void { $subtaxa_cvterm_id = $this->getCVtermID('TAXRANK', '0000023'); $this->importer->setCVterm('accession', 9); $this->importer->setCVterm('subtaxa', $subtaxa_cvterm_id); + $this->importer->setCVterm('institute_code', 10); + $this->importer->setCVterm('institute_name', 11); + $this->importer->setCVterm('country_of_origin_code',12); + $this->importer->setCVterm('biological_status_of_accession_code', 13); + $this->importer->setCVterm('breeding_method_DbId', 14); + $this->importer->setCVterm('pedigree', 15); } /** @@ -383,14 +389,90 @@ public function testGermplasmAccessionImporterloadStockProperties() { ]) ->execute(); - // Try running the function with no properties. Expecting a notice but should - // return true - $stock_props = []; - ob_start(); - $this->importer->loadStockProperties($stock_id, $stock_props); - $printed_output = ob_get_clean(); - $this->assertTrue($printed_output == 'There are no stock properties to insert for stock ID "1".', "Expected a notice message when trying to insert 0 stock properties for a stock."); - + // Declare our stock property variables with empty values + $empty_string = ''; + $stock_empty_props = [ + 'institute_code' => $empty_string, + 'institute_name' => $empty_string, + 'country_of_origin_code' => $empty_string, + 'biological_status_of_accession_code' => $empty_string, + 'breeding_method_DbId' => $empty_string, + 'pedigree' => $empty_string + ]; + + // "Load" our empty values and check that the stockprop table is empty + // before and after + $sp_initial_count = $this->connection->select('1:stockprop', 'sp') + ->condition('sp.stock_id', $stock_id, '=') + ->countQuery()->execute()->fetchField(); + $this->importer->loadStockProperties($stock_id, $stock_empty_props); + $sp_empty_count = $this->connection->select('1:stockprop', 'sp') + ->condition('sp.stock_id', $stock_id, '=') + ->countQuery()->execute()->fetchField(); + $this->assertEquals($sp_initial_count, $sp_empty_count, "The row count of the stockprop table before and after inserting empty values is not the same."); + + // Now load in all 6 stock properties for this stock_id + $stock_props = [ + 'institute_code' => 'CUAC', + 'institute_name' => 'Crop Development Center, University of Saskatchewan', + 'country_of_origin_code' => 124, + 'biological_status_of_accession_code' => 410, + 'breeding_method_DbId' => 'Recurrent selection', + 'pedigree' => '1049F^3/819-5R' + ]; + $stock_prop_count = count($stock_props); + $this->importer->loadStockProperties($stock_id, $stock_props); + $sp_six_count = $this->connection->select('1:stockprop', 'sp') + ->condition('sp.stock_id', $stock_id, '=') + ->countQuery()->execute()->fetchField(); + $this->assertEquals($sp_six_count, $stock_prop_count, "The row count of the stockprop table after inserting 6 values is not correct."); + + // Select a random property to see if it inserted correctly + $sp_six_institute_name = $this->connection->select('1:stockprop', 'sp') + ->fields('sp', ['value']) + ->condition('sp.stock_id', $stock_id, '=') + ->condition('sp.type_id', 11, '='); + $sp_six_institute_name_record = $sp_six_institute_name->execute()->fetchAll(); + $this->assertEquals($sp_six_institute_name_record[0]->value,'Crop Development Center, University of Saskatchewan', "The selected stockprop value for institute name does not match what was inserted."); + + // Now add some new properties for the same stock_id + $new_stock_props = [ + 'biological_status_of_accession_code' => 500, // New value + 'breeding_method_DbId' => 'Breeder line', // New value + 'pedigree' => '1049F^3/819-5R' // Old value + ]; + $new_sp_count = count($new_stock_props); + // 2 should be added to the stockprop table, 1 should not + $total_expected_sp_count = $sp_six_count + $new_sp_count - 1; + + $this->importer->loadStockProperties($stock_id, $new_stock_props); + $sp_eight_count = $this->connection->select('1:stockprop', 'sp') + ->condition('sp.stock_id', $stock_id, '=') + ->countQuery()->execute()->fetchField(); + $this->assertEquals($sp_eight_count, $total_expected_sp_count, "The row count of the stockprop table after inserting 2 additional values is not correct."); + + // Select one of our new properties and compare the ranks + $sp_eight_breeding_method_DbId = $this->connection->select('1:stockprop', 'sp') + ->fields('sp', ['value', 'rank']) + ->condition('sp.stock_id', $stock_id, '=') + ->condition('sp.type_id', 14, '='); + $sp_eight_breeding_method_DbId_record = $sp_eight_breeding_method_DbId->execute()->fetchAll(); + $first_value = $sp_eight_breeding_method_DbId_record[0]->value; + $this->assertEquals($first_value,'Recurrent selection', "The selected stockprop value for the first breeding_method_db_id does not match what was inserted."); + $second_value = $sp_eight_breeding_method_DbId_record[1]->value; + $this->assertEquals($second_value,'Breeder line', "The selected stockprop value for the second breeding_method_db_id does not match what was inserted."); + + $first_rank = $sp_eight_breeding_method_DbId_record[0]->rank; + $second_rank = $sp_eight_breeding_method_DbId_record[1]->rank; + $this->assertGreaterThan($first_rank, $second_rank, "The rank of the second inserted stockprop for breeding_method_db_id is not greater than the first one."); + + // Ensure only one record is retrieved for pedigree since the second array + // contained an identical value for it + $sp_eight_pedigree_count = $this->connection->select('1:stockprop', 'sp') + ->condition('sp.stock_id', $stock_id, '=') + ->condition('sp.type_id', 15, '=') + ->countQuery()->execute()->fetchField(); + $this->assertEquals($sp_eight_pedigree_count, 1, "The number of records for stockprop pedigree is not 1."); } } From 22ea136bc1e2d94e1b6102c89db8864b89d4e4f3 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 6 Nov 2023 17:26:31 -0600 Subject: [PATCH 22/48] Started the loadSynonyms function --- .../GermplasmAccessionImporter.php | 46 +++++++++++++++++-- .../GermplasmAccessionImporterTest.php | 45 +++++++++++++++++- 2 files changed, 86 insertions(+), 5 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index ab58a84..fc61d08 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -118,7 +118,7 @@ public function describeUploadFileFormat() { 'Biological Status of Accession' => 'The 3 digit code representing the biological status of the accession (e.g. 410)', 'Breeding Method' => 'The unique identifier for the breeding method used to create this germplasm (e.g. "Recurrent selection")', 'Pedigree' => 'The cross name and optional selection history (e.g. 1049F^3/819-5R)', - 'Synonyms' => 'Any synonyms of the accession. (e.g. Redberry)', + 'Synonyms' => 'Any synonyms of the accession, separated by a comma. (e.g. Redberry)', ]; $required_col = ['Germplasm Name', 'External Database', 'Accession Number', 'Germplasm Species']; @@ -288,10 +288,10 @@ public function run(){ // STEP 4: Load stock properties $load_props = $this->loadStockProperties($stock_id, $stock_properties); - } - - // STEP 5: Load synonyms + // STEP 5: Load synonyms + $load_synonyms = $this->loadSynonyms($stock_id, $synonyms); + } } } @@ -617,6 +617,44 @@ public function loadStockProperties($stock_id, $stock_properties) { } } } + + /** + * Loads each synonym into the chado.stock_synonym table + * Returns true if the insert was successful. + * + * @param int $stock_id + * The value of the primary key for the stock record in Chado. + * @param string $stock_properties + * The name that is a synonym of the current germplasm. Multiple + * synonyms may be specified, in which case they are expected to + * be separated using a comma or semicolon. + * @return boolean + * Returns true if inserting all the properties was successful, + * including if there are no properties + */ + public function loadSynonyms($stock_id, $synonyms) { + // First check if we a synonym to deal with + if ($synonyms) { + // Separate out multiple synonyms if we have them by either + // semicolons or commas. Whitespace is optional + $all_synonyms = preg_split("/[;,]\s*/", $synonyms); + + foreach ($all_synonyms as $synonym) { + $synonym = trim($synonym); + $synonym_type_id = $this->getCVterm('synonym'); + + // Check for and load any synonyms to chado.synonym + $synonym_query = $this->connection->select('1:synonym', 's') + ->fields('s', ['synonym_id']) + ->condition('s.name', $synonym, '=') + ->condition('s.type_id', $synonym_type_id, '='); + $stockprop_record = $synonym_query->execute()->fetchAll(); + + // Make sure there aren't 2 or more records for this synonym + + } + } + } /* * {@inheritdoc} */ diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index a257c07..0c001cf 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -91,6 +91,7 @@ protected function setUp(): void { $this->importer->setCVterm('biological_status_of_accession_code', 13); $this->importer->setCVterm('breeding_method_DbId', 14); $this->importer->setCVterm('pedigree', 15); + $this->importer->setCVterm('synonym', 16); } /** @@ -366,7 +367,7 @@ public function testGermplasmAccessionImporterGetDbxrefID() { * * @group germ_accession_importer */ - public function testGermplasmAccessionImporterloadStockProperties() { + public function testGermplasmAccessionImporterLoadStockProperties() { // Insert an organism $subtaxa_cvterm_id = $this->getCVtermID('TAXRANK', '0000023'); @@ -475,4 +476,46 @@ public function testGermplasmAccessionImporterloadStockProperties() { ->countQuery()->execute()->fetchField(); $this->assertEquals($sp_eight_pedigree_count, 1, "The number of records for stockprop pedigree is not 1."); } + + /** + * Tests focusing on the Germplasm Accession Importer loadSynonyms() function + * + * @group germ_accession_importer + */ + public function testGermplasmAccessionImporterLoadSynonyms() { + + // Insert an organism + $subtaxa_cvterm_id = $this->getCVtermID('TAXRANK', '0000023'); + + $organism_id = $this->connection->insert('1:organism') + ->fields([ + 'genus' => 'Tripalus', + 'species' => 'databasica', + 'infraspecific_name' => 'chadoii', + 'type_id' => $subtaxa_cvterm_id, + ]) + ->execute(); + + // Insert a stock + $stock_id = $this->connection->insert('1:stock') + ->fields([ + 'organism_id' => $organism_id, + 'name' => 'stock1', + 'uniquename' => 'TEST:1', + 'type_id' => 9, + ]) + ->execute(); + + // Attempt to load an empty string (ie. an empty column in the file) + $stock1_synonym = ''; + $this->importer->loadSynonyms($stock_id, $stock1_synonym); + + // Make sure no synonyms were entered + $synonym_empty_count = $this->connection->select('1:synonym', 's') + ->fields('s', ['name']) + ->condition('s.name', $stock1_synonym, '=') + ->countQuery()->execute()->fetchField(); + + $this->assertEquals($synonym_empty_count, 0, "The number of record in the synonym table is not zero despite trying to add an empty string."); + } } From 8a29b393764c4ed5229b3f3bdd7bb343e20fb05f Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Tue, 7 Nov 2023 16:47:48 -0600 Subject: [PATCH 23/48] Finish chado.stock lookup and insert for loadSynonyms --- .../GermplasmAccessionImporter.php | 51 +++++++++++++++---- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index fc61d08..770758d 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -141,7 +141,7 @@ public function describeUploadFileFormat() { /** * Set a cvterm with its cvterm_id - * + * * @param string $key * A key used in the config settings.yml * @param int $cvterm_id @@ -154,10 +154,10 @@ public function setCVterm($key, $cvterm_id) { /** * Get a cvterm ID, given a key that maps to the config settings.yml - * + * * @param string $key * The cvterm name - * @return int + * @return int * The cvterm ID */ public function getCVterm($key) { @@ -561,10 +561,10 @@ public function loadStockProperties($stock_id, $stock_properties) { $maxrank = 0; foreach ($stockprop_record as $record) { $found = false; - if ($record->value == $prop_value) { - $found = true; + if ($record->value == $prop_value) { + $found = true; break; - } + } else { $rank = $record->rank; if ($rank > $maxrank) { $maxrank = $rank; } @@ -633,9 +633,9 @@ public function loadStockProperties($stock_id, $stock_properties) { * including if there are no properties */ public function loadSynonyms($stock_id, $synonyms) { - // First check if we a synonym to deal with + if ($synonyms) { - // Separate out multiple synonyms if we have them by either + // Separate out multiple synonyms if we have them by either // semicolons or commas. Whitespace is optional $all_synonyms = preg_split("/[;,]\s*/", $synonyms); @@ -648,10 +648,41 @@ public function loadSynonyms($stock_id, $synonyms) { ->fields('s', ['synonym_id']) ->condition('s.name', $synonym, '=') ->condition('s.type_id', $synonym_type_id, '='); - $stockprop_record = $synonym_query->execute()->fetchAll(); + $synonym_record = $synonym_query->execute()->fetchAll(); // Make sure there aren't 2 or more records for this synonym - + if (sizeof($synonym_record) >= 2) { + $this->logger->error("Found more than one synonym for \"@synonym\" in chado.synonym.", ['@synonym' => $synonym]); + $this->error_tracker = TRUE; + return false; + } + elseif (sizeof($synonym_record) == 1) { + $synonym_id = $synonym_record[0]->synonym_id; + } + // Can't find a synonym in the chado.synonym table, so insert it + else { + $values = [ + 'name' => $synonym, + 'type_id' => $synonym_type_id, + ]; + $result = $this->connection->insert('1:synonym') + ->fields($values) + ->execute(); + + // If the primary key is not available, then the insert failed + if (!$result) { + $this->logger->error("Insertion of \"@synonym\" into chado.synonym failed.", ['@synonym' => $synonym]); + $this->error_tracker = TRUE; + return false; + } + else { + $synonym_id = $result; + } + } + // ------------------------------------------------------------------------ + // Create a synonym-stock relationship via chado.stock_synonym + // ------------------------------------------------------------------------ + } } } From 32b5f02d3571198feb7d45c8074f0a1f530969c5 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Wed, 8 Nov 2023 17:08:30 -0600 Subject: [PATCH 24/48] Added table definition for stock_synonym to install --- .../install/trpcultivate_germplasm.install | 107 ++++++++++++++++++ .../GermplasmAccessionImporter.php | 29 +++++ .../GermplasmAccessionImporterTest.php | 12 +- 3 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 trpcultivate_germplasm/config/install/trpcultivate_germplasm.install diff --git a/trpcultivate_germplasm/config/install/trpcultivate_germplasm.install b/trpcultivate_germplasm/config/install/trpcultivate_germplasm.install new file mode 100644 index 0000000..00e07c4 --- /dev/null +++ b/trpcultivate_germplasm/config/install/trpcultivate_germplasm.install @@ -0,0 +1,107 @@ + 'stock_synonym', + 'description' => 'Linking table between stock and synonym.', + 'fields' => [ + 'stock_synonym_id' => [ + 'type' => 'serial', + 'not null' => TRUE, + ], + 'synonym_id' => [ + 'size' => 'big', + 'type' => 'int', + 'not null' => TRUE, + ], + 'stock_id' => [ + 'size' => 'big', + 'type' => 'int', + 'not null' => TRUE, + ], + 'pub_id' => [ + 'size' => 'big', + 'type' => 'int', + 'not null' => TRUE, + ], + 'is_current' => [ + 'type' => 'int', + 'default' => 0, + ], + 'is_internal' => [ + 'type' => 'int', + 'default' => 0, + ], + ], + 'primary key' => [ + 'stock_synonym_id', + ], + 'indexes' => [ + 'stock_synonym_idx1' => [ + 0 => 'synonym_id', + ], + 'stock_synonym_idx2' => [ + 0 => 'stock_id', + ], + 'stock_synonym_idx3' => [ + 0 => 'pub_id', + ], + ], + 'foreign keys' => [ + 'synonym' => [ + 'table' => 'synonym', + 'columns' => [ + 'synonym_id' => 'synonym_id', + ], + ], + 'stock' => [ + 'table' => 'stock', + 'columns' => [ + 'stock_id' => 'stock_id', + ], + ], + 'pub' => [ + 'table' => 'pub', + 'columns' => [ + 'pub_id' => 'pub_id', + ], + ], + ], + ); + + $custom_tables = \Drupal::service('tripal_chado.custom_tables'); + $custom_table = $custom_tables->create($table, $this->chado_schema_main); + $custom_table->setTableSchema($schema); + $custom_table->setHidden(True); + + // Relationship verb and Default db set to 0, frontend will notify user + // to configure these variables before any upload. + + // Create a settings/configuration variable used by + // Germplasm Population Importer to limit/filter relationship verb + // field to specific controlled vocabulary. + //variable_set('germplasm_population_importer_verb_cv', 0); + // Default prefix used in germplasm names. + //variable_set('germplasm_population_importer_default_prefix', 'GERM'); + // Default db used by chado stock.dbxref_id when inserting germplasm. + //variable_set('germplasm_population_importer_db', 0); +} diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 770758d..9bee54a 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -682,7 +682,36 @@ public function loadSynonyms($stock_id, $synonyms) { // ------------------------------------------------------------------------ // Create a synonym-stock relationship via chado.stock_synonym // ------------------------------------------------------------------------ + // First check if this stock-synonym relationship already exists + $synonym_stock_query = $this->connection->select('1:stock_synonym', 'ss') + ->fields('ss', ['stock_synonym_id']) + ->condition('ss.synonym_id', $synonym_id, '=') + ->condition('ss.stock_id', $stock_id, '='); + $synonym_stock_record = $synonym_stock_query->execute()->fetchAll(); + + // Make sure there aren't 2 or more records + if (sizeof($synonym_stock_record) >= 2) { + $this->logger->error("Found more than one stock-synonym relationship for stock ID \"@stock\" and synonym \"@synonym\" in chado.stock_synonym.", ['@stock' => $stock_id, '@synonym' => $synonym]); + $this->error_tracker = TRUE; + return false; + } + elseif (sizeof($synonym_stock_record) == 0) { + $values = [ + 'synonym_id' => $synonym_id, + 'stock_id'=> $stock_id, + 'pub_id'=> '1', // Set to the NULL publication and hopefully someone will update it later :) + ]; + $result = $this->connection->insert('1:stock_synonym') + ->fields($values) + ->execute(); + // If the primary key is not available, then the insert failed + if (!$result) { + $this->logger->error("Insertion of stock ID \"@stock\" and synonym \"@synonym\" into chado.stock_synonym failed.", ['@stock' => $stock_id, '@synonym' => $synonym]); + $this->error_tracker = TRUE; + return false; + } + } } } } diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index 0c001cf..6117bcc 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -401,7 +401,7 @@ public function testGermplasmAccessionImporterLoadStockProperties() { 'pedigree' => $empty_string ]; - // "Load" our empty values and check that the stockprop table is empty + // "Load" our empty values and check that the stockprop table is empty // before and after $sp_initial_count = $this->connection->select('1:stockprop', 'sp') ->condition('sp.stock_id', $stock_id, '=') @@ -436,7 +436,7 @@ public function testGermplasmAccessionImporterLoadStockProperties() { ->condition('sp.type_id', 11, '='); $sp_six_institute_name_record = $sp_six_institute_name->execute()->fetchAll(); $this->assertEquals($sp_six_institute_name_record[0]->value,'Crop Development Center, University of Saskatchewan', "The selected stockprop value for institute name does not match what was inserted."); - + // Now add some new properties for the same stock_id $new_stock_props = [ 'biological_status_of_accession_code' => 500, // New value @@ -446,7 +446,7 @@ public function testGermplasmAccessionImporterLoadStockProperties() { $new_sp_count = count($new_stock_props); // 2 should be added to the stockprop table, 1 should not $total_expected_sp_count = $sp_six_count + $new_sp_count - 1; - + $this->importer->loadStockProperties($stock_id, $new_stock_props); $sp_eight_count = $this->connection->select('1:stockprop', 'sp') ->condition('sp.stock_id', $stock_id, '=') @@ -463,7 +463,7 @@ public function testGermplasmAccessionImporterLoadStockProperties() { $this->assertEquals($first_value,'Recurrent selection', "The selected stockprop value for the first breeding_method_db_id does not match what was inserted."); $second_value = $sp_eight_breeding_method_DbId_record[1]->value; $this->assertEquals($second_value,'Breeder line', "The selected stockprop value for the second breeding_method_db_id does not match what was inserted."); - + $first_rank = $sp_eight_breeding_method_DbId_record[0]->rank; $second_rank = $sp_eight_breeding_method_DbId_record[1]->rank; $this->assertGreaterThan($first_rank, $second_rank, "The rank of the second inserted stockprop for breeding_method_db_id is not greater than the first one."); @@ -517,5 +517,9 @@ public function testGermplasmAccessionImporterLoadSynonyms() { ->countQuery()->execute()->fetchField(); $this->assertEquals($synonym_empty_count, 0, "The number of record in the synonym table is not zero despite trying to add an empty string."); + + // Now attempt to load a single synonym + $stock1_synonym = 's1'; + $this->importer->loadSynonyms($stock_id, $stock1_synonym); } } From f5b464cbedb4e9d8aaf28d8151529edee85f4e3e Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 13 Nov 2023 18:05:00 -0600 Subject: [PATCH 25/48] Added definition for the stock_synonym custom table --- .../GermplasmAccessionImporter.php | 40 +++++- .../GermplasmAccessionImporterTest.php | 125 +++++++++++++++++- .../trpcultivate_germplasm.install | 3 +- 3 files changed, 159 insertions(+), 9 deletions(-) rename trpcultivate_germplasm/{config/install => }/trpcultivate_germplasm.install (98%) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 9bee54a..e1b833d 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -290,7 +290,7 @@ public function run(){ $load_props = $this->loadStockProperties($stock_id, $stock_properties); // STEP 5: Load synonyms - $load_synonyms = $this->loadSynonyms($stock_id, $synonyms); + $load_synonyms = $this->loadSynonyms($stock_id, $synonyms, $organism_id); } } } @@ -619,8 +619,8 @@ public function loadStockProperties($stock_id, $stock_properties) { } /** - * Loads each synonym into the chado.stock_synonym table - * Returns true if the insert was successful. + * Loads each synonym into the chado.synonym and chado.stock_synonym + * tables. Returns true if the insert was successful. * * @param int $stock_id * The value of the primary key for the stock record in Chado. @@ -632,7 +632,7 @@ public function loadStockProperties($stock_id, $stock_properties) { * Returns true if inserting all the properties was successful, * including if there are no properties */ - public function loadSynonyms($stock_id, $synonyms) { + public function loadSynonyms($stock_id, $synonyms, $organism_id) { if ($synonyms) { // Separate out multiple synonyms if we have them by either @@ -664,6 +664,7 @@ public function loadSynonyms($stock_id, $synonyms) { $values = [ 'name' => $synonym, 'type_id' => $synonym_type_id, + 'synonym_sgml' => '' ]; $result = $this->connection->insert('1:synonym') ->fields($values) @@ -695,6 +696,7 @@ public function loadSynonyms($stock_id, $synonyms) { $this->error_tracker = TRUE; return false; } + // If 1 result was returned, just ignore it and move on elseif (sizeof($synonym_stock_record) == 0) { $values = [ 'synonym_id' => $synonym_id, @@ -712,6 +714,36 @@ public function loadSynonyms($stock_id, $synonyms) { return false; } } + // ------------------------------------------------------------------------ + // Lastly, check if our synonym is in the stock table. If yes, THEN create + // a stock_relationship to connect this stock_id to the synonym. + // NOTE: Lookup the type_id for relationship in schema:alternateName + // ------------------------------------------------------------------------ + $stock_query = $this->connection->select('1:stock', 'st') + ->fields('st', ['stock_id']) + ->condition('st.name', $synonym, '=') + ->condition('st.organism_id', $organism_id, '='); + $stock_record = $stock_query->execute()->fetchAll(); + + // Make sure there aren't 2 or more records + if (sizeof($stock_record) >= 2) { + $this->logger->error("Found more than one match for synonym name \"@synonym\" in chado.stock.", ['@synonym' => $synonym]); + $this->error_tracker = TRUE; + return false; + } + elseif (sizeof($stock_record) == 1) { + $stock_id_of_synonym = $synonym_record[0]->synonym_id; + /** + $values = [ + 'subject_id' => $stock_id_of_synonym, + 'type_id' => $stock_relationship_type_id, + 'object_id' => $stock_id + ]; + $result = $this->connection->insert('1:synonym') + ->fields($values) + ->execute(); + */ + } } } } diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index 6117bcc..0b55217 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -55,6 +55,7 @@ protected function setUp(): void { // ... Finally the file module + tables itself. $this->installEntitySchema('file'); $this->installSchema('file', ['file_usage']); + $this->installSchema('tripal_chado', ['tripal_custom_tables']); // We need to mock the logger to test the progress reporting. $container = \Drupal::getContainer(); @@ -477,6 +478,81 @@ public function testGermplasmAccessionImporterLoadStockProperties() { $this->assertEquals($sp_eight_pedigree_count, 1, "The number of records for stockprop pedigree is not 1."); } + public function createStockSynonymTable() { + $table = 'stock_synonym'; + $schema = [ + 'table' => 'stock_synonym', + 'description' => 'Linking table between stock and synonym.', + 'fields' => [ + 'stock_synonym_id' => [ + 'type' => 'serial', + 'not null' => TRUE, + ], + 'synonym_id' => [ + 'size' => 'big', + 'type' => 'int', + 'not null' => TRUE, + ], + 'stock_id' => [ + 'size' => 'big', + 'type' => 'int', + 'not null' => TRUE, + ], + 'pub_id' => [ + 'size' => 'big', + 'type' => 'int', + 'not null' => TRUE, + ], + 'is_current' => [ + 'type' => 'int', + 'default' => 0, + ], + 'is_internal' => [ + 'type' => 'int', + 'default' => 0, + ], + ], + 'primary key' => [ + 'stock_synonym_id', + ], + 'indexes' => [ + 'stock_synonym_idx1' => [ + 0 => 'synonym_id', + ], + 'stock_synonym_idx2' => [ + 0 => 'stock_id', + ], + 'stock_synonym_idx3' => [ + 0 => 'pub_id', + ], + ], + 'foreign keys' => [ + 'synonym' => [ + 'table' => 'synonym', + 'columns' => [ + 'synonym_id' => 'synonym_id', + ], + ], + 'stock' => [ + 'table' => 'stock', + 'columns' => [ + 'stock_id' => 'stock_id', + ], + ], + 'pub' => [ + 'table' => 'pub', + 'columns' => [ + 'pub_id' => 'pub_id', + ], + ], + ], + ]; + + $custom_tables = \Drupal::service('tripal_chado.custom_tables'); + $custom_table = $custom_tables->create($table, $this->connection->getSchemaName()); + $custom_table->setTableSchema($schema); + } + /** * Tests focusing on the Germplasm Accession Importer loadSynonyms() function * @@ -484,6 +560,8 @@ public function testGermplasmAccessionImporterLoadStockProperties() { */ public function testGermplasmAccessionImporterLoadSynonyms() { + $this->createStockSynonymTable(); + // Insert an organism $subtaxa_cvterm_id = $this->getCVtermID('TAXRANK', '0000023'); @@ -508,7 +586,7 @@ public function testGermplasmAccessionImporterLoadSynonyms() { // Attempt to load an empty string (ie. an empty column in the file) $stock1_synonym = ''; - $this->importer->loadSynonyms($stock_id, $stock1_synonym); + $this->importer->loadSynonyms($stock_id, $stock1_synonym, $organism_id); // Make sure no synonyms were entered $synonym_empty_count = $this->connection->select('1:synonym', 's') @@ -518,8 +596,49 @@ public function testGermplasmAccessionImporterLoadSynonyms() { $this->assertEquals($synonym_empty_count, 0, "The number of record in the synonym table is not zero despite trying to add an empty string."); - // Now attempt to load a single synonym + // ------------------------------------------------------------------------ + // Load a single synonym $stock1_synonym = 's1'; - $this->importer->loadSynonyms($stock_id, $stock1_synonym); + $this->importer->loadSynonyms($stock_id, $stock1_synonym, $organism_id); + + // STEP 1: Check the synonym table + $stock1_synonym_query = $this->connection->select('1:synonym', 's') + ->fields('s', ['synonym_id', 'name']) + ->condition('s.name', $stock1_synonym, '='); + $stock1_synonym_record = $stock1_synonym_query->execute()->fetchAll(); + + $this->assertEquals($stock1_synonym_record[0]->name, $stock1_synonym, "The selected synonym in the synonym table does not match what was just inserted."); + + // STEP 2: Check the stock_synonym table + // Grab the synonym_id to pull it out of the stock_synonym table + $s1_synonym_id = $stock1_synonym_record[0]->synonym_id; + + $stock1_stock_synonym_query = $this->connection->select('1:stock_synonym', 'ss') + ->fields('ss', ['stock_id', 'synonym_id']) + ->condition('ss.synonym_id', $s1_synonym_id, '='); + $stock1_stock_synonym_record = $stock1_stock_synonym_query->execute()->fetchAll(); + + $this->assertEquals($stock1_stock_synonym_record[0]->stock_id, $stock_id, "The synonym s1 in the stock_synonym table does not contain the correct stock_id."); + + // STEP 3: Check the stock_relationship table + // $stock1_stock_relationship_count = $this->connection->select('1:stock_relationship', 'sr') + // ->fields('sr', ['subject_id', 'object_id']) + // ->condition('sr.subject_id', $s1_synonym_id, '=') + // ->condition('sr.object_id', $stock_id, '=') + // ->countQuery()->execute()->fetchField(); + + // $this->assertEquals($stock1_stock_relationship_count, 1, "A stock_relationship was not created for stock1 and its synonym, s1."); + // ------------------------------------------------------------------------ + + // Attempt to insert another synonym for stock1 + $stock1_synonym_2 = 's1_2'; + $this->importer->loadSynonyms($stock_id, $stock1_synonym_2, $organism_id); + + $stock1_synonym_2_query = $this->connection->select('1:synonym', 's') + ->fields('s', ['name']) + ->condition('s.name', $stock1_synonym_2, '='); + $stock1_synonym_2_record = $stock1_synonym_2_query->execute()->fetchAll(); + + $this->assertEquals($stock1_synonym_2_record[0]->name, $stock1_synonym_2, "The second synonym added to the synonym table does not match what was just inserted."); } } diff --git a/trpcultivate_germplasm/config/install/trpcultivate_germplasm.install b/trpcultivate_germplasm/trpcultivate_germplasm.install similarity index 98% rename from trpcultivate_germplasm/config/install/trpcultivate_germplasm.install rename to trpcultivate_germplasm/trpcultivate_germplasm.install index 00e07c4..4a0ff02 100644 --- a/trpcultivate_germplasm/config/install/trpcultivate_germplasm.install +++ b/trpcultivate_germplasm/trpcultivate_germplasm.install @@ -86,12 +86,11 @@ function trpcultivate_germplasm_enable(){ ], ], ], - ); + ]; $custom_tables = \Drupal::service('tripal_chado.custom_tables'); $custom_table = $custom_tables->create($table, $this->chado_schema_main); $custom_table->setTableSchema($schema); - $custom_table->setHidden(True); // Relationship verb and Default db set to 0, frontend will notify user // to configure these variables before any upload. From 12703aff24a2e879b31ba170186887cfe5c407eb Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 20 Nov 2023 17:10:52 -0600 Subject: [PATCH 26/48] Completed loadSynonyms function and its tests --- .../trpcultivate_germplasm.settings.yml | 1 + .../schema/trpcultivate_germplasm.schema.yml | 3 + .../GermplasmAccessionImporter.php | 25 +++++-- .../GermplasmAccessionImporterTest.php | 66 ++++++++++++++----- 4 files changed, 75 insertions(+), 20 deletions(-) diff --git a/trpcultivate_germplasm/config/install/trpcultivate_germplasm.settings.yml b/trpcultivate_germplasm/config/install/trpcultivate_germplasm.settings.yml index 558ff03..b97f76c 100644 --- a/trpcultivate_germplasm/config/install/trpcultivate_germplasm.settings.yml +++ b/trpcultivate_germplasm/config/install/trpcultivate_germplasm.settings.yml @@ -11,3 +11,4 @@ terms: biological_status_of_accession_code: 0 breeding_method_DbId: 0 subtaxa: 0 + stock_relationship_type_synonym: 0 \ No newline at end of file diff --git a/trpcultivate_germplasm/config/schema/trpcultivate_germplasm.schema.yml b/trpcultivate_germplasm/config/schema/trpcultivate_germplasm.schema.yml index 9d09f64..77701f4 100644 --- a/trpcultivate_germplasm/config/schema/trpcultivate_germplasm.schema.yml +++ b/trpcultivate_germplasm/config/schema/trpcultivate_germplasm.schema.yml @@ -42,3 +42,6 @@ trpcultivate_germplasm.settings: subtaxa: type: integer label: 'The cvterm ID of the subtaxa of a germplasm' + stock_relationship_type_synonym: + type: integer + label: 'The cvterm ID of the stock_relationship synonym' diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index e1b833d..86cecf8 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -230,6 +230,8 @@ public function run(){ $this->setCVterm('biological_status_of_accession_code', $germplasm_config->get('terms.biological_status_of_accession_code')); $this->setCVterm('breeding_method_DbId', $germplasm_config->get('terms.breeding_method_DbId')); $this->setCVterm('pedigree', $germplasm_config->get('terms.pedigree')); + $this->setCVterm('synonym', $germplasm_config->get('terms.synonym')); + $this->setCVterm('stock_relationship_type_synonym', $germplasm_config->get('terms.stock_relationship_type_synonym')); // Set up the ability to track progress so we can report it to the user $filesize = filesize($file_path); @@ -714,11 +716,13 @@ public function loadSynonyms($stock_id, $synonyms, $organism_id) { return false; } } + // ------------------------------------------------------------------------ // Lastly, check if our synonym is in the stock table. If yes, THEN create // a stock_relationship to connect this stock_id to the synonym. - // NOTE: Lookup the type_id for relationship in schema:alternateName // ------------------------------------------------------------------------ + $stock_relationship_type_id = $this->getCVterm('stock_relationship_type_synonym'); + $stock_query = $this->connection->select('1:stock', 'st') ->fields('st', ['stock_id']) ->condition('st.name', $synonym, '=') @@ -732,19 +736,30 @@ public function loadSynonyms($stock_id, $synonyms, $organism_id) { return false; } elseif (sizeof($stock_record) == 1) { - $stock_id_of_synonym = $synonym_record[0]->synonym_id; - /** + $stock_id_of_synonym = $stock_record[0]->stock_id; $values = [ 'subject_id' => $stock_id_of_synonym, 'type_id' => $stock_relationship_type_id, 'object_id' => $stock_id ]; - $result = $this->connection->insert('1:synonym') + $result = $this->connection->insert('1:stock_relationship') ->fields($values) ->execute(); - */ + + // If the primary key is not available, then the insert failed + if (!$result) { + $this->logger->error("Insertion of stock ID \"@stock\" and stock ID of its synonym \"@sid_synonym\" into chado.stock_relationship failed.", ['@stock' => $stock_id, '@sid_synonym' => $stock_id_of_synonym]); + $this->error_tracker = TRUE; + return false; + } + } + else { + $this->logger->notice("Synonym \"@synonym\" was not found in the stock table, so no stock_relationship was made with stock ID \"@stock\".", ['@synonym' => $synonym, '@stock' => $stock_id]); } } + // Cycled through all the synonyms by this point, and if false hasn't been + // returned, then return true + return true; } } /* diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index 0b55217..d6ff575 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -93,6 +93,7 @@ protected function setUp(): void { $this->importer->setCVterm('breeding_method_DbId', 14); $this->importer->setCVterm('pedigree', 15); $this->importer->setCVterm('synonym', 16); + $this->importer->setCVterm('stock_relationship_type_synonym', 17); } /** @@ -585,29 +586,31 @@ public function testGermplasmAccessionImporterLoadSynonyms() { ->execute(); // Attempt to load an empty string (ie. an empty column in the file) - $stock1_synonym = ''; - $this->importer->loadSynonyms($stock_id, $stock1_synonym, $organism_id); + $stock1_synonym_empty = ''; + $this->importer->loadSynonyms($stock_id, $stock1_synonym_empty, $organism_id); // Make sure no synonyms were entered $synonym_empty_count = $this->connection->select('1:synonym', 's') ->fields('s', ['name']) - ->condition('s.name', $stock1_synonym, '=') + ->condition('s.name', $stock1_synonym_empty, '=') ->countQuery()->execute()->fetchField(); $this->assertEquals($synonym_empty_count, 0, "The number of record in the synonym table is not zero despite trying to add an empty string."); // ------------------------------------------------------------------------ // Load a single synonym - $stock1_synonym = 's1'; - $this->importer->loadSynonyms($stock_id, $stock1_synonym, $organism_id); + $stock1_synonym_1 = 's1'; + ob_start(); + $this->importer->loadSynonyms($stock_id, $stock1_synonym_1, $organism_id); + $printed_output = ob_get_clean(); // STEP 1: Check the synonym table $stock1_synonym_query = $this->connection->select('1:synonym', 's') ->fields('s', ['synonym_id', 'name']) - ->condition('s.name', $stock1_synonym, '='); + ->condition('s.name', $stock1_synonym_1, '='); $stock1_synonym_record = $stock1_synonym_query->execute()->fetchAll(); - $this->assertEquals($stock1_synonym_record[0]->name, $stock1_synonym, "The selected synonym in the synonym table does not match what was just inserted."); + $this->assertEquals($stock1_synonym_record[0]->name, $stock1_synonym_1, "The selected synonym in the synonym table does not match what was just inserted."); // STEP 2: Check the stock_synonym table // Grab the synonym_id to pull it out of the stock_synonym table @@ -620,25 +623,58 @@ public function testGermplasmAccessionImporterLoadSynonyms() { $this->assertEquals($stock1_stock_synonym_record[0]->stock_id, $stock_id, "The synonym s1 in the stock_synonym table does not contain the correct stock_id."); - // STEP 3: Check the stock_relationship table - // $stock1_stock_relationship_count = $this->connection->select('1:stock_relationship', 'sr') - // ->fields('sr', ['subject_id', 'object_id']) - // ->condition('sr.subject_id', $s1_synonym_id, '=') - // ->condition('sr.object_id', $stock_id, '=') - // ->countQuery()->execute()->fetchField(); + // STEP 3: Check the stock_relationship table. We expect to have 0 records + // since the stock_relationship only gets created if the synonym itself is + // in the stock table + $stock1_stock_relationship_count = $this->connection->select('1:stock_relationship', 'sr') + ->fields('sr', ['subject_id', 'object_id']) + ->condition('sr.subject_id', $s1_synonym_id, '=') + ->condition('sr.object_id', $stock_id, '=') + ->countQuery()->execute()->fetchField(); - // $this->assertEquals($stock1_stock_relationship_count, 1, "A stock_relationship was not created for stock1 and its synonym, s1."); + $this->assertEquals($stock1_stock_relationship_count, 0, "Did not expect for one or more stock_relationships to be present in the stock_relationship table."); + + // We can also check the output of our command as we expect a notice to be given + $this->assertTrue($printed_output == 'Synonym "s1" was not found in the stock table, so no stock_relationship was made with stock ID "1".', "Did not get the expected notice message when adding a synonym that does not exist in the stock table."); // ------------------------------------------------------------------------ - // Attempt to insert another synonym for stock1 + // Attempt to insert another synonym for stock1. This time, also add the + // synonym to the stock table and confirm that we have a stock_relationship $stock1_synonym_2 = 's1_2'; + $stock_id_of_synonym = $this->connection->insert('1:stock') + ->fields([ + 'organism_id' => $organism_id, + 'name' => $stock1_synonym_2, + 'uniquename' => 'TEST:2', + 'type_id' => 9, + ]) + ->execute(); + $this->importer->loadSynonyms($stock_id, $stock1_synonym_2, $organism_id); + // Make sure this synonym was added to chado.synonym $stock1_synonym_2_query = $this->connection->select('1:synonym', 's') ->fields('s', ['name']) ->condition('s.name', $stock1_synonym_2, '='); $stock1_synonym_2_record = $stock1_synonym_2_query->execute()->fetchAll(); $this->assertEquals($stock1_synonym_2_record[0]->name, $stock1_synonym_2, "The second synonym added to the synonym table does not match what was just inserted."); + + // Check that we now have 2 records in chado.stock_synonym + $stock_synonym_count = $this->connection->select('1:stock_synonym', 'ss') + ->fields('ss', ['stock_synonym_id']) + ->countQuery()->execute()->fetchField(); + + $this->assertEquals($stock_synonym_count, 2, "The stock_synonym table does not contain the expected 2 records."); + + // Check that we have exactly 1 record in the stock_relationship table now + $stock1_stock_relationship_synonym2_count = $this->connection->select('1:stock_relationship', 'sr') + ->fields('sr', ['subject_id', 'object_id']) + ->condition('sr.subject_id', $stock_id_of_synonym, '=') + ->condition('sr.object_id', $stock_id, '=') + ->condition('sr.type_id', $this->importer->getCVterm('stock_relationship_type_synonym', '=')) + ->countQuery()->execute()->fetchField(); + + $this->assertEquals($stock1_stock_relationship_synonym2_count, 1, "Did not count the expected single stock_relationship between stock1 and its 2nd synonym, s1_2."); } } From 4dfd70dc76c65f613ba70d4c8a972539fce5e8c7 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Wed, 22 Nov 2023 15:25:52 -0600 Subject: [PATCH 27/48] Included tests for comma and semicolon-separated synonyms --- .../GermplasmAccessionImporterTest.php | 65 ++++++++++++++++++- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index d6ff575..79c7117 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -624,7 +624,7 @@ public function testGermplasmAccessionImporterLoadSynonyms() { $this->assertEquals($stock1_stock_synonym_record[0]->stock_id, $stock_id, "The synonym s1 in the stock_synonym table does not contain the correct stock_id."); // STEP 3: Check the stock_relationship table. We expect to have 0 records - // since the stock_relationship only gets created if the synonym itself is + // since the stock_relationship only gets created if the synonym itself is // in the stock table $stock1_stock_relationship_count = $this->connection->select('1:stock_relationship', 'sr') ->fields('sr', ['subject_id', 'object_id']) @@ -664,7 +664,7 @@ public function testGermplasmAccessionImporterLoadSynonyms() { $stock_synonym_count = $this->connection->select('1:stock_synonym', 'ss') ->fields('ss', ['stock_synonym_id']) ->countQuery()->execute()->fetchField(); - + $this->assertEquals($stock_synonym_count, 2, "The stock_synonym table does not contain the expected 2 records."); // Check that we have exactly 1 record in the stock_relationship table now @@ -676,5 +676,66 @@ public function testGermplasmAccessionImporterLoadSynonyms() { ->countQuery()->execute()->fetchField(); $this->assertEquals($stock1_stock_relationship_synonym2_count, 1, "Did not count the expected single stock_relationship between stock1 and its 2nd synonym, s1_2."); + + // Add a comma separated list of synonyms + $stock_comma = 'stock-comma'; + $stock_comma_id = $this->connection->insert('1:stock') + ->fields([ + 'organism_id' => $organism_id, + 'name' => $stock_comma, + 'uniquename' => 'TEST:3', + 'type_id' => 9, + ]) + ->execute(); + + $synonyms_comma = 'syn1, syn2'; + $syn1 = 'syn1'; + $syn2 = 'syn2'; + ob_start(); + $this->importer->loadSynonyms($stock_comma_id, $synonyms_comma, $organism_id); + ob_end_clean(); + + // Check both synonyms are in chado.synonym + $synonyms_comma_query = $this->connection->select('1:synonym', 's') + ->fields('s', ['name']); + $group = $synonyms_comma_query->orConditionGroup() + ->condition('s.name', $syn1, '=') + ->condition('s.name', $syn2, '='); + $synonyms_comma_query->condition($group); + $synonyms_comma_record = $synonyms_comma_query->execute()->fetchAll(); + + $this->assertEquals($synonyms_comma_record[0]->name, $syn1, "The synonym table does not contain the expected synonym syn1"); + $this->assertEquals($synonyms_comma_record[1]->name, $syn2, "The synonym table does not contain the expected synonym syn2"); + + // Add a semicolon separated list of synonyms + $stock_semicolon = 'stock-semicolon'; + $stock_semicolon_id = $this->connection->insert('1:stock') + ->fields([ + 'organism_id' => $organism_id, + 'name' => $stock_semicolon, + 'uniquename' => 'TEST:4', + 'type_id' => 9, + ]) + ->execute(); + + $synonyms_semicolon = 'syn3;syn4'; + $syn3 = 'syn3'; + $syn4 = 'syn4'; + ob_start(); + $this->importer->loadSynonyms($stock_semicolon_id, $synonyms_semicolon, $organism_id); + ob_end_clean(); + + // Check both synonyms are in chado.synonym + $synonyms_semicolon_query = $this->connection->select('1:synonym', 's') + ->fields('s', ['name']); + $group = $synonyms_semicolon_query->orConditionGroup() + ->condition('s.name', $syn3, '=') + ->condition('s.name', $syn4, '='); + $synonyms_semicolon_query->condition($group); + $synonyms_semicolon_record = $synonyms_semicolon_query->execute()->fetchAll(); + + $this->assertEquals($synonyms_semicolon_record[0]->name, $syn3, "The synonym table does not contain the expected synonym syn3"); + $this->assertEquals($synonyms_semicolon_record[1]->name, $syn4, "The synonym table does not contain the expected synonym syn4"); + } } From 38d2e5365f38c9df357ecbcd0b5447ec2a81a978 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Thu, 23 Nov 2023 17:15:08 -0600 Subject: [PATCH 28/48] Started test and example file for run method --- .../tests/src/Fixtures/simple_example.txt | 3 +++ .../GermplasmAccessionImporterTest.php | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 trpcultivate_germplasm/tests/src/Fixtures/simple_example.txt diff --git a/trpcultivate_germplasm/tests/src/Fixtures/simple_example.txt b/trpcultivate_germplasm/tests/src/Fixtures/simple_example.txt new file mode 100644 index 0000000..2f0ba19 --- /dev/null +++ b/trpcultivate_germplasm/tests/src/Fixtures/simple_example.txt @@ -0,0 +1,3 @@ +Germplasm Name External Database Accession Number Germplasm Species Germplasm Subtaxa Institute Code Institute Name Country of Origin Code Biological Status of Accession Breeding Method Pedigree Synonyms +Test1 TestDB T1 databasica subspecies chadoii +Test2 TestDB T2 databasica subspecies chadoii \ No newline at end of file diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index 79c7117..aef8240 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -736,6 +736,19 @@ public function testGermplasmAccessionImporterLoadSynonyms() { $this->assertEquals($synonyms_semicolon_record[0]->name, $syn3, "The synonym table does not contain the expected synonym syn3"); $this->assertEquals($synonyms_semicolon_record[1]->name, $syn4, "The synonym table does not contain the expected synonym syn4"); + } + + /** + * Tests focusing on the Germplasm Accession Importer run() function + * + * @group germ_accession_importer + */ + public function testGermplasmAccessionImporterRun() { + + $simple_example_file = __DIR__ . '/../../Fixtures/simple_example.txt'; + $this->importer->prepareFiles(); + + $this->importer->run(); } } From b18e121396195582832b2ff1dc8dda01ff4453c3 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Fri, 24 Nov 2023 16:29:48 -0600 Subject: [PATCH 29/48] Improvements made to the run method and test --- .../GermplasmAccessionImporter.php | 38 ++++-- .../tests/src/Fixtures/simple_example.txt | 4 +- .../GermplasmAccessionImporterRunTest.php | 125 ++++++++++++++++++ .../GermplasmAccessionImporterTest.php | 14 -- 4 files changed, 151 insertions(+), 30 deletions(-) create mode 100644 trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 86cecf8..7dde385 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -208,9 +208,9 @@ public function formValidate($form, &$form_state){ * @see TripalImporter::run() */ public function run(){ - // All values provided by the user in the Importer's form widgets are - // made available to us here by the Class' arguments member variable. - $arguments = $this->arguments['run_args']; + + // Grabbing our arguments from the form + $arguments = $this->getArguments(); // The path to the uploaded file is always made available using the // 'files' argument. The importer can support multiple files, therefore @@ -219,7 +219,7 @@ public function run(){ $file_path = $arguments['files'][0]['file_path']; // Grab the genus name - $genus_name = $arguments['genus_name']; + $genus_name = $arguments['run_args']['genus_name']; // Set up our array of cv terms $germplasm_config = $this->config_factory->get('trpcultivate_germplasm.settings'); @@ -241,13 +241,17 @@ public function run(){ // Open the file and start iterating through each line $GERMPLASM_FILE = fopen($file_path, 'r'); + if(!$GERMPLASM_FILE) { + $this->logger->error("Could not open file: @file", ['@file' => $file_path]); + } + while (!feof($GERMPLASM_FILE)){ - $current_line = fgetcsv($GERMPLASM_FILE, 0, "\t"); + $current_line = fgets($GERMPLASM_FILE); // Calculate how many bytes we have read from the file and let the // importer know how many have been processed so it can provide a // progress indicator. - $bytes_read += drupal_strlen($current_line); + $bytes_read += mb_strlen($current_line); $this->setItemsHandled($bytes_read); // Check for empty lines, comment lines and a header line @@ -259,22 +263,28 @@ public function run(){ // into an array $current_line = trim($current_line); $germplasm_columns = explode("\t", $current_line); + $num_columns = count($germplasm_columns); + if (count($germplasm_columns) < 4) { + $this->logger->error("Insufficient number of columns detected (<4) for the current line: @current", ['@current' => $current_line]); + $this->error_tracker = TRUE; + } // Collect our values from our current line into variables + // Since the 1st 4 columns are required, make sure there are values there $germplasm_name = $germplasm_columns[0]; $external_database = $germplasm_columns[1]; $accession_number = $germplasm_columns[2]; $germplasm_species = $germplasm_columns[3]; - $germplasm_subtaxa = $germplasm_columns[4]; + $germplasm_subtaxa = $germplasm_columns[4] ?? ''; $stock_properties = [ - 'institute_code' => $germplasm_columns[5], - 'institute_name' => $germplasm_columns[6], - 'country_of_origin_code' => $germplasm_columns[7], - 'biological_status_of_accession_code' => $germplasm_columns[8], - 'breeding_method_DbId' => $germplasm_columns[9], - 'pedigree' => $germplasm_columns[10] + 'institute_code' => $germplasm_columns[5] ?? '', + 'institute_name' => $germplasm_columns[6] ?? '', + 'country_of_origin_code' => $germplasm_columns[7] ?? '', + 'biological_status_of_accession_code' => $germplasm_columns[8] ?? '', + 'breeding_method_DbId' => $germplasm_columns[9] ?? '', + 'pedigree' => $germplasm_columns[10] ?? '' ]; - $synonyms = $germplasm_columns[11]; + $synonyms = $germplasm_columns[11] ?? ''; // STEP 1: Pull out the organism ID for the current germplasm $organism_id = $this->getOrganismID($genus_name, $germplasm_species, $germplasm_subtaxa); diff --git a/trpcultivate_germplasm/tests/src/Fixtures/simple_example.txt b/trpcultivate_germplasm/tests/src/Fixtures/simple_example.txt index 2f0ba19..10547ac 100644 --- a/trpcultivate_germplasm/tests/src/Fixtures/simple_example.txt +++ b/trpcultivate_germplasm/tests/src/Fixtures/simple_example.txt @@ -1,3 +1,3 @@ Germplasm Name External Database Accession Number Germplasm Species Germplasm Subtaxa Institute Code Institute Name Country of Origin Code Biological Status of Accession Breeding Method Pedigree Synonyms -Test1 TestDB T1 databasica subspecies chadoii -Test2 TestDB T2 databasica subspecies chadoii \ No newline at end of file +Test1 TestDB T1 databasica subspecies chadoii +Test2 TestDB T2 databasica subspecies chadoii \ No newline at end of file diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php new file mode 100644 index 0000000..75de51a --- /dev/null +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php @@ -0,0 +1,125 @@ + [ + 'id' => 'trpcultivate-germplasm-accession', + 'label' => 'Tripal Cultivate: Germplasm Accessions', + 'description' => 'Imports germplasm accessions into Chado with metadata meeting BrAPI standards.', + 'file_types' => ["tsv", "txt"], + 'use_analysis' => FALSE, + 'require_analysis' => FALSE, + 'upload_title' => 'Germplasm Accession Import', + 'upload_description' => 'This should not be visible!', + 'button_text' => 'Import Germplasm Accessions', + 'file_upload' => True, + 'file_load' => True, + 'file_remote' => True, + 'file_required' => True, + 'cardinality' => 1, + ], + ]; + + /** + * {@inheritdoc} + */ + protected function setUp(): void { + parent::setUp(); + + // Ensure we see all logging in tests. + \Drupal::state()->set('is_a_test_environment', TRUE); + + // Open connection to Chado + $this->connection = $this->getTestSchema(ChadoTestKernelBase::PREPARE_TEST_CHADO); + + // Ensure we can access file_managed related functionality from Drupal. + // ... users need access to system.action config? + $this->installConfig(['system', 'trpcultivate_germplasm']); + // ... managed files are associated with a user. + $this->installEntitySchema('user'); + // ... Finally the file module + tables itself. + $this->installEntitySchema('file'); + $this->installSchema('file', ['file_usage']); + $this->installSchema('tripal_chado', ['tripal_custom_tables']); + // Ensure we have our tripal import tables. + $this->installSchema('tripal', ['tripal_import', 'tripal_jobs']); + // Create and log-in a user. + $this->setUpCurrentUser(); + + // We need to mock the logger to test the progress reporting. + $container = \Drupal::getContainer(); + $mock_logger = $this->getMockBuilder(\Drupal\tripal\Services\TripalLogger::class) + ->onlyMethods(['notice','error']) + ->getMock(); + $mock_logger->method('notice') + ->willReturnCallback(function($message, $context, $options) { + print str_replace(array_keys($context), $context, $message); + return NULL; + }); + $mock_logger->method('error') + ->willReturnCallback(function($message, $context, $options) { + print str_replace(array_keys($context), $context, $message); + return NULL; + }); + $container->set('tripal.logger', $mock_logger); + + $this->config_factory = \Drupal::configFactory(); + $this->importer = new \Drupal\trpcultivate_germplasm\Plugin\TripalImporter\GermplasmAccessionImporter( + [], + 'trpcultivate-germplasm-accession', + $this->definitions, + $this->connection, + $this->config_factory + ); + + $subtaxa_cvterm_id = $this->getCVtermID('TAXRANK', '0000023'); + $this->importer->setCVterm('accession', 9); + $this->importer->setCVterm('subtaxa', $subtaxa_cvterm_id); + $this->importer->setCVterm('institute_code', 10); + $this->importer->setCVterm('institute_name', 11); + $this->importer->setCVterm('country_of_origin_code',12); + $this->importer->setCVterm('biological_status_of_accession_code', 13); + $this->importer->setCVterm('breeding_method_DbId', 14); + $this->importer->setCVterm('pedigree', 15); + $this->importer->setCVterm('synonym', 16); + $this->importer->setCVterm('stock_relationship_type_synonym', 17); + } + + /** + * Tests focusing on the Germplasm Accession Importer run() function + * + * @group germ_accession_importer + */ + public function testGermplasmAccessionImporterRun() { + + // Test 1: Test using a file with only the required columns + $simple_example_file = __DIR__ . '/../../../Fixtures/simple_example.txt'; + + $genus = 'Tripalus'; + $run_args = ['genus_name' => $genus]; + $file_details = ['file_local' => $simple_example_file]; + + $this->importer->createImportJob($run_args, $file_details); + $this->importer->prepareFiles(); + //$this->importer->run(); + + } +} \ No newline at end of file diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index aef8240..de47699 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -737,18 +737,4 @@ public function testGermplasmAccessionImporterLoadSynonyms() { $this->assertEquals($synonyms_semicolon_record[0]->name, $syn3, "The synonym table does not contain the expected synonym syn3"); $this->assertEquals($synonyms_semicolon_record[1]->name, $syn4, "The synonym table does not contain the expected synonym syn4"); } - - /** - * Tests focusing on the Germplasm Accession Importer run() function - * - * @group germ_accession_importer - */ - public function testGermplasmAccessionImporterRun() { - - $simple_example_file = __DIR__ . '/../../Fixtures/simple_example.txt'; - $this->importer->prepareFiles(); - - $this->importer->run(); - - } } From 983271c59213306c1bf00d534e15df56d4b5000f Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 27 Nov 2023 17:04:20 -0600 Subject: [PATCH 30/48] Simplified setting up of CVterms in run, and added first tests --- .../GermplasmAccessionImporter.php | 50 +++++++++++----- .../GermplasmAccessionImporterRunTest.php | 58 ++++++++++++++++++- 2 files changed, 93 insertions(+), 15 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 7dde385..f417628 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -45,7 +45,18 @@ class GermplasmAccessionImporter extends ChadoImporterBase { /** * An associative array of cvterms as the key and the cvterm_id as the value. */ - protected $cvterms = []; + protected $cvterms = [ + 'accession', + 'subtaxa', + 'institute_code', + 'institute_name', + 'country_of_origin_code', + 'biological_status_of_accession_code', + 'breeding_method_DbId', + 'pedigree', + 'synonym', + 'stock_relationship_type_synonym' + ]; /** * Implements ContainerFactoryPluginInterface->create(). @@ -204,6 +215,19 @@ public function formValidate($form, &$form_state){ // Nothing to validate since the genus field is set to "required". } + public function setUpCVterms(){ + + $germplasm_config = $this->config_factory->get('trpcultivate_germplasm.settings'); + // Iterate through our cvterms + // If it hasn't been set before, set it now + foreach($this->cvterms as $term){ + if (!isset($this->cvterms[$term])){ + $terms_string = 'terms.' . $term; + $this->setCVterm($term, $germplasm_config->get($terms_string)); + } + } + } + /** * @see TripalImporter::run() */ @@ -221,23 +245,15 @@ public function run(){ // Grab the genus name $genus_name = $arguments['run_args']['genus_name']; - // Set up our array of cv terms - $germplasm_config = $this->config_factory->get('trpcultivate_germplasm.settings'); - $this->setCVterm('accession', $germplasm_config->get('terms.accession')); - $this->setCVterm('institute_code', $germplasm_config->get('terms.institute_code')); - $this->setCVterm('institute_name', $germplasm_config->get('terms.institute_name')); - $this->setCVterm('country_of_origin_code', $germplasm_config->get('terms.country_of_origin_code')); - $this->setCVterm('biological_status_of_accession_code', $germplasm_config->get('terms.biological_status_of_accession_code')); - $this->setCVterm('breeding_method_DbId', $germplasm_config->get('terms.breeding_method_DbId')); - $this->setCVterm('pedigree', $germplasm_config->get('terms.pedigree')); - $this->setCVterm('synonym', $germplasm_config->get('terms.synonym')); - $this->setCVterm('stock_relationship_type_synonym', $germplasm_config->get('terms.stock_relationship_type_synonym')); + // Make sure our CVterms are all set + $this->setUpCVterms(); // Set up the ability to track progress so we can report it to the user $filesize = filesize($file_path); $this->setTotalItems($filesize); $this->setItemsHandled(0); $bytes_read = 0; + $line_count = 0; // Open the file and start iterating through each line $GERMPLASM_FILE = fopen($file_path, 'r'); @@ -247,6 +263,7 @@ public function run(){ while (!feof($GERMPLASM_FILE)){ $current_line = fgets($GERMPLASM_FILE); + $line_count++; // Calculate how many bytes we have read from the file and let the // importer know how many have been processed so it can provide a @@ -265,12 +282,19 @@ public function run(){ $germplasm_columns = explode("\t", $current_line); $num_columns = count($germplasm_columns); if (count($germplasm_columns) < 4) { - $this->logger->error("Insufficient number of columns detected (<4) for the current line: @current", ['@current' => $current_line]); + $this->logger->error("Insufficient number of columns detected (<4) for line # @line", ['@line' => $line_count]); $this->error_tracker = TRUE; } // Collect our values from our current line into variables // Since the 1st 4 columns are required, make sure there are values there + for($i=0; $i<4; $i++) { + if ($germplasm_columns[$i] == '') { + $column = $i+1; + $this->logger->error("Column @column is required and cannot be empty for line # @line", ['@column' => $column, '@line' => $line_count]); + $this->error_tracker = TRUE; + } + } $germplasm_name = $germplasm_columns[0]; $external_database = $germplasm_columns[1]; $accession_number = $germplasm_columns[2]; diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php index 75de51a..80ffb8b 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php @@ -110,7 +110,26 @@ protected function setUp(): void { */ public function testGermplasmAccessionImporterRun() { - // Test 1: Test using a file with only the required columns + // Insert our organism + $subtaxa_cvterm_id = $this->importer->getCVterm('subtaxa'); + $organism_id = $this->connection->insert('1:organism') + ->fields([ + 'genus' => 'Tripalus', + 'species' => 'databasica', + 'infraspecific_name' => 'chadoii', + 'type_id' => $subtaxa_cvterm_id, + ]) + ->execute(); + + // Insert an external db + $db_id = $this->connection->insert('1:db') + ->fields([ + 'name' => 'TestDB', + ]) + ->execute(); + + // ----------------------------- ROUND 1 ------------------------------- + // Test using a simple file with only the required columns $simple_example_file = __DIR__ . '/../../../Fixtures/simple_example.txt'; $genus = 'Tripalus'; @@ -119,7 +138,42 @@ public function testGermplasmAccessionImporterRun() { $this->importer->createImportJob($run_args, $file_details); $this->importer->prepareFiles(); - //$this->importer->run(); + ob_start(); + $this->importer->run(); + $printed_output = ob_get_clean(); + $this->assertStringContainsString('Inserting "Test2".', $printed_output, "Did not get the expected output when running the run() method during ROUND 1."); + + // Now check the db for our 2 new stocks + $stock_query = $this->connection->select('1:stock', 's') + ->fields('s', ['organism_id', 'name', 'uniquename', 'type_id']); + $stock_record = $stock_query->execute()->fetchAll(); + + // Stock: Test1 + $this->assertEquals($stock_record[0]->organism_id, $organism_id, "The inserted organism ID and the selected organism ID for stock Test1 don't match."); + $this->assertEquals($stock_record[0]->name, 'Test1', "The inserted stock.name and the selected name for stock Test1 don't match."); + $this->assertEquals($stock_record[0]->uniquename, 'T1', "The inserted stock.uniquename and the selected uniquename for stock Test1 don't match."); + $this->assertEquals($stock_record[0]->type_id, 9, "The inserted type_id and the selected type_id for stock Test1 don't match."); + // Stock: Test2 + $this->assertEquals($stock_record[1]->organism_id, $organism_id, "The inserted organism ID and the selected organism ID for stock Test2 don't match."); + $this->assertEquals($stock_record[1]->name, 'Test2', "The inserted stock.name and the selected name for stock Test2 don't match."); + $this->assertEquals($stock_record[1]->uniquename, 'T2', "The inserted stock.uniquename and the selected uniquename for stock Test2 don't match."); + $this->assertEquals($stock_record[1]->type_id, 9, "The inserted type_id and the selected type_id for stock Test2 don't match."); + + // Make sure that the stockprop and synonyms table are empty + $stockprop_count_query = $this->connection->select('1:stockprop', 'sp') + ->countQuery()->execute()->fetchField(); + $this->assertEquals($stockprop_count_query, 0, "The row count of the stockprop table is not empty, despite there be no stock properties to insert during ROUND 1."); + + $synonym_count_query = $this->connection->select('1:synonym', 'syn') + ->countQuery()->execute()->fetchField(); + $this->assertEquals($synonym_count_query, 0, "The row count of the synonym table is not empty, despite there be no snyonyms to insert during ROUND 1."); + + // ----------------------------- ROUND 2 ------------------------------- + // Test a file with missing required columns + // Also ensure lines that are empty or begin with "#" and "Germplasm" + // are skipped + // ----------------------------- ROUND 3 ------------------------------- + // Test using a complex file with some/all optional columns } } \ No newline at end of file From a2f2895b2116c952a2f3b2506ac9c0066a0a5cb8 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Wed, 29 Nov 2023 17:16:58 -0600 Subject: [PATCH 31/48] Added additional test files and tests, including a test trait to add stock_synonym table --- .../GermplasmAccessionImporter.php | 35 ++++--- .../src/Fixtures/missing_required_example.txt | 8 ++ .../tests/src/Fixtures/props_syns_example.txt | 7 ++ .../GermplasmAccessionImporterRunTest.php | 98 ++++++++++++++----- .../GermplasmAccessionImporterTest.php | 82 +--------------- .../GermplasmAccessionImporterTestTrait.php | 86 ++++++++++++++++ 6 files changed, 199 insertions(+), 117 deletions(-) create mode 100644 trpcultivate_germplasm/tests/src/Fixtures/missing_required_example.txt create mode 100644 trpcultivate_germplasm/tests/src/Fixtures/props_syns_example.txt create mode 100644 trpcultivate_germplasm/tests/src/Traits/GermplasmAccessionImporterTestTrait.php diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index f417628..e45dbbc 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -246,12 +246,12 @@ public function run(){ $genus_name = $arguments['run_args']['genus_name']; // Make sure our CVterms are all set - $this->setUpCVterms(); + $this->setUpCVterms(); // Set up the ability to track progress so we can report it to the user $filesize = filesize($file_path); - $this->setTotalItems($filesize); - $this->setItemsHandled(0); + //$this->setTotalItems($filesize); + //$this->setItemsHandled(0); $bytes_read = 0; $line_count = 0; @@ -269,30 +269,35 @@ public function run(){ // importer know how many have been processed so it can provide a // progress indicator. $bytes_read += mb_strlen($current_line); - $this->setItemsHandled($bytes_read); + //$this->setItemsHandled($bytes_read); // Check for empty lines, comment lines and a header line - if (empty($current_line)) continue; + $current_line = trim($current_line); + if ($current_line == '') continue; if (preg_match('/^#/', $current_line)) continue; - if (preg_match('/^Germplasm/', $current_line)) continue; + if (preg_match('/^Germplasm/i', $current_line)) continue; - // Trim the current line for trailing whitespace and split columns - // into an array - $current_line = trim($current_line); + // Split our columns into an array for easier processing $germplasm_columns = explode("\t", $current_line); $num_columns = count($germplasm_columns); if (count($germplasm_columns) < 4) { $this->logger->error("Insufficient number of columns detected (<4) for line # @line", ['@line' => $line_count]); $this->error_tracker = TRUE; + // Continue to next line since we already know this will cascade into + // further issues + continue; } // Collect our values from our current line into variables // Since the 1st 4 columns are required, make sure there are values there for($i=0; $i<4; $i++) { + $column = $i+1; if ($germplasm_columns[$i] == '') { - $column = $i+1; $this->logger->error("Column @column is required and cannot be empty for line # @line", ['@column' => $column, '@line' => $line_count]); $this->error_tracker = TRUE; + // Continue to next line since we already know this will cascade into + // further issues + continue 2; } } $germplasm_name = $germplasm_columns[0]; @@ -655,7 +660,7 @@ public function loadStockProperties($stock_id, $stock_properties) { } /** - * Loads each synonym into the chado.synonym and chado.stock_synonym + * Loads each synonym into the chado.synonym and chado.stock_synonym * tables. Returns true if the insert was successful. * * @param int $stock_id @@ -752,8 +757,8 @@ public function loadSynonyms($stock_id, $synonyms, $organism_id) { } // ------------------------------------------------------------------------ - // Lastly, check if our synonym is in the stock table. If yes, THEN create - // a stock_relationship to connect this stock_id to the synonym. + // Lastly, check if our synonym is in the stock table. If yes, THEN create + // a stock_relationship to connect this stock_id to the synonym. // ------------------------------------------------------------------------ $stock_relationship_type_id = $this->getCVterm('stock_relationship_type_synonym'); @@ -779,7 +784,7 @@ public function loadSynonyms($stock_id, $synonyms, $organism_id) { $result = $this->connection->insert('1:stock_relationship') ->fields($values) ->execute(); - + // If the primary key is not available, then the insert failed if (!$result) { $this->logger->error("Insertion of stock ID \"@stock\" and stock ID of its synonym \"@sid_synonym\" into chado.stock_relationship failed.", ['@stock' => $stock_id, '@sid_synonym' => $stock_id_of_synonym]); @@ -791,7 +796,7 @@ public function loadSynonyms($stock_id, $synonyms, $organism_id) { $this->logger->notice("Synonym \"@synonym\" was not found in the stock table, so no stock_relationship was made with stock ID \"@stock\".", ['@synonym' => $synonym, '@stock' => $stock_id]); } } - // Cycled through all the synonyms by this point, and if false hasn't been + // Cycled through all the synonyms by this point, and if false hasn't been // returned, then return true return true; } diff --git a/trpcultivate_germplasm/tests/src/Fixtures/missing_required_example.txt b/trpcultivate_germplasm/tests/src/Fixtures/missing_required_example.txt new file mode 100644 index 0000000..0b297ba --- /dev/null +++ b/trpcultivate_germplasm/tests/src/Fixtures/missing_required_example.txt @@ -0,0 +1,8 @@ +# These +# are +# header +# lines + +Germplasm Name External Database Accession Number Germplasm Species Germplasm Subtaxa Institute Code Institute Name Country of Origin Code Biological Status of Accession Breeding Method Pedigree Synonyms +Test4 T4 databasica subspecies chadoii +Test3 TestDB diff --git a/trpcultivate_germplasm/tests/src/Fixtures/props_syns_example.txt b/trpcultivate_germplasm/tests/src/Fixtures/props_syns_example.txt new file mode 100644 index 0000000..8952927 --- /dev/null +++ b/trpcultivate_germplasm/tests/src/Fixtures/props_syns_example.txt @@ -0,0 +1,7 @@ +# These +# are +# header +# lines + +Germplasm Name External Database Accession Number Germplasm Species Germplasm Subtaxa Institute Code Institute Name Country of Origin Code Biological Status of Accession Breeding Method Pedigree Synonyms +Test5 TestDB T5 databasica subspecies chadoii 500 Breeder line synonym1;synonym2;synonym3 diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php index 80ffb8b..f7da2fe 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php @@ -4,7 +4,8 @@ use Drupal\Core\Url; use Drupal\Tests\tripal_chado\Kernel\ChadoTestKernelBase; -use \Drupal\Tests\user\Traits\UserCreationTrait; +use Drupal\Tests\user\Traits\UserCreationTrait; +use Drupal\Tests\trpcultivate_germplasm\Traits\GermplasmAccessionImporterTestTrait; /** * Tests the functionality of the Germplasm Accession Importer. @@ -16,6 +17,7 @@ class GermplasmAccessionImporterRunTest extends ChadoTestKernelBase { protected static $modules = ['system', 'user', 'file', 'tripal', 'tripal_chado', 'trpcultivate_germplasm']; use UserCreationTrait; + use GermplasmAccessionImporterTestTrait; protected $importer; @@ -101,15 +103,10 @@ protected function setUp(): void { $this->importer->setCVterm('pedigree', 15); $this->importer->setCVterm('synonym', 16); $this->importer->setCVterm('stock_relationship_type_synonym', 17); - } - - /** - * Tests focusing on the Germplasm Accession Importer run() function - * - * @group germ_accession_importer - */ - public function testGermplasmAccessionImporterRun() { - + + // Create the stock_synonym table + $this->createStockSynonymTable(); + // Insert our organism $subtaxa_cvterm_id = $this->importer->getCVterm('subtaxa'); $organism_id = $this->connection->insert('1:organism') @@ -128,20 +125,28 @@ public function testGermplasmAccessionImporterRun() { ]) ->execute(); - // ----------------------------- ROUND 1 ------------------------------- - // Test using a simple file with only the required columns + } + + /** + * Tests focusing on the Germplasm Accession Importer run() function + * using a simple example file that only populates required columns + * + * @group germ_accession_importer + */ + public function testGermplasmAccessionImporterRunSimple() { + $simple_example_file = __DIR__ . '/../../../Fixtures/simple_example.txt'; $genus = 'Tripalus'; $run_args = ['genus_name' => $genus]; $file_details = ['file_local' => $simple_example_file]; - + $this->importer->createImportJob($run_args, $file_details); $this->importer->prepareFiles(); ob_start(); $this->importer->run(); $printed_output = ob_get_clean(); - $this->assertStringContainsString('Inserting "Test2".', $printed_output, "Did not get the expected output when running the run() method during ROUND 1."); + $this->assertStringContainsString('Inserting "Test2".', $printed_output, "Did not get the expected output when running the run() method on simple_example.txt."); // Now check the db for our 2 new stocks $stock_query = $this->connection->select('1:stock', 's') @@ -162,18 +167,61 @@ public function testGermplasmAccessionImporterRun() { // Make sure that the stockprop and synonyms table are empty $stockprop_count_query = $this->connection->select('1:stockprop', 'sp') ->countQuery()->execute()->fetchField(); - $this->assertEquals($stockprop_count_query, 0, "The row count of the stockprop table is not empty, despite there be no stock properties to insert during ROUND 1."); + $this->assertEquals($stockprop_count_query, 0, "The row count of the stockprop table is not empty, despite there be no stock properties to insert from simple_example.txt."); $synonym_count_query = $this->connection->select('1:synonym', 'syn') ->countQuery()->execute()->fetchField(); - $this->assertEquals($synonym_count_query, 0, "The row count of the synonym table is not empty, despite there be no snyonyms to insert during ROUND 1."); - - // ----------------------------- ROUND 2 ------------------------------- - // Test a file with missing required columns - // Also ensure lines that are empty or begin with "#" and "Germplasm" - // are skipped - - // ----------------------------- ROUND 3 ------------------------------- - // Test using a complex file with some/all optional columns + $this->assertEquals($synonym_count_query, 0, "The row count of the synonym table is not empty, despite there be no syonyms to insert from simple_example.txt."); + } + + /** + * Tests focusing on the Germplasm Accession Importer run() function + * using a file where some required columns are missing + * + * @group germ_accession_importer + */ + public function testGermplasmAccessionImporterRunMissing() { + + $problem_example_file = __DIR__ . '/../../../Fixtures/missing_required_example.txt'; + + $genus = 'Tripalus'; + $run_args = ['genus_name' => $genus]; + $file_details = ['file_local' => $problem_example_file]; + + $this->importer->createImportJob($run_args, $file_details); + $this->importer->prepareFiles(); + ob_start(); + $this->importer->run(); + $printed_output = ob_get_clean(); + $this->assertStringContainsString('Column 2 is required and cannot be empty for line # 7', $printed_output, "Did not get the expected output regarding line #7 when running the run() method on missing_required_example.txt."); + $this->assertStringContainsString('Insufficient number of columns detected (<4) for line # 8', $printed_output, "Did not get the expected output regarding line #8 when running the run() method on missing_required_example.txt."); + + // Double check that neither germplasm made it to the database + $stock_count_query = $this->connection->select('1:stock', 's') + ->countQuery()->execute()->fetchField(); + $this->assertEquals($stock_count_query, 0, "The row count of the stock table is not empty, despite expecting to skip stocks in missing_required_example.txt."); + } + + /** + * Tests focusing on the Germplasm Accession Importer run() function + * using a more complicated file where some optional columns are specified + * + * @group germ_accession_importer + */ + public function testGermplasmAccessionImporterRunComplex() { + + $problem_example_file = __DIR__ . '/../../../Fixtures/props_syns_example.txt'; + + $genus = 'Tripalus'; + $run_args = ['genus_name' => $genus]; + $file_details = ['file_local' => $problem_example_file]; + + $this->importer->createImportJob($run_args, $file_details); + $this->importer->prepareFiles(); + ob_start(); + $this->importer->run(); + $printed_output = ob_get_clean(); + //$this->assertStringContainsString('Column 2 is required and cannot be empty for line # 7', $printed_output, "Did not get the expected output regarding line #7 when running the run() method on missing_required_example.txt."); + //$this->assertStringContainsString('Insufficient number of columns detected (<4) for line # 8', $printed_output, "Did not get the expected output regarding line #8 when running the run() method on missing_required_example.txt."); } -} \ No newline at end of file +} diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php index de47699..d44e88e 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php @@ -4,6 +4,7 @@ use Drupal\Core\Url; use Drupal\Tests\tripal_chado\Kernel\ChadoTestKernelBase; +use Drupal\Tests\trpcultivate_germplasm\Traits\GermplasmAccessionImporterTestTrait; /** * Tests the functionality of the Germplasm Accession Importer. @@ -14,6 +15,8 @@ class GermplasmAccessionImporterTest extends ChadoTestKernelBase { protected static $modules = ['system', 'user', 'file', 'tripal', 'tripal_chado', 'trpcultivate_germplasm']; + use GermplasmAccessionImporterTestTrait; + protected $importer; protected $definitions = [ @@ -94,6 +97,8 @@ protected function setUp(): void { $this->importer->setCVterm('pedigree', 15); $this->importer->setCVterm('synonym', 16); $this->importer->setCVterm('stock_relationship_type_synonym', 17); + + $this->createStockSynonymTable(); } /** @@ -479,81 +484,6 @@ public function testGermplasmAccessionImporterLoadStockProperties() { $this->assertEquals($sp_eight_pedigree_count, 1, "The number of records for stockprop pedigree is not 1."); } - public function createStockSynonymTable() { - $table = 'stock_synonym'; - $schema = [ - 'table' => 'stock_synonym', - 'description' => 'Linking table between stock and synonym.', - 'fields' => [ - 'stock_synonym_id' => [ - 'type' => 'serial', - 'not null' => TRUE, - ], - 'synonym_id' => [ - 'size' => 'big', - 'type' => 'int', - 'not null' => TRUE, - ], - 'stock_id' => [ - 'size' => 'big', - 'type' => 'int', - 'not null' => TRUE, - ], - 'pub_id' => [ - 'size' => 'big', - 'type' => 'int', - 'not null' => TRUE, - ], - 'is_current' => [ - 'type' => 'int', - 'default' => 0, - ], - 'is_internal' => [ - 'type' => 'int', - 'default' => 0, - ], - ], - 'primary key' => [ - 'stock_synonym_id', - ], - 'indexes' => [ - 'stock_synonym_idx1' => [ - 0 => 'synonym_id', - ], - 'stock_synonym_idx2' => [ - 0 => 'stock_id', - ], - 'stock_synonym_idx3' => [ - 0 => 'pub_id', - ], - ], - 'foreign keys' => [ - 'synonym' => [ - 'table' => 'synonym', - 'columns' => [ - 'synonym_id' => 'synonym_id', - ], - ], - 'stock' => [ - 'table' => 'stock', - 'columns' => [ - 'stock_id' => 'stock_id', - ], - ], - 'pub' => [ - 'table' => 'pub', - 'columns' => [ - 'pub_id' => 'pub_id', - ], - ], - ], - ]; - - $custom_tables = \Drupal::service('tripal_chado.custom_tables'); - $custom_table = $custom_tables->create($table, $this->connection->getSchemaName()); - $custom_table->setTableSchema($schema); - } - /** * Tests focusing on the Germplasm Accession Importer loadSynonyms() function * @@ -561,8 +491,6 @@ public function createStockSynonymTable() { */ public function testGermplasmAccessionImporterLoadSynonyms() { - $this->createStockSynonymTable(); - // Insert an organism $subtaxa_cvterm_id = $this->getCVtermID('TAXRANK', '0000023'); diff --git a/trpcultivate_germplasm/tests/src/Traits/GermplasmAccessionImporterTestTrait.php b/trpcultivate_germplasm/tests/src/Traits/GermplasmAccessionImporterTestTrait.php new file mode 100644 index 0000000..cd2237c --- /dev/null +++ b/trpcultivate_germplasm/tests/src/Traits/GermplasmAccessionImporterTestTrait.php @@ -0,0 +1,86 @@ + 'stock_synonym', + 'description' => 'Linking table between stock and synonym.', + 'fields' => [ + 'stock_synonym_id' => [ + 'type' => 'serial', + 'not null' => TRUE, + ], + 'synonym_id' => [ + 'size' => 'big', + 'type' => 'int', + 'not null' => TRUE, + ], + 'stock_id' => [ + 'size' => 'big', + 'type' => 'int', + 'not null' => TRUE, + ], + 'pub_id' => [ + 'size' => 'big', + 'type' => 'int', + 'not null' => TRUE, + ], + 'is_current' => [ + 'type' => 'int', + 'default' => 0, + ], + 'is_internal' => [ + 'type' => 'int', + 'default' => 0, + ], + ], + 'primary key' => [ + 'stock_synonym_id', + ], + 'indexes' => [ + 'stock_synonym_idx1' => [ + 0 => 'synonym_id', + ], + 'stock_synonym_idx2' => [ + 0 => 'stock_id', + ], + 'stock_synonym_idx3' => [ + 0 => 'pub_id', + ], + ], + 'foreign keys' => [ + 'synonym' => [ + 'table' => 'synonym', + 'columns' => [ + 'synonym_id' => 'synonym_id', + ], + ], + 'stock' => [ + 'table' => 'stock', + 'columns' => [ + 'stock_id' => 'stock_id', + ], + ], + 'pub' => [ + 'table' => 'pub', + 'columns' => [ + 'pub_id' => 'pub_id', + ], + ], + ], + ]; + + $custom_tables = \Drupal::service('tripal_chado.custom_tables'); + $custom_table = $custom_tables->create($table, $this->connection->getSchemaName()); + $custom_table->setTableSchema($schema); + } +} From 38677311d9e4bc4f1b73b0784ed58ef847bf0fe6 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Thu, 30 Nov 2023 16:57:38 -0600 Subject: [PATCH 32/48] Finished tests for run() --- .../GermplasmAccessionImporter.php | 15 +++- .../GermplasmAccessionImporterRunTest.php | 86 +++++++++++++++++-- 2 files changed, 92 insertions(+), 9 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index e45dbbc..07be98f 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -284,7 +284,7 @@ public function run(){ $this->logger->error("Insufficient number of columns detected (<4) for line # @line", ['@line' => $line_count]); $this->error_tracker = TRUE; // Continue to next line since we already know this will cascade into - // further issues + // further errors continue; } @@ -296,7 +296,7 @@ public function run(){ $this->logger->error("Column @column is required and cannot be empty for line # @line", ['@column' => $column, '@line' => $line_count]); $this->error_tracker = TRUE; // Continue to next line since we already know this will cascade into - // further issues + // further errors continue 2; } } @@ -334,6 +334,17 @@ public function run(){ $load_synonyms = $this->loadSynonyms($stock_id, $synonyms, $organism_id); } } + // Check the error flag + // If true, throw an exception explaining that nothing will be added to the database + // unless errors are resolved + if ($this->error_tracker) { + throw new \Exception( + t("The database transaction was not commited due to the presence of one or more errors. Please fix all errors and try the import again.") + ); + } + else { + $this->logger->notice("Reached end of file without encountering any errors. Transaction will be committed to the database."); + } } /** diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php index f7da2fe..843300a 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php @@ -40,6 +40,9 @@ class GermplasmAccessionImporterRunTest extends ChadoTestKernelBase { ], ]; + // Make the organism ID accessible by all the functions + public $organism_id; + /** * {@inheritdoc} */ @@ -109,7 +112,7 @@ protected function setUp(): void { // Insert our organism $subtaxa_cvterm_id = $this->importer->getCVterm('subtaxa'); - $organism_id = $this->connection->insert('1:organism') + $this->organism_id = $this->connection->insert('1:organism') ->fields([ 'genus' => 'Tripalus', 'species' => 'databasica', @@ -154,12 +157,12 @@ public function testGermplasmAccessionImporterRunSimple() { $stock_record = $stock_query->execute()->fetchAll(); // Stock: Test1 - $this->assertEquals($stock_record[0]->organism_id, $organism_id, "The inserted organism ID and the selected organism ID for stock Test1 don't match."); + $this->assertEquals($stock_record[0]->organism_id, $this->organism_id, "The inserted organism ID and the selected organism ID for stock Test1 don't match."); $this->assertEquals($stock_record[0]->name, 'Test1', "The inserted stock.name and the selected name for stock Test1 don't match."); $this->assertEquals($stock_record[0]->uniquename, 'T1', "The inserted stock.uniquename and the selected uniquename for stock Test1 don't match."); $this->assertEquals($stock_record[0]->type_id, 9, "The inserted type_id and the selected type_id for stock Test1 don't match."); // Stock: Test2 - $this->assertEquals($stock_record[1]->organism_id, $organism_id, "The inserted organism ID and the selected organism ID for stock Test2 don't match."); + $this->assertEquals($stock_record[1]->organism_id, $this->organism_id, "The inserted organism ID and the selected organism ID for stock Test2 don't match."); $this->assertEquals($stock_record[1]->name, 'Test2', "The inserted stock.name and the selected name for stock Test2 don't match."); $this->assertEquals($stock_record[1]->uniquename, 'T2', "The inserted stock.uniquename and the selected uniquename for stock Test2 don't match."); $this->assertEquals($stock_record[1]->type_id, 9, "The inserted type_id and the selected type_id for stock Test2 don't match."); @@ -190,9 +193,18 @@ public function testGermplasmAccessionImporterRunMissing() { $this->importer->createImportJob($run_args, $file_details); $this->importer->prepareFiles(); - ob_start(); - $this->importer->run(); + + // Need a try-catch since errors in this file will trigger the error flag exception + $exception_caught = FALSE; + try { + ob_start(); + $this->importer->run(); + } + catch ( \Exception $e ) { + $exception_caught = TRUE; + } $printed_output = ob_get_clean(); + $this->assertTrue($exception_caught, 'Did not catch exception that should have occurred due to missing required columns.'); $this->assertStringContainsString('Column 2 is required and cannot be empty for line # 7', $printed_output, "Did not get the expected output regarding line #7 when running the run() method on missing_required_example.txt."); $this->assertStringContainsString('Insufficient number of columns detected (<4) for line # 8', $printed_output, "Did not get the expected output regarding line #8 when running the run() method on missing_required_example.txt."); @@ -216,12 +228,72 @@ public function testGermplasmAccessionImporterRunComplex() { $run_args = ['genus_name' => $genus]; $file_details = ['file_local' => $problem_example_file]; + $stock_type_id = $this->importer->getCVterm('accession'); + $stockprop_bsoac_type_id = $this->importer->getCVterm('biological_status_of_accession_code'); + $stockprop_bmDbId_type_id = $this->importer->getCVterm('breeding_method_DbId'); + $stock_relationship_type_id = $this->importer->getCVterm('stock_relationship_type_synonym'); + + // Insert one of the synonyms in our test file into the database as a stock + // This way we can ensure a stock_relationship record is created + $stock_id_of_synonym = $this->connection->insert('1:stock') + ->fields([ + 'organism_id' => $this->organism_id, + 'name' => 'synonym2', + 'uniquename' => 'synonym2', + 'type_id' => $stock_type_id, + ]) + ->execute(); + $this->importer->createImportJob($run_args, $file_details); $this->importer->prepareFiles(); ob_start(); $this->importer->run(); $printed_output = ob_get_clean(); - //$this->assertStringContainsString('Column 2 is required and cannot be empty for line # 7', $printed_output, "Did not get the expected output regarding line #7 when running the run() method on missing_required_example.txt."); - //$this->assertStringContainsString('Insufficient number of columns detected (<4) for line # 8', $printed_output, "Did not get the expected output regarding line #8 when running the run() method on missing_required_example.txt."); + $this->assertStringContainsString('Inserting "Test5".Synonym "synonym1" was not found in the stock table, so no stock_relationship was made with stock ID "2".Synonym "synonym3" was not found in the stock table, so no stock_relationship was made with stock ID "2".', $printed_output, "Did not get the expected output regarding synonyms when running the run() method on props_syns_example.txt."); + + // Now check that the stock properties inserted correctly + $stockprop_count_query = $this->connection->select('1:stockprop', 'sp') + ->countQuery()->execute()->fetchField(); + $this->assertEquals($stockprop_count_query, 2, "The row count of the stockprop table after inserting 2 stock properties values is not correct."); + + // Grab the stock ID + $stock_query = $this->connection->select('1:stock', 's') + ->fields('s', ['stock_id']) + ->condition('name', 'Test5'); + $stock_record = $stock_query->execute()->fetchAll(); + $stock_id = $stock_record[0]->stock_id; + + $stockprop_query = $this->connection->select('1:stockprop', 'sp') + ->fields('sp', ['stock_id', 'type_id', 'value']); + $stockprop_records = $stockprop_query->execute()->fetchAll(); + $this->assertEquals($stockprop_records[0]->stock_id, $stock_id, 'The inserted stock_id and the existing stock_id for the first stockprop does not match for stock Test5.'); + $this->assertEquals($stockprop_records[0]->type_id, $stockprop_bsoac_type_id, 'The inserted type_id and the existing type_id for the first stockprop does not match for stock Test5.'); + $this->assertEquals($stockprop_records[0]->value, 500, 'The value of the inserted stock property "Biological Status of Accession" does not match what was in the file for stock Test5.'); + $this->assertEquals($stockprop_records[1]->stock_id, $stock_id, 'The inserted stock_id and the existing stock_id for the second stockprop does not match for stock Test5.'); + $this->assertEquals($stockprop_records[1]->type_id, $stockprop_bmDbId_type_id, 'The inserted type_id and the existing type_id for the second stockprop does not match for stock Test5.'); + $this->assertEquals($stockprop_records[1]->value, 'Breeder line', 'The value of the inserted stock property "Breeding Method" does not match what was in the file for stock Test5.'); + + // Lastly, check on our synonyms + // Count number of synonyms in the synonym table + $synonym_count_query = $this->connection->select('1:synonym', 'sy') + ->countQuery()->execute()->fetchField(); + $this->assertEquals($synonym_count_query, 3, "Expected there to be 3 synonyms in the synonym table after inserting stock Test5."); + + // Count the number of records in stock_synonym + $stock_synonym_count_query = $this->connection->select('1:stock_synonym', 'ssy') + ->countQuery()->execute()->fetchField(); + $this->assertEquals($stock_synonym_count_query, 3, "Expected there to be 3 records in the stock_synonym table after inserting stock Test5."); + + // Count the number of record in stock_relationship + $stock_relationship_count_query = $this->connection->select('1:stock_relationship', 'sr') + ->countQuery()->execute()->fetchField(); + $this->assertEquals($stock_relationship_count_query, 1, "Expected there to be 1 record in the stock_relationship table after inserting stock Test5."); + + $stock_relationship_query = $this->connection->select('1:stock_relationship', 'sr') + ->fields('sr', ['subject_id', 'object_id', 'type_id', 'value']); + $stock_relationship_record = $stock_relationship_query->execute()->fetchAll(); + $this->assertEquals($stock_relationship_record[0]->subject_id, $stock_id_of_synonym, 'The subject ID of the stock_relationship that was inserted is not the expected stock ID of synonym2'); + $this->assertEquals($stock_relationship_record[0]->object_id, $stock_id, 'The object ID of the stock_relationship that was inserted is not the expected stock ID of Test5'); + $this->assertEquals($stock_relationship_record[0]->type_id, $stock_relationship_type_id, 'The type ID of the stock_relationship that was inserted is not of type synonym'); } } From 8ba2144ee072b5140b4b9ff9bf616e679ce083df Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 4 Dec 2023 16:06:20 -0600 Subject: [PATCH 33/48] Now throws an exception if the stock_synonym table or input file doesn't exist --- .../TripalImporter/GermplasmAccessionImporter.php | 11 ++++++++++- .../GermplasmAccessionImporterRunTest.php | 1 - 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 07be98f..28ce6b7 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -248,6 +248,13 @@ public function run(){ // Make sure our CVterms are all set $this->setUpCVterms(); + // Check if the stock_synonym table exists before moving forward + if (!$this->connection->schema()->tableExists('stock_synonym')) { + throw new \Exception( + t("Could not find stock_synonym table in the current database schema") + ); + } + // Set up the ability to track progress so we can report it to the user $filesize = filesize($file_path); //$this->setTotalItems($filesize); @@ -258,7 +265,9 @@ public function run(){ // Open the file and start iterating through each line $GERMPLASM_FILE = fopen($file_path, 'r'); if(!$GERMPLASM_FILE) { - $this->logger->error("Could not open file: @file", ['@file' => $file_path]); + throw new \Exception( + t("Could not open file: @file", ['@file' => $file_path]) + ); } while (!feof($GERMPLASM_FILE)){ diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php index 843300a..ec8a631 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php @@ -127,7 +127,6 @@ protected function setUp(): void { 'name' => 'TestDB', ]) ->execute(); - } /** From 15d5b8fd06a494c296bc451e1ed0a1b28e6af11b Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 4 Dec 2023 16:46:34 -0600 Subject: [PATCH 34/48] Changed hook_enable to hook_install and made sure to grab the schema --- trpcultivate_germplasm/trpcultivate_germplasm.install | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/trpcultivate_germplasm/trpcultivate_germplasm.install b/trpcultivate_germplasm/trpcultivate_germplasm.install index 4a0ff02..a3cf81a 100644 --- a/trpcultivate_germplasm/trpcultivate_germplasm.install +++ b/trpcultivate_germplasm/trpcultivate_germplasm.install @@ -5,9 +5,9 @@ */ /** - * Implement hook_enable() + * Implement hook_install() */ -function trpcultivate_germplasm_enable(){ +function trpcultivate_germplasm_install(){ // Load required ontologies and cvterms for this module, which includes: // - MCPD (Multi-Crop Passport Ontology) @@ -88,8 +88,10 @@ function trpcultivate_germplasm_enable(){ ], ]; + $connection = \Drupal::service('tripal_chado.database'); + $schema_name = $connection->getSchemaName(); $custom_tables = \Drupal::service('tripal_chado.custom_tables'); - $custom_table = $custom_tables->create($table, $this->chado_schema_main); + $custom_table = $custom_tables->create($table, $schema_name); $custom_table->setTableSchema($schema); // Relationship verb and Default db set to 0, frontend will notify user From 3388c75942bba2ed00ab67a3d7995bce2e8cb10a Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 4 Dec 2023 16:47:41 -0600 Subject: [PATCH 35/48] Cleaned up .install file --- .../trpcultivate_germplasm.install | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/trpcultivate_germplasm/trpcultivate_germplasm.install b/trpcultivate_germplasm/trpcultivate_germplasm.install index a3cf81a..a9df7d4 100644 --- a/trpcultivate_germplasm/trpcultivate_germplasm.install +++ b/trpcultivate_germplasm/trpcultivate_germplasm.install @@ -9,15 +9,6 @@ */ function trpcultivate_germplasm_install(){ - // Load required ontologies and cvterms for this module, which includes: - // - MCPD (Multi-Crop Passport Ontology) - // - GCP germplasm ontology. - // - a collection of custom terms required for germplasm. - // For a full list, see uofspb_germplasm.module:uofspb_germplasm_ontology_list(). - // global $user; - // require_once('includes/job.load_ontolgies.inc'); - // load_ontologies_job(); - // Now create the stock_synonym linking table. $table = 'stock_synonym'; $schema = [ @@ -93,16 +84,4 @@ function trpcultivate_germplasm_install(){ $custom_tables = \Drupal::service('tripal_chado.custom_tables'); $custom_table = $custom_tables->create($table, $schema_name); $custom_table->setTableSchema($schema); - - // Relationship verb and Default db set to 0, frontend will notify user - // to configure these variables before any upload. - - // Create a settings/configuration variable used by - // Germplasm Population Importer to limit/filter relationship verb - // field to specific controlled vocabulary. - //variable_set('germplasm_population_importer_verb_cv', 0); - // Default prefix used in germplasm names. - //variable_set('germplasm_population_importer_default_prefix', 'GERM'); - // Default db used by chado stock.dbxref_id when inserting germplasm. - //variable_set('germplasm_population_importer_db', 0); } From f81e23c16fe3a0cc977544483b8b0d4a01221044 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 4 Dec 2023 17:13:31 -0600 Subject: [PATCH 36/48] Some general cleanup --- .../GermplasmAccessionImporter.php | 13 ++++++++++--- .../GermplasmAccessionImporterRunTest.php | 17 ++++++++++++----- .../GermplasmAccessionImporterTest.php | 0 3 files changed, 22 insertions(+), 8 deletions(-) rename trpcultivate_germplasm/tests/src/Kernel/TripalImporter/{ => GermplasmAccessionImporter}/GermplasmAccessionImporterTest.php (100%) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 28ce6b7..305bd8b 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -215,6 +215,13 @@ public function formValidate($form, &$form_state){ // Nothing to validate since the genus field is set to "required". } + /** + * Checks if our terms have been set already from the config file. + * This is helpful for automated test functionality where terms are + * set there using our setCVterm() function. + * If not already set, then the value of the term is set using setCVterm() + * here. + */ public function setUpCVterms(){ $germplasm_config = $this->config_factory->get('trpcultivate_germplasm.settings'); @@ -257,8 +264,8 @@ public function run(){ // Set up the ability to track progress so we can report it to the user $filesize = filesize($file_path); - //$this->setTotalItems($filesize); - //$this->setItemsHandled(0); + $this->setTotalItems($filesize); + $this->setItemsHandled(0); $bytes_read = 0; $line_count = 0; @@ -278,7 +285,7 @@ public function run(){ // importer know how many have been processed so it can provide a // progress indicator. $bytes_read += mb_strlen($current_line); - //$this->setItemsHandled($bytes_read); + $this->setItemsHandled($bytes_read); // Check for empty lines, comment lines and a header line $current_line = trim($current_line); diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php index ec8a631..f1cc2f3 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php @@ -232,8 +232,10 @@ public function testGermplasmAccessionImporterRunComplex() { $stockprop_bmDbId_type_id = $this->importer->getCVterm('breeding_method_DbId'); $stock_relationship_type_id = $this->importer->getCVterm('stock_relationship_type_synonym'); - // Insert one of the synonyms in our test file into the database as a stock - // This way we can ensure a stock_relationship record is created + // -------------------------------------------------------------------- + // 1. Insert one of the synonyms in our test file into the database as + // a stock. This way we can ensure a stock_relationship record is created + $stock_id_of_synonym = $this->connection->insert('1:stock') ->fields([ 'organism_id' => $this->organism_id, @@ -250,7 +252,10 @@ public function testGermplasmAccessionImporterRunComplex() { $printed_output = ob_get_clean(); $this->assertStringContainsString('Inserting "Test5".Synonym "synonym1" was not found in the stock table, so no stock_relationship was made with stock ID "2".Synonym "synonym3" was not found in the stock table, so no stock_relationship was made with stock ID "2".', $printed_output, "Did not get the expected output regarding synonyms when running the run() method on props_syns_example.txt."); - // Now check that the stock properties inserted correctly + // -------------------------------------------------------------------- + // 2. Check that the stock properties inserted correctly + + // Count the number of stock properties in the database $stockprop_count_query = $this->connection->select('1:stockprop', 'sp') ->countQuery()->execute()->fetchField(); $this->assertEquals($stockprop_count_query, 2, "The row count of the stockprop table after inserting 2 stock properties values is not correct."); @@ -272,7 +277,9 @@ public function testGermplasmAccessionImporterRunComplex() { $this->assertEquals($stockprop_records[1]->type_id, $stockprop_bmDbId_type_id, 'The inserted type_id and the existing type_id for the second stockprop does not match for stock Test5.'); $this->assertEquals($stockprop_records[1]->value, 'Breeder line', 'The value of the inserted stock property "Breeding Method" does not match what was in the file for stock Test5.'); - // Lastly, check on our synonyms + // -------------------------------------------------------------------- + // 3. Check on our synonyms + // Count number of synonyms in the synonym table $synonym_count_query = $this->connection->select('1:synonym', 'sy') ->countQuery()->execute()->fetchField(); @@ -283,7 +290,7 @@ public function testGermplasmAccessionImporterRunComplex() { ->countQuery()->execute()->fetchField(); $this->assertEquals($stock_synonym_count_query, 3, "Expected there to be 3 records in the stock_synonym table after inserting stock Test5."); - // Count the number of record in stock_relationship + // Count the number of records in stock_relationship $stock_relationship_count_query = $this->connection->select('1:stock_relationship', 'sr') ->countQuery()->execute()->fetchField(); $this->assertEquals($stock_relationship_count_query, 1, "Expected there to be 1 record in the stock_relationship table after inserting stock Test5."); diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterTest.php similarity index 100% rename from trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporterTest.php rename to trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterTest.php From 8bb301b2c079cdd8d679ca99fa875f66658dc1fd Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Tue, 5 Dec 2023 16:33:42 -0600 Subject: [PATCH 37/48] Fixed InstallTest so that module is installed after setUp --- .../tests/src/Functional/InstallTest.php | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/trpcultivate_germplasm/tests/src/Functional/InstallTest.php b/trpcultivate_germplasm/tests/src/Functional/InstallTest.php index c0a47e2..a445bc0 100644 --- a/trpcultivate_germplasm/tests/src/Functional/InstallTest.php +++ b/trpcultivate_germplasm/tests/src/Functional/InstallTest.php @@ -20,7 +20,7 @@ class InstallTest extends ChadoTestBrowserBase { * * @var array */ - protected static $modules = ['help', 'trpcultivate_germplasm']; + protected static $modules = ['help', 'tripal_chado']; /** * The name of your module in the .info.yml @@ -38,6 +38,25 @@ class InstallTest extends ChadoTestBrowserBase { */ protected static $help_text_excerpt = 'specialized Tripal fields and importers for germplasm'; + /** + * {@inheritdoc} + */ + protected function setUp() :void { + + parent::setUp(); + + // Ensure we see all logging in tests. + \Drupal::state()->set('is_a_test_environment', TRUE); + + // Open connection to Chado + $this->connection = $this->getTestSchema(ChadoTestBrowserBase::PREPARE_TEST_CHADO); + + $moduleHandler = $this->container->get('module_handler'); + $moduleInstaller = $this->container->get('module_installer'); + $this->assertFalse($moduleHandler->moduleExists('trpcultivate_germplasm')); + $this->assertTrue($moduleInstaller->install(['trpcultivate_germplasm'])); + } + /** * Tests that a specific set of pages load with a 200 response. */ From 5e1ef623813ee56f41190f41717c8f312033d946 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Fri, 15 Dec 2023 11:43:13 -0600 Subject: [PATCH 38/48] Addressed Lacey's wording suggestings to the form --- .../Plugin/TripalImporter/GermplasmAccessionImporter.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 305bd8b..d209ef6 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -115,7 +115,9 @@ public function describeUploadFileFormat() { $file_types = $this->plugin_definition['file_types']; - $output = "Germplasm file should be a tab separated file (" . implode(', ', $file_types) . ") with the following columns:"; + $output = "The input file should be a tab separated file (" . implode(', ', $file_types) . ") with the following columns. "; + $output .= "For more detailed information on this format including links to lookup various codes, please see "; + $output .= 'the official documentation.'; $columns = [ 'Germplasm Name' => 'Name of this germplasm accession (e.g. CDC Redberry)', @@ -191,8 +193,8 @@ public function form($form, &$form_state) { $form['instructions'] = [ '#weight' => -99, '#markup' => ' -

          Load germplasm into database

          -

          Please confirm the file format and column order before upload.

          +

          Import Germplasm Accessions

          +

          Use this form to import germplasm accessions into Chado with metadata that meet the BrAPI standards. Please confirm the file format and column order before upload as this will insert records into your Chado database.

          ', ]; From 19e91ce047099feb5880c8684d729eb7ad9f5d2a Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Fri, 15 Dec 2023 15:55:24 -0600 Subject: [PATCH 39/48] Changed the query in getStockID and updated error handling for duplicate germplasm accessions --- .../GermplasmAccessionImporter.php | 38 +++++++++++++++---- .../GermplasmAccessionImporterTest.php | 2 +- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index d209ef6..9afc49a 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -424,34 +424,58 @@ public function getStockID($germplasm_name, $accession_number, $organism_id) { $accession_type_id = $this->getCVterm('accession'); - // First query the stock table just using the germplasm name and organism ID + // First query the stock table: + // 1. Using a regular condition to ensure the organism_id is a match + // 2. Create an OR condition group to look for records that match germplasm name OR + // the uniquename. Since the unique constraint is organism_id/uniquename/type_id, + // we have to make sure this combo doesn't already exist with a different germplasm + // name. $query = $this->connection->select('1:stock', 's') - ->fields('s', ['stock_id', 'uniquename', 'type_id']); - $query->condition('s.name', $germplasm_name, '=') + ->fields('s', ['stock_id', 'name', 'uniquename', 'type_id']) ->condition('s.organism_id', $organism_id, '='); + + $orGroup = $query->orConditionGroup() + ->condition('s.name', $germplasm_name, '=') + ->condition('s.uniquename', $accession_number, '='); + + // Now add the OR condition group to the query + $query->condition($orGroup); $record = $query->execute()->fetchAll(); + // We may have retrieved 1+ records that share the germplasm name and/or 1+ records that + // share the accession_number. In this case, throw an error since there's no way to + // enter a new record with a unique organism_id/uniquename/type_id combo in this scenario if (sizeof($record) >= 2) { - $this->logger->error("Found more than one stock ID for \"@germplasm_name\".", ['@germplasm_name' => $germplasm_name]); + $this->logger->error("Found more than one stock ID for \"@germplasm_name\" and/or \"@accession\".", ['@germplasm_name' => $germplasm_name, '@accession' => $accession_number]); $this->error_tracker = TRUE; return false; } elseif (sizeof($record) == 1) { // Handle the situation where a stock record exists - // Check the uniquename matches the accession_number column in the file + // Here we are individually checking that our uniquename, name and type_id all match + // what is in the input file. This is to provide an informative error message if one + // of these don't match. In the future, we may want to handle each case differently. + // For example, some groups may want to allow the same germplasm name but a different + // type_id to be allowed. + // 1. Check the uniquename matches the accession_number column in the file if ($accession_number != $record[0]->uniquename) { $this->logger->error("A stock already exists for \"@germplasm_name\" but with an accession of \"@accession\" which does not match the input file.", ['@germplasm_name' => $germplasm_name, '@accession' => $record[0]->uniquename]); $this->error_tracker = TRUE; return false; } - // Check the type_id is of type accession + // 2. Check that our germplasm name matches + if ($germplasm_name != $record[0]->name) { + $this->logger->error("A stock already exists for accession \"@accession\" but with a germplasm name of \"@germplasm_name\" which does not match the input file.", ['@germplasm_name' => $record[0]->name, '@accession' => $accession_number]); + $this->error_tracker = TRUE; + return false; + } + // 3. Check the type_id is of type accession if ($accession_type_id != $record[0]->type_id) { $this->logger->error("A stock already exists for \"@germplasm_name\" but with a type ID of \"@type\" which is not of type \"accession\".", ['@germplasm_name' => $germplasm_name, '@type' => $accession_type_id]); $this->error_tracker = TRUE; return false; } - // Confirmed that the selected record matches what's in the upload file, so return the stock_id return $record[0]->stock_id; } diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterTest.php index d44e88e..51116ec 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterTest.php @@ -249,7 +249,7 @@ public function testGermplasmAccessionImporterGetStockID() { ob_start(); $grabbed_dup_stock_id = $this->importer->getStockID('stock1', 'TEST:1', $organism_id); $printed_output = ob_get_clean(); - $this->assertTrue($printed_output == 'Found more than one stock ID for "stock1".', "Did not get the expected error message when testing for duplicate stock IDs."); + $this->assertStringContainsString('Found more than one stock ID for "stock1"', $printed_output, "Did not get the expected error message when testing for duplicate stock IDs."); } /** From 707ceed99125eaf19cac2be7183290abc1a85dd6 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Mon, 18 Dec 2023 16:59:33 -0600 Subject: [PATCH 40/48] Added additional tests for things that would trigger an exception, and better handling of exceptions in general. --- .../GermplasmAccessionImporter.php | 45 +++++++---- .../tests/src/Fixtures/incomplete_example.txt | 3 + .../GermplasmAccessionImporterRunTest.php | 74 ++++++++++++++++++- 3 files changed, 105 insertions(+), 17 deletions(-) create mode 100644 trpcultivate_germplasm/tests/src/Fixtures/incomplete_example.txt diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 9afc49a..7e8461e 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -250,6 +250,11 @@ public function run(){ // this is an array of files, where each has a 'file_path' key specifying // where the file is located on the server. $file_path = $arguments['files'][0]['file_path']; + if (!file_exists($file_path)) { + throw new \Exception( + t("File does not exist: @file", ['@file' => $file_path]) + ); + } // Grab the genus name $genus_name = $arguments['run_args']['genus_name']; @@ -260,7 +265,7 @@ public function run(){ // Check if the stock_synonym table exists before moving forward if (!$this->connection->schema()->tableExists('stock_synonym')) { throw new \Exception( - t("Could not find stock_synonym table in the current database schema") + t("Could not find stock_synonym table in the current database schema.") ); } @@ -333,23 +338,33 @@ public function run(){ ]; $synonyms = $germplasm_columns[11] ?? ''; - // STEP 1: Pull out the organism ID for the current germplasm - $organism_id = $this->getOrganismID($genus_name, $germplasm_species, $germplasm_subtaxa); - - // STEP 2: Check/Insert this germplasm into the Chado stock table - if ($organism_id) { - $stock_id = $this->getStockID($germplasm_name, $accession_number, $organism_id); - } + // Here we are calling 5 separate functions to check for and insert various + // parts of the input file. Everything is wrapped in a try-catch to ensure + // a useful error message can be passed onto the user and that all errors + // that the file encounters can be reported at one time and not committed + // to the database. + try { + // STEP 1: Pull out the organism ID for the current germplasm + $organism_id = $this->getOrganismID($genus_name, $germplasm_species, $germplasm_subtaxa); + + // STEP 2: Check/Insert this germplasm into the Chado stock table + if ($organism_id) { + $stock_id = $this->getStockID($germplasm_name, $accession_number, $organism_id); + } - if ($stock_id) { - // STEP 3: Load the external database info into Chado dbxref table - $dbxref_id = $this->getDbxrefID($external_database, $stock_id, $accession_number); + if (isset($stock_id) && ($stock_id != null)) { + // STEP 3: Load the external database info into Chado dbxref table + $dbxref_id = $this->getDbxrefID($external_database, $stock_id, $accession_number); - // STEP 4: Load stock properties - $load_props = $this->loadStockProperties($stock_id, $stock_properties); + // STEP 4: Load stock properties + $load_props = $this->loadStockProperties($stock_id, $stock_properties); - // STEP 5: Load synonyms - $load_synonyms = $this->loadSynonyms($stock_id, $synonyms, $organism_id); + // STEP 5: Load synonyms + $load_synonyms = $this->loadSynonyms($stock_id, $synonyms, $organism_id); + } + } catch ( \Exception $e ) { + $this->logger->error("An unusual error occurred when processing germplasm \"@germplasm\". Here is the stack trace: \n" . $e->getMessage() . "\n", ['@germplasm' => $germplasm_name] ); + $this->error_tracker = TRUE; } } // Check the error flag diff --git a/trpcultivate_germplasm/tests/src/Fixtures/incomplete_example.txt b/trpcultivate_germplasm/tests/src/Fixtures/incomplete_example.txt new file mode 100644 index 0000000..7e05a4f --- /dev/null +++ b/trpcultivate_germplasm/tests/src/Fixtures/incomplete_example.txt @@ -0,0 +1,3 @@ +Germplasm Name External Database Accession Number Germplasm Species Germplasm Subtaxa Institute Code Institute Name Country of Origin Code Biological Status of Accession Breeding Method Pedigree Synonyms +Test1 TestDB T1 databasica subspecies chadoii +Test2 TestDB T1 databasica subspecies chadoii \ No newline at end of file diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php index f1cc2f3..8c4f140 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php @@ -221,11 +221,11 @@ public function testGermplasmAccessionImporterRunMissing() { */ public function testGermplasmAccessionImporterRunComplex() { - $problem_example_file = __DIR__ . '/../../../Fixtures/props_syns_example.txt'; + $complex_example_file = __DIR__ . '/../../../Fixtures/props_syns_example.txt'; $genus = 'Tripalus'; $run_args = ['genus_name' => $genus]; - $file_details = ['file_local' => $problem_example_file]; + $file_details = ['file_local' => $complex_example_file]; $stock_type_id = $this->importer->getCVterm('accession'); $stockprop_bsoac_type_id = $this->importer->getCVterm('biological_status_of_accession_code'); @@ -302,4 +302,74 @@ public function testGermplasmAccessionImporterRunComplex() { $this->assertEquals($stock_relationship_record[0]->object_id, $stock_id, 'The object ID of the stock_relationship that was inserted is not the expected stock ID of Test5'); $this->assertEquals($stock_relationship_record[0]->type_id, $stock_relationship_type_id, 'The type ID of the stock_relationship that was inserted is not of type synonym'); } + + /** + * Tests focusing on the Germplasm Accession Importer run() function + * using an example file that should specifically cause an exception + * to occur due to the following cases: + * 1. A non-existant organism in the database + * 2. Attempt to insert a duplicate stock accession + * + * @group germ_accession_importer + */ + public function testGermplasmAccessionImporterRunIncomplete() { + + // Test for a non-existant file + $non_existant_file = __DIR__ . '/does_not_exist.txt'; + + $genus = 'Sally'; + $run_args = ['genus_name' => $genus]; + $file_details = ['file_local' => $non_existant_file]; + + $this->importer->createImportJob($run_args, $file_details); + $this->importer->prepareFiles(); + + $exception_caught = FALSE; + try { + $this->importer->run(); + } catch ( \Exception $e ) { + $exception_caught = TRUE; + } + $this->assertStringContainsString("File does not exist:", $e->getMessage(), "Expected an exception message that file \"does_not_exist.txt\", does not, in fact, exist."); + $this->assertTrue($exception_caught, "Did not catch exception for a non-existant file as input."); + + $incomplete_example_file = __DIR__ . '/../../../Fixtures/incomplete_example.txt'; + + // Test for a non-existant organism + $genus = 'Sally'; + $run_args = ['genus_name' => $genus]; + $file_details = ['file_local' => $incomplete_example_file]; + + $this->importer->createImportJob($run_args, $file_details); + $this->importer->prepareFiles(); + + $exception_caught = FALSE; + ob_start(); + try { + $this->importer->run(); + } catch ( \Exception $e ) { + $exception_caught = TRUE; + } + $printed_output = ob_get_clean(); + $this->assertStringContainsString("Could not find an organism", $printed_output, "Expected an error that an organism in the file 'incomplete_example.txt' could not be found in the database."); + $this->assertTrue($exception_caught, "Did not catch exception for trying to insert germplasm with a non-existant organism."); + + // Test for existing organism, but non-existing stockID + $genus = 'Tripalus'; + $run_args = ['genus_name' => $genus]; + + $this->importer->createImportJob($run_args, $file_details); + $this->importer->prepareFiles(); + + $exception_caught = FALSE; + ob_start(); + try { + $this->importer->run(); + } catch ( \Exception $e ) { + $exception_caught = TRUE; + } + $printed_output = ob_get_clean(); + $this->assertStringContainsString('A stock already exists for accession "T1" but with a germplasm name of "Test1" which does not match the input file.', $printed_output, "Expected an error that a stock accession 'T1' already exists in the file 'incomplete_example.txt'"); + $this->assertTrue($exception_caught, "Did not catch exception for trying to insert duplicate stock accession numbers."); + } } From 03b4faef250f4d01dc053e0495b71dab1e74a8a7 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Thu, 1 Feb 2024 14:03:29 -0600 Subject: [PATCH 41/48] Added a query + assert to ensure no stock records inserted when an error occurred --- .../GermplasmAccessionImporterRunTest.php | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php index 8c4f140..8b97150 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php @@ -233,7 +233,7 @@ public function testGermplasmAccessionImporterRunComplex() { $stock_relationship_type_id = $this->importer->getCVterm('stock_relationship_type_synonym'); // -------------------------------------------------------------------- - // 1. Insert one of the synonyms in our test file into the database as + // 1. Insert one of the synonyms in our test file into the database as // a stock. This way we can ensure a stock_relationship record is created $stock_id_of_synonym = $this->connection->insert('1:stock') @@ -276,7 +276,7 @@ public function testGermplasmAccessionImporterRunComplex() { $this->assertEquals($stockprop_records[1]->stock_id, $stock_id, 'The inserted stock_id and the existing stock_id for the second stockprop does not match for stock Test5.'); $this->assertEquals($stockprop_records[1]->type_id, $stockprop_bmDbId_type_id, 'The inserted type_id and the existing type_id for the second stockprop does not match for stock Test5.'); $this->assertEquals($stockprop_records[1]->value, 'Breeder line', 'The value of the inserted stock property "Breeding Method" does not match what was in the file for stock Test5.'); - + // -------------------------------------------------------------------- // 3. Check on our synonyms @@ -354,7 +354,8 @@ public function testGermplasmAccessionImporterRunIncomplete() { $this->assertStringContainsString("Could not find an organism", $printed_output, "Expected an error that an organism in the file 'incomplete_example.txt' could not be found in the database."); $this->assertTrue($exception_caught, "Did not catch exception for trying to insert germplasm with a non-existant organism."); - // Test for existing organism, but non-existing stockID + // Test for existing organism, but try to enter 2 germplasm with the same name but separate accession numbers + // Note: File used is still incomplete_example.txt $genus = 'Tripalus'; $run_args = ['genus_name' => $genus]; @@ -371,5 +372,11 @@ public function testGermplasmAccessionImporterRunIncomplete() { $printed_output = ob_get_clean(); $this->assertStringContainsString('A stock already exists for accession "T1" but with a germplasm name of "Test1" which does not match the input file.', $printed_output, "Expected an error that a stock accession 'T1' already exists in the file 'incomplete_example.txt'"); $this->assertTrue($exception_caught, "Did not catch exception for trying to insert duplicate stock accession numbers."); + + // Now query stock table to ensure the database transaction was + // successfully rolled back + $stock_count_query = $this->connection->select('1:stock', 's') + ->countQuery()->execute()->fetchField(); + $this->assertEquals($stock_count_query, 0, 'The chado.stock table is not empty despite a database rollback being triggered by an error.'); } } From 5028b0fc4bcbb3ffae4acf5246f455e0bc9d777c Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Thu, 1 Feb 2024 16:27:59 -0600 Subject: [PATCH 42/48] Expanded the error message for pre-existing duplicate stocks to provide more information --- .../GermplasmAccessionImporter.php | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 7e8461e..dfd15b8 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -221,7 +221,7 @@ public function formValidate($form, &$form_state){ * Checks if our terms have been set already from the config file. * This is helpful for automated test functionality where terms are * set there using our setCVterm() function. - * If not already set, then the value of the term is set using setCVterm() + * If not already set, then the value of the term is set using setCVterm() * here. */ public function setUpCVterms(){ @@ -441,9 +441,9 @@ public function getStockID($germplasm_name, $accession_number, $organism_id) { // First query the stock table: // 1. Using a regular condition to ensure the organism_id is a match - // 2. Create an OR condition group to look for records that match germplasm name OR + // 2. Create an OR condition group to look for records that match germplasm name OR // the uniquename. Since the unique constraint is organism_id/uniquename/type_id, - // we have to make sure this combo doesn't already exist with a different germplasm + // we have to make sure this combo doesn't already exist with a different germplasm // name. $query = $this->connection->select('1:stock', 's') ->fields('s', ['stock_id', 'name', 'uniquename', 'type_id']) @@ -452,7 +452,7 @@ public function getStockID($germplasm_name, $accession_number, $organism_id) { $orGroup = $query->orConditionGroup() ->condition('s.name', $germplasm_name, '=') ->condition('s.uniquename', $accession_number, '='); - + // Now add the OR condition group to the query $query->condition($orGroup); $record = $query->execute()->fetchAll(); @@ -461,7 +461,13 @@ public function getStockID($germplasm_name, $accession_number, $organism_id) { // share the accession_number. In this case, throw an error since there's no way to // enter a new record with a unique organism_id/uniquename/type_id combo in this scenario if (sizeof($record) >= 2) { - $this->logger->error("Found more than one stock ID for \"@germplasm_name\" and/or \"@accession\".", ['@germplasm_name' => $germplasm_name, '@accession' => $accession_number]); + $stock_string_array = []; + foreach ($record as $stock_hit) { + $stock_string = $stock_hit->name . " (uniquename=" . $stock_hit->uniquename . "; stock_id=" . $stock_hit->stock_id . ")"; + array_push($stock_string_array, $stock_string); + } + + $this->logger->error("Found more than one stock ID for \"@germplasm_name\" and/or \"@accession\". The existing stocks are: @stock_list", ['@germplasm_name' => $germplasm_name, '@accession' => $accession_number, '@stock_list' => implode(", ", $stock_string_array)]); $this->error_tracker = TRUE; return false; } From 708d2e58ff090b637959455e75a9f375f317c513 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Thu, 1 Feb 2024 17:17:45 -0600 Subject: [PATCH 43/48] Expanded the error which reports multiple synonym ids --- .../TripalImporter/GermplasmAccessionImporter.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index dfd15b8..3aa42ba 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -763,16 +763,16 @@ public function loadSynonyms($stock_id, $synonyms, $organism_id) { ->fields('s', ['synonym_id']) ->condition('s.name', $synonym, '=') ->condition('s.type_id', $synonym_type_id, '='); - $synonym_record = $synonym_query->execute()->fetchAll(); + $synonym_ids = $synonym_query->execute()->fetchCol(); // Make sure there aren't 2 or more records for this synonym - if (sizeof($synonym_record) >= 2) { - $this->logger->error("Found more than one synonym for \"@synonym\" in chado.synonym.", ['@synonym' => $synonym]); + if (sizeof($synonym_ids) >= 2) { + $this->logger->error("Found more than one synonym for \"@synonym\" in chado.synonym (synonym_ids @ids).", ['@synonym' => $synonym, '@ids' => implode(', ', $synonym_ids)]); $this->error_tracker = TRUE; return false; } - elseif (sizeof($synonym_record) == 1) { - $synonym_id = $synonym_record[0]->synonym_id; + elseif (sizeof($synonym_ids) == 1) { + $synonym_id = $synonym_ids[0]; } // Can't find a synonym in the chado.synonym table, so insert it else { From e07af905ce9d79b575f7e78490e9231a887dc09c Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Fri, 2 Feb 2024 11:37:16 -0600 Subject: [PATCH 44/48] Updated importer run command to use the api method for tripal importer in tests --- .../GermplasmAccessionImporterRunTest.php | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php index 8b97150..f1ea599 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php @@ -21,6 +21,8 @@ class GermplasmAccessionImporterRunTest extends ChadoTestKernelBase { protected $importer; + protected $logger; + protected $definitions = [ 'test-germplasm-accession' => [ 'id' => 'trpcultivate-germplasm-accession', @@ -85,6 +87,7 @@ protected function setUp(): void { return NULL; }); $container->set('tripal.logger', $mock_logger); + $this->logger = $mock_logger; $this->config_factory = \Drupal::configFactory(); $this->importer = new \Drupal\trpcultivate_germplasm\Plugin\TripalImporter\GermplasmAccessionImporter( @@ -140,13 +143,13 @@ public function testGermplasmAccessionImporterRunSimple() { $simple_example_file = __DIR__ . '/../../../Fixtures/simple_example.txt'; $genus = 'Tripalus'; - $run_args = ['genus_name' => $genus]; + $run_args = ['genus_name' => $genus, 'schema_name' => $this->testSchemaName]; $file_details = ['file_local' => $simple_example_file]; $this->importer->createImportJob($run_args, $file_details); $this->importer->prepareFiles(); ob_start(); - $this->importer->run(); + tripal_run_importer_run($this->importer, $this->logger); $printed_output = ob_get_clean(); $this->assertStringContainsString('Inserting "Test2".', $printed_output, "Did not get the expected output when running the run() method on simple_example.txt."); @@ -187,7 +190,7 @@ public function testGermplasmAccessionImporterRunMissing() { $problem_example_file = __DIR__ . '/../../../Fixtures/missing_required_example.txt'; $genus = 'Tripalus'; - $run_args = ['genus_name' => $genus]; + $run_args = ['genus_name' => $genus, 'schema_name' => $this->testSchemaName]; $file_details = ['file_local' => $problem_example_file]; $this->importer->createImportJob($run_args, $file_details); @@ -197,7 +200,7 @@ public function testGermplasmAccessionImporterRunMissing() { $exception_caught = FALSE; try { ob_start(); - $this->importer->run(); + tripal_run_importer_run($this->importer, $this->logger); } catch ( \Exception $e ) { $exception_caught = TRUE; @@ -224,7 +227,7 @@ public function testGermplasmAccessionImporterRunComplex() { $complex_example_file = __DIR__ . '/../../../Fixtures/props_syns_example.txt'; $genus = 'Tripalus'; - $run_args = ['genus_name' => $genus]; + $run_args = ['genus_name' => $genus, 'schema_name' => $this->testSchemaName]; $file_details = ['file_local' => $complex_example_file]; $stock_type_id = $this->importer->getCVterm('accession'); @@ -248,7 +251,7 @@ public function testGermplasmAccessionImporterRunComplex() { $this->importer->createImportJob($run_args, $file_details); $this->importer->prepareFiles(); ob_start(); - $this->importer->run(); + tripal_run_importer_run($this->importer, $this->logger); $printed_output = ob_get_clean(); $this->assertStringContainsString('Inserting "Test5".Synonym "synonym1" was not found in the stock table, so no stock_relationship was made with stock ID "2".Synonym "synonym3" was not found in the stock table, so no stock_relationship was made with stock ID "2".', $printed_output, "Did not get the expected output regarding synonyms when running the run() method on props_syns_example.txt."); @@ -318,7 +321,7 @@ public function testGermplasmAccessionImporterRunIncomplete() { $non_existant_file = __DIR__ . '/does_not_exist.txt'; $genus = 'Sally'; - $run_args = ['genus_name' => $genus]; + $run_args = ['genus_name' => $genus, 'schema_name' => $this->testSchemaName]; $file_details = ['file_local' => $non_existant_file]; $this->importer->createImportJob($run_args, $file_details); @@ -326,7 +329,7 @@ public function testGermplasmAccessionImporterRunIncomplete() { $exception_caught = FALSE; try { - $this->importer->run(); + tripal_run_importer_run($this->importer, $this->logger); } catch ( \Exception $e ) { $exception_caught = TRUE; } @@ -337,7 +340,7 @@ public function testGermplasmAccessionImporterRunIncomplete() { // Test for a non-existant organism $genus = 'Sally'; - $run_args = ['genus_name' => $genus]; + $run_args = ['genus_name' => $genus, 'schema_name' => $this->testSchemaName]; $file_details = ['file_local' => $incomplete_example_file]; $this->importer->createImportJob($run_args, $file_details); @@ -346,7 +349,7 @@ public function testGermplasmAccessionImporterRunIncomplete() { $exception_caught = FALSE; ob_start(); try { - $this->importer->run(); + tripal_run_importer_run($this->importer, $this->logger); } catch ( \Exception $e ) { $exception_caught = TRUE; } @@ -357,7 +360,7 @@ public function testGermplasmAccessionImporterRunIncomplete() { // Test for existing organism, but try to enter 2 germplasm with the same name but separate accession numbers // Note: File used is still incomplete_example.txt $genus = 'Tripalus'; - $run_args = ['genus_name' => $genus]; + $run_args = ['genus_name' => $genus, 'schema_name' => $this->testSchemaName]; $this->importer->createImportJob($run_args, $file_details); $this->importer->prepareFiles(); @@ -365,7 +368,7 @@ public function testGermplasmAccessionImporterRunIncomplete() { $exception_caught = FALSE; ob_start(); try { - $this->importer->run(); + tripal_run_importer_run($this->importer, $this->logger); } catch ( \Exception $e ) { $exception_caught = TRUE; } From 3a1392c49d2e75ece7176518b4af69cacdf9e5c7 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Fri, 2 Feb 2024 12:00:48 -0600 Subject: [PATCH 45/48] Fixed ordering for assertEquals in the Run tests --- .gitignore | 1 + .../GermplasmAccessionImporterRunTest.php | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 9d44d4a..b3db51d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ supervisord* .phpunit.result.cache +.DS_Store diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php index f1ea599..2fb690a 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterRunTest.php @@ -172,11 +172,11 @@ public function testGermplasmAccessionImporterRunSimple() { // Make sure that the stockprop and synonyms table are empty $stockprop_count_query = $this->connection->select('1:stockprop', 'sp') ->countQuery()->execute()->fetchField(); - $this->assertEquals($stockprop_count_query, 0, "The row count of the stockprop table is not empty, despite there be no stock properties to insert from simple_example.txt."); + $this->assertEquals(0, $stockprop_count_query, "The row count of the stockprop table is not empty, despite there be no stock properties to insert from simple_example.txt."); $synonym_count_query = $this->connection->select('1:synonym', 'syn') ->countQuery()->execute()->fetchField(); - $this->assertEquals($synonym_count_query, 0, "The row count of the synonym table is not empty, despite there be no syonyms to insert from simple_example.txt."); + $this->assertEquals(0, $synonym_count_query, "The row count of the synonym table is not empty, despite there be no syonyms to insert from simple_example.txt."); } /** @@ -213,7 +213,7 @@ public function testGermplasmAccessionImporterRunMissing() { // Double check that neither germplasm made it to the database $stock_count_query = $this->connection->select('1:stock', 's') ->countQuery()->execute()->fetchField(); - $this->assertEquals($stock_count_query, 0, "The row count of the stock table is not empty, despite expecting to skip stocks in missing_required_example.txt."); + $this->assertEquals(0, $stock_count_query, "The row count of the stock table is not empty, despite expecting to skip stocks in missing_required_example.txt."); } /** @@ -261,7 +261,7 @@ public function testGermplasmAccessionImporterRunComplex() { // Count the number of stock properties in the database $stockprop_count_query = $this->connection->select('1:stockprop', 'sp') ->countQuery()->execute()->fetchField(); - $this->assertEquals($stockprop_count_query, 2, "The row count of the stockprop table after inserting 2 stock properties values is not correct."); + $this->assertEquals(2, $stockprop_count_query, "The row count of the stockprop table after inserting 2 stock properties values is not correct."); // Grab the stock ID $stock_query = $this->connection->select('1:stock', 's') @@ -286,17 +286,17 @@ public function testGermplasmAccessionImporterRunComplex() { // Count number of synonyms in the synonym table $synonym_count_query = $this->connection->select('1:synonym', 'sy') ->countQuery()->execute()->fetchField(); - $this->assertEquals($synonym_count_query, 3, "Expected there to be 3 synonyms in the synonym table after inserting stock Test5."); + $this->assertEquals(3, $synonym_count_query, "Expected there to be 3 synonyms in the synonym table after inserting stock Test5."); // Count the number of records in stock_synonym $stock_synonym_count_query = $this->connection->select('1:stock_synonym', 'ssy') ->countQuery()->execute()->fetchField(); - $this->assertEquals($stock_synonym_count_query, 3, "Expected there to be 3 records in the stock_synonym table after inserting stock Test5."); + $this->assertEquals(3, $stock_synonym_count_query, "Expected there to be 3 records in the stock_synonym table after inserting stock Test5."); // Count the number of records in stock_relationship $stock_relationship_count_query = $this->connection->select('1:stock_relationship', 'sr') ->countQuery()->execute()->fetchField(); - $this->assertEquals($stock_relationship_count_query, 1, "Expected there to be 1 record in the stock_relationship table after inserting stock Test5."); + $this->assertEquals(1, $stock_relationship_count_query, "Expected there to be 1 record in the stock_relationship table after inserting stock Test5."); $stock_relationship_query = $this->connection->select('1:stock_relationship', 'sr') ->fields('sr', ['subject_id', 'object_id', 'type_id', 'value']); @@ -380,6 +380,6 @@ public function testGermplasmAccessionImporterRunIncomplete() { // successfully rolled back $stock_count_query = $this->connection->select('1:stock', 's') ->countQuery()->execute()->fetchField(); - $this->assertEquals($stock_count_query, 0, 'The chado.stock table is not empty despite a database rollback being triggered by an error.'); + $this->assertEquals(0, $stock_count_query, 'The chado.stock table is not empty despite a database rollback being triggered by an error.'); } } From 371d157617971458b627935d837d61c31d7aeae9 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Fri, 2 Feb 2024 14:55:58 -0600 Subject: [PATCH 46/48] Added a check in loadSynonym for if there is a pre-existing stock_relationship --- .../GermplasmAccessionImporter.php | 60 ++++++++++++------- .../GermplasmAccessionImporterTest.php | 12 +++- 2 files changed, 51 insertions(+), 21 deletions(-) diff --git a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php index 3aa42ba..5692a1e 100644 --- a/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php +++ b/trpcultivate_germplasm/src/Plugin/TripalImporter/GermplasmAccessionImporter.php @@ -796,9 +796,9 @@ public function loadSynonyms($stock_id, $synonyms, $organism_id) { } } // ------------------------------------------------------------------------ - // Create a synonym-stock relationship via chado.stock_synonym + // Create a synonym-stock connection via chado.stock_synonym // ------------------------------------------------------------------------ - // First check if this stock-synonym relationship already exists + // First check if this stock-synonym connection already exists $synonym_stock_query = $this->connection->select('1:stock_synonym', 'ss') ->fields('ss', ['stock_synonym_id']) ->condition('ss.synonym_id', $synonym_id, '=') @@ -806,12 +806,15 @@ public function loadSynonyms($stock_id, $synonyms, $organism_id) { $synonym_stock_record = $synonym_stock_query->execute()->fetchAll(); // Make sure there aren't 2 or more records + // Should not be possible due to a unique constraint if (sizeof($synonym_stock_record) >= 2) { - $this->logger->error("Found more than one stock-synonym relationship for stock ID \"@stock\" and synonym \"@synonym\" in chado.stock_synonym.", ['@stock' => $stock_id, '@synonym' => $synonym]); + $this->logger->error("Found more than one stock-synonym connection for stock ID \"@stock\" and synonym \"@synonym\" in chado.stock_synonym.", ['@stock' => $stock_id, '@synonym' => $synonym]); $this->error_tracker = TRUE; return false; } // If 1 result was returned, just ignore it and move on + + // Otherwise, create it elseif (sizeof($synonym_stock_record) == 0) { $values = [ 'synonym_id' => $synonym_id, @@ -831,8 +834,9 @@ public function loadSynonyms($stock_id, $synonyms, $organism_id) { } // ------------------------------------------------------------------------ - // Lastly, check if our synonym is in the stock table. If yes, THEN create - // a stock_relationship to connect this stock_id to the synonym. + // Lastly, check if our synonym name is in the stock table. If yes, THEN create + // a stock_relationship to connect these 2 stocks (ie: the current stock and the + // stock matching the name of the synonym). // ------------------------------------------------------------------------ $stock_relationship_type_id = $this->getCVterm('stock_relationship_type_synonym'); @@ -844,27 +848,43 @@ public function loadSynonyms($stock_id, $synonyms, $organism_id) { // Make sure there aren't 2 or more records if (sizeof($stock_record) >= 2) { - $this->logger->error("Found more than one match for synonym name \"@synonym\" in chado.stock.", ['@synonym' => $synonym]); - $this->error_tracker = TRUE; - return false; + $this->logger->notice("Found more than one match for synonym name \"@synonym\" in chado.stock.", ['@synonym' => $synonym]); } elseif (sizeof($stock_record) == 1) { + // Query the stock_relationship table to see if this relationship already exists $stock_id_of_synonym = $stock_record[0]->stock_id; - $values = [ - 'subject_id' => $stock_id_of_synonym, - 'type_id' => $stock_relationship_type_id, - 'object_id' => $stock_id - ]; - $result = $this->connection->insert('1:stock_relationship') - ->fields($values) - ->execute(); - - // If the primary key is not available, then the insert failed - if (!$result) { - $this->logger->error("Insertion of stock ID \"@stock\" and stock ID of its synonym \"@sid_synonym\" into chado.stock_relationship failed.", ['@stock' => $stock_id, '@sid_synonym' => $stock_id_of_synonym]); + $stock_relationship_query = $this->connection->select('1:stock_relationship', 'str') + ->fields('str', ['stock_relationship_id']) + ->condition('str.subject_id', $stock_id_of_synonym, '=') + ->condition('str.object_id', $stock_id, '=') + ->condition('type_id', $stock_relationship_type_id, '='); + $stock_relationship_record = $stock_relationship_query->execute()->fetchAll(); + // If 2+ relationships exist, then report an error + if (sizeof($stock_relationship_record) >= 2) { + $this->logger->error("Found more than one stock relationship for synonym name \"@synonym\" and stock ID \"@stock\" in chado.stock_relationship.", ['@synonym' => $synonym, '@stock' => $stock_id]); $this->error_tracker = TRUE; return false; } + // If 1 result, carry on + + // If no results, create the stock relationship + if (sizeof($stock_relationship_record) == 0) { + $values = [ + 'subject_id' => $stock_id_of_synonym, + 'type_id' => $stock_relationship_type_id, + 'object_id' => $stock_id + ]; + $result = $this->connection->insert('1:stock_relationship') + ->fields($values) + ->execute(); + + // If the primary key is not available, then the insert failed + if (!$result) { + $this->logger->error("Insertion of stock ID \"@stock\" and stock ID of its synonym \"@sid_synonym\" into chado.stock_relationship failed.", ['@stock' => $stock_id, '@sid_synonym' => $stock_id_of_synonym]); + $this->error_tracker = TRUE; + return false; + } + } } else { $this->logger->notice("Synonym \"@synonym\" was not found in the stock table, so no stock_relationship was made with stock ID \"@stock\".", ['@synonym' => $synonym, '@stock' => $stock_id]); diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterTest.php index 51116ec..5aaa7d5 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterTest.php @@ -512,7 +512,7 @@ public function testGermplasmAccessionImporterLoadSynonyms() { 'type_id' => 9, ]) ->execute(); - + // Attempt to load an empty string (ie. an empty column in the file) $stock1_synonym_empty = ''; $this->importer->loadSynonyms($stock_id, $stock1_synonym_empty, $organism_id); @@ -664,5 +664,15 @@ public function testGermplasmAccessionImporterLoadSynonyms() { $this->assertEquals($synonyms_semicolon_record[0]->name, $syn3, "The synonym table does not contain the expected synonym syn3"); $this->assertEquals($synonyms_semicolon_record[1]->name, $syn4, "The synonym table does not contain the expected synonym syn4"); + + // @todo: Test the case when a stock_relationship previously exists + + // Do my final queries to ensure everything we expect to be in the tables is there + // stock table + // $stock_table_query = $this->connection->select('1:stock', 'st') + // ->fields('st', ['stock_id', 'name', 'uniquename', 'organism_id']); + // $stock_table_records = $stock_table_query->execute(); + // $stock_table_array_actual = $stock_table_records->fetchAllAssoc('stock_id'); + // print_r($stock_table_records->fetchAllAssoc('stock_id')); } } From 6c363dacdc437d980191d6e09612d709fbdc33f7 Mon Sep 17 00:00:00 2001 From: Carolyn Caron Date: Fri, 2 Feb 2024 15:14:55 -0600 Subject: [PATCH 47/48] Added a unique key to stock_synonym table definition --- .../tests/src/Traits/GermplasmAccessionImporterTestTrait.php | 3 +++ trpcultivate_germplasm/trpcultivate_germplasm.install | 3 +++ 2 files changed, 6 insertions(+) diff --git a/trpcultivate_germplasm/tests/src/Traits/GermplasmAccessionImporterTestTrait.php b/trpcultivate_germplasm/tests/src/Traits/GermplasmAccessionImporterTestTrait.php index cd2237c..b2811b5 100644 --- a/trpcultivate_germplasm/tests/src/Traits/GermplasmAccessionImporterTestTrait.php +++ b/trpcultivate_germplasm/tests/src/Traits/GermplasmAccessionImporterTestTrait.php @@ -46,6 +46,9 @@ protected function createStockSynonymTable() { 'primary key' => [ 'stock_synonym_id', ], + 'unique_keys' => [ + 'stock_synonym_c1' => ['synonym_id', 'stock_id', 'pub_id'], + ], 'indexes' => [ 'stock_synonym_idx1' => [ 0 => 'synonym_id', diff --git a/trpcultivate_germplasm/trpcultivate_germplasm.install b/trpcultivate_germplasm/trpcultivate_germplasm.install index a9df7d4..1fd7fab 100644 --- a/trpcultivate_germplasm/trpcultivate_germplasm.install +++ b/trpcultivate_germplasm/trpcultivate_germplasm.install @@ -46,6 +46,9 @@ function trpcultivate_germplasm_install(){ 'primary key' => [ 'stock_synonym_id', ], + 'unique_keys' => [ + 'stock_synonym_c1' => ['synonym_id', 'stock_id', 'pub_id'], + ], 'indexes' => [ 'stock_synonym_idx1' => [ 0 => 'synonym_id', From 09f6fe740b3e2f7ed6125a37e8a51425eef17072 Mon Sep 17 00:00:00 2001 From: Lacey Sanderson Date: Fri, 9 Feb 2024 13:12:32 -0600 Subject: [PATCH 48/48] Switch from assertTrue to assertEquals so you see the two things being compared if a test fails. --- .../GermplasmAccessionImporterTest.php | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterTest.php b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterTest.php index 5aaa7d5..dac947f 100644 --- a/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterTest.php +++ b/trpcultivate_germplasm/tests/src/Kernel/TripalImporter/GermplasmAccessionImporter/GermplasmAccessionImporterTest.php @@ -175,7 +175,8 @@ public function testGermplasmAccessionImporterGetOrganismID() { ob_start(); $non_existent_organism_id = $this->importer->getOrganismID('Nullus', 'organismus', ''); $printed_output = ob_get_clean(); - $this->assertTrue($printed_output == 'Could not find an organism "Nullus organismus" in the database.', "Did not get the expected error message when testing for a non-existant organism."); + $this->assertEquals('Could not find an organism "Nullus organismus" in the database.', $printed_output, + "Did not get the expected error message when testing for a non-existant organism."); // Not testing if multiple organisms are retrieved, since Chado should be preventing such a situation } @@ -217,7 +218,8 @@ public function testGermplasmAccessionImporterGetStockID() { ob_start(); $created_stock_id = $this->importer->getStockID('stock2', 'TEST:2', $organism_id); $printed_output = ob_get_clean(); - $this->assertTrue($printed_output == 'Inserting "stock2".', "Did not get the expected notice message when inserting a new stock."); + $this->assertEquals('Inserting "stock2".', $printed_output, + "Did not get the expected notice message when inserting a new stock."); $stock2_query = $this->connection->select('1:stock', 's') ->fields('s', ['stock_id']) @@ -234,7 +236,8 @@ public function testGermplasmAccessionImporterGetStockID() { ob_start(); $grabbed_dup_stock_name = $this->importer->getStockID('stock1', 'TEST:1000', $organism_id); $printed_output = ob_get_clean(); - $this->assertTrue($printed_output == 'A stock already exists for "stock1" but with an accession of "TEST:1" which does not match the input file.', "Did not get the expected error message when testing for duplicate stock names."); + $this->assertEquals('A stock already exists for "stock1" but with an accession of "TEST:1" which does not match the input file.', $printed_output, + "Did not get the expected error message when testing for duplicate stock names."); // Now test for multiple stocks with the same name and accession, but different type_id $stock_id = $this->connection->insert('1:stock') @@ -287,7 +290,8 @@ public function testGermplasmAccessionImporterGetDbxrefID() { ob_start(); $non_existing_external_db = $this->importer->getDbxrefID('PRETEND', $stock_id, $accession); $printed_output = ob_get_clean(); - $this->assertTrue($printed_output == 'Unable to find "PRETEND" in chado.db.', "Did not get the expected error message when looking up an external database that does not yet exist."); + $this->assertEquals('Unable to find "PRETEND" in chado.db.', $printed_output, + "Did not get the expected error message when looking up an external database that does not yet exist."); // Verify that the stock has an empty dbxref_id $empty_stock_query = $this->connection->select('1:stock', 's') @@ -366,7 +370,8 @@ public function testGermplasmAccessionImporterGetDbxrefID() { ob_start(); $multiple_dbxref_accessions = $this->importer->getDbxrefID($second_db_name, $stock_id, $accession); $printed_output = ob_get_clean(); - $this->assertTrue($printed_output == 'There is already a primary dbxref_id for stock ID "1" that does not match the external database and accession provided in the file (Second Test DB:TEST:1).', "Did not get the expected error message when inserting a dbxref with an existing accession with a different db."); + $this->assertEquals('There is already a primary dbxref_id for stock ID "1" that does not match the external database and accession provided in the file (Second Test DB:TEST:1).', $printed_output, + "Did not get the expected error message when inserting a dbxref with an existing accession with a different db."); } /** @@ -512,7 +517,7 @@ public function testGermplasmAccessionImporterLoadSynonyms() { 'type_id' => 9, ]) ->execute(); - + // Attempt to load an empty string (ie. an empty column in the file) $stock1_synonym_empty = ''; $this->importer->loadSynonyms($stock_id, $stock1_synonym_empty, $organism_id); @@ -563,7 +568,9 @@ public function testGermplasmAccessionImporterLoadSynonyms() { $this->assertEquals($stock1_stock_relationship_count, 0, "Did not expect for one or more stock_relationships to be present in the stock_relationship table."); // We can also check the output of our command as we expect a notice to be given - $this->assertTrue($printed_output == 'Synonym "s1" was not found in the stock table, so no stock_relationship was made with stock ID "1".', "Did not get the expected notice message when adding a synonym that does not exist in the stock table."); + $this->assertEquals('Synonym "s1" was not found in the stock table, so no stock_relationship was made with stock ID "1".', $printed_output, + "Did not get the expected notice message when adding a synonym that does not exist in the stock table."); + // ------------------------------------------------------------------------ // Attempt to insert another synonym for stock1. This time, also add the