From a49027f35d01cb672ff86ad2e38286babc20f4a9 Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Wed, 20 Feb 2019 12:35:15 -0500 Subject: [PATCH 01/21] Add an SPL loader --- .../ESInstance.php => Common/Instance.php} | 92 ++--- .../ESQueue.php => Common/Queue.php} | 10 +- .../ESRequest.php => Common/Request.php} | 6 +- .../ESResponse.php => Common/Response.php} | 0 .../{Elasticsearch/ESJob.php => Jobs/Job.php} | 12 +- tests/DatabaseSeeders/UsersTableSeeder.php | 28 -- tests/ExampleTest.php | 25 -- ...cSearchServerTest.php => InstanceTest.php} | 11 +- tripal_elasticsearch.module | 343 ++++++++++++------ 9 files changed, 303 insertions(+), 224 deletions(-) rename includes/{Elasticsearch/ESInstance.php => Common/Instance.php} (92%) rename includes/{Elasticsearch/ESQueue.php => Common/Queue.php} (97%) rename includes/{Elasticsearch/ESRequest.php => Common/Request.php} (98%) rename includes/{Elasticsearch/ESResponse.php => Common/Response.php} (100%) rename includes/{Elasticsearch/ESJob.php => Jobs/Job.php} (91%) delete mode 100644 tests/DatabaseSeeders/UsersTableSeeder.php delete mode 100644 tests/ExampleTest.php rename tests/Feature/{ConnectToElasticSearchServerTest.php => InstanceTest.php} (75%) diff --git a/includes/Elasticsearch/ESInstance.php b/includes/Common/Instance.php similarity index 92% rename from includes/Elasticsearch/ESInstance.php rename to includes/Common/Instance.php index 484a8801..d82eb376 100644 --- a/includes/Elasticsearch/ESInstance.php +++ b/includes/Common/Instance.php @@ -75,7 +75,7 @@ public function __construct($host = NULL) { * * @return mixed */ - protected function sanitizeQuery($query) { + public function sanitizeQuery($query) { $query = stripslashes($query); $query = str_replace('\\', ' ', $query); $query = str_replace('+', ' ', $query); @@ -96,19 +96,12 @@ protected function sanitizeQuery($query) { * * @return $this */ - public function setWebsiteSearchParams( - $search_terms, - $node_type = '', - $index = 'website', - $index_type = '', - $offset = [], - $force_entities_only = FALSE - ) { + public function setWebsiteSearchParams($search_terms, $node_type = '', $index = 'website', $index_type = '', $offset = [], $force_entities_only = FALSE) { $queries = []; $queries[] = [ 'query_string' => [ - 'default_field' => 'content', + //'default_field' => 'content.taxrank__genus', 'query' => $this->sanitizeQuery($search_terms), 'default_operator' => 'AND', ], @@ -178,6 +171,17 @@ public function setWebsiteSearchParams( return $this; } + /** + * @param $index + * + * @return bool + */ + public function hasIndex($index) { + $indices = $this->getIndices(); + + return in_array($index, $indices); + } + /** * Build table search params. * USe this method if not searching the website or entities indices. @@ -190,13 +194,7 @@ public function setWebsiteSearchParams( * * @return $this */ - public function setTableSearchParams( - $index, - $type, - $query, - $offset = [], - $highlight = FALSE - ) { + public function setTableSearchParams($index, $type, $query, $offset = [], $highlight = FALSE) { $params = []; $params['index'] = $index; $params['type'] = $type; @@ -253,14 +251,7 @@ public function setTableSearchParams( * * @return $this */ - public function setIndexParams( - $index_name, - $shards = 5, - $replicas = 0, - $tokenizer = 'standard', - $token_filters = [], - $field_mapping_types = [] - ) { + public function setIndexParams($index_name, $shards = 5, $replicas = 0, $tokenizer = 'standard', $token_filters = [], $field_mapping_types = []) { $analysis = [ 'analyzer' => [ $index_name => [ @@ -281,13 +272,13 @@ public function setIndexParams( $properties = []; foreach ($field_mapping_types as $field => $mapping_type) { $properties[$field] = [ - 'type' => $mapping_type, - 'fields' => [ - 'raw' => [ - 'type' => $mapping_type, - //'index' => 'not_analyzed', - ], - ], + //'type' => $mapping_type, + //'fields' => [ + // 'raw' => [ + // 'type' => $mapping_type, + // //'index' => 'not_analyzed', + // ], + //], ]; } @@ -331,6 +322,17 @@ public function search($return_source = FALSE) { return $hits; } + return $this->formatHits($hits); + } + + /** + * Format hits. + * + * @param array $hits The hits returned from the search operation. + * + * @return array + */ + public function formatHits($hits) { $results = []; foreach ($hits['hits']['hits'] as $hit) { if (isset($hit['highlight'])) { @@ -489,13 +491,7 @@ public function bulkUpdate($index, $entries, $type = NULL, $id_key = NULL) { * * @return array */ - public function bulk( - $operation, - $index, - $entries, - $type = NULL, - $id_key = NULL - ) { + public function bulk($operation, $index, $entries, $type = NULL, $id_key = NULL) { if (count($entries) === 0) { return []; } @@ -542,8 +538,8 @@ public function bulk( * @return array */ public function paginate($per_page) { - $total = $this->count(); - $total = min($total, 10 * 100000); + $count = $this->count(); + $total = min($count, 1000000); $current_page = pager_default_initialize($total, $per_page); // Set the offset. @@ -555,6 +551,7 @@ public function paginate($per_page) { return [ 'results' => $results, 'total' => $total, + 'count' => $count, 'page' => $current_page + 1, 'pages' => ceil($total / $per_page), 'pager' => theme('pager', ['quantity', $total]), @@ -583,11 +580,7 @@ public function getIndices() { * If count is requested, 2 arrays will be returned. * Otherwise, the structure is $array[$type_label] = $type_label */ - public function getAllCategories( - $version = NULL, - $get_count = FALSE, - $keyword = '*' - ) { + public function getAllCategories($version = NULL, $get_count = FALSE, $keyword = '*') { $types = []; $indices = $this->getIndices(); $search_index = []; @@ -777,12 +770,7 @@ public function getRecord($index, $type, $id) { * @throws \Exception * @return array */ - public function putMapping( - $index_name, - $field_name, - $field_type, - $index_type = NULL - ) { + public function putMapping($index_name, $field_name, $field_type, $index_type = NULL) { if ($index_type === NULL) { $index_type = $index_name; } diff --git a/includes/Elasticsearch/ESQueue.php b/includes/Common/Queue.php similarity index 97% rename from includes/Elasticsearch/ESQueue.php rename to includes/Common/Queue.php index d6bda793..75e904eb 100644 --- a/includes/Elasticsearch/ESQueue.php +++ b/includes/Common/Queue.php @@ -124,7 +124,7 @@ public static function fixProgress(&$queue) { * Dispatch a new job. Uses the queue that has minimum items if queue name is * not provided. * - * @param \ESJob $job + * @param \ES\Common\Job $job * @param string $queue_name * * @return boolean @@ -142,12 +142,12 @@ public static function dispatch($job, $queue_name = NULL) { /** * Execute a given job. * - * @param \ESJob $job + * @param \ES\Common\Job $job * * @throws \Exception */ public static function run($job) { - if ($job instanceof ESJob) { + if ($job instanceof \ES\Common\Job) { try { $job->handle(); $total = $job->total(); @@ -164,7 +164,7 @@ public static function run($job) { return; } - throw new Exception('Elasticsearch Queue: ' . get_class($job) . ' is an invalid job type. Jobs must extend the ESJob class'); + throw new Exception('Elasticsearch Queue: ' . get_class($job) . ' is an invalid job type. Jobs must extend the ES\Common\Job class'); } /** @@ -301,4 +301,4 @@ protected function generateQueueCount() { } } } -} \ No newline at end of file +} diff --git a/includes/Elasticsearch/ESRequest.php b/includes/Common/Request.php similarity index 98% rename from includes/Elasticsearch/ESRequest.php rename to includes/Common/Request.php index eca6e283..feea4b1e 100644 --- a/includes/Elasticsearch/ESRequest.php +++ b/includes/Common/Request.php @@ -1,11 +1,13 @@ 'test user', - 'pass' => 'secret', - 'mail' => 'test@example.com', - 'status' => 1, - 'init' => 'Email', - 'roles' => [ - DRUPAL_AUTHENTICATED_RID => 'authenticated user', - ], - ]; - - // The first parameter is sent blank so a new user is created. - user_save(new \stdClass(), $new_user); - } -} diff --git a/tests/ExampleTest.php b/tests/ExampleTest.php deleted file mode 100644 index e216e59b..00000000 --- a/tests/ExampleTest.php +++ /dev/null @@ -1,25 +0,0 @@ -assertTrue(true); - } -} diff --git a/tests/Feature/ConnectToElasticSearchServerTest.php b/tests/Feature/InstanceTest.php similarity index 75% rename from tests/Feature/ConnectToElasticSearchServerTest.php rename to tests/Feature/InstanceTest.php index 7a335c66..4d9125c5 100644 --- a/tests/Feature/ConnectToElasticSearchServerTest.php +++ b/tests/Feature/InstanceTest.php @@ -2,10 +2,11 @@ namespace Test\Feature; +use ES\Common\Instance; use StatonLab\TripalTestSuite\DBTransaction; -use StatonLab\TripalTestSuite\TripalTestCase; +use Tests\TestCase; -class ConnectToElasticSearchServerTest extends TripalTestCase +class ConnectToElasticSearchServerTest extends TestCase { use DBTransaction; @@ -21,7 +22,7 @@ public function should_fail_to_connect_to_server_before_specifying_host() $this->expectException('Exception'); - new \ESInstance(); + new \ES\Common\Instance(); } /** @@ -34,8 +35,8 @@ public function should_successfully_connect_to_server_after_specifying_host() { variable_set('elasticsearch_host', 'http://localhost:9200'); - $es = new \ESInstance(); + $es = new \ES\Common\Instance(); - $this->assertInstanceOf('\\ESInstance', $es); + $this->assertInstanceOf(Instance::class, $es); } } diff --git a/tripal_elasticsearch.module b/tripal_elasticsearch.module index 62f137ab..c266167e 100644 --- a/tripal_elasticsearch.module +++ b/tripal_elasticsearch.module @@ -16,7 +16,7 @@ require 'includes/tripal_elasticsearch.fields.inc'; require 'includes/tripal_elasticsearch.collections.form.inc'; // Auto discover and include jobs and ES classes. -tripal_elasticsearch_auto_discover_classes(); +//tripal_elasticsearch_auto_discover_classes(); /** * Implements hook_init(). @@ -27,15 +27,22 @@ function tripal_elasticsearch_init() { $library = libraries_detect('elasticsearch-php'); if (user_access('administer tripal elasticsearch', $user)) { if (!$library) { - drupal_set_message(t('The Elastichsearch-PHP library is not installed. - Please install this library first.'), 'warning'); + drupal_set_message( + t( + 'The Elastichsearch-PHP library is not installed. + Please install this library first.' + ), + 'warning' + ); } // Try to load the library and check if that worked. $library = libraries_load('elasticsearch-php'); if (empty($library['loaded'])) { - drupal_set_message(t('The Elasticsearch-PHP library loading failed!'), - 'warning'); + drupal_set_message( + t('The Elasticsearch-PHP library loading failed!'), + 'warning' + ); } } } @@ -62,7 +69,9 @@ function tripal_elasticsearch_menu() { 'access arguments' => ['administer tripal elasticsearch'], 'page callback' => 'drupal_get_form', 'page arguments' => ['elasticsearch_connection_form'], - 'description' => t('Add or edit Elasticsearch connections for this site or remote sites.'), + 'description' => t( + 'Add or edit Elasticsearch connections for this site or remote sites.' + ), 'file' => 'includes/tripal_elasticsearch.connection.form.inc', 'file_path' => drupal_get_path('module', 'tripal_elasticsearch'), 'type' => MENU_LOCAL_TASK, @@ -96,7 +105,9 @@ function tripal_elasticsearch_menu() { 'access arguments' => ['administer tripal elasticsearch'], 'file' => 'includes/tripal_elasticsearch.indices.form.inc', 'file_path' => drupal_get_path('module', 'tripal_elasticsearch'), - 'description' => t('Manage your Elasticsearch indices. Create, edit, or delete indices, check their status, or manage settings such as cross-site querying.'), + 'description' => t( + 'Manage your Elasticsearch indices. Create, edit, or delete indices, check their status, or manage settings such as cross-site querying.' + ), 'type' => MENU_LOCAL_TASK, ]; @@ -168,7 +179,9 @@ function tripal_elasticsearch_menu() { 'access arguments' => ['administer tripal elasticsearch'], 'file' => 'includes/tripal_elasticsearch.search_forms.form.inc', 'file_path' => drupal_get_path('module', 'tripal_elasticsearch'), - 'description' => t('Create or edit search forms for custom database table indices.'), + 'description' => t( + 'Create or edit search forms for custom database table indices.' + ), 'type' => MENU_LOCAL_TASK, ]; @@ -196,7 +209,9 @@ function tripal_elasticsearch_menu() { 'page callback' => 'drupal_get_form', 'page arguments' => ['tripal_elasticsearch_tuning_form'], 'access arguments' => ['administer tripal elasticsearch'], - 'description' => t('Make fine changes to what fields are indexed. Applies to Tripal 3 entities index only.'), + 'description' => t( + 'Make fine changes to what fields are indexed. Applies to Tripal 3 entities index only.' + ), 'type' => MENU_LOCAL_TASK, ]; @@ -303,7 +318,9 @@ function tripal_elasticsearch_permission() { return [ 'administer tripal elasticsearch' => [ 'title' => t('Administer Tripal Elasticsearch module'), - 'description' => t('Perform administration tasks for Tripal Elasticsearch'), + 'description' => t( + 'Perform administration tasks for Tripal Elasticsearch' + ), ], ]; } @@ -382,8 +399,12 @@ function tripal_elasticsearch_theme($existing, $type, $theme, $path) { * Add JS file when the elastic modal theme function is called. */ function tripal_elasticsearch_preprocess_elastic_modal(&$variables) { - drupal_add_js(drupal_get_path('module', - 'tripal_elasticsearch') . '/js/ESModal.js'); + drupal_add_js( + drupal_get_path( + 'module', + 'tripal_elasticsearch' + ) . '/js/ESModal.js' + ); } /** @@ -424,13 +445,13 @@ function tripal_elasticsearch_cron_queue_info() { for ($n = 1; $n <= $queue_number; $n++) { $queues['elasticsearch_queue_' . $n] = [ - 'worker callback' => 'ESQueue::run', + 'worker callback' => '\ES\Common\Queue::run', 'time' => 60 * 2, ]; } $queues['elasticsearch_dispatcher'] = [ - 'worker callback' => 'ESQueue::run', + 'worker callback' => '\ES\Common\Queue::run', // Let worker take as much time required to dispatch all jobs 'time' => 60 * 20, ]; @@ -520,31 +541,45 @@ function tripal_elasticsearch_generate_block($delta) { // Use index name obtained from block name and query the database. $sql = "SELECT DISTINCT index_name FROM {tripal_elasticsearch} WHERE index_name = :index_name"; - $result = db_query($sql, - [':index_name' => $index_name])->fetchCol('index_name'); + $result = db_query( + $sql, + [':index_name' => $index_name] + )->fetchCol('index_name'); // If query result is not empty, display the block. if (!empty($result)) { - $block['subject'] = t('Search block form for index: ' . $index_name . ''); + $block['subject'] = t( + 'Search block form for index: ' . $index_name . '' + ); - $page['form'] = drupal_get_form('tripal_elasticsearch_build_search_block_form', - $index_name); + $page['form'] = drupal_get_form( + 'tripal_elasticsearch_build_search_block_form', + $index_name + ); if (isset($_GET['op'])) { - drupal_add_js(drupal_get_path('module', - 'tripal_elasticsearch') . '/js/table_search_results_datatable.js'); + drupal_add_js( + drupal_get_path( + 'module', + 'tripal_elasticsearch' + ) . '/js/table_search_results_datatable.js' + ); $search_results = tripal_elasticsearch_paginate(10); if (empty($search_results)) { $page['content'] = ['#markup' => '0 results found.']; } else { - $markup = tripal_elasticsearch_get_table_search_result_table($search_results['results'], - $index_name, $search_results['total']); + $markup = tripal_elasticsearch_get_table_search_result_table( + $search_results['results'], + $index_name, + $search_results['total'] + ); $page_number = $search_results['page']; $total_pages = ceil($search_results['total'] / 10); $page['download'] = [ - '#markup' => '

' . 'Download all results in csv format' . '

', + '#markup' => '

' . 'Download all results in csv format' . '

', ]; $page['count'] = [ '#markup' => "
" . "

Showing page {$page_number} out of {$total_pages} pages.

" . "

Found {$search_results['total']} results.

" . "
", @@ -585,24 +620,34 @@ function tripal_elasticsearch_block_view($delta = '') { $block['subject'] = ''; $keyword = isset($_GET['search_box']) ? $_GET['search_box'] : ''; - if (strstr(current_path(), - 'tripal_elasticsearch/search_website') !== FALSE) { - $block['content'] = tripal_elasticsearch_get_website_search_results_category_list($keyword); + if (strstr( + current_path(), + 'tripal_elasticsearch/search_website' + ) !== FALSE) { + $block['content'] = tripal_elasticsearch_get_website_search_results_category_list( + $keyword + ); } break; case 'cross_site_search_block': $block['subject'] = ''; - $block['content'] = drupal_get_form('tripal_elasticsearch_cross_site_search_form'); + $block['content'] = drupal_get_form( + 'tripal_elasticsearch_cross_site_search_form' + ); break; case 'elasticsearch_gene_search_block': $block['subject'] = ''; - $block['content'] = drupal_get_form('tripal_elasticsearch_gene_search_form', - FALSE); + $block['content'] = drupal_get_form( + 'tripal_elasticsearch_gene_search_form', + FALSE + ); break; case 'es_local_gene_search_block': $block['subject'] = ''; - $block['content'] = drupal_get_form('tripal_elasticsearch_gene_search_form', - TRUE); + $block['content'] = drupal_get_form( + 'tripal_elasticsearch_gene_search_form', + TRUE + ); break; } @@ -616,8 +661,7 @@ function tripal_elasticsearch_block_view($delta = '') { * * @throws Exception */ -function tripal_elasticsearch_web_search_results_page_callback($node_type = '' -) { +function tripal_elasticsearch_web_search_results_page_callback($node_type = '') { $keyword = isset($_GET['search_box']) ? $_GET['search_box'] : ''; if (empty($keyword)) { @@ -629,7 +673,7 @@ function tripal_elasticsearch_web_search_results_page_callback($node_type = '' // Run Elasticsearch. try { - $es = new ESInstance(); + $es = new Instance(); $indices = $es->getIndices(); $index_name = []; if (in_array('website', $indices)) { @@ -645,8 +689,12 @@ function tripal_elasticsearch_web_search_results_page_callback($node_type = '' } $per_page = 10; - $results = $es->setWebsiteSearchParams($keyword, $node_type, - implode(',', $index_name), '')->paginate($per_page); + $results = $es->setWebsiteSearchParams( + $keyword, + $node_type, + implode(',', $index_name), + '' + )->paginate($per_page); } catch (\Exception $e) { tripal_report_error('tripal_elasticsearch', TRIPAL_ERROR, $e->getMessage()); @@ -667,23 +715,32 @@ function tripal_elasticsearch_web_search_results_page_callback($node_type = '' $content = tripal_elasticsearch_create_collection_button(); } $content .= '

' . $results['total'] . ' results found Page ' . $current_page . ' out of ' . $pages . '

'; - $content .= tripal_elasticsearch_get_website_search_result_table($results['results'], FALSE); + $content .= tripal_elasticsearch_get_website_search_result_table( + $results['results'], + FALSE + ); $content .= $results['pager']; if (function_exists('tripal_create_collection')) { $form = drupal_get_form('tripal_elasticsearch_collections_form'); try { - $content .= theme('elastic_modal', [ - 'trigger' => '#create-collection-btn', - 'title' => 'Create Collection', - 'submit' => 'Save Collection', - 'content' => [ - 'form' => $form, - ], - ]); + $content .= theme( + 'elastic_modal', + [ + 'trigger' => '#create-collection-btn', + 'title' => 'Create Collection', + 'submit' => 'Save Collection', + 'content' => [ + 'form' => $form, + ], + ] + ); } catch (Exception $exception) { - tripal_report_error('tripal_elasticsearch', TRIPAL_ERROR, - $exception->getMessage()); + tripal_report_error( + 'tripal_elasticsearch', + TRIPAL_ERROR, + $exception->getMessage() + ); } } @@ -701,13 +758,16 @@ function tripal_elasticsearch_create_collection_button() { return ''; } - return '

' . l('Create Downloadable Collection', - 'tripal_elasticsearch/collections/create', [ + return '

' . l( + 'Create Downloadable Collection', + 'tripal_elasticsearch/collections/create', + [ 'attributes' => [ 'id' => 'create-collection-btn', 'class' => 'btn btn-primary', ], - ]) . '

'; + ] + ) . '

'; } /** @@ -723,22 +783,30 @@ function tripal_elasticsearch_table_search_page_callback() { /** * Implements hook_form_FORM_ID_alter(). */ -function tripal_elasticsearch_form_tripal_elasticsearch_build_search_block_form_alter( - &$form, - &$form_state, - $form_id -) { +function tripal_elasticsearch_form_tripal_elasticsearch_build_search_block_form_alter(&$form, &$form_state, $form_id) { if (!isset($form_state['values']['op'])) { return; } elseif ($form_state['values']['op'] !== 'Download') { // add js and css files - drupal_add_css(drupal_get_path('module', - 'tripal_elasticsearch') . '/css/jquery.dataTables.min.css'); - drupal_add_js(drupal_get_path('module', - 'tripal_elasticsearch') . '/js/jquery.dataTables.min.js'); - drupal_add_js(drupal_get_path('module', - 'tripal_elasticsearch') . '/js/table_search_results_datatable.js'); + drupal_add_css( + drupal_get_path( + 'module', + 'tripal_elasticsearch' + ) . '/css/jquery.dataTables.min.css' + ); + drupal_add_js( + drupal_get_path( + 'module', + 'tripal_elasticsearch' + ) . '/js/jquery.dataTables.min.js' + ); + drupal_add_js( + drupal_get_path( + 'module', + 'tripal_elasticsearch' + ) . '/js/table_search_results_datatable.js' + ); /** * build an associated array in which keys are field names and values are user input contents. @@ -757,21 +825,31 @@ function tripal_elasticsearch_form_tripal_elasticsearch_build_search_block_form_ // get index type $index_type = $record->table_name; // Build search query for table search. - $query = tripal_elasticsearch_build_search_query_from_field_content_pairs($index_fields); + $query = tripal_elasticsearch_build_search_query_from_field_content_pairs( + $index_fields + ); // Build table search params. $select_window = isset($form_state['values']['select_window']) ? $form_state['values']['select_window'] : ''; $from = empty($select_window) ? 0 : 1000 * ($select_window - 1); // Run Elasticsearch and return search results into an array. $results = []; try { - $es = new ESInstance(); - $results = $es->setTableSearchParams($index_name, $index_type, $query, [ - $from, - 1000, - ])->search(); + $es = new Instance(); + $results = $es->setTableSearchParams( + $index_name, + $index_type, + $query, + [ + $from, + 1000, + ] + )->search(); } catch (\Exception $e) { - tripal_report_error('tripal_elasticsearch', 'TRIPAL_ERROR', - $e->getMessage()); + tripal_report_error( + 'tripal_elasticsearch', + 'TRIPAL_ERROR', + $e->getMessage() + ); } // Theme search results @@ -779,7 +857,11 @@ function tripal_elasticsearch_form_tripal_elasticsearch_build_search_block_form_ $total = count($results); if ($total > 0) { $output = '
'; - $output .= tripal_elasticsearch_get_table_search_result_table($results, $index_name, $total); + $output .= tripal_elasticsearch_get_table_search_result_table( + $results, + $index_name, + $total + ); $output .= '
'; } @@ -807,15 +889,19 @@ function tripal_elasticsearch_node_update($node) { static $tripal_elasticsearch_errors = FALSE; try { - $es = new ESInstance(); + $es = new Instance(); $indices = $es->getIndices(); if (!in_array('website', $indices)) { return; } } catch (Exception $exception) { if (!$tripal_elasticsearch_errors) { - watchdog('tripal_elasticsearch', $exception->getMessage(), [], - WATCHDOG_ERROR); + watchdog( + 'tripal_elasticsearch', + $exception->getMessage(), + [], + WATCHDOG_ERROR + ); $tripal_elasticsearch_errors = TRUE; } } @@ -845,11 +931,12 @@ function tripal_elasticsearch_node_delete($node) { static $tripal_elasticsearch_errors = FALSE; try { - $es = new ESInstance(); + $es = new Instance(); $es->deleteEntry('website', 'website', $node->nid); } catch (\Exception $e) { if (!$tripal_elasticsearch_errors) { - $message = $e->getMessage() . ' Failed to delete indexed node ' . $node->nid; + $message = $e->getMessage( + ) . ' Failed to delete indexed node ' . $node->nid; tripal_report_error('tripal_elasticsearch', TRIPAL_ERROR, $message); $tripal_elasticsearch_errors = TRUE; } @@ -870,12 +957,13 @@ function tripal_elasticsearch_entity_update($entity, $entity_type) { static $tripal_elasticsearch_errors = FALSE; try { - $es = new ESInstance(); + $es = new Instance(); $indices = $es->getIndices(); if (in_array('gene_search_index', $indices)) { if (isset($entity->chado_table) && $entity->chado_table === 'feature') { - $job = new GeneSearchIndexJob('chado_' . $entity->bundle, 3, - $entity->id); + $job = new GeneSearchIndexJob( + 'chado_' . $entity->bundle, 3, $entity->id + ); $job->dispatch(); } } @@ -885,8 +973,12 @@ function tripal_elasticsearch_entity_update($entity, $entity_type) { } } catch (Exception $exception) { if (!$tripal_elasticsearch_errors) { - watchdog('tripal_elasticsearch', $exception->getMessage(), [], - WATCHDOG_ERROR); + watchdog( + 'tripal_elasticsearch', + $exception->getMessage(), + [], + WATCHDOG_ERROR + ); $tripal_elasticsearch_errors = TRUE; } } @@ -938,19 +1030,25 @@ function tripal_elasticsearch_entity_delete($entity, $type) { static $tripal_elasticsearch_errors = FALSE; try { - $es = new ESInstance(); + $es = new Instance(); if (in_array('entities', $es->getIndices())) { $es->deleteEntry('entities', 'entities', $entity->id); } - if ($entity->chado_table === 'feature' && in_array('gene_search_index', - $es->getIndices())) { - $es->deleteEntry('gene_search_index', 'chado.feature', - $entity->chado_record_id); + if ($entity->chado_table === 'feature' && in_array( + 'gene_search_index', + $es->getIndices() + )) { + $es->deleteEntry( + 'gene_search_index', + 'chado.feature', + $entity->chado_record_id + ); } } catch (\Exception $e) { if (!$tripal_elasticsearch_errors) { - $message = $e->getMessage() . ' Failed to delete indexed entity ' . $entity->id; + $message = $e->getMessage( + ) . ' Failed to delete indexed entity ' . $entity->id; watchdog('tripal_elasticsearch', $message, WATCHDOG_WARNING); $tripal_elasticsearch_errors = TRUE; } @@ -990,11 +1088,13 @@ function tripal_elasticsearch_paginate($per_page) { $index_type = $record->table_name; // Build search query for table search. - $query = tripal_elasticsearch_build_search_query_from_field_content_pairs($index_fields); + $query = tripal_elasticsearch_build_search_query_from_field_content_pairs( + $index_fields + ); // Run Elasticsearch and return search results into an array. try { - $es = new ESInstance(); + $es = new Instance(); return $es->setTableSearchParams($index_name, $index_type, $query) ->paginate($per_page); @@ -1033,7 +1133,9 @@ function tripal_elasticsearch_table_search_download() { $index_type = $record->table_name; // Build search query for table search. - $query = tripal_elasticsearch_build_search_query_from_field_content_pairs($index_fields); + $query = tripal_elasticsearch_build_search_query_from_field_content_pairs( + $index_fields + ); // loop through window and write returned search results into a file. $directory = 'public://table_search_results'; @@ -1041,13 +1143,16 @@ function tripal_elasticsearch_table_search_download() { $temp_file = drupal_tempnam($directory, 'search_results_') . '.csv'; try { - $es = new ESInstance(); - $count = $es->setTableSearchParams($index_name, $index_type, $query) - ->count(); + $es = new Instance(); + $count = $es->setTableSearchParams($index_name, $index_type, $query)->count( + ); $window_range = range(1, ceil($count / 1000)); } catch (Exception $exception) { - tripal_report_error('tripal_elasticsearch', 'TRIPAL_ERROR', - $exception->getMessage()); + tripal_report_error( + 'tripal_elasticsearch', + 'TRIPAL_ERROR', + $exception->getMessage() + ); return; } @@ -1056,8 +1161,12 @@ function tripal_elasticsearch_table_search_download() { $offset = [1000 * ($window - 1), 1000]; try { - $result = $es->setTableSearchParams($index_name, $index_type, $query, - $offset)->search(); + $result = $es->setTableSearchParams( + $index_name, + $index_type, + $query, + $offset + )->search(); } catch (Exception $exception) { drupal_set_message($exception->getMessage(), 'error'); @@ -1075,14 +1184,44 @@ function tripal_elasticsearch_table_search_download() { /** * Auto discover and include ES classes and Jobs. + * + * After registering this autoload function with SPL, the following line + * would cause the function to attempt to load the \Foo\Bar\Baz\Qux class + * from includes/Indices/Entities.php: + * + * new \ES\Indices\Entity; + * + * @author https://www.php-fig.org/psr/psr-4/examples/ + * + * @param string $class The fully-qualified class name. + * + * @return void */ -function tripal_elasticsearch_auto_discover_classes() { - $path = drupal_get_path('module', 'tripal_elasticsearch'); - foreach (glob("{$path}/includes/Elasticsearch/*.php", GLOB_NOSORT) as $file) { - require_once $file; +spl_autoload_register('tripal_elasticsearch_auto_discover_classes'); +function tripal_elasticsearch_auto_discover_classes($class) { + // project-specific namespace prefix + $prefix = 'ES\\'; + + // base directory for the namespace prefix + $base_dir = __DIR__ . '/includes/'; + + // does the class use the namespace prefix? + $len = strlen($prefix); + if (strncmp($prefix, $class, $len) !== 0) { + // no, move to the next registered autoloader + return; } - foreach (glob("{$path}/includes/Jobs/*.php", GLOB_NOSORT) as $file) { + // get the relative class name + $relative_class = substr($class, $len); + + // replace the namespace prefix with the base directory, replace namespace + // separators with directory separators in the relative class name, append + // with .php + $file = $base_dir . str_replace('\\', '/', $relative_class) . '.php'; + + // if the file exists, require it + if (file_exists($file)) { require_once $file; } } From 3b4f802b317fcf454714922ea9c5f353bbb8faa1 Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Wed, 20 Feb 2019 12:36:19 -0500 Subject: [PATCH 02/21] Create environment variables --- .travis.yml | 1 + tests/example.env | 1 + tests/travis.env | 4 ++++ 3 files changed, 6 insertions(+) create mode 100644 tests/travis.env diff --git a/.travis.yml b/.travis.yml index 04fa8900..057bd8a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ env: before_script: - docker-compose up -d - sleep 60 + - cp tests/travis.env tests/.env script: - docker-compose exec app drush en -y tripal_elasticsearch diff --git a/tests/example.env b/tests/example.env index 75e21212..b1ab060a 100644 --- a/tests/example.env +++ b/tests/example.env @@ -1,3 +1,4 @@ BASE_URL=http://localhost DRUPAL_ROOT=/var/www/html FAKER_LOCALE=en_US +ES_HOST=http://localhost:9201 diff --git a/tests/travis.env b/tests/travis.env new file mode 100644 index 00000000..b1ab060a --- /dev/null +++ b/tests/travis.env @@ -0,0 +1,4 @@ +BASE_URL=http://localhost +DRUPAL_ROOT=/var/www/html +FAKER_LOCALE=en_US +ES_HOST=http://localhost:9201 From 42a9cc956871f744eb65b6c9cd90dee675b1ab65 Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Wed, 20 Feb 2019 12:36:42 -0500 Subject: [PATCH 03/21] Create a new query class --- includes/Query/Builder.php | 138 +++++++++++++++++++++++++++++++++++++ includes/Query/Clause.php | 87 +++++++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 includes/Query/Builder.php create mode 100644 includes/Query/Clause.php diff --git a/includes/Query/Builder.php b/includes/Query/Builder.php new file mode 100644 index 00000000..55123b36 --- /dev/null +++ b/includes/Query/Builder.php @@ -0,0 +1,138 @@ +index = $index; + } + + /** + * Range setter. + * + * @param int $from The offset parameter. + * @param int $size The limit parameter. Defaults to 10. + * + * @return $this + * An Instance of this object. + */ + public function range($from, $size = 10) { + $this->from = $from; + $this->size = $size; + + return $this; + } + + /** + * Add a clause. + * + * @param $query + * @param null $fields + */ + public function query($query, array $fields = NULL) { + $query = [ + 'simple_query_string' => [ + 'query' => $query, + ], + ]; + + if (!is_null($fields)) { + if (!is_array($fields)) { + $fields = [$fields]; + } + + $query['simple_query_string']['fields'] = $fields; + } + + $this->queries[] = $query; + } + + /** + * + * @param string|array $fields + * + * @return $this The current object. + */ + public function highlight($fields) { + if (is_array($fields)) { + foreach ($fields as $field) { + $this->addHighlightField($field); + } + } + else { + $this->addHighlightField($fields); + } + + return $this; + } + + /** + * Highlight a field. + * + * @param string $field The name of the field + */ + private function addHighlightField($field) { + $this->highlight[$field] = ['fragment_size' => 150]; + } + + /** + * Build and return the parameters. + * + * @return array + * The params array. + */ + public function build() { + $params = []; + + if ($this->size) { + $params['size'] = $this->size; + } + + if ($this['from']) { + $params['from'] = $this->from; + } + + $params['body'] = [ + 'query' => $this->queries, + ]; + + if ($this->highlight) { + $params['body']['highlight'] = [ + 'fields' => $this->highlight, + ]; + } + + return $params; + } +} diff --git a/includes/Query/Clause.php b/includes/Query/Clause.php new file mode 100644 index 00000000..f43e8eb0 --- /dev/null +++ b/includes/Query/Clause.php @@ -0,0 +1,87 @@ +build(); + $query .= ')'; + return $query; + } + + if (is_null($value)) { + return $field; + } + + return $field . ':' . $value; + } + + /** + * Add a where clause. + * + * @param string $field The value to query if a field is not specified or the + * field name. + * @param string $data The value to query if a field is specified. + * + * @return $this + * The object. + */ + public function where($field, $data = NULL) { + $query = $this->makeQuery($field, $data); + if (!empty($this->query)) { + $this->query .= " AND $query"; + } + else { + $this->query = $query; + } + + return $this; + } + + /** + * Add an or where clause. + * + * @param string $field The value to query if a field is not specified or the + * field name. + * @param string $data The value to query if a field is specified. + * + * @return $this + * The object. + */ + public function orWhere($field, $data = NULL) { + $query = $this->makeQuery($field, $data); + if (!empty($this->query)) { + $this->query .= " OR $query"; + } + else { + $this->query = $query; + } + + return $this; + } + + /** + * @return string + */ + public function build() { + return $this->query; + } +} From de82e5160b460b5625f06cc816fadfc8606de9c1 Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Wed, 20 Feb 2019 12:37:00 -0500 Subject: [PATCH 04/21] Create tests for the Clause class --- tests/Feature/ClauseTest.php | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 tests/Feature/ClauseTest.php diff --git a/tests/Feature/ClauseTest.php b/tests/Feature/ClauseTest.php new file mode 100644 index 00000000..9b1c5c85 --- /dev/null +++ b/tests/Feature/ClauseTest.php @@ -0,0 +1,30 @@ +where('field', 'value')->where('value')->orWhere('value'); + $this->assertEquals('field:value AND value OR value', $query->build()); + } + + /** @test */ + public function testThatClosuresGenerateEnclosedParameters() { + $clause = new Clause(); + $query = $clause->where(function(Clause $query) { + $query->where('f', 'v'); + $query->where('f', 'v'); + })->orWhere(function (Clause $query) { + $query->where('f', 'v'); + $query->orWhere('v'); + })->build(); + + $this->assertEquals('(f:v AND f:v) OR (f:v OR v)', $query); + } +} From ab5901e11ee9dba8c43aabd9f1a14f490b8b5501 Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Wed, 20 Feb 2019 12:37:17 -0500 Subject: [PATCH 05/21] Update deps --- composer.json | 9 ++ composer.lock | 411 ++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 305 insertions(+), 115 deletions(-) diff --git a/composer.json b/composer.json index 0416a8bf..05f59b16 100644 --- a/composer.json +++ b/composer.json @@ -8,7 +8,16 @@ "phpunit/phpunit": "~7.0", "statonlab/tripal-test-suite": "~1.1.2" }, + "require": { + "ext-curl": "*", + "ext-json": "*" + }, "license": "GPLv3", + "autoload": { + "files": [ + "tests/TestCase.php" + ] + }, "authors": [ { "name": "Staton Lab", diff --git a/composer.lock b/composer.lock index bf2c6a5e..d10c9d83 100644 --- a/composer.lock +++ b/composer.lock @@ -4,21 +4,21 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "9815c8c22cd2727f03b0fded72d1f2aa", + "content-hash": "659c79abed2accea365e00d3efd955c4", "packages": [], "packages-dev": [ { "name": "composer/installers", - "version": "v1.5.0", + "version": "v1.6.0", "source": { "type": "git", "url": "https://github.com/composer/installers.git", - "reference": "049797d727261bf27f2690430d935067710049c2" + "reference": "cfcca6b1b60bc4974324efb5783c13dca6932b5b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/installers/zipball/049797d727261bf27f2690430d935067710049c2", - "reference": "049797d727261bf27f2690430d935067710049c2", + "url": "https://api.github.com/repos/composer/installers/zipball/cfcca6b1b60bc4974324efb5783c13dca6932b5b", + "reference": "cfcca6b1b60bc4974324efb5783c13dca6932b5b", "shasum": "" }, "require": { @@ -125,7 +125,7 @@ "zend", "zikula" ], - "time": "2017-12-29T09:13:20+00:00" + "time": "2018-08-27T06:10:37+00:00" }, { "name": "doctrine/instantiator", @@ -183,29 +183,31 @@ }, { "name": "elasticsearch/elasticsearch", - "version": "v5.3.2", + "version": "v6.1.0", "source": { "type": "git", "url": "https://github.com/elastic/elasticsearch-php.git", - "reference": "4b29a4121e790bbfe690d5ee77da348b62d48eb8" + "reference": "b237a37b2cdf23a5a17fd3576cdea771394ad00d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/elastic/elasticsearch-php/zipball/4b29a4121e790bbfe690d5ee77da348b62d48eb8", - "reference": "4b29a4121e790bbfe690d5ee77da348b62d48eb8", + "url": "https://api.github.com/repos/elastic/elasticsearch-php/zipball/b237a37b2cdf23a5a17fd3576cdea771394ad00d", + "reference": "b237a37b2cdf23a5a17fd3576cdea771394ad00d", "shasum": "" }, "require": { + "ext-json": ">=1.3.7", "guzzlehttp/ringphp": "~1.0", - "php": "^5.6|^7.0", + "php": "^7.0", "psr/log": "~1.0" }, "require-dev": { "cpliakas/git-wrapper": "~1.0", "doctrine/inflector": "^1.1", "mockery/mockery": "0.9.4", - "phpunit/phpunit": "^4.7|^5.4", - "sami/sami": "~3.2", + "phpstan/phpstan-shim": "0.8.3", + "phpunit/phpunit": "6.3.0", + "squizlabs/php_codesniffer": "3.0.2", "symfony/finder": "^2.8", "symfony/yaml": "^2.8" }, @@ -234,7 +236,7 @@ "elasticsearch", "search" ], - "time": "2017-11-08T17:04:47+00:00" + "time": "2019-01-08T18:53:46+00:00" }, { "name": "fzaninotto/faker", @@ -404,32 +406,33 @@ }, { "name": "guzzlehttp/psr7", - "version": "1.4.2", + "version": "1.5.2", "source": { "type": "git", "url": "https://github.com/guzzle/psr7.git", - "reference": "f5b8a8512e2b58b0071a7280e39f14f72e05d87c" + "reference": "9f83dded91781a01c63574e387eaa769be769115" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/psr7/zipball/f5b8a8512e2b58b0071a7280e39f14f72e05d87c", - "reference": "f5b8a8512e2b58b0071a7280e39f14f72e05d87c", + "url": "https://api.github.com/repos/guzzle/psr7/zipball/9f83dded91781a01c63574e387eaa769be769115", + "reference": "9f83dded91781a01c63574e387eaa769be769115", "shasum": "" }, "require": { "php": ">=5.4.0", - "psr/http-message": "~1.0" + "psr/http-message": "~1.0", + "ralouphie/getallheaders": "^2.0.5" }, "provide": { "psr/http-message-implementation": "1.0" }, "require-dev": { - "phpunit/phpunit": "~4.0" + "phpunit/phpunit": "~4.8.36 || ^5.7.27 || ^6.5.8" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.4-dev" + "dev-master": "1.5-dev" } }, "autoload": { @@ -459,26 +462,27 @@ "keywords": [ "http", "message", + "psr-7", "request", "response", "stream", "uri", "url" ], - "time": "2017-03-20T17:10:46+00:00" + "time": "2018-12-04T20:46:45+00:00" }, { "name": "guzzlehttp/ringphp", - "version": "1.1.0", + "version": "1.1.1", "source": { "type": "git", "url": "https://github.com/guzzle/RingPHP.git", - "reference": "dbbb91d7f6c191e5e405e900e3102ac7f261bc0b" + "reference": "5e2a174052995663dd68e6b5ad838afd47dd615b" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/RingPHP/zipball/dbbb91d7f6c191e5e405e900e3102ac7f261bc0b", - "reference": "dbbb91d7f6c191e5e405e900e3102ac7f261bc0b", + "url": "https://api.github.com/repos/guzzle/RingPHP/zipball/5e2a174052995663dd68e6b5ad838afd47dd615b", + "reference": "5e2a174052995663dd68e6b5ad838afd47dd615b", "shasum": "" }, "require": { @@ -516,7 +520,7 @@ } ], "description": "Provides a simple API and specification that abstracts away the details of HTTP into a single PHP function.", - "time": "2015-05-20T03:37:09+00:00" + "time": "2018-07-31T13:22:33+00:00" }, { "name": "guzzlehttp/streams", @@ -872,16 +876,16 @@ }, { "name": "phpspec/prophecy", - "version": "1.7.6", + "version": "1.8.0", "source": { "type": "git", "url": "https://github.com/phpspec/prophecy.git", - "reference": "33a7e3c4fda54e912ff6338c48823bd5c0f0b712" + "reference": "4ba436b55987b4bf311cb7c6ba82aa528aac0a06" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/33a7e3c4fda54e912ff6338c48823bd5c0f0b712", - "reference": "33a7e3c4fda54e912ff6338c48823bd5c0f0b712", + "url": "https://api.github.com/repos/phpspec/prophecy/zipball/4ba436b55987b4bf311cb7c6ba82aa528aac0a06", + "reference": "4ba436b55987b4bf311cb7c6ba82aa528aac0a06", "shasum": "" }, "require": { @@ -893,12 +897,12 @@ }, "require-dev": { "phpspec/phpspec": "^2.5|^3.2", - "phpunit/phpunit": "^4.8.35 || ^5.7 || ^6.5" + "phpunit/phpunit": "^4.8.35 || ^5.7 || ^6.5 || ^7.1" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.7.x-dev" + "dev-master": "1.8.x-dev" } }, "autoload": { @@ -931,20 +935,20 @@ "spy", "stub" ], - "time": "2018-04-18T13:57:24+00:00" + "time": "2018-08-05T17:53:17+00:00" }, { "name": "phpunit/php-code-coverage", - "version": "6.0.7", + "version": "6.1.4", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-code-coverage.git", - "reference": "865662550c384bc1db7e51d29aeda1c2c161d69a" + "reference": "807e6013b00af69b6c5d9ceb4282d0393dbb9d8d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/865662550c384bc1db7e51d29aeda1c2c161d69a", - "reference": "865662550c384bc1db7e51d29aeda1c2c161d69a", + "url": "https://api.github.com/repos/sebastianbergmann/php-code-coverage/zipball/807e6013b00af69b6c5d9ceb4282d0393dbb9d8d", + "reference": "807e6013b00af69b6c5d9ceb4282d0393dbb9d8d", "shasum": "" }, "require": { @@ -955,7 +959,7 @@ "phpunit/php-text-template": "^1.2.1", "phpunit/php-token-stream": "^3.0", "sebastian/code-unit-reverse-lookup": "^1.0.1", - "sebastian/environment": "^3.1", + "sebastian/environment": "^3.1 || ^4.0", "sebastian/version": "^2.0.1", "theseer/tokenizer": "^1.1" }, @@ -968,7 +972,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "6.0-dev" + "dev-master": "6.1-dev" } }, "autoload": { @@ -994,25 +998,28 @@ "testing", "xunit" ], - "time": "2018-06-01T07:51:50+00:00" + "time": "2018-10-31T16:06:48+00:00" }, { "name": "phpunit/php-file-iterator", - "version": "2.0.1", + "version": "2.0.2", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-file-iterator.git", - "reference": "cecbc684605bb0cc288828eb5d65d93d5c676d3c" + "reference": "050bedf145a257b1ff02746c31894800e5122946" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/cecbc684605bb0cc288828eb5d65d93d5c676d3c", - "reference": "cecbc684605bb0cc288828eb5d65d93d5c676d3c", + "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/050bedf145a257b1ff02746c31894800e5122946", + "reference": "050bedf145a257b1ff02746c31894800e5122946", "shasum": "" }, "require": { "php": "^7.1" }, + "require-dev": { + "phpunit/phpunit": "^7.1" + }, "type": "library", "extra": { "branch-alias": { @@ -1041,7 +1048,7 @@ "filesystem", "iterator" ], - "time": "2018-06-11T11:44:00+00:00" + "time": "2018-09-13T20:33:42+00:00" }, { "name": "phpunit/php-text-template", @@ -1086,16 +1093,16 @@ }, { "name": "phpunit/php-timer", - "version": "2.0.0", + "version": "2.1.1", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-timer.git", - "reference": "8b8454ea6958c3dee38453d3bd571e023108c91f" + "reference": "8b389aebe1b8b0578430bda0c7c95a829608e059" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/8b8454ea6958c3dee38453d3bd571e023108c91f", - "reference": "8b8454ea6958c3dee38453d3bd571e023108c91f", + "url": "https://api.github.com/repos/sebastianbergmann/php-timer/zipball/8b389aebe1b8b0578430bda0c7c95a829608e059", + "reference": "8b389aebe1b8b0578430bda0c7c95a829608e059", "shasum": "" }, "require": { @@ -1107,7 +1114,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "2.0-dev" + "dev-master": "2.1-dev" } }, "autoload": { @@ -1131,20 +1138,20 @@ "keywords": [ "timer" ], - "time": "2018-02-01T13:07:23+00:00" + "time": "2019-02-20T10:12:59+00:00" }, { "name": "phpunit/php-token-stream", - "version": "3.0.0", + "version": "3.0.1", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/php-token-stream.git", - "reference": "21ad88bbba7c3d93530d93994e0a33cd45f02ace" + "reference": "c99e3be9d3e85f60646f152f9002d46ed7770d18" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-token-stream/zipball/21ad88bbba7c3d93530d93994e0a33cd45f02ace", - "reference": "21ad88bbba7c3d93530d93994e0a33cd45f02ace", + "url": "https://api.github.com/repos/sebastianbergmann/php-token-stream/zipball/c99e3be9d3e85f60646f152f9002d46ed7770d18", + "reference": "c99e3be9d3e85f60646f152f9002d46ed7770d18", "shasum": "" }, "require": { @@ -1180,20 +1187,20 @@ "keywords": [ "tokenizer" ], - "time": "2018-02-01T13:16:43+00:00" + "time": "2018-10-30T05:52:18+00:00" }, { "name": "phpunit/phpunit", - "version": "7.2.7", + "version": "7.5.6", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/phpunit.git", - "reference": "8e878aff7917ef66e702e03d1359b16eee254e2c" + "reference": "09c85e14994df92e5ff1f5ec0b481bdb7d3d3df9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/8e878aff7917ef66e702e03d1359b16eee254e2c", - "reference": "8e878aff7917ef66e702e03d1359b16eee254e2c", + "url": "https://api.github.com/repos/sebastianbergmann/phpunit/zipball/09c85e14994df92e5ff1f5ec0b481bdb7d3d3df9", + "reference": "09c85e14994df92e5ff1f5ec0b481bdb7d3d3df9", "shasum": "" }, "require": { @@ -1214,11 +1221,11 @@ "phpunit/php-timer": "^2.0", "sebastian/comparator": "^3.0", "sebastian/diff": "^3.0", - "sebastian/environment": "^3.1", + "sebastian/environment": "^4.0", "sebastian/exporter": "^3.1", "sebastian/global-state": "^2.0", "sebastian/object-enumerator": "^3.0.3", - "sebastian/resource-operations": "^1.0", + "sebastian/resource-operations": "^2.0", "sebastian/version": "^2.0.1" }, "conflict": { @@ -1238,7 +1245,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "7.2-dev" + "dev-master": "7.5-dev" } }, "autoload": { @@ -1264,7 +1271,7 @@ "testing", "xunit" ], - "time": "2018-07-15T05:20:50+00:00" + "time": "2019-02-18T09:24:50+00:00" }, { "name": "psr/http-message", @@ -1318,16 +1325,16 @@ }, { "name": "psr/log", - "version": "1.0.2", + "version": "1.1.0", "source": { "type": "git", "url": "https://github.com/php-fig/log.git", - "reference": "4ebe3a8bf773a19edfe0a84b6585ba3d401b724d" + "reference": "6c001f1daafa3a3ac1d8ff69ee4db8e799a654dd" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/php-fig/log/zipball/4ebe3a8bf773a19edfe0a84b6585ba3d401b724d", - "reference": "4ebe3a8bf773a19edfe0a84b6585ba3d401b724d", + "url": "https://api.github.com/repos/php-fig/log/zipball/6c001f1daafa3a3ac1d8ff69ee4db8e799a654dd", + "reference": "6c001f1daafa3a3ac1d8ff69ee4db8e799a654dd", "shasum": "" }, "require": { @@ -1361,20 +1368,60 @@ "psr", "psr-3" ], - "time": "2016-10-10T12:19:37+00:00" + "time": "2018-11-20T15:27:04+00:00" + }, + { + "name": "ralouphie/getallheaders", + "version": "2.0.5", + "source": { + "type": "git", + "url": "https://github.com/ralouphie/getallheaders.git", + "reference": "5601c8a83fbba7ef674a7369456d12f1e0d0eafa" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/ralouphie/getallheaders/zipball/5601c8a83fbba7ef674a7369456d12f1e0d0eafa", + "reference": "5601c8a83fbba7ef674a7369456d12f1e0d0eafa", + "shasum": "" + }, + "require": { + "php": ">=5.3" + }, + "require-dev": { + "phpunit/phpunit": "~3.7.0", + "satooshi/php-coveralls": ">=1.0" + }, + "type": "library", + "autoload": { + "files": [ + "src/getallheaders.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Ralph Khattar", + "email": "ralph.khattar@gmail.com" + } + ], + "description": "A polyfill for getallheaders.", + "time": "2016-02-11T07:05:27+00:00" }, { "name": "react/promise", - "version": "v2.7.0", + "version": "v2.7.1", "source": { "type": "git", "url": "https://github.com/reactphp/promise.git", - "reference": "f4edc2581617431aea50430749db55cc3fc031b3" + "reference": "31ffa96f8d2ed0341a57848cbb84d88b89dd664d" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/reactphp/promise/zipball/f4edc2581617431aea50430749db55cc3fc031b3", - "reference": "f4edc2581617431aea50430749db55cc3fc031b3", + "url": "https://api.github.com/repos/reactphp/promise/zipball/31ffa96f8d2ed0341a57848cbb84d88b89dd664d", + "reference": "31ffa96f8d2ed0341a57848cbb84d88b89dd664d", "shasum": "" }, "require": { @@ -1407,7 +1454,7 @@ "promise", "promises" ], - "time": "2018-06-13T15:59:06+00:00" + "time": "2019-01-07T21:25:54+00:00" }, { "name": "sebastian/code-unit-reverse-lookup", @@ -1520,23 +1567,23 @@ }, { "name": "sebastian/diff", - "version": "3.0.1", + "version": "3.0.2", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/diff.git", - "reference": "366541b989927187c4ca70490a35615d3fef2dce" + "reference": "720fcc7e9b5cf384ea68d9d930d480907a0c1a29" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/366541b989927187c4ca70490a35615d3fef2dce", - "reference": "366541b989927187c4ca70490a35615d3fef2dce", + "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/720fcc7e9b5cf384ea68d9d930d480907a0c1a29", + "reference": "720fcc7e9b5cf384ea68d9d930d480907a0c1a29", "shasum": "" }, "require": { "php": "^7.1" }, "require-dev": { - "phpunit/phpunit": "^7.0", + "phpunit/phpunit": "^7.5 || ^8.0", "symfony/process": "^2 || ^3.3 || ^4" }, "type": "library", @@ -1572,32 +1619,35 @@ "unidiff", "unified diff" ], - "time": "2018-06-10T07:54:39+00:00" + "time": "2019-02-04T06:01:07+00:00" }, { "name": "sebastian/environment", - "version": "3.1.0", + "version": "4.1.0", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/environment.git", - "reference": "cd0871b3975fb7fc44d11314fd1ee20925fce4f5" + "reference": "6fda8ce1974b62b14935adc02a9ed38252eca656" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/cd0871b3975fb7fc44d11314fd1ee20925fce4f5", - "reference": "cd0871b3975fb7fc44d11314fd1ee20925fce4f5", + "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/6fda8ce1974b62b14935adc02a9ed38252eca656", + "reference": "6fda8ce1974b62b14935adc02a9ed38252eca656", "shasum": "" }, "require": { - "php": "^7.0" + "php": "^7.1" }, "require-dev": { - "phpunit/phpunit": "^6.1" + "phpunit/phpunit": "^7.5" + }, + "suggest": { + "ext-posix": "*" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "3.1.x-dev" + "dev-master": "4.1-dev" } }, "autoload": { @@ -1622,7 +1672,7 @@ "environment", "hhvm" ], - "time": "2017-07-01T08:51:00+00:00" + "time": "2019-02-01T05:27:49+00:00" }, { "name": "sebastian/exporter", @@ -1889,25 +1939,25 @@ }, { "name": "sebastian/resource-operations", - "version": "1.0.0", + "version": "2.0.1", "source": { "type": "git", "url": "https://github.com/sebastianbergmann/resource-operations.git", - "reference": "ce990bb21759f94aeafd30209e8cfcdfa8bc3f52" + "reference": "4d7a795d35b889bf80a0cc04e08d77cedfa917a9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/resource-operations/zipball/ce990bb21759f94aeafd30209e8cfcdfa8bc3f52", - "reference": "ce990bb21759f94aeafd30209e8cfcdfa8bc3f52", + "url": "https://api.github.com/repos/sebastianbergmann/resource-operations/zipball/4d7a795d35b889bf80a0cc04e08d77cedfa917a9", + "reference": "4d7a795d35b889bf80a0cc04e08d77cedfa917a9", "shasum": "" }, "require": { - "php": ">=5.6.0" + "php": "^7.1" }, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.0.x-dev" + "dev-master": "2.0-dev" } }, "autoload": { @@ -1927,7 +1977,7 @@ ], "description": "Provides a list of PHP built-in functions that operate on resources", "homepage": "https://www.github.com/sebastianbergmann/resource-operations", - "time": "2015-07-28T20:34:47+00:00" + "time": "2018-10-04T04:07:39+00:00" }, { "name": "sebastian/version", @@ -2022,26 +2072,30 @@ }, { "name": "symfony/console", - "version": "v4.1.2", + "version": "v4.2.3", "source": { "type": "git", "url": "https://github.com/symfony/console.git", - "reference": "5c31f6a97c1c240707f6d786e7e59bfacdbc0219" + "reference": "1f0ad51dfde4da8a6070f06adc58b4e37cbb37a4" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/console/zipball/5c31f6a97c1c240707f6d786e7e59bfacdbc0219", - "reference": "5c31f6a97c1c240707f6d786e7e59bfacdbc0219", + "url": "https://api.github.com/repos/symfony/console/zipball/1f0ad51dfde4da8a6070f06adc58b4e37cbb37a4", + "reference": "1f0ad51dfde4da8a6070f06adc58b4e37cbb37a4", "shasum": "" }, "require": { "php": "^7.1.3", + "symfony/contracts": "^1.0", "symfony/polyfill-mbstring": "~1.0" }, "conflict": { "symfony/dependency-injection": "<3.4", "symfony/process": "<3.3" }, + "provide": { + "psr/log-implementation": "1.0" + }, "require-dev": { "psr/log": "~1.0", "symfony/config": "~3.4|~4.0", @@ -2051,7 +2105,7 @@ "symfony/process": "~3.4|~4.0" }, "suggest": { - "psr/log-implementation": "For using the console logger", + "psr/log": "For using the console logger", "symfony/event-dispatcher": "", "symfony/lock": "", "symfony/process": "" @@ -2059,7 +2113,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "4.1-dev" + "dev-master": "4.2-dev" } }, "autoload": { @@ -2086,20 +2140,146 @@ ], "description": "Symfony Console Component", "homepage": "https://symfony.com", - "time": "2018-07-16T14:05:40+00:00" + "time": "2019-01-25T14:35:16+00:00" + }, + { + "name": "symfony/contracts", + "version": "v1.0.2", + "source": { + "type": "git", + "url": "https://github.com/symfony/contracts.git", + "reference": "1aa7ab2429c3d594dd70689604b5cf7421254cdf" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/contracts/zipball/1aa7ab2429c3d594dd70689604b5cf7421254cdf", + "reference": "1aa7ab2429c3d594dd70689604b5cf7421254cdf", + "shasum": "" + }, + "require": { + "php": "^7.1.3" + }, + "require-dev": { + "psr/cache": "^1.0", + "psr/container": "^1.0" + }, + "suggest": { + "psr/cache": "When using the Cache contracts", + "psr/container": "When using the Service contracts", + "symfony/cache-contracts-implementation": "", + "symfony/service-contracts-implementation": "", + "symfony/translation-contracts-implementation": "" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0-dev" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Contracts\\": "" + }, + "exclude-from-classmap": [ + "**/Tests/" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "A set of abstractions extracted out of the Symfony components", + "homepage": "https://symfony.com", + "keywords": [ + "abstractions", + "contracts", + "decoupling", + "interfaces", + "interoperability", + "standards" + ], + "time": "2018-12-05T08:06:11+00:00" + }, + { + "name": "symfony/polyfill-ctype", + "version": "v1.10.0", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-ctype.git", + "reference": "e3d826245268269cd66f8326bd8bc066687b4a19" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/e3d826245268269cd66f8326bd8bc066687b4a19", + "reference": "e3d826245268269cd66f8326bd8bc066687b4a19", + "shasum": "" + }, + "require": { + "php": ">=5.3.3" + }, + "suggest": { + "ext-ctype": "For best performance" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.9-dev" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Polyfill\\Ctype\\": "" + }, + "files": [ + "bootstrap.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + }, + { + "name": "Gert de Pagter", + "email": "BackEndTea@gmail.com" + } + ], + "description": "Symfony polyfill for ctype functions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "ctype", + "polyfill", + "portable" + ], + "time": "2018-08-06T14:22:27+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "v1.8.0", + "version": "v1.10.0", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "3296adf6a6454a050679cde90f95350ad604b171" + "reference": "c79c051f5b3a46be09205c73b80b346e4153e494" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/3296adf6a6454a050679cde90f95350ad604b171", - "reference": "3296adf6a6454a050679cde90f95350ad604b171", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/c79c051f5b3a46be09205c73b80b346e4153e494", + "reference": "c79c051f5b3a46be09205c73b80b346e4153e494", "shasum": "" }, "require": { @@ -2111,7 +2291,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "1.8-dev" + "dev-master": "1.9-dev" } }, "autoload": { @@ -2145,7 +2325,7 @@ "portable", "shim" ], - "time": "2018-04-26T10:06:28+00:00" + "time": "2018-09-21T13:07:52+00:00" }, { "name": "theseer/tokenizer", @@ -2189,20 +2369,21 @@ }, { "name": "webmozart/assert", - "version": "1.3.0", + "version": "1.4.0", "source": { "type": "git", "url": "https://github.com/webmozart/assert.git", - "reference": "0df1908962e7a3071564e857d86874dad1ef204a" + "reference": "83e253c8e0be5b0257b881e1827274667c5c17a9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/webmozart/assert/zipball/0df1908962e7a3071564e857d86874dad1ef204a", - "reference": "0df1908962e7a3071564e857d86874dad1ef204a", + "url": "https://api.github.com/repos/webmozart/assert/zipball/83e253c8e0be5b0257b881e1827274667c5c17a9", + "reference": "83e253c8e0be5b0257b881e1827274667c5c17a9", "shasum": "" }, "require": { - "php": "^5.3.3 || ^7.0" + "php": "^5.3.3 || ^7.0", + "symfony/polyfill-ctype": "^1.8" }, "require-dev": { "phpunit/phpunit": "^4.6", @@ -2235,7 +2416,7 @@ "check", "validate" ], - "time": "2018-01-29T19:49:41+00:00" + "time": "2018-12-25T11:19:39+00:00" } ], "aliases": [], From 01bfdb9e1f8a8b43bbc753a4ba2b659a2229337e Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Wed, 20 Feb 2019 12:37:56 -0500 Subject: [PATCH 06/21] Restructure classes and replace all instances with new naming conventions --- includes/Common/ESQuery.php | 282 ++++++++++++++++++ includes/Common/Instance.php | 136 +++++---- includes/Common/Queue.php | 117 +++++--- includes/Common/Request.php | 7 + includes/Common/Response.php | 6 +- includes/Indices/HandlesIndices.php | 20 ++ includes/Indices/Index.php | 128 ++++++++ includes/Jobs/DispatcherJob.php | 15 +- includes/Jobs/EntitiesIndexJob.php | 21 +- includes/Jobs/GeneSearchIndexJob.php | 4 +- includes/Jobs/Job.php | 2 +- includes/Jobs/NodesIndexJob.php | 4 +- includes/Jobs/TableIndexJob.php | 6 +- .../local__feature_search_formatter.inc | 4 +- .../tripal_elasticsearch.collections.form.inc | 2 +- .../tripal_elasticsearch.connection.form.inc | 2 +- .../tripal_elasticsearch.gene_search.form.inc | 6 +- .../tripal_elasticsearch.indices.form.inc | 48 ++- ...tripal_elasticsearch.search_forms.form.inc | 4 +- ...ipal_elasticsearch.website_search.form.inc | 2 +- tests/Feature/InstanceTest.php | 33 +- tests/TestCase.php | 60 ++++ tripal_elasticsearch.api.inc | 2 +- tripal_elasticsearch.install | 8 +- tripal_elasticsearch.ws.inc | 58 ++-- 25 files changed, 768 insertions(+), 209 deletions(-) create mode 100644 includes/Common/ESQuery.php create mode 100644 includes/Indices/HandlesIndices.php create mode 100644 includes/Indices/Index.php create mode 100644 tests/TestCase.php diff --git a/includes/Common/ESQuery.php b/includes/Common/ESQuery.php new file mode 100644 index 00000000..cb9c13c2 --- /dev/null +++ b/includes/Common/ESQuery.php @@ -0,0 +1,282 @@ +es = $es; + $this->index = $index; + } + + /** + * Set the category. + * + * @param string $category + * + * @return $this + */ + public function setCategory($category) { + $this->category = $category; + + return $this; + } + + /** + * Set the type. + * + * @param string $type + * + * @return $this + */ + public function setType($type) { + $this->type = $type; + return $this; + } + + /** + * Set the index. + * + * @param string $index + * + * @return $this + */ + public function setIndex($index) { + $this->index = $index; + return $this; + } + + /** + * Get the field as an array. + * + * @return array + */ + public function getField() { + return is_array($this->field) ? $this->field : [$this->field]; + } + + /** + * Specify which field to search, + * + * @param array|string $field The name of the field or an array of field + * names. + * + * @return $this + */ + public function setField($field) { + $this->field = $field; + + return $this; + } + + /** + * Get the fields to search when specifying a category. + * + * @return array + */ + protected function getCategoryFields() { + $fields = []; + if ($this->es->hasIndex('website')) { + $fields[] = 'type'; + } + + if ($this->es->hasIndex('entities')) { + $fields[] = 'bundle_label'; + } + + return $fields; + } + + /** + * Validate a query. + * + * @throws \Exception + */ + protected function validateQuery() { + if (empty($this->index)) { + throw new Exception('Please provide an index name'); + } + + if (!$this->es->hasIndex($this->index)) { + throw new Exception( + 'Index ' . $this->index . ' does not exist. Please provide a valid index name.' + ); + } + + if (!empty($this->category) && !in_array( + $this->index, + ['website', 'entities'] + )) { + throw new Exception( + 'When specifying a category, the index name must be either website or entities. Currently, it is set to ' . $this->index . '.' + ); + } + } + + /** + * Set the range for the query. + * + * @param int $from + * @param int $size + * + * @return $this + */ + public function range($from, $size) { + $this->range = [$from, $size]; + + return $this; + } + + /** + * @param $terms + * @param $per_page + * + * @return array + * @throws \Exception + */ + public function paginate($terms, $per_page) { + $count = $this->count($terms); + $total = min($count, 1000000); + $current_page = pager_default_initialize($total, $per_page); + + // Set the offset. + $this->range($per_page * $current_page, $per_page); + + $results = $this->search($terms); + + return [ + 'results' => $results, + 'total' => $total, + 'count' => $count, + 'page' => $current_page + 1, + 'pages' => ceil($total / $per_page), + 'pager' => theme('pager', ['quantity', $total]), + ]; + } + + /** + * @param string $terms Query string + * + * @return int + */ + public function count($terms) { + $params = $this->buildQuery($terms, FALSE); + return (int) $this->es->client->count($params)['count']; + } + + /** + * @param $terms + * @param bool $with_range + * + * @return array + */ + protected function buildQuery($terms, $with_range = TRUE) { + // Initialize the query + $query = []; + $query[] = [ + 'simple_query_string' => [ + 'fields' => ['content.*'], + 'query' => $terms, + ], + ]; + + // Set searchable fields + $fields = $this->getField(); + if (!empty($fields)) { + $query[0]['simple_query_string']['fields'] = $fields; + } + + // Add query + if (!empty($this->category)) { + $query[] = [ + 'query_string' => [ + 'fields' => $this->getCategoryFields(), + 'query' => '"' . $this->category . '"', + ], + ]; + } + + $params = [ + 'index' => $this->index, + 'type' => $this->type, + 'body' => ['query' => $query], + ]; + + if ($with_range && !empty($this->range)) { + $params['from'] = $this->range[0]; + $params['size'] = $this->range[1]; + } + + return $params; + } + + /** + * @param $terms + * + * @throws \Exception + * @return array + */ + public function search($terms) { + $this->validateQuery(); + + $hits = $this->es->client->search($this->buildQuery($terms)); + + return $this->es->formatHits($hits); + } +} diff --git a/includes/Common/Instance.php b/includes/Common/Instance.php index d82eb376..d9e7aa8f 100644 --- a/includes/Common/Instance.php +++ b/includes/Common/Instance.php @@ -1,13 +1,18 @@ client = Elasticsearch\ClientBuilder::create() + $this->client = ClientBuilder::create() ->setHosts($host) ->build(); } @@ -126,8 +135,10 @@ public function setWebsiteSearchParams($search_terms, $node_type = '', $index = ]; } - if (in_array('entities', $indices) && in_array('website', - $indices) && !$force_entities_only) { + if (in_array('entities', $indices) && in_array( + 'website', + $indices + ) && !$force_entities_only) { $queries[1]['query_string'] = [ 'fields' => ['type', 'bundle_label'], 'query' => '"' . $node_type . '"', // Gene or mRNA (feature,Gene) @@ -305,15 +316,17 @@ public function setIndexParams($index_name, $shards = 5, $replicas = 0, $tokeniz * * @param bool $return_source whether to format the results or not. * - * @see \ESInstance::setTableSearchParams() - * @see \ESInstance::setWebsiteSearchParams() + * @see \ES\Common\Instance::setTableSearchParams() + * @see \ES\Common\Instance::setWebsiteSearchParams() * * @return array * @throws \Exception */ public function search($return_source = FALSE) { if (empty($this->searchParams)) { - throw new Exception('Please build search parameters before attempting to search.'); + throw new Exception( + 'Please build search parameters before attempting to search.' + ); } $hits = $this->client->search($this->searchParams); @@ -357,7 +370,9 @@ public function formatHits($hits) { */ public function count() { if (empty($this->searchParams)) { - throw new Exception('Please build search parameters before attempting to count results.'); + throw new Exception( + 'Please build search parameters before attempting to count results.' + ); } // Get the search query @@ -375,15 +390,16 @@ public function count() { * Create a new index. * Use this function after building the index parameters. * - * @see \ESInstance::setIndexParams() - * - * @param $params + * @see \ES\Common\Instance::setIndexParams() * * @return array + * @throws \Exception */ public function createIndex() { if (empty($this->indexParams)) { - throw new Exception('Please set the index parameters before attempting to create a new index.'); + throw new Exception( + 'Please set the index parameters before attempting to create a new index.' + ); } return $this->client->indices()->create($this->indexParams); @@ -584,8 +600,10 @@ public function getAllCategories($version = NULL, $get_count = FALSE, $keyword = $types = []; $indices = $this->getIndices(); $search_index = []; - if (in_array('website', - $indices) && ($version === NULL || $version === 2)) { + if (in_array( + 'website', + $indices + ) && ($version === NULL || $version === 2)) { // Get all node types from the node table. $node_types = db_query("SELECT name, type FROM {node_type}")->fetchAll(); foreach ($node_types as $type) { @@ -595,10 +613,14 @@ public function getAllCategories($version = NULL, $get_count = FALSE, $keyword = $search_index[] = 'website'; } - if (in_array('entities', - $indices) && ($version === NULL || $version === 3)) { + if (in_array( + 'entities', + $indices + ) && ($version === NULL || $version === 3)) { // Get all tripal entity types from the tripal_bundle table. - $entity_types = db_query("SELECT name, label FROM {tripal_bundle}")->fetchAll(); + $entity_types = db_query( + "SELECT name, label FROM {tripal_bundle}" + )->fetchAll(); foreach ($entity_types as $type) { $types[$type->name] = $type->label; } @@ -721,22 +743,26 @@ public function getIndexFields($index) { */ public function deleteAllRecords($index_name, $type = NULL) { if (empty($index_name)) { - throw new Exception('Please provide an index name when deleting records from an index'); + throw new Exception( + 'Please provide an index name when deleting records from an index' + ); } if ($type === NULL) { $type = $index_name; } - $this->client->deleteByQuery([ - 'index' => $index_name, - 'type' => $type, - 'body' => [ - 'query' => [ - 'match_all' => (object) [], + $this->client->deleteByQuery( + [ + 'index' => $index_name, + 'type' => $type, + 'body' => [ + 'query' => [ + 'match_all' => (object) [], + ], ], - ], - ]); + ] + ); } /** @@ -750,11 +776,13 @@ public function deleteAllRecords($index_name, $type = NULL) { */ public function getRecord($index, $type, $id) { try { - return $this->client->get([ - 'index' => $index, - 'type' => $type, - 'id' => $id, - ]); + return $this->client->get( + [ + 'index' => $index, + 'type' => $type, + 'id' => $id, + ] + ); } catch (Exception $exception) { return ['found' => FALSE]; } @@ -787,13 +815,15 @@ public function putMapping($index_name, $field_name, $field_type, $index_type = ], ]; - return $this->client->indices()->putMapping([ - 'index' => $index_name, - 'type' => $index_type, - 'body' => [ - 'properties' => $properties, - ], - ]); + return $this->client->indices()->putMapping( + [ + 'index' => $index_name, + 'type' => $index_type, + 'body' => [ + 'properties' => $properties, + ], + ] + ); } /** @@ -808,14 +838,16 @@ public function putMapping($index_name, $field_name, $field_type, $index_type = * @return array */ public function createOrUpdate($index, $index_type, $id, $item) { - return $this->client->update([ - 'index' => $index, - 'type' => $index_type, - 'id' => $id, - 'body' => [ - 'doc' => $item, - 'upsert' => $item, - ], - ]); + return $this->client->update( + [ + 'index' => $index, + 'type' => $index_type, + 'id' => $id, + 'body' => [ + 'doc' => $item, + 'upsert' => $item, + ], + ] + ); } } diff --git a/includes/Common/Queue.php b/includes/Common/Queue.php index 75e904eb..e0e8057b 100644 --- a/includes/Common/Queue.php +++ b/includes/Common/Queue.php @@ -1,6 +1,12 @@ completed > $queue->total) { + if ($queue->completed > $queue->total) { static::fixProgress($queue); } @@ -91,16 +98,21 @@ public static function progress() { $total += $queue->total; $completed += $queue->completed; - $round_name = ' Round: ' . (intval($queue->priority) === 1 ? 'High' : 'Low'); + $round_name = ' Round: ' . (intval( + $queue->priority + ) === 1 ? 'High' : 'Low'); $progress[$queue->index_name . $round_name] = (object) [ 'total' => $queue->total, 'completed' => $queue->completed, 'remaining' => $queue->total - $queue->completed, - 'percent' => number_format(($queue->completed / ($queue->total ?: 1)) * 100, 2), + 'percent' => number_format( + ($queue->completed / ($queue->total ?: 1)) * 100, + 2 + ), 'last_run_at' => $last_run, 'started_at' => $started_at, 'time' => $queue->last_run_at - $queue->started_at, - 'priority' => $queue->priority + 'priority' => $queue->priority, ]; } @@ -110,21 +122,25 @@ public static function progress() { 'completed' => $completed, 'remaining' => $total - $completed, 'percent' => number_format(($completed / ($total ?: 1)) * 100, 2), - 'time' => count($progress) > 0 ? $progress_last_run_at - $progress_started_at : 0, + 'time' => count( + $progress + ) > 0 ? $progress_last_run_at - $progress_started_at : 0, ]; } public static function fixProgress(&$queue) { $queue->completed = $queue->total; - db_query('UPDATE {'.self::QUEUE_TABLE.'} SET completed=total WHERE completed > total'); + db_query( + 'UPDATE {' . self::QUEUE_TABLE . '} SET completed=total WHERE completed > total' + ); } /** * Dispatch a new job. Uses the queue that has minimum items if queue name is * not provided. * - * @param \ES\Common\Job $job + * @param \ES\Jobs $job * @param string $queue_name * * @return boolean @@ -142,17 +158,21 @@ public static function dispatch($job, $queue_name = NULL) { /** * Execute a given job. * - * @param \ES\Common\Job $job + * @param \ES\Jobs $job * * @throws \Exception */ public static function run($job) { - if ($job instanceof \ES\Common\Job) { + if ($job instanceof Job) { try { $job->handle(); $total = $job->total(); } catch (Exception $exception) { - tripal_report_error('tripal_elasticsearch', TRIPAL_ERROR, $exception->getMessage()); + tripal_report_error( + 'tripal_elasticsearch', + TRIPAL_ERROR, + $exception->getMessage() + ); $total = $job->chunk ? $job->chunk : 1; } @@ -164,7 +184,11 @@ public static function run($job) { return; } - throw new Exception('Elasticsearch Queue: ' . get_class($job) . ' is an invalid job type. Jobs must extend the ES\Common\Job class'); + throw new Exception( + 'Elasticsearch Queue: ' . get_class( + $job + ) . ' is an invalid job type. Jobs must extend the ES\Jobs class' + ); } /** @@ -174,48 +198,50 @@ public static function run($job) { * @param string $index_name Name of the index. * @param int $total the total number of records going to the queue (not * number of jobs). + * @param int $priority The priority round. * - * @return DatabaseStatementInterface + * @return \DatabaseStatementInterface */ - public static function initProgress( - $type, - $index_name, - $total = 1, - $priority = 1 - ) { + public static function initProgress($type, $index_name, $total = 1, $priority = 1) { $counter_table = self::QUEUE_TABLE; $query = 'SELECT total, completed FROM {' . $counter_table . '} WHERE type=:type'; $queue = db_query($query, [':type' => $type])->fetchObject(); // If type already exists, reset progress if ($queue) { - return db_query('UPDATE {' . $counter_table . '} SET total=:total, last_run_at=:time, completed=:completed, started_at=:started_at WHERE type=:type', [ - ':type' => $type, - ':total' => $total, - ':completed' => 0, - ':time' => time(), - ':started_at' => time(), - ]); + return db_query( + 'UPDATE {' . $counter_table . '} SET total=:total, last_run_at=:time, completed=:completed, started_at=:started_at WHERE type=:type', + [ + ':type' => $type, + ':total' => $total, + ':completed' => 0, + ':time' => time(), + ':started_at' => time(), + ] + ); } // Initialize a new progress report for index name - return db_query('INSERT INTO {' . $counter_table . '} (index_name, type, total, completed, last_run_at, started_at, priority) VALUES (:index_name, :type, :total, 0, :last_run_at, :started_at, :priority)', [ - ':type' => $type, - ':index_name' => $index_name, - ':total' => $total, - ':last_run_at' => time(), - ':started_at' => time(), - ':priority' => $priority, - ]); + return db_query( + 'INSERT INTO {' . $counter_table . '} (index_name, type, total, completed, last_run_at, started_at, priority) VALUES (:index_name, :type, :total, 0, :last_run_at, :started_at, :priority)', + [ + ':type' => $type, + ':index_name' => $index_name, + ':total' => $total, + ':last_run_at' => time(), + ':started_at' => time(), + ':priority' => $priority, + ] + ); } /** * Update the number of items in the counter. * - * @param string $index_name the index name. + * @param string $type the index type. * @param int $by the number to decrement by. * - * @return DatabaseStatementInterface|boolean + * @return \DatabaseStatementInterface|boolean */ public static function updateProgress($type, $by = 1) { $counter_table = self::QUEUE_TABLE; @@ -223,11 +249,14 @@ public static function updateProgress($type, $by = 1) { $queue = db_query($query, [':type' => $type])->fetchObject(); if ($queue) { - db_query('UPDATE {' . $counter_table . '} SET completed=:completed, last_run_at=:last_run_at WHERE type=:type', [ - ':type' => $type, - ':completed' => $queue->completed + $by, - ':last_run_at' => time(), - ]); + db_query( + 'UPDATE {' . $counter_table . '} SET completed=:completed, last_run_at=:last_run_at WHERE type=:type', + [ + ':type' => $type, + ':completed' => $queue->completed + $by, + ':last_run_at' => time(), + ] + ); return $queue->total - ($queue->completed + $by); } @@ -271,7 +300,9 @@ protected function generateQueueCount() { $max = NULL; $min = NULL; - $queues = db_query("SELECT name, COUNT(name) as count FROM {queue} WHERE name LIKE 'elasticsearch%' GROUP BY name ORDER BY count ASC")->fetchAll(); + $queues = db_query( + "SELECT name, COUNT(name) as count FROM {queue} WHERE name LIKE 'elasticsearch%' GROUP BY name ORDER BY count ASC" + )->fetchAll(); foreach ($queues as $queue) { if (!isset($this->queues[$queue->name])) { diff --git a/includes/Common/Request.php b/includes/Common/Request.php index feea4b1e..cfc7317a 100644 --- a/includes/Common/Request.php +++ b/includes/Common/Request.php @@ -1,6 +1,8 @@ index ?? strtolower(get_class($this)); + } + + /** + * @return mixed + */ + private function getIndexType() { + return $this->type; + } +} diff --git a/includes/Indices/Index.php b/includes/Indices/Index.php new file mode 100644 index 00000000..86363f6d --- /dev/null +++ b/includes/Indices/Index.php @@ -0,0 +1,128 @@ +instance = new \ES\Common\Instance(); + $this->fill($data); + } + + /** + * @param $data + * @param bool $id + */ + public function create($data, $id = FALSE) { + $this->instance->createEntry( + $this->getIndexName(), + $this->getIndexType(), + $id, + $data + ); + } + + /** + * Fills the object attributes with values. + * + * @param array $data + */ + public function fill(array $data = []) { + foreach ($data as $key => $value) { + if (isset($this->fields[$key])) { + $this->attributes[$key] = $value; + } + } + } + + public function save() { + + } + + /** + * Get an attribute. + * + * @param string $name The name of attribute. + * + * @return mixed + * The value of the attribute if it exists. + */ + public function __get($name) { + if (array_key_exists($name, $this->attributes)) { + return $this->attributes[$name]; + } + + if (method_exists(static::class, $name)) { + return; + } + + return $this->{$name}; + } + + /** + * Check if an attribute isset. + * + * @param string $name The name of the attribute. + * @return bool + * Whether the attribute has been set. + */ + public function __isset($name) { + return isset($this->attributes[$name]); + } +} diff --git a/includes/Jobs/DispatcherJob.php b/includes/Jobs/DispatcherJob.php index 48e2eed1..213ea2d3 100644 --- a/includes/Jobs/DispatcherJob.php +++ b/includes/Jobs/DispatcherJob.php @@ -1,17 +1,18 @@ job = $job; @@ -45,7 +46,7 @@ public function handle() { $chunk = $this->job->chunk; $this->total = $this->job->count(); $round = $this->job->hasRounds() ? $this->job->currentRound() : 1; - ESQueue::initProgress($this->job->type, $this->job->index, $this->total, + \ES\Common\Queue::initProgress($this->job->type, $this->job->index, $this->total, $round); for ($offset = 0; $offset < $this->total; $offset += $chunk) { @@ -59,7 +60,7 @@ public function handle() { * @param string $queue_name * Defaults to the dispatcher queue. * - * @see ESJob::dispatch + * @see Job::dispatch */ public function dispatch($queue_name = NULL) { if ($queue_name === NULL) { @@ -71,7 +72,7 @@ public function dispatch($queue_name = NULL) { /** * Get the job. * - * @return \ESJob + * @return \ES\Jobs */ public function job() { return $this->job; diff --git a/includes/Jobs/EntitiesIndexJob.php b/includes/Jobs/EntitiesIndexJob.php index 74dfd7ff..d2f35103 100644 --- a/includes/Jobs/EntitiesIndexJob.php +++ b/includes/Jobs/EntitiesIndexJob.php @@ -1,6 +1,6 @@ es = new ESInstance(); + $this->es = new Instance(); $entities = $this->get(); + $this->total = count($entities); $records = $this->loadContent($entities); @@ -135,20 +136,22 @@ protected function loadContent($records) { if (property_exists($entity, $field) && isset($entity->{$field}['und'])) { + $content[$field] = []; foreach ($entity->{$field}['und'] as $elements) { if (!isset($elements['value'])) { continue; } - $value = $this->extractValue($elements['value']); + $value = $this->extractValue($elements['value']); if (empty($value)) { continue; } - $content[] = $value; + $content[$field][] = $value; + //$content[] = $value; } } } @@ -403,7 +406,8 @@ public static function generateDispatcherJobs($round = 1, $clear_queue = FALSE, if ($clear_queue) { // Clear all entries from the queue $sql = 'SELECT item_id, data FROM queue q WHERE name LIKE :name'; - $results = db_query($sql, [':name' => db_like('elasticsearch') . '%'])->fetchAll(); + $results = db_query($sql, + [':name' => db_like('elasticsearch') . '%'])->fetchAll(); $delete = []; foreach ($results as $result) { @@ -417,7 +421,8 @@ public static function generateDispatcherJobs($round = 1, $clear_queue = FALSE, } if (!empty($delete)) { - $dsql = 'DELETE FROM queue WHERE item_id IN (' . implode(',', $delete) . ')'; + $dsql = 'DELETE FROM queue WHERE item_id IN (' . implode(',', + $delete) . ')'; db_query($dsql)->execute(); } } @@ -439,7 +444,7 @@ public static function generateDispatcherJobs($round = 1, $clear_queue = FALSE, } /** - * Tells the ESQueue class whether this job implements + * Tells the \ES\Common\Queue class whether this job implements * priority queues. * * @return bool diff --git a/includes/Jobs/GeneSearchIndexJob.php b/includes/Jobs/GeneSearchIndexJob.php index 6eee50e4..fdb8d3e6 100644 --- a/includes/Jobs/GeneSearchIndexJob.php +++ b/includes/Jobs/GeneSearchIndexJob.php @@ -1,6 +1,6 @@ total = count($records); try { - $es = new ESInstance(); + $es = new Instance(); // Can't use bulk indexing since we are using array data // type (ES error not our fault) diff --git a/includes/Jobs/Job.php b/includes/Jobs/Job.php index e86ee725..41dc301e 100644 --- a/includes/Jobs/Job.php +++ b/includes/Jobs/Job.php @@ -1,6 +1,6 @@ get(); $this->total = count($records); diff --git a/includes/Jobs/TableIndexJob.php b/includes/Jobs/TableIndexJob.php index f58f76ba..0a339a87 100644 --- a/includes/Jobs/TableIndexJob.php +++ b/includes/Jobs/TableIndexJob.php @@ -1,6 +1,6 @@ get(); - $es = new ESInstance(); + $es = new Instance(); $this->total = count($records); if ($this->total > 1) { @@ -112,4 +112,4 @@ public function total() { public function count() { return db_query('SELECT COUNT(*) FROM {' . db_escape_table($this->table) . '}')->fetchField(); } -} \ No newline at end of file +} diff --git a/includes/TripalFields/local__feature_search/local__feature_search_formatter.inc b/includes/TripalFields/local__feature_search/local__feature_search_formatter.inc index 312c975c..f16d06f7 100644 --- a/includes/TripalFields/local__feature_search/local__feature_search_formatter.inc +++ b/includes/TripalFields/local__feature_search/local__feature_search_formatter.inc @@ -81,7 +81,7 @@ class local__feature_search_formatter extends ChadoFieldFormatter { $organism = $entity->chado_record; try { - $es = new ESInstance(); + $es = new Instance(); $query = tripal_elasticsearch_gene_search_index_query_mapper([ 'organism' => "$organism->genus $organism->species", "search_term" => '*', @@ -150,4 +150,4 @@ class local__feature_search_formatter extends ChadoFieldFormatter { public function settingsSummary($view_mode) { return ''; } -} \ No newline at end of file +} diff --git a/includes/tripal_elasticsearch.collections.form.inc b/includes/tripal_elasticsearch.collections.form.inc index 2f17ba49..24081490 100644 --- a/includes/tripal_elasticsearch.collections.form.inc +++ b/includes/tripal_elasticsearch.collections.form.inc @@ -148,7 +148,7 @@ function tripal_elasticsearch_collections_form_submit($form, &$form_state) { $entities = []; try { - $es = new ESInstance(); + $es = new Instance(); $es->setWebsiteSearchParams($query, $bundle_label, '', '', [0, 10], TRUE); $count = $es->count(); diff --git a/includes/tripal_elasticsearch.connection.form.inc b/includes/tripal_elasticsearch.connection.form.inc index b3c1fe87..6b6d7315 100644 --- a/includes/tripal_elasticsearch.connection.form.inc +++ b/includes/tripal_elasticsearch.connection.form.inc @@ -135,7 +135,7 @@ function elasticsearch_connection_form($form, &$form_state) { ]; try { - $client = (new ESInstance())->client; + $client = (new Instance())->client; // Obtain cluster health information. $params['v'] = TRUE; $health = $client->cat()->health($params)[0]; diff --git a/includes/tripal_elasticsearch.gene_search.form.inc b/includes/tripal_elasticsearch.gene_search.form.inc index 839a2876..9c04199a 100644 --- a/includes/tripal_elasticsearch.gene_search.form.inc +++ b/includes/tripal_elasticsearch.gene_search.form.inc @@ -238,7 +238,7 @@ function tripal_elasticsearch_gene_search_index_results($display_download = FALS } try { - $es = new ESInstance(); + $es = new Instance(); } catch (Exception $exception) { drupal_set_message('The search service is currently unavailable. Please try again later.', 'error'); @@ -318,7 +318,7 @@ function tripal_elasticsearch_gene_search_download() { } try { - $es = new ESInstance(); + $es = new Instance(); } catch (Exception $exception) { drupal_set_message('Error occurred while generating records for download. Please try again later.', 'error'); @@ -413,7 +413,7 @@ function tripal_elasticsearch_get_gene_search_organisms() { $organism_list = chado_query($sql)->fetchAll(); try { - $es = new ESInstance(); + $es = new Instance(); $organisms = []; foreach ($organism_list as $organism) { $name = "{$organism->genus} {$organism->species}"; diff --git a/includes/tripal_elasticsearch.indices.form.inc b/includes/tripal_elasticsearch.indices.form.inc index 20495ad0..d8b8a6b7 100644 --- a/includes/tripal_elasticsearch.indices.form.inc +++ b/includes/tripal_elasticsearch.indices.form.inc @@ -14,7 +14,7 @@ */ function tripal_elasticsearch_indexing_form($form, &$form_state) { try { - $es = new ESInstance(); + $es = new Instance(); // associate index name with indexed table. $indices = $es->getIndices(); @@ -285,7 +285,7 @@ function tripal_elasticsearch_indexing_form_submit($form, &$form_state) { */ function tripal_elasticsearch_delete_indices_form($form, &$form_state) { try { - $client = (new ESInstance())->client; + $client = (new Instance())->client; // associate index name with indexed table. $mappings = $client->indices()->getMapping(); @@ -341,7 +341,7 @@ function tripal_elasticsearch_delete_indices_form_submit($form, &$form_state) { if (!empty($delete_indices)) { foreach ($delete_indices as $index) { try { - $es = new ESInstance(); + $es = new Instance(); $es->deleteIndex($index); db_query('DELETE FROM {tripal_elasticsearch_indices} WHERE index_name=:index_name', [ @@ -361,7 +361,7 @@ function tripal_elasticsearch_delete_indices_form_submit($form, &$form_state) { */ function tripal_elasticsearch_indices_list_page() { try { - $es = new ESInstance(); + $es = new Instance(); // associate index name with indexed table. $indices = $es->getIndices(); @@ -446,11 +446,7 @@ function tripal_elasticsearch_indices_list_page() { * * @return mixed */ -function tripal_elasticsearch_index_edit_confirm( - $form, - &$form_state, - $index_name -) { +function tripal_elasticsearch_index_edit_confirm($form, &$form_state, $index_name) { $form = []; $exposed = FALSE; $url = NULL; @@ -485,7 +481,7 @@ function tripal_elasticsearch_index_edit_confirm( $result = db_query($sql, [':index_name' => $index_name])->fetchObject(); if (!$result) { try { - $es = new ESInstance(); + $es = new Instance(); $mappings = $es->getIndexMappings($index_name); $table_name = array_keys($mappings); if (count($table_name) > 1) { @@ -532,7 +528,7 @@ function tripal_elasticsearch_index_edit_confirm( // Get existing index settings and mappings. // TODO: what about default tables? try { - $es = new ESInstance(); + $es = new Instance(); $index_settings = $es->getIndexSettings($index_name); $index_mappings = $es->getIndexMappings($index_name); } catch (Exception $exception) { @@ -739,11 +735,7 @@ function tripal_elasticsearch_index_edit_confirm_submit($form, &$form_state) { * * @return mixed */ -function tripal_elasticsearch_index_delete_confirm( - $form, - &$form_state, - $index_name -) { +function tripal_elasticsearch_index_delete_confirm($form, &$form_state, $index_name) { $form = []; $description = 'Are you sure you want to delete the ' . $index_name . ' index?'; $cancel_path = 'admin/tripal/extension/tripal_elasticsearch/indices'; @@ -808,7 +800,10 @@ function tripal_elasticsearch_create_index(&$values) { 'entity_id' => 'integer', 'bundle_label' => 'text', 'title' => 'text', - 'content' => 'text', + 'content' => [ + 'dynamic' => TRUE, + 'properties' => [], + ], ]; } elseif ($index_type === 'gene_search') { @@ -875,7 +870,7 @@ function tripal_elasticsearch_create_index(&$values) { // Create the index. try { - $es = new ESInstance(); + $es = new Instance(); $es->setIndexParams($index_name, 5, 0, $tokenizer, $token_filters, $field_mapping_types)->createIndex(); } catch (Exception $exception) { @@ -928,7 +923,7 @@ function tripal_elasticsearch_create_index(&$values) { function tripal_elasticsearch_delete_index($index) { // Find the index and delete it from ES try { - $es = new ESInstance(); + $es = new Instance(); $es->deleteIndex($index); } catch (Exception $exception) { drupal_set_message($exception->getMessage(), 'error'); @@ -937,17 +932,13 @@ function tripal_elasticsearch_delete_index($index) { } // Find the index and delete it from the DB. this db is for the search blocks - $result = db_query("DELETE FROM {tripal_elasticsearch} WHERE index_name=:index_name", - [':index_name' => $index])->fetchObject(); - - if (!$result) { - // It's not in the DB so there is nothing to delete here. - return; - } + db_query("DELETE FROM {tripal_elasticsearch} WHERE index_name=:index_name", + [':index_name' => $index]); // Now delete the table info db_query("DELETE FROM {tripal_elasticsearch_indices} WHERE index_name=:index_name", [':index_name' => $index]); + // Delete any progress indicator db_query('DELETE FROM {tripal_elasticsearch_queues} WHERE index_name=:index_name', [':index_name' => $index]); @@ -961,6 +952,7 @@ function tripal_elasticsearch_delete_index($index) { function tripal_elasticsearch_update_index($index) { $result = db_query("SELECT * FROM {tripal_elasticsearch_indices} WHERE index_name=:index_name", [':index_name' => $index])->fetchObject(); + if (!$result) { drupal_set_message("Index \"{$index}\" not found", 'error'); return drupal_goto('admin/tripal/extension/tripal_elasticsearch/indices'); @@ -982,6 +974,7 @@ function tripal_elasticsearch_update_index($index) { } drupal_set_message("{$index} updating jobs have been dispatched successfully. Please run `drush cron` to process items in the queue."); + drupal_goto('admin/tripal/extension/tripal_elasticsearch/indices'); } /** @@ -997,7 +990,7 @@ function tripal_elasticsearch_progress_page($callback = FALSE) { $content .= '

Indexing Progress Tracker

'; - $data = ESQueue::progress(); + $data = \ES\Common\Queue::progress(); $content .= '

Overall Progress

'; @@ -1100,7 +1093,6 @@ function tripal_elasticsearch_compute_ETR($total, $completed, $time) { return number_format($remaining, 2) . " {$unit}"; } - function tripal_elasticsearch_update_entities_form($form, &$form_state) { // get bundles $bundles = db_query('SELECT * FROM {tripal_bundle}')->fetchAll(); diff --git a/includes/tripal_elasticsearch.search_forms.form.inc b/includes/tripal_elasticsearch.search_forms.form.inc index 3a3224cb..29608f9a 100644 --- a/includes/tripal_elasticsearch.search_forms.form.inc +++ b/includes/tripal_elasticsearch.search_forms.form.inc @@ -26,7 +26,7 @@ function table_search_interface_building_form($form, &$form_state) { ]; try { - $client = (new ESInstance())->client; + $client = (new Instance())->client; $existing_indices = $client->indices()->getMapping(); } catch (\Exception $e) { $form['markup'] = [ @@ -394,4 +394,4 @@ function tripal_elasticsearch_form_table_search_interface_building_form_alter( } } } -} \ No newline at end of file +} diff --git a/includes/tripal_elasticsearch.website_search.form.inc b/includes/tripal_elasticsearch.website_search.form.inc index 1ddcfe9e..25d7ae41 100644 --- a/includes/tripal_elasticsearch.website_search.form.inc +++ b/includes/tripal_elasticsearch.website_search.form.inc @@ -159,7 +159,7 @@ function tripal_elasticsearch_build_search_block_form($form, &$form_state, $inde // Build search query for table search. $query = tripal_elasticsearch_build_search_query_from_field_content_pairs($index_fields); try { - $es = new ESInstance(); + $es = new Instance(); $results_count = $es->setTableSearchParams($index_name, $index_type, $query) ->count(); $count = '' . $results_count . ' results match your search.'; diff --git a/tests/Feature/InstanceTest.php b/tests/Feature/InstanceTest.php index 4d9125c5..c617889d 100644 --- a/tests/Feature/InstanceTest.php +++ b/tests/Feature/InstanceTest.php @@ -6,37 +6,36 @@ use StatonLab\TripalTestSuite\DBTransaction; use Tests\TestCase; -class ConnectToElasticSearchServerTest extends TestCase +class InstanceTest extends TestCase { use DBTransaction; - /** - * Tests that an exception is thrown when no host is specified. - * - * @throws \Exception - * @test - */ - public function should_fail_to_connect_to_server_before_specifying_host() + /** @test */ + public function testThatConnectionToAnInvalidHostFails() { variable_del('elasticsearch_host'); - $this->expectException('Exception'); + $this->expectException(\Exception::class); new \ES\Common\Instance(); } - /** - * Tests whether connecting to ES server is possible. - * - * @throws \Exception - * @test - */ - public function should_successfully_connect_to_server_after_specifying_host() + /** @test */ + public function testThatWeCanSuccessfullyConnectToNonSpecifiedHost() { - variable_set('elasticsearch_host', 'http://localhost:9200'); + variable_set('elasticsearch_host', getenv('ES_HOST')); $es = new \ES\Common\Instance(); $this->assertInstanceOf(Instance::class, $es); } + + /** @test */ + public function testThatCreatingAnIndexSucceeds() { + $name = uniqid(); + $index = $this->makeIndex($name); + + $this->assertTrue($index['acknowledged']); + $this->assertEquals($name, $index['index']); + } } diff --git a/tests/TestCase.php b/tests/TestCase.php new file mode 100644 index 00000000..6e3be9b5 --- /dev/null +++ b/tests/TestCase.php @@ -0,0 +1,60 @@ +_indices as $index) { + if (is_array($index) && isset($index['index'])) { + $this->deleteIndex($index['index']); + } + } + + parent::tearDown(); + } + + /** + * @param string $name + * + * @throws \Exception + */ + public function deleteIndex($name) { + $es = new \ES\Common\Instance(getenv('ES_HOST')); + + $es->deleteIndex($name); + } + + /** + * @param string $name + * + * @throws \Exception + */ + public function makeIndex($name = NULL) { + $es = new \ES\Common\Instance(getenv('ES_HOST')); + + if (is_null($name)) { + return $name; + } + + $index = $es->setIndexParams($name)->createIndex(); + + $this->_indices[] = $index; + + return $index; + } +} diff --git a/tripal_elasticsearch.api.inc b/tripal_elasticsearch.api.inc index 108f365e..03834fda 100644 --- a/tripal_elasticsearch.api.inc +++ b/tripal_elasticsearch.api.inc @@ -130,7 +130,7 @@ function tripal_elasticsearch_build_search_query_from_field_content_pairs(array */ function tripal_elasticsearch_get_website_search_results_category_list($keyword) { try { - $es = new ESInstance(); + $es = new Instance(); $indices = $es->getIndices(); $types = $es->getAllCategories(NULL, TRUE, $keyword); } catch (Exception $exception) { diff --git a/tripal_elasticsearch.install b/tripal_elasticsearch.install index 1fc14805..4ad19d1e 100644 --- a/tripal_elasticsearch.install +++ b/tripal_elasticsearch.install @@ -369,7 +369,7 @@ function tripal_elasticsearch_update_7208() { */ function tripal_elasticsearch_update_7209() { try { - $es = new ESInstance(); + $es = new Instance(); $es->putMapping('gene_search_index', 'entity_id', 'integer'); $es->putMapping('gene_search_index', 'node_id', 'integer'); } catch (Exception $exception) { @@ -385,7 +385,7 @@ function tripal_elasticsearch_update_7209() { */ function tripal_elasticsearch_update_7210() { try { - $es = new ESInstance(); + $es = new Instance(); if (in_array('website', $es->getIndices())) { return; } @@ -433,7 +433,7 @@ function tripal_elasticsearch_update_7210() { */ function tripal_elasticsearch_update_7211() { try { - $es = new ESInstance(); + $es = new Instance(); if (!in_array('website', $es->getIndices())) { return; @@ -482,7 +482,7 @@ function tripal_elasticsearch_update_7211() { */ function tripal_elasticsearch_update_7212() { try { - $es = new ESInstance(); + $es = new Instance(); if (in_array('gene_search_index', $es->getIndices())) { $es->putMapping('gene_search_index', 'related_features', 'text', 'chado.feature'); } diff --git a/tripal_elasticsearch.ws.inc b/tripal_elasticsearch.ws.inc index 22aee742..dd30ce62 100644 --- a/tripal_elasticsearch.ws.inc +++ b/tripal_elasticsearch.ws.inc @@ -14,10 +14,10 @@ function tripal_elasticsearch_api_v1_status() { try { $params['v'] = TRUE; - $es = new ESInstance(); + $es = new Instance(); $health = current($es->client->cat()->health($params)); } catch (Exception $exception) { - return ESResponse::error( + return \ES\Common\Response::error( [ 'status' => 'Inactive', 'healthy' => FALSE, @@ -26,7 +26,7 @@ function tripal_elasticsearch_api_v1_status() { } if (empty($health)) { - return ESResponse::error( + return \ES\Common\Response::error( [ 'status' => 'Inactive', 'healthy' => FALSE, @@ -34,7 +34,7 @@ function tripal_elasticsearch_api_v1_status() { ); } - return ESResponse::success( + return \ES\Common\Response::success( [ 'status' => 'Active', 'healthy' => TRUE, @@ -53,7 +53,7 @@ function tripal_elasticsearch_api_v1_remote_status($remote_id) { $remote_id = abs(intval(trim($remote_id))); if (!is_int($remote_id)) { - return ESResponse::error( + return \ES\Common\Response::error( [ 'remote_id' => ['Invalid remote server id'], ] @@ -66,7 +66,7 @@ function tripal_elasticsearch_api_v1_remote_status($remote_id) { [':id' => $remote_id] )->fetchObject(); if (!$host) { - return ESResponse::error( + return \ES\Common\Response::error( [ 'remote_id' => ['Remote server record does not exist'], ] @@ -76,12 +76,12 @@ function tripal_elasticsearch_api_v1_remote_status($remote_id) { $url = trim(trim($host->url), '/') . '/elasticsearch/api/v1/status'; try { - $response = ESRequest::get($url); + $response = \ES\Common\Request::get($url); } catch (Exception $exception) { - return ESResponse::error(['server' => $exception->getMessage()], 500); + return \ES\Common\Response::error(['server' => $exception->getMessage()], 500); } - return ESResponse::success($response->data); + return \ES\Common\Response::success($response->data); } /** @@ -95,7 +95,7 @@ function tripal_elasticsearch_api_v1_search($remote_host_id) { $remote_host_id = abs(intval(trim($remote_host_id))); if (!isset($_GET['terms'])) { - return ESResponse::error(['terms' => ['Please provide search terms']]); + return \ES\Common\Response::error(['terms' => ['Please provide search terms']]); } $size = 0; @@ -115,7 +115,7 @@ function tripal_elasticsearch_api_v1_search($remote_host_id) { $url = rtrim(trim($host->url), '/') . '/elasticsearch/api/v1/local-search'; try { - $response = ESRequest::get( + $response = \ES\Common\Request::get( $url, [ 'terms' => $_GET['terms'], 'category' => isset($_GET['category']) ? $_GET['category'] : '', @@ -123,7 +123,7 @@ function tripal_elasticsearch_api_v1_search($remote_host_id) { ] ); } catch (Exception $exception) { - return ESResponse::error(['server' => $exception->getMessage()], 500); + return \ES\Common\Response::error(['server' => $exception->getMessage()], 500); } } @@ -132,7 +132,7 @@ function tripal_elasticsearch_api_v1_search($remote_host_id) { ); // Render results into fields and send them back - return ESResponse::success( + return \ES\Common\Response::success( [ 'count' => $response->data->count, 'url' => $response->data->url, @@ -157,7 +157,7 @@ function tripal_elasticsearch_api_v1_local_search($http = TRUE) { $exposed = tripal_elasticsearch_is_index_exposed(['website', 'entities']); if (!$exposed) { - return ESResponse::error( + return \ES\Common\Response::error( '403 Forbidden. The request index is not available for cross site search. Contact the admin to mark the index as public', 403 ); @@ -169,7 +169,7 @@ function tripal_elasticsearch_api_v1_local_search($http = TRUE) { $category = isset($_GET['category']) ? $_GET['category'] : NULL; if ($size !== 0 && $size > 20) { - return ESResponse::error( + return \ES\Common\Response::error( [ 'size' => ['Please provide a valid size between 1 and 20'], ] @@ -181,13 +181,13 @@ function tripal_elasticsearch_api_v1_local_search($http = TRUE) { } try { - $es = new ESInstance(); + $es = new Instance(); $results = $es->searchWebIndices($terms, $size, $category); } catch (Exception $exception) { - return ESResponse::error($exception->getMessage(), 500); + return \ES\Common\Response::error($exception->getMessage(), 500); } - return ESResponse::success( + return \ES\Common\Response::success( array_merge( $results, [ 'url' => url( @@ -218,7 +218,7 @@ function tripal_elasticsearch_api_v1_table_index_search( $remote_id = abs(intval(trim($remote_host_id))); if (!tripal_elasticsearch_is_index_exposed($index_name)) { - return ESResponse::error( + return \ES\Common\Response::error( '403 Forbidden. The request index is not available for cross site search. Contact the admin to mark the index as public', 403 ); @@ -231,7 +231,7 @@ function tripal_elasticsearch_api_v1_table_index_search( $index_name, FALSE ); } catch (Exception $exception) { - return ESResponse::error( + return \ES\Common\Response::error( [ 'results' => [$exception->getMessage()], ] @@ -245,7 +245,7 @@ function tripal_elasticsearch_api_v1_table_index_search( )->fetchObject(); if (!$host) { - return ESResponse::error( + return \ES\Common\Response::error( [ 'remote_id' => ['Invalid remote host provided. ID: ' . $remote_id], ] @@ -259,9 +259,9 @@ function tripal_elasticsearch_api_v1_table_index_search( if (isset($data['q'])) { unset($data['q']); } - $results = ESRequest::get($url, $data); + $results = \ES\Common\Request::get($url, $data); } catch (Exception $exception) { - return ESResponse::error( + return \ES\Common\Response::error( [ 'remote_id' => [ 'The host returned an invalid response', @@ -275,7 +275,7 @@ function tripal_elasticsearch_api_v1_table_index_search( $markup = tripal_elasticsearch_results_formatter( $results->data->results, $index_name, $host ); - return ESResponse::success( + return \ES\Common\Response::success( [ 'url' => $results->data->url, 'markup' => $markup, @@ -297,7 +297,7 @@ function tripal_elasticsearch_api_v1_table_index_local_search( $index_name, $http = TRUE ) { try { - $es = new ESInstance(); + $es = new Instance(); $fields = $es->getIndexFields($index_name); $field_content_pairs = []; @@ -324,7 +324,7 @@ function tripal_elasticsearch_api_v1_table_index_local_search( $results = $es->setTableSearchParams($index_name, '', $query, [0, 2], TRUE); } catch (Exception $exception) { if ($http) { - return ESResponse::error( + return \ES\Common\Response::error( [ 'results' => [$exception->getMessage()], ] @@ -349,7 +349,7 @@ function tripal_elasticsearch_api_v1_table_index_local_search( global $base_url; $query = $_GET; unset($query['q']); - return ESResponse::success( + return \ES\Common\Response::success( [ 'count' => $count, 'url' => $url ? url($base_url . '/' . $url, ['query' => $query]) @@ -450,7 +450,7 @@ function elasticsearch_recursive_flatten($array, &$result) { * @return array|bool|object */ function tripal_elasticsearch_api_v1_categories() { - return ESResponse::success( + return \ES\Common\Response::success( [ [ 'label' => 'Gene', @@ -499,7 +499,7 @@ function tripal_elasticsearch_cross_site_search_form($form, &$form_state) { $default_category = ['Any Type' => 'Any Type']; try { - $es = new ESInstance(); + $es = new Instance(); $categories = drupal_map_assoc($es->getAllCategories()); } catch (Exception $exception) { $categories = []; From e245e1ac13cb01e0800599e41fb2353b39c1ace9 Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Thu, 21 Feb 2019 08:00:59 -0500 Subject: [PATCH 07/21] Add query builder methods to the index --- includes/Common/Instance.php | 28 +++++- includes/Indices/Entities.php | 8 ++ includes/Indices/Index.php | 151 ++++++++++++++++++++++++++--- includes/Query/Builder.php | 84 +++++++++++----- includes/Query/BuilderContract.php | 26 +++++ includes/Query/Clause.php | 25 ++++- includes/Query/ClauseSanitizer.php | 15 +++ tests/Feature/ClauseTest.php | 20 ++-- tests/Feature/InstanceTest.php | 72 ++++++++++---- tests/TestCase.php | 7 +- 10 files changed, 362 insertions(+), 74 deletions(-) create mode 100644 includes/Indices/Entities.php create mode 100644 includes/Query/BuilderContract.php create mode 100644 includes/Query/ClauseSanitizer.php diff --git a/includes/Common/Instance.php b/includes/Common/Instance.php index d9e7aa8f..7d3f04cd 100644 --- a/includes/Common/Instance.php +++ b/includes/Common/Instance.php @@ -72,9 +72,7 @@ public function __construct($host = NULL) { ); } - $this->client = ClientBuilder::create() - ->setHosts($host) - ->build(); + $this->client = ClientBuilder::create()->setHosts($host)->build(); } /** @@ -488,6 +486,30 @@ public function bulkUpdate($index, $entries, $type = NULL, $id_key = NULL) { return $this->bulk('update', $index, $entries, $type, $id_key); } + /** + * Update a document. + * + * @param string $index The index name. + * @param string $type The type name. + * @param string $id The id of the document to update. + * @param array $data The data to replace. + * + * @return array + * The returned array from the client. + */ + public function update($index, $type, $id, $data) { + $params = [ + 'index' => $index, + 'type' => $type, + 'id' => $id, + 'body' => [ + 'doc' => $data, + ], + ]; + + return $this->client->update($params); + } + /** * @param string $operation * @param string $index diff --git a/includes/Indices/Entities.php b/includes/Indices/Entities.php new file mode 100644 index 00000000..56f29f91 --- /dev/null +++ b/includes/Indices/Entities.php @@ -0,0 +1,8 @@ +instance = new \ES\Common\Instance(); + public function __construct($data = [], Instance $instance = NULL) { + $this->instance = $instance ?? new Instance(); $this->fill($data); } /** - * @param $data - * @param bool $id + * Add a where clause. + * + * @param string $field The field. + * @param string $value The value + * + * @return $this + * The object. + * @see \ES\Query\Clause::where() */ - public function create($data, $id = FALSE) { - $this->instance->createEntry( - $this->getIndexName(), - $this->getIndexType(), - $id, - $data - ); + public function where($field, $value = NULL) { + $this->builder->where($field, $value); + + return $this; + } + + /** + * Add an or clause. + * + * @param string $field The field. + * @param string $value The value + * + * @return $this + * The object. + * @see \ES\Query\Clause::orWhere() + */ + public function orWhere($field, $value = NULL) { + $this->builder->where($field, $value); + + return $this; } /** @@ -91,8 +124,97 @@ public function fill(array $data = []) { } } + /** + * Save the data. + * + * @return $this + */ public function save() { + if ($this->exists()) { + $data = $this->instance->update( + $this->getIndexName(), + $this->getIndexType(), + $this->id ?? FALSE, + $this->attributes + ); + $this->id = $data['_id']; + + return $this; + } + + $data = $this->instance->createEntry( + $this->getIndexName(), + $this->getIndexType(), + $this->id ?? FALSE, + $this->attributes + ); + + $this->id = $data['_id']; + + return $this; + } + + /** + * Check whether the record exists. + * + * @return bool + */ + public function exists() { + $this->exists = $this->count() > 0; + + return $this->exists; + } + + /** + * Count the number of documents in the index. + * + * @return int + * The number of documents in the index. + */ + public function count() { + return (int) $this->instance->client->count($this->builder->build(FALSE)); + } + + /** + * @param $data + * + * @throws \Exception + */ + public static function createOrUpdate($data, $id = FALSE) { + $record = new static($data); + + $record->setID($id); + $record->save(); + + return $record; + } + + /** + * @param $id + */ + public function setID($id) { + $this->id = $id; + + $this->builder->setID($id); + + return $this; + } + + /** + * @param array $data + * @param bool $id + * + * @return \ES\Indices\Index + * @throws \Exception + */ + public static function create(array $data, $id = FALSE) { + $index = new static($data); + + $index->setID($id); + $index->save(); + + return $index; } /** @@ -119,6 +241,7 @@ public function __get($name) { * Check if an attribute isset. * * @param string $name The name of the attribute. + * * @return bool * Whether the attribute has been set. */ diff --git a/includes/Query/Builder.php b/includes/Query/Builder.php index 55123b36..1bcbd974 100644 --- a/includes/Query/Builder.php +++ b/includes/Query/Builder.php @@ -2,7 +2,7 @@ namespace ES\Query; -class Builder{ +class Builder implements BuilderContract{ /** * @var string @@ -22,12 +22,21 @@ class Builder{ /** * @var array */ - protected $queries = []; + protected $highlight; /** - * @var array + * The query clause builder. + * + * @var \ES\Query\Clause */ - protected $highlight; + protected $query; + + /** + * ES generated id if exists. + * + * @var string + */ + protected $id = FALSE; /** * Builder constructor. @@ -36,6 +45,7 @@ class Builder{ */ public function __construct($index) { $this->index = $index; + $this->query = new Clause(); } /** @@ -55,27 +65,29 @@ public function range($from, $size = 10) { } /** - * Add a clause. + * Build a where clause. * - * @param $query - * @param null $fields + * @param string $field + * @param string $value + * + * @return $this */ - public function query($query, array $fields = NULL) { - $query = [ - 'simple_query_string' => [ - 'query' => $query, - ], - ]; - - if (!is_null($fields)) { - if (!is_array($fields)) { - $fields = [$fields]; - } - - $query['simple_query_string']['fields'] = $fields; - } + public function where($field, $value = NULL) { + $this->query->where($field, $value); + return $this; + } - $this->queries[] = $query; + /** + * Build an or where clause. + * + * @param string $field + * @param string $value + * + * @return $this + */ + public function orWhere($field, $value = NULL) { + $this->query->orWhere($field, $value); + return $this; } /** @@ -106,25 +118,41 @@ private function addHighlightField($field) { $this->highlight[$field] = ['fragment_size' => 150]; } + /** + * @param $id + * + * @return $this + */ + public function setID($id) { + $this->id = $id; + return $this; + } + /** * Build and return the parameters. * + * @param bool $with_range Whether to include the range. + * * @return array * The params array. */ - public function build() { + public function build($with_range = TRUE) { $params = []; - if ($this->size) { + if ($this->size && $with_range) { $params['size'] = $this->size; } - if ($this['from']) { + if ($this->from && $with_range) { $params['from'] = $this->from; } $params['body'] = [ - 'query' => $this->queries, + 'query' => [ + 'simple_query_string' => [ + 'query' => $this->query->build(), + ], + ], ]; if ($this->highlight) { @@ -133,6 +161,10 @@ public function build() { ]; } + if ($this->id !== FALSE) { + $params['id'] = $this->id; + } + return $params; } } diff --git a/includes/Query/BuilderContract.php b/includes/Query/BuilderContract.php new file mode 100644 index 00000000..8e181275 --- /dev/null +++ b/includes/Query/BuilderContract.php @@ -0,0 +1,26 @@ +sanitizer = new ClauseSanitizer(); + } + + /** + * Recursively build the query. + * + * @param string $field + * @param string $value * * @return string */ @@ -28,9 +42,12 @@ protected function makeQuery($field, $value = NULL) { } if (is_null($value)) { - return $field; + return $this->sanitizer->escape($field); } + $field = $this->sanitizer->escape($field); + $value = $this->sanitizer->escape($value); + return $field . ':' . $value; } diff --git a/includes/Query/ClauseSanitizer.php b/includes/Query/ClauseSanitizer.php new file mode 100644 index 00000000..207d45e0 --- /dev/null +++ b/includes/Query/ClauseSanitizer.php @@ -0,0 +1,15 @@ +where(function(Clause $query) { - $query->where('f', 'v'); - $query->where('f', 'v'); - })->orWhere(function (Clause $query) { - $query->where('f', 'v'); - $query->orWhere('v'); - })->build(); + + $query = $clause->where( + function (BuilderContract $query) { + $query->where('f', 'v'); + $query->where('f', 'v'); + } + )->orWhere( + function (BuilderContract $query) { + $query->where('f', 'v'); + $query->orWhere('v'); + } + )->build(); $this->assertEquals('(f:v AND f:v) OR (f:v OR v)', $query); } diff --git a/tests/Feature/InstanceTest.php b/tests/Feature/InstanceTest.php index c617889d..1cb03eec 100644 --- a/tests/Feature/InstanceTest.php +++ b/tests/Feature/InstanceTest.php @@ -6,29 +6,27 @@ use StatonLab\TripalTestSuite\DBTransaction; use Tests\TestCase; -class InstanceTest extends TestCase -{ - use DBTransaction; +class InstanceTest extends TestCase{ - /** @test */ - public function testThatConnectionToAnInvalidHostFails() - { - variable_del('elasticsearch_host'); + use DBTransaction; - $this->expectException(\Exception::class); + /** @test */ + public function testThatConnectionToAnInvalidHostFails() { + variable_del('elasticsearch_host'); + + $this->expectException(\Exception::class); - new \ES\Common\Instance(); - } + new \ES\Common\Instance(); + } - /** @test */ - public function testThatWeCanSuccessfullyConnectToNonSpecifiedHost() - { - variable_set('elasticsearch_host', getenv('ES_HOST')); + /** @test */ + public function testThatWeCanSuccessfullyConnectToNonSpecifiedHost() { + variable_set('elasticsearch_host', getenv('ES_HOST')); - $es = new \ES\Common\Instance(); + $es = new \ES\Common\Instance(); - $this->assertInstanceOf(Instance::class, $es); - } + $this->assertInstanceOf(Instance::class, $es); + } /** @test */ public function testThatCreatingAnIndexSucceeds() { @@ -38,4 +36,44 @@ public function testThatCreatingAnIndexSucceeds() { $this->assertTrue($index['acknowledged']); $this->assertEquals($name, $index['index']); } + + /** @test */ + public function testCreatingAndUpdatingDocuments() { + $name = uniqid(); + + $this->makeIndex($name, ['content' => 'text']); + + $es = new Instance(); + $data = $es->createEntry( + $name, + $name, + FALSE, + [ + 'content' => 'some text', + ] + ); + + $this->assertTrue(is_array($data)); + $this->assertEquals('created', $data['result']); + + $id = $data['_id']; + + $data = $es->update( + $name, + $name, + $id, + [ + 'content' => 'updated text!', + ] + ); + + $this->assertTrue(is_array($data)); + $this->assertEquals('updated', $data['result']); + + // Verify that the record got updated + $data = $es->getRecord($name, $name, $id); + $this->assertTrue(is_array($data)); + $this->assertTrue($data['found']); + $this->assertEquals('updated text!', $data['_source']['content']); + } } diff --git a/tests/TestCase.php b/tests/TestCase.php index 6e3be9b5..cbe4857f 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -44,14 +44,15 @@ public function deleteIndex($name) { * * @throws \Exception */ - public function makeIndex($name = NULL) { + public function makeIndex($name = NULL, $fields = []) { $es = new \ES\Common\Instance(getenv('ES_HOST')); if (is_null($name)) { - return $name; + $name = uniqid(); } - $index = $es->setIndexParams($name)->createIndex(); + $index = $es->setIndexParams($name, 1, 0, 'standard', [], $fields) + ->createIndex(); $this->_indices[] = $index; From a08b5b9201e05b576395113665d759b9ec27af5d Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Thu, 21 Feb 2019 08:11:21 -0500 Subject: [PATCH 08/21] Debug travis --- tests/TestCase.php | 3 +++ tests/travis.env | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/TestCase.php b/tests/TestCase.php index cbe4857f..9a961e20 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -45,6 +45,9 @@ public function deleteIndex($name) { * @throws \Exception */ public function makeIndex($name = NULL, $fields = []) { + var_dump(getenv('ES_HOST')); + var_dump(file_get_contents('http://127.0.0.1:9201')); + $es = new \ES\Common\Instance(getenv('ES_HOST')); if (is_null($name)) { diff --git a/tests/travis.env b/tests/travis.env index b1ab060a..5f60d4ef 100644 --- a/tests/travis.env +++ b/tests/travis.env @@ -1,4 +1,4 @@ -BASE_URL=http://localhost +BASE_URL=http://127.0.0.1 DRUPAL_ROOT=/var/www/html FAKER_LOCALE=en_US -ES_HOST=http://localhost:9201 +ES_HOST=http://127.0.0.1:9201 From 209f78fbe79faf38f0e16c9816ce945b568d8fa7 Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Thu, 21 Feb 2019 08:24:02 -0500 Subject: [PATCH 09/21] fix travis --- composer.lock | 11 +++++++---- tests/TestCase.php | 3 --- tests/travis.env | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/composer.lock b/composer.lock index d10c9d83..f6d606a7 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "659c79abed2accea365e00d3efd955c4", + "content-hash": "3a8bf0a4e263278d88698c4969217b0b", "packages": [], "packages-dev": [ { @@ -2027,12 +2027,12 @@ "version": "1.1.2", "source": { "type": "git", - "url": "https://github.com/statonlab/TripalTestSuite.git", + "url": "https://github.com/tripal/TripalTestSuite.git", "reference": "ca85284fc1da9c9f2863353dcb6eefe6d4b39343" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/statonlab/TripalTestSuite/zipball/ca85284fc1da9c9f2863353dcb6eefe6d4b39343", + "url": "https://api.github.com/repos/tripal/TripalTestSuite/zipball/ca85284fc1da9c9f2863353dcb6eefe6d4b39343", "reference": "ca85284fc1da9c9f2863353dcb6eefe6d4b39343", "shasum": "" }, @@ -2424,6 +2424,9 @@ "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, - "platform": [], + "platform": { + "ext-curl": "*", + "ext-json": "*" + }, "platform-dev": [] } diff --git a/tests/TestCase.php b/tests/TestCase.php index 9a961e20..cbe4857f 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -45,9 +45,6 @@ public function deleteIndex($name) { * @throws \Exception */ public function makeIndex($name = NULL, $fields = []) { - var_dump(getenv('ES_HOST')); - var_dump(file_get_contents('http://127.0.0.1:9201')); - $es = new \ES\Common\Instance(getenv('ES_HOST')); if (is_null($name)) { diff --git a/tests/travis.env b/tests/travis.env index 5f60d4ef..6591b689 100644 --- a/tests/travis.env +++ b/tests/travis.env @@ -1,4 +1,4 @@ BASE_URL=http://127.0.0.1 DRUPAL_ROOT=/var/www/html FAKER_LOCALE=en_US -ES_HOST=http://127.0.0.1:9201 +ES_HOST=http://elasticsearch:9201 From 2f0efba0452d90063d802787bf9cbb31d0cfa804 Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Thu, 21 Feb 2019 08:33:18 -0500 Subject: [PATCH 10/21] Wrong travis port --- tests/travis.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/travis.env b/tests/travis.env index 6591b689..d89ee421 100644 --- a/tests/travis.env +++ b/tests/travis.env @@ -1,4 +1,4 @@ BASE_URL=http://127.0.0.1 DRUPAL_ROOT=/var/www/html FAKER_LOCALE=en_US -ES_HOST=http://elasticsearch:9201 +ES_HOST=http://elasticsearch:9200 From a9b8c6963679535a2c34264544cc417ce5ee5e1c Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Thu, 21 Feb 2019 08:43:38 -0500 Subject: [PATCH 11/21] Add Query Builder Test --- includes/Query/Builder.php | 75 +++++++++++++++++++++++------- tests/Feature/QueryBuilderTest.php | 59 +++++++++++++++++++++++ 2 files changed, 118 insertions(+), 16 deletions(-) create mode 100644 tests/Feature/QueryBuilderTest.php diff --git a/includes/Query/Builder.php b/includes/Query/Builder.php index 1bcbd974..880508ba 100644 --- a/includes/Query/Builder.php +++ b/includes/Query/Builder.php @@ -12,12 +12,12 @@ class Builder implements BuilderContract{ /** * @var int */ - protected $from; + protected $from = NULL; /** * @var int */ - protected $size; + protected $size = NULL; /** * @var array @@ -38,12 +38,17 @@ class Builder implements BuilderContract{ */ protected $id = FALSE; + /** + * @var string + */ + protected $type; + /** * Builder constructor. * * @param $index */ - public function __construct($index) { + public function __construct($index = NULL) { $this->index = $index; $this->query = new Clause(); } @@ -128,6 +133,28 @@ public function setID($id) { return $this; } + /** + * @param string $index + * + * @return $this + */ + public function setIndex($index) { + $this->index = $index; + + return $this; + } + + /** + * @param string $type + * + * @return $this + */ + public function setType($type) { + $this->type = $type; + + return $this; + } + /** * Build and return the parameters. * @@ -135,36 +162,52 @@ public function setID($id) { * * @return array * The params array. + * + * @throws \Exception */ public function build($with_range = TRUE) { - $params = []; - - if ($this->size && $with_range) { - $params['size'] = $this->size; - } - - if ($this->from && $with_range) { - $params['from'] = $this->from; + if (is_null($this->index)) { + throw new \Exception('Index name must be set to build a query'); } - $params['body'] = [ - 'query' => [ - 'simple_query_string' => [ - 'query' => $this->query->build(), + // Initialize params + $params = [ + 'index' => $this->index, + 'body' => [ + 'query' => [ + 'simple_query_string' => [ + 'query' => $this->query->build(), + ], ], ], ]; + // Set range + if (!is_null($this->from) && $with_range) { + $params['from'] = $this->from; + } + + if (!is_null($this->size) && $with_range) { + $params['size'] = $this->size; + } + + // Set the highlighted field if ($this->highlight) { $params['body']['highlight'] = [ 'fields' => $this->highlight, ]; } - if ($this->id !== FALSE) { + // Set the id + if ($this->id !== FALSE && $this->id !== NULL) { $params['id'] = $this->id; } + // Set the index type + if ($this->type) { + $params['type'] = $this->type; + } + return $params; } } diff --git a/tests/Feature/QueryBuilderTest.php b/tests/Feature/QueryBuilderTest.php new file mode 100644 index 00000000..fb11faa6 --- /dev/null +++ b/tests/Feature/QueryBuilderTest.php @@ -0,0 +1,59 @@ +setID('some_id')->where('field', 'value')->orWhere( + 'other', + 'value' + )->setType('type')->highlight('highlighted_field')->range(0, 100); + + // Build with range + $params = $builder->build(); + + $this->assertArrayHasKey('body', $params); + $this->assertArrayHasKey('from', $params); + $this->assertArrayHasKey('size', $params); + $this->assertArrayHasKey('type', $params); + $this->assertArrayHasKey('index', $params); + $this->assertArrayHasKey('query', $params['body']); + $this->assertArrayHasKey( + 'query', + $params['body']['query']['simple_query_string'] + ); + $this->assertArrayHasKey('highlight', $params['body']); + $this->assertArrayHasKey('fields', $params['body']['highlight']); + $this->assertArrayHasKey( + 'highlighted_field', + $params['body']['highlight']['fields'] + ); + + // Check query value + $this->assertEquals( + 'field:value OR other:value', + $params['body']['query']['simple_query_string']['query'] + ); + + // Build without range + $params = $builder->build(FALSE); + + // Verify range doesn't exist + $this->assertArrayNotHasKey('from', $params); + $this->assertArrayNotHasKey('size', $params); + } + + public function testThatAnExceptionIsThrownWhenIndexIsNotAvailable() { + $builder = new Builder(); + + $this->expectException(\Exception::class); + $builder->build(); + } +} From 10fd6b83b14d5063c4207d904c884323de1342b8 Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Thu, 21 Feb 2019 08:46:06 -0500 Subject: [PATCH 12/21] Add Query Builder Test --- includes/Query/Builder.php | 23 ++++++++++++++++++++--- tests/Feature/QueryBuilderTest.php | 11 ++++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/includes/Query/Builder.php b/includes/Query/Builder.php index 880508ba..5f8bf70d 100644 --- a/includes/Query/Builder.php +++ b/includes/Query/Builder.php @@ -155,6 +155,25 @@ public function setType($type) { return $this; } + /** + * Validate the query. + * + * @throws \Exception + */ + public function validate() { + if (is_null($this->index)) { + throw new \Exception( + 'Index name must be set to build request parameters.' + ); + } + + if (empty($this->query->build())) { + throw new \Exception( + 'Query string must be provided in order to build request parameters.' + ); + } + } + /** * Build and return the parameters. * @@ -166,9 +185,7 @@ public function setType($type) { * @throws \Exception */ public function build($with_range = TRUE) { - if (is_null($this->index)) { - throw new \Exception('Index name must be set to build a query'); - } + $this->validate(); // Initialize params $params = [ diff --git a/tests/Feature/QueryBuilderTest.php b/tests/Feature/QueryBuilderTest.php index fb11faa6..a71f65a5 100644 --- a/tests/Feature/QueryBuilderTest.php +++ b/tests/Feature/QueryBuilderTest.php @@ -50,10 +50,19 @@ public function testThatParamsBuildCorrectly() { $this->assertArrayNotHasKey('size', $params); } - public function testThatAnExceptionIsThrownWhenIndexIsNotAvailable() { + /** @test */ + public function testThatAnExceptionIsThrownWhenIndexIsNotProvided() { $builder = new Builder(); $this->expectException(\Exception::class); $builder->build(); } + + /** @test */ + public function testThatAnExceptionIsThrownWhenQueryIsNotProvided() { + $builder = new Builder('test'); + + $this->expectException(\Exception::class); + $builder->build(); + } } From 08e6b61a3f660710b07c30c852550f6d5131072a Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Thu, 21 Feb 2019 09:10:41 -0500 Subject: [PATCH 13/21] Add addon query builders --- includes/Query/Builder.php | 3 +- includes/Query/BuilderContract.php | 142 +++++++++++++++++++++++++++-- includes/Query/Clause.php | 2 +- tests/Feature/ClauseTest.php | 55 +++++++++++ tests/Feature/QueryBuilderTest.php | 2 +- 5 files changed, 194 insertions(+), 10 deletions(-) diff --git a/includes/Query/Builder.php b/includes/Query/Builder.php index 5f8bf70d..02cb8310 100644 --- a/includes/Query/Builder.php +++ b/includes/Query/Builder.php @@ -2,7 +2,7 @@ namespace ES\Query; -class Builder implements BuilderContract{ +class Builder extends BuilderContract{ /** * @var string @@ -203,7 +203,6 @@ public function build($with_range = TRUE) { if (!is_null($this->from) && $with_range) { $params['from'] = $this->from; } - if (!is_null($this->size) && $with_range) { $params['size'] = $this->size; } diff --git a/includes/Query/BuilderContract.php b/includes/Query/BuilderContract.php index 8e181275..b3ef3c0a 100644 --- a/includes/Query/BuilderContract.php +++ b/includes/Query/BuilderContract.php @@ -2,25 +2,155 @@ namespace ES\Query; -interface BuilderContract{ +abstract class BuilderContract{ /** * Add a match condition joined with ANDs. * - * @param string $field The value to match against or the name of the field. + * @param string|\Closure $field The value to match against or the name of + * the field. * @param string $value The value if the field is specified. * - * @return mixed + * @return $this */ - public function where($field, $value = NULL); + abstract public function where($field, $value = NULL); /** * Add a match condition joined with ORs. * + * @param string|\Closure $field The value to match against or the name of + * the field. + * @param string $value The value if the field is specified. + * + * @return $this + */ + abstract public function orWhere($field, $value = NULL); + + /** + * Add a begins with condition. + * + * @param string $field The value to match against or the name of the field. + * @param string $value The value if the field is specified. + * + * @return $this + */ + public function beginsWith($field, $value = NULL) { + if (is_null($value)) { + return $this->where("{$field}*"); + } + + return $this->where($field, "{$value}*"); + } + + /** + * Add a ends with condition. + * + * @param string $field The value to match against or the name of the field. + * @param string $value The value if the field is specified. + * + * @return $this + */ + public function endsWith($field, $value = NULL) { + if (is_null($value)) { + return $this->where("*{$field}"); + } + + return $this->where($field, "*{$value}"); + } + + /** + * Add a contains condition. + * + * @param string $field The value to match against or the name of the field. + * @param string $value The value if the field is specified. + * + * @return $this + */ + public function contains($field, $value = NULL) { + if (is_null($value)) { + return $this->where("*{$field}*"); + } + + return $this->where($field, "*{$value}*"); + } + + /** + * Add a fuzzy condition. + * + * @param string $field The value to match against or the name of the field. + * @param string $value The value if the field is specified. + * + * @return $this + */ + public function fuzzy($field, $value = NULL) { + if (is_null($value)) { + return $this->where("{$field}~"); + } + + return $this->where($field, "{$value}~"); + } + + /** + * Add a begins with condition. + * + * @param string $field The value to match against or the name of the field. + * @param string $value The value if the field is specified. + * + * @return $this + */ + public function orBeginsWith($field, $value = NULL) { + if (is_null($value)) { + return $this->orWhere("{$field}*"); + } + + return $this->orWhere($field, "{$value}*"); + } + + /** + * Add a ends with condition. + * + * @param string $field The value to match against or the name of the field. + * @param string $value The value if the field is specified. + * + * @return $this + */ + public function orEndsWith($field, $value = NULL) { + if (is_null($value)) { + return $this->orWhere("*{$field}"); + } + + return $this->orWhere($field, "*{$value}"); + } + + /** + * Add a contains condition. + * + * @param string $field The value to match against or the name of the field. + * @param string $value The value if the field is specified. + * + * @return $this + */ + public function orContains($field, $value = NULL) { + if (is_null($value)) { + return $this->orWhere("*{$field}*"); + } + + return $this->orWhere($field, "*{$value}*"); + } + + /** + * Add a fuzzy condition. + * * @param string $field The value to match against or the name of the field. * @param string $value The value if the field is specified. * - * @return mixed + * @return $this */ - public function orWhere($field, $value = NULL); + public function orFuzzy($field, $value = NULL) { + if (is_null($value)) { + return $this->orWhere("{$field}~"); + } + + return $this->orWhere($field, "{$value}~"); + } } diff --git a/includes/Query/Clause.php b/includes/Query/Clause.php index e845974c..b321a9f3 100644 --- a/includes/Query/Clause.php +++ b/includes/Query/Clause.php @@ -2,7 +2,7 @@ namespace ES\Query; -class Clause implements BuilderContract{ +class Clause extends BuilderContract{ /** * The built query. diff --git a/tests/Feature/ClauseTest.php b/tests/Feature/ClauseTest.php index 33b7a3b9..3a28dbcf 100644 --- a/tests/Feature/ClauseTest.php +++ b/tests/Feature/ClauseTest.php @@ -33,4 +33,59 @@ function (BuilderContract $query) { $this->assertEquals('(f:v AND f:v) OR (f:v OR v)', $query); } + + /** @test */ + public function testAddonBuilders() { + $clause = new Clause(); + + $query = $clause->beginsWith('test') + ->contains('test') + ->endsWith('test') + ->fuzzy('test') + ->build(); + + $this->assertEquals('test* AND *test* AND *test AND test~', $query); + } + + /** @test */ + public function testAddonBuildersThatUseOr() { + $clause = new Clause(); + + $query = $clause->beginsWith('test')->orBeginsWith('test')->orContains( + 'test' + )->orEndsWith('test')->orFuzzy('test')->build(); + + $this->assertEquals('test* OR test* OR *test* OR *test OR test~', $query); + } + + public function testAddonsWithFields() { + $clause = new Clause(); + + $query = $clause->beginsWith('field', 'test') + ->contains('field', 'test') + ->endsWith('field', 'test') + ->fuzzy('field', 'test') + ->build(); + + $this->assertEquals( + 'field:test* AND field:*test* AND field:*test AND field:test~', + $query + ); + } + + public function testAddonsWithFieldsUsingOr() { + $clause = new Clause(); + + $query = $clause->beginsWith('field', 'test') + ->orBeginsWith('field', 'test') + ->orContains('field', 'test') + ->orEndsWith('field', 'test') + ->orFuzzy('field', 'test') + ->build(); + + $this->assertEquals( + 'field:test* OR field:test* OR field:*test* OR field:*test OR field:test~', + $query + ); + } } diff --git a/tests/Feature/QueryBuilderTest.php b/tests/Feature/QueryBuilderTest.php index a71f65a5..e5bbae69 100644 --- a/tests/Feature/QueryBuilderTest.php +++ b/tests/Feature/QueryBuilderTest.php @@ -61,7 +61,7 @@ public function testThatAnExceptionIsThrownWhenIndexIsNotProvided() { /** @test */ public function testThatAnExceptionIsThrownWhenQueryIsNotProvided() { $builder = new Builder('test'); - + $this->expectException(\Exception::class); $builder->build(); } From 953027d8b2d1b0cb24a41b78eaafe32d29adb7ef Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Thu, 21 Feb 2019 09:21:49 -0500 Subject: [PATCH 14/21] Fix sanitizer and add raw query method --- includes/Query/Builder.php | 26 +++++++++++++++++++++++ includes/Query/BuilderContract.php | 3 +++ includes/Query/Clause.php | 33 ++++++++++++++++++++++++++++++ includes/Query/ClauseSanitizer.php | 7 +------ tests/Feature/ClauseTest.php | 11 ++++++++++ 5 files changed, 74 insertions(+), 6 deletions(-) diff --git a/includes/Query/Builder.php b/includes/Query/Builder.php index 02cb8310..cf4d38cc 100644 --- a/includes/Query/Builder.php +++ b/includes/Query/Builder.php @@ -95,6 +95,32 @@ public function orWhere($field, $value = NULL) { return $this; } + /** + * @param $query + * @param string $op + * + * @return $this + * + * @see \ES\Query\Clause::raw() + */ + public function raw($query, $op = 'AND') { + $this->query->raw($query, $op); + + return $this; + } + + /** + * @param $query + * + * @return $this + * @see \ES\Query\Clause::orRaw() + */ + public function orRaw($query) { + $this->query->orRaw($query); + + return $this; + } + /** * * @param string|array $fields diff --git a/includes/Query/BuilderContract.php b/includes/Query/BuilderContract.php index b3ef3c0a..cf3aae51 100644 --- a/includes/Query/BuilderContract.php +++ b/includes/Query/BuilderContract.php @@ -26,6 +26,9 @@ abstract public function where($field, $value = NULL); */ abstract public function orWhere($field, $value = NULL); + abstract public function raw($query, $op = 'AND'); + abstract public function orRaw($query); + /** * Add a begins with condition. * diff --git a/includes/Query/Clause.php b/includes/Query/Clause.php index b321a9f3..07c8d0ad 100644 --- a/includes/Query/Clause.php +++ b/includes/Query/Clause.php @@ -101,4 +101,37 @@ public function orWhere($field, $data = NULL) { public function build() { return $this->query; } + + /** + * Add a raw query joined with $op. + * + * @param string $query + * @param string $op + * + * @return $this + */ + public function raw($query, $op = 'AND') { + $query = $this->sanitizer->escape($query); + + if (empty($this->query)) { + $this->query = $query; + + return $this; + } + + $this->query .= " {$op} {$query}"; + + return $this; + } + + /** + * Add a raw query joined with OR. + * + * @param string $query + * + * @return \ES\Query\Clause + */ + public function orRaw($query) { + return $this->raw($query, 'OR'); + } } diff --git a/includes/Query/ClauseSanitizer.php b/includes/Query/ClauseSanitizer.php index 207d45e0..2b61861b 100644 --- a/includes/Query/ClauseSanitizer.php +++ b/includes/Query/ClauseSanitizer.php @@ -5,11 +5,6 @@ class ClauseSanitizer{ public function escape($value) { - $value = stripslashes($value); - $value = str_replace('\\', ' ', $value); - $value = str_replace('+', ' ', $value); - $value = str_replace('-', ' ', $value); - $value = str_replace('^', '', $value); - return str_replace(':', '\\:', $value); + return stripslashes($value); } } diff --git a/tests/Feature/ClauseTest.php b/tests/Feature/ClauseTest.php index 3a28dbcf..0db1c2d6 100644 --- a/tests/Feature/ClauseTest.php +++ b/tests/Feature/ClauseTest.php @@ -58,6 +58,7 @@ public function testAddonBuildersThatUseOr() { $this->assertEquals('test* OR test* OR *test* OR *test OR test~', $query); } + /** @test */ public function testAddonsWithFields() { $clause = new Clause(); @@ -73,6 +74,7 @@ public function testAddonsWithFields() { ); } + /** @test */ public function testAddonsWithFieldsUsingOr() { $clause = new Clause(); @@ -88,4 +90,13 @@ public function testAddonsWithFieldsUsingOr() { $query ); } + + /** @test */ + public function testRawQuery() { + $clause = new Clause(); + + $query = $clause->raw('test:value* AND test')->orRaw('test')->build(); + + $this->assertEquals('test:value* AND test OR test', $query); + } } From df0a33b5618314611ec9be3e2ff024bb122c338e Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Thu, 21 Feb 2019 12:49:19 -0500 Subject: [PATCH 15/21] Fix ESInstance reference to use the full namespace --- includes/Common/Instance.php | 39 +-- includes/Exceptions/IndexExistsException.php | 7 + includes/Exceptions/IndexMissingException.php | 7 + includes/Indices/Entities.php | 8 - includes/Indices/HandlesIndices.php | 20 -- includes/Indices/Index.php | 245 +++++--------- includes/Jobs/DispatcherJob.php | 2 +- includes/Jobs/EntitiesIndexJob.php | 4 +- includes/Jobs/GeneSearchIndexJob.php | 5 +- includes/Jobs/NodesIndexJob.php | 4 +- includes/Jobs/TableIndexJob.php | 5 +- includes/Models/Model.php | 310 ++++++++++++++++++ includes/Query/ClauseSanitizer.php | 14 + .../local__feature_search_formatter.inc | 2 +- .../tripal_elasticsearch.collections.form.inc | 2 +- .../tripal_elasticsearch.connection.form.inc | 2 +- .../tripal_elasticsearch.gene_search.form.inc | 6 +- .../tripal_elasticsearch.indices.form.inc | 16 +- ...tripal_elasticsearch.search_forms.form.inc | 2 +- ...ipal_elasticsearch.website_search.form.inc | 2 +- tripal_elasticsearch.api.inc | 2 +- tripal_elasticsearch.install | 8 +- tripal_elasticsearch.module | 16 +- tripal_elasticsearch.ws.inc | 8 +- 24 files changed, 490 insertions(+), 246 deletions(-) create mode 100644 includes/Exceptions/IndexExistsException.php create mode 100644 includes/Exceptions/IndexMissingException.php delete mode 100644 includes/Indices/Entities.php delete mode 100644 includes/Indices/HandlesIndices.php create mode 100644 includes/Models/Model.php diff --git a/includes/Common/Instance.php b/includes/Common/Instance.php index 7d3f04cd..2779431c 100644 --- a/includes/Common/Instance.php +++ b/includes/Common/Instance.php @@ -251,22 +251,23 @@ public function setTableSearchParams($index, $type, $query, $offset = [], $highl /** * Build a new index parameters. * - * @param $index_name - * @param int $shards - * @param int $replicas - * @param string $tokenizer - * @param array $token_filters - * @param array $field_mapping_types + * @param string $index_name The index name. + * @param int $shards Number of shards. + * @param int $replicas Number of replicas. + * @param string $tokenizer The type of tokenizer. + * @param array $filters The filters. + * @param array $fields The fields as ['field' => 'type', ...]. * * @return $this + * The current object. */ - public function setIndexParams($index_name, $shards = 5, $replicas = 0, $tokenizer = 'standard', $token_filters = [], $field_mapping_types = []) { + public function setIndexParams($index_name, $shards = 5, $replicas = 0, $tokenizer = 'standard', $filters = [], $fields = []) { $analysis = [ 'analyzer' => [ $index_name => [ 'type' => 'custom', 'tokenizer' => $tokenizer, - 'filter' => array_keys($token_filters), + 'filter' => array_keys($filters), ], ], ]; @@ -279,16 +280,15 @@ public function setIndexParams($index_name, $shards = 5, $replicas = 0, $tokeniz ]; $properties = []; - foreach ($field_mapping_types as $field => $mapping_type) { - $properties[$field] = [ - //'type' => $mapping_type, - //'fields' => [ - // 'raw' => [ - // 'type' => $mapping_type, - // //'index' => 'not_analyzed', - // ], - //], - ]; + foreach ($fields as $field => $mapping_type) { + if ($mapping_type === 'dynamic') { + $properties[$field] = []; + } + else { + $properties[$field] = [ + 'type' => $mapping_type, + ]; + } } $mappings = [ @@ -312,12 +312,13 @@ public function setIndexParams($index_name, $shards = 5, $replicas = 0, $tokeniz * Perform the actual search. * Use this function after setting the search params. * - * @param bool $return_source whether to format the results or not. + * @param bool $return_source Whether to format the results or not. * * @see \ES\Common\Instance::setTableSearchParams() * @see \ES\Common\Instance::setWebsiteSearchParams() * * @return array + * The search results. * @throws \Exception */ public function search($return_source = FALSE) { diff --git a/includes/Exceptions/IndexExistsException.php b/includes/Exceptions/IndexExistsException.php new file mode 100644 index 00000000..1ea2f1b3 --- /dev/null +++ b/includes/Exceptions/IndexExistsException.php @@ -0,0 +1,7 @@ +index ?? strtolower(get_class($this)); - } - - /** - * @return mixed - */ - private function getIndexType() { - return $this->type; - } -} diff --git a/includes/Indices/Index.php b/includes/Indices/Index.php index cc8a1ce1..993e0f2d 100644 --- a/includes/Indices/Index.php +++ b/includes/Indices/Index.php @@ -3,20 +3,17 @@ namespace ES\Indices; use ES\Common\Instance; +use ES\Exceptions\IndexExistsException; +use ES\Exceptions\IndexMissingException; +use ES\Models\Model; +use mysql_xdevapi\Exception; /** * Class Index * * @package ES\Indices */ -abstract class Index{ - - use HandlesIndices; - - /** - * @var array - */ - protected $attributes = []; +class Index{ /** * The ES instance. @@ -30,14 +27,7 @@ abstract class Index{ * * @var string */ - protected $index; - - /** - * Optional index type. - * - * @var string - */ - protected $type; + protected $index = NULL; /** * Fields mappings. @@ -46,206 +36,139 @@ abstract class Index{ */ protected $fields = []; - /** - * The id of the entry. - * - * @var int - */ - protected $id; - - /** - * Query Builder. - * - * @var \ES\Query\Builder - */ - protected $builder; - - /** - * Whether the entry exists in the index. - * - * @var bool - */ - protected $exists = FALSE; - /** * Index constructor. * * @param array $data Fill the object with data. + * @param string $id The ES given id for the object. + * @param Instance $instance The ES instance. * * @throws \Exception */ - public function __construct($data = [], Instance $instance = NULL) { + public function __construct(Instance $instance = NULL) { $this->instance = $instance ?? new Instance(); - $this->fill($data); } /** - * Add a where clause. + * Creates an index from a given model. * - * @param string $field The field. - * @param string $value The value + * @param \ES\Models\Model $model The model to create an index for. * - * @return $this - * The object. - * @see \ES\Query\Clause::where() + * @return array + * @throws \Exception + * @throws \ES\Exceptions\IndexExistsException */ - public function where($field, $value = NULL) { - $this->builder->where($field, $value); + public function createFromModel(Model $model) { + $this->setIndexName($model->getIndexName()); + $this->setFields($model->getFields()); - return $this; + return $this->createIndex(); } /** - * Add an or clause. - * - * @param string $field The field. - * @param string $value The value + * Create the index. * - * @return $this - * The object. - * @see \ES\Query\Clause::orWhere() + * @throws \Exception + * @throws \ES\Exceptions\IndexExistsException */ - public function orWhere($field, $value = NULL) { - $this->builder->where($field, $value); + public function createIndex() { + if (!$this->index || empty($this->index)) { + throw new \Exception('Index name must be specified to create the index.'); + } - return $this; - } + if (empty($this->fields)) { + throw new \Exception( + 'Indices cannot be created if fields are not specified.' + ); + } - /** - * Fills the object attributes with values. - * - * @param array $data - */ - public function fill(array $data = []) { - foreach ($data as $key => $value) { - if (isset($this->fields[$key])) { - $this->attributes[$key] = $value; - } + if ($this->instance->hasIndex($this->index)) { + throw new IndexExistsException( + 'Index ' . $this->index . ' already exists.' + ); } + + $shards = 5; + $replicas = 0; + $tokenizer = 'standard'; + $filters = []; + + return $this->instance->setIndexParams( + $this->index, + $shards, + $replicas, + $tokenizer, + $filters, + $this->fields + )->createIndex(); } /** - * Save the data. + * Delete the index. * - * @return $this + * @return array|bool + * @throws \ES\Exceptions\IndexMissingException + * @throws \Exception */ - public function save() { - if ($this->exists()) { - $data = $this->instance->update( - $this->getIndexName(), - $this->getIndexType(), - $this->id ?? FALSE, - $this->attributes - ); - - $this->id = $data['_id']; - - return $this; + public function deleteIndex() { + if (!$this->index || empty($this->index)) { + throw new \Exception('Index name must be specified to create the index.'); } - $data = $this->instance->createEntry( - $this->getIndexName(), - $this->getIndexType(), - $this->id ?? FALSE, - $this->attributes + db_query( + 'DELETE FROM tripal_elasticsearch_indices WHERE index_name = :name', + [':name' => $this->index] ); - $this->id = $data['_id']; - - return $this; - } - - /** - * Check whether the record exists. - * - * @return bool - */ - public function exists() { - $this->exists = $this->count() > 0; + if (!$this->instance->hasIndex($this->index)) { + throw new IndexMissingException( + 'The index ' . $this->index . ' cannot be deleted because it does not exist.' + ); + } - return $this->exists; + return $this->instance->deleteIndex($this->index); } /** - * Count the number of documents in the index. + * Set the index name. * - * @return int - * The number of documents in the index. - */ - public function count() { - return (int) $this->instance->client->count($this->builder->build(FALSE)); - } - - /** - * @param $data + * @param $index * - * @throws \Exception - */ - public static function createOrUpdate($data, $id = FALSE) { - $record = new static($data); - - $record->setID($id); - $record->save(); - - return $record; - } - - /** - * @param $id + * @return $this */ - public function setID($id) { - $this->id = $id; - - $this->builder->setID($id); + public function setIndexName($index) { + $this->index = $index; return $this; } /** - * @param array $data - * @param bool $id + * Get the index name. * - * @return \ES\Indices\Index - * @throws \Exception + * @return string */ - public static function create(array $data, $id = FALSE) { - $index = new static($data); + public function getIndexName() { + return $this->index; + } - $index->setID($id); - $index->save(); + public function setFields(array $fields) { + $this->fields = $fields; - return $index; + return $this; } /** - * Get an attribute. - * - * @param string $name The name of attribute. - * - * @return mixed - * The value of the attribute if it exists. + * @return array */ - public function __get($name) { - if (array_key_exists($name, $this->attributes)) { - return $this->attributes[$name]; + public function getFields($refresh = FALSE) { + if (!$refresh && !empty($this->fields)) { + return array_keys($this->fields); } - if (method_exists(static::class, $name)) { - return; + if (empty($this->index)) { + throw new Exception('Index name must be provided before getting fields'); } - return $this->{$name}; - } - - /** - * Check if an attribute isset. - * - * @param string $name The name of the attribute. - * - * @return bool - * Whether the attribute has been set. - */ - public function __isset($name) { - return isset($this->attributes[$name]); + return $this->instance->getIndexMappings($this->index); } } diff --git a/includes/Jobs/DispatcherJob.php b/includes/Jobs/DispatcherJob.php index 213ea2d3..daadf28c 100644 --- a/includes/Jobs/DispatcherJob.php +++ b/includes/Jobs/DispatcherJob.php @@ -6,7 +6,7 @@ * ====================================== * Dispatch jobs for bulk indexing. */ -class DispatcherJob extends \ES\Jobs { +class DispatcherJob extends Job{ /** * Dispatcher job type. diff --git a/includes/Jobs/EntitiesIndexJob.php b/includes/Jobs/EntitiesIndexJob.php index d2f35103..b30b3440 100644 --- a/includes/Jobs/EntitiesIndexJob.php +++ b/includes/Jobs/EntitiesIndexJob.php @@ -1,6 +1,8 @@ instance = $instance ?? new Instance(); + $this->fill($data); + + $this->id = $id; + + $this->builder = new Builder($this->index); + } + + /** + * Add a where clause. + * + * @param string $field The field. + * @param string $value The value + * + * @return $this + * The object. + * @see \ES\Query\Clause::where() + */ + public function where($field, $value = NULL) { + $this->builder->where($field, $value); + + return $this; + } + + /** + * Add an or clause. + * + * @param string $field The field. + * @param string $value The value + * + * @return $this + * The object. + * @see \ES\Query\Clause::orWhere() + */ + public function orWhere($field, $value = NULL) { + $this->builder->where($field, $value); + + return $this; + } + + /** + * Fills the object attributes with values. + * + * @param array $data + */ + public function fill(array $data = []) { + foreach ($data as $key => $value) { + if (isset($this->fields[$key])) { + $this->attributes[$key] = $value; + } + } + } + + /** + * Save the data. + * + * @return $this + * The object. + * @throws \Exception + */ + public function save() { + if ($this->exists()) { + $data = $this->instance->update( + $this->getIndexName(), + $this->getIndexType(), + $this->id ?? FALSE, + $this->attributes + ); + + $this->id = $data['_id']; + + return $this; + } + + // There data does not exist so + $data = $this->instance->createEntry( + $this->getIndexName(), + $this->getIndexType(), + $this->id ?? FALSE, + $this->attributes + ); + + $this->id = $data['_id']; + + return $this; + } + + /** + * Check whether the record exists. + * + * @return bool + * Whether the record exists. + * @throws \Exception + */ + public function exists() { + $this->exists = $this->count() > 0; + + return $this->exists; + } + + /** + * Count the number of documents in the index. + * + * @return int + * The number of documents in the index. + * @throws \Exception + */ + public function count() { + return (int) $this->instance->client->count($this->builder->build(FALSE)); + } + + /** + * @param $data + * + * @throws \Exception + * @return \ES\Models\Model + */ + public static function createOrUpdate($data, $id = FALSE) { + $record = new static($data); + + $record->setID($id); + $record->save(); + + return $record; + } + + /** + * @param $id + * + * @return $this + */ + public function setID($id) { + $this->id = $id; + + $this->builder->setID($id); + + return $this; + } + + /** + * @param array $data + * @param bool $id + * + * @return \ES\Models\Model + * @throws \Exception + */ + public static function create(array $data, $id = FALSE) { + $index = new static($data); + + $index->setID($id); + $index->save(); + + return $index; + } + + /** + * Set the index name. + * + * @param $index + * + * @return $this + */ + public function setIndexName($index) { + $this->index = $index; + + return $this; + } + + /** + * Get the index name. + * + * @return string + */ + public function getIndexName() { + return $this->index; + } + + /** + * Set the type of the index. + * + * @param string $type + * + * @return $this + */ + public function setIndexType($type) { + $this->type = $type; + + return $this; + } + + /** + * Get the index type. + * + * @return string + */ + public function getIndexType() { + return $this->type; + } + + /** + * Get the fields for the model. + * + * @return array + */ + public function getFields() { + return $this->fields; + } + + /** + * Get an attribute. + * + * @param string $name The name of attribute. + * + * @return mixed + * The value of the attribute if it exists. + */ + public function __get($name) { + if (array_key_exists($name, $this->attributes)) { + return $this->attributes[$name]; + } + + if (method_exists(static::class, $name)) { + return; + } + + return $this->{$name}; + } + + /** + * Check if an attribute isset. + * + * @param string $name The name of the attribute. + * + * @return bool + * Whether the attribute has been set. + */ + public function __isset($name) { + return isset($this->attributes[$name]); + } +} diff --git a/includes/Query/ClauseSanitizer.php b/includes/Query/ClauseSanitizer.php index 2b61861b..322ddc15 100644 --- a/includes/Query/ClauseSanitizer.php +++ b/includes/Query/ClauseSanitizer.php @@ -4,7 +4,21 @@ class ClauseSanitizer{ + /** + * @param $value + * + * @return string + */ public function escape($value) { return stripslashes($value); } + + /** + * @param $field + * + * @return mixed + */ + public function escapeField($field) { + return str_replace('.*', '.\\*', $field); + } } diff --git a/includes/TripalFields/local__feature_search/local__feature_search_formatter.inc b/includes/TripalFields/local__feature_search/local__feature_search_formatter.inc index f16d06f7..83bdb3d3 100644 --- a/includes/TripalFields/local__feature_search/local__feature_search_formatter.inc +++ b/includes/TripalFields/local__feature_search/local__feature_search_formatter.inc @@ -81,7 +81,7 @@ class local__feature_search_formatter extends ChadoFieldFormatter { $organism = $entity->chado_record; try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $query = tripal_elasticsearch_gene_search_index_query_mapper([ 'organism' => "$organism->genus $organism->species", "search_term" => '*', diff --git a/includes/tripal_elasticsearch.collections.form.inc b/includes/tripal_elasticsearch.collections.form.inc index 24081490..72151571 100644 --- a/includes/tripal_elasticsearch.collections.form.inc +++ b/includes/tripal_elasticsearch.collections.form.inc @@ -148,7 +148,7 @@ function tripal_elasticsearch_collections_form_submit($form, &$form_state) { $entities = []; try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $es->setWebsiteSearchParams($query, $bundle_label, '', '', [0, 10], TRUE); $count = $es->count(); diff --git a/includes/tripal_elasticsearch.connection.form.inc b/includes/tripal_elasticsearch.connection.form.inc index 6b6d7315..75e9ddbf 100644 --- a/includes/tripal_elasticsearch.connection.form.inc +++ b/includes/tripal_elasticsearch.connection.form.inc @@ -135,7 +135,7 @@ function elasticsearch_connection_form($form, &$form_state) { ]; try { - $client = (new Instance())->client; + $client = (new \ES\Common\Instance())->client; // Obtain cluster health information. $params['v'] = TRUE; $health = $client->cat()->health($params)[0]; diff --git a/includes/tripal_elasticsearch.gene_search.form.inc b/includes/tripal_elasticsearch.gene_search.form.inc index 9c04199a..9783230f 100644 --- a/includes/tripal_elasticsearch.gene_search.form.inc +++ b/includes/tripal_elasticsearch.gene_search.form.inc @@ -238,7 +238,7 @@ function tripal_elasticsearch_gene_search_index_results($display_download = FALS } try { - $es = new Instance(); + $es = new \ES\Common\Instance(); } catch (Exception $exception) { drupal_set_message('The search service is currently unavailable. Please try again later.', 'error'); @@ -318,7 +318,7 @@ function tripal_elasticsearch_gene_search_download() { } try { - $es = new Instance(); + $es = new \ES\Common\Instance(); } catch (Exception $exception) { drupal_set_message('Error occurred while generating records for download. Please try again later.', 'error'); @@ -413,7 +413,7 @@ function tripal_elasticsearch_get_gene_search_organisms() { $organism_list = chado_query($sql)->fetchAll(); try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $organisms = []; foreach ($organism_list as $organism) { $name = "{$organism->genus} {$organism->species}"; diff --git a/includes/tripal_elasticsearch.indices.form.inc b/includes/tripal_elasticsearch.indices.form.inc index d8b8a6b7..f86b1398 100644 --- a/includes/tripal_elasticsearch.indices.form.inc +++ b/includes/tripal_elasticsearch.indices.form.inc @@ -14,7 +14,7 @@ */ function tripal_elasticsearch_indexing_form($form, &$form_state) { try { - $es = new Instance(); + $es = new \ES\Common\Instance(); // associate index name with indexed table. $indices = $es->getIndices(); @@ -285,7 +285,7 @@ function tripal_elasticsearch_indexing_form_submit($form, &$form_state) { */ function tripal_elasticsearch_delete_indices_form($form, &$form_state) { try { - $client = (new Instance())->client; + $client = (new \ES\Common\Instance())->client; // associate index name with indexed table. $mappings = $client->indices()->getMapping(); @@ -341,7 +341,7 @@ function tripal_elasticsearch_delete_indices_form_submit($form, &$form_state) { if (!empty($delete_indices)) { foreach ($delete_indices as $index) { try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $es->deleteIndex($index); db_query('DELETE FROM {tripal_elasticsearch_indices} WHERE index_name=:index_name', [ @@ -361,7 +361,7 @@ function tripal_elasticsearch_delete_indices_form_submit($form, &$form_state) { */ function tripal_elasticsearch_indices_list_page() { try { - $es = new Instance(); + $es = new \ES\Common\Instance(); // associate index name with indexed table. $indices = $es->getIndices(); @@ -481,7 +481,7 @@ function tripal_elasticsearch_index_edit_confirm($form, &$form_state, $index_nam $result = db_query($sql, [':index_name' => $index_name])->fetchObject(); if (!$result) { try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $mappings = $es->getIndexMappings($index_name); $table_name = array_keys($mappings); if (count($table_name) > 1) { @@ -528,7 +528,7 @@ function tripal_elasticsearch_index_edit_confirm($form, &$form_state, $index_nam // Get existing index settings and mappings. // TODO: what about default tables? try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $index_settings = $es->getIndexSettings($index_name); $index_mappings = $es->getIndexMappings($index_name); } catch (Exception $exception) { @@ -870,7 +870,7 @@ function tripal_elasticsearch_create_index(&$values) { // Create the index. try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $es->setIndexParams($index_name, 5, 0, $tokenizer, $token_filters, $field_mapping_types)->createIndex(); } catch (Exception $exception) { @@ -923,7 +923,7 @@ function tripal_elasticsearch_create_index(&$values) { function tripal_elasticsearch_delete_index($index) { // Find the index and delete it from ES try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $es->deleteIndex($index); } catch (Exception $exception) { drupal_set_message($exception->getMessage(), 'error'); diff --git a/includes/tripal_elasticsearch.search_forms.form.inc b/includes/tripal_elasticsearch.search_forms.form.inc index 29608f9a..8b5c3504 100644 --- a/includes/tripal_elasticsearch.search_forms.form.inc +++ b/includes/tripal_elasticsearch.search_forms.form.inc @@ -26,7 +26,7 @@ function table_search_interface_building_form($form, &$form_state) { ]; try { - $client = (new Instance())->client; + $client = (new \ES\Common\Instance())->client; $existing_indices = $client->indices()->getMapping(); } catch (\Exception $e) { $form['markup'] = [ diff --git a/includes/tripal_elasticsearch.website_search.form.inc b/includes/tripal_elasticsearch.website_search.form.inc index 25d7ae41..547cf088 100644 --- a/includes/tripal_elasticsearch.website_search.form.inc +++ b/includes/tripal_elasticsearch.website_search.form.inc @@ -159,7 +159,7 @@ function tripal_elasticsearch_build_search_block_form($form, &$form_state, $inde // Build search query for table search. $query = tripal_elasticsearch_build_search_query_from_field_content_pairs($index_fields); try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $results_count = $es->setTableSearchParams($index_name, $index_type, $query) ->count(); $count = '' . $results_count . ' results match your search.'; diff --git a/tripal_elasticsearch.api.inc b/tripal_elasticsearch.api.inc index 03834fda..6cfc9edf 100644 --- a/tripal_elasticsearch.api.inc +++ b/tripal_elasticsearch.api.inc @@ -130,7 +130,7 @@ function tripal_elasticsearch_build_search_query_from_field_content_pairs(array */ function tripal_elasticsearch_get_website_search_results_category_list($keyword) { try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $indices = $es->getIndices(); $types = $es->getAllCategories(NULL, TRUE, $keyword); } catch (Exception $exception) { diff --git a/tripal_elasticsearch.install b/tripal_elasticsearch.install index 4ad19d1e..8f4b4e46 100644 --- a/tripal_elasticsearch.install +++ b/tripal_elasticsearch.install @@ -369,7 +369,7 @@ function tripal_elasticsearch_update_7208() { */ function tripal_elasticsearch_update_7209() { try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $es->putMapping('gene_search_index', 'entity_id', 'integer'); $es->putMapping('gene_search_index', 'node_id', 'integer'); } catch (Exception $exception) { @@ -385,7 +385,7 @@ function tripal_elasticsearch_update_7209() { */ function tripal_elasticsearch_update_7210() { try { - $es = new Instance(); + $es = new \ES\Common\Instance(); if (in_array('website', $es->getIndices())) { return; } @@ -433,7 +433,7 @@ function tripal_elasticsearch_update_7210() { */ function tripal_elasticsearch_update_7211() { try { - $es = new Instance(); + $es = new \ES\Common\Instance(); if (!in_array('website', $es->getIndices())) { return; @@ -482,7 +482,7 @@ function tripal_elasticsearch_update_7211() { */ function tripal_elasticsearch_update_7212() { try { - $es = new Instance(); + $es = new \ES\Common\Instance(); if (in_array('gene_search_index', $es->getIndices())) { $es->putMapping('gene_search_index', 'related_features', 'text', 'chado.feature'); } diff --git a/tripal_elasticsearch.module b/tripal_elasticsearch.module index c266167e..95adf1f3 100644 --- a/tripal_elasticsearch.module +++ b/tripal_elasticsearch.module @@ -673,7 +673,7 @@ function tripal_elasticsearch_web_search_results_page_callback($node_type = '') // Run Elasticsearch. try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $indices = $es->getIndices(); $index_name = []; if (in_array('website', $indices)) { @@ -834,7 +834,7 @@ function tripal_elasticsearch_form_tripal_elasticsearch_build_search_block_form_ // Run Elasticsearch and return search results into an array. $results = []; try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $results = $es->setTableSearchParams( $index_name, $index_type, @@ -889,7 +889,7 @@ function tripal_elasticsearch_node_update($node) { static $tripal_elasticsearch_errors = FALSE; try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $indices = $es->getIndices(); if (!in_array('website', $indices)) { return; @@ -931,7 +931,7 @@ function tripal_elasticsearch_node_delete($node) { static $tripal_elasticsearch_errors = FALSE; try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $es->deleteEntry('website', 'website', $node->nid); } catch (\Exception $e) { if (!$tripal_elasticsearch_errors) { @@ -957,7 +957,7 @@ function tripal_elasticsearch_entity_update($entity, $entity_type) { static $tripal_elasticsearch_errors = FALSE; try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $indices = $es->getIndices(); if (in_array('gene_search_index', $indices)) { if (isset($entity->chado_table) && $entity->chado_table === 'feature') { @@ -1030,7 +1030,7 @@ function tripal_elasticsearch_entity_delete($entity, $type) { static $tripal_elasticsearch_errors = FALSE; try { - $es = new Instance(); + $es = new \ES\Common\Instance(); if (in_array('entities', $es->getIndices())) { $es->deleteEntry('entities', 'entities', $entity->id); } @@ -1094,7 +1094,7 @@ function tripal_elasticsearch_paginate($per_page) { // Run Elasticsearch and return search results into an array. try { - $es = new Instance(); + $es = new \ES\Common\Instance(); return $es->setTableSearchParams($index_name, $index_type, $query) ->paginate($per_page); @@ -1143,7 +1143,7 @@ function tripal_elasticsearch_table_search_download() { $temp_file = drupal_tempnam($directory, 'search_results_') . '.csv'; try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $count = $es->setTableSearchParams($index_name, $index_type, $query)->count( ); $window_range = range(1, ceil($count / 1000)); diff --git a/tripal_elasticsearch.ws.inc b/tripal_elasticsearch.ws.inc index dd30ce62..fbfd2757 100644 --- a/tripal_elasticsearch.ws.inc +++ b/tripal_elasticsearch.ws.inc @@ -14,7 +14,7 @@ function tripal_elasticsearch_api_v1_status() { try { $params['v'] = TRUE; - $es = new Instance(); + $es = new \ES\Common\Instance(); $health = current($es->client->cat()->health($params)); } catch (Exception $exception) { return \ES\Common\Response::error( @@ -181,7 +181,7 @@ function tripal_elasticsearch_api_v1_local_search($http = TRUE) { } try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $results = $es->searchWebIndices($terms, $size, $category); } catch (Exception $exception) { return \ES\Common\Response::error($exception->getMessage(), 500); @@ -297,7 +297,7 @@ function tripal_elasticsearch_api_v1_table_index_local_search( $index_name, $http = TRUE ) { try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $fields = $es->getIndexFields($index_name); $field_content_pairs = []; @@ -499,7 +499,7 @@ function tripal_elasticsearch_cross_site_search_form($form, &$form_state) { $default_category = ['Any Type' => 'Any Type']; try { - $es = new Instance(); + $es = new \ES\Common\Instance(); $categories = drupal_map_assoc($es->getAllCategories()); } catch (Exception $exception) { $categories = []; From cd84622d50b125ecf2a87dca0a67f4b6d9e47fd8 Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Thu, 21 Feb 2019 13:07:02 -0500 Subject: [PATCH 16/21] Fix job definition errors --- includes/Common/Instance.php | 2 +- includes/Common/Queue.php | 7 +- includes/Jobs/DispatcherJob.php | 4 +- includes/Jobs/Job.php | 2 + .../tripal_elasticsearch.indices.form.inc | 466 ++++++++++++------ tests/Feature/IndexTest.php | 32 ++ tests/TestCase.php | 12 +- tripal_elasticsearch.module | 4 +- 8 files changed, 366 insertions(+), 163 deletions(-) create mode 100644 tests/Feature/IndexTest.php diff --git a/includes/Common/Instance.php b/includes/Common/Instance.php index 2779431c..bbb30e4f 100644 --- a/includes/Common/Instance.php +++ b/includes/Common/Instance.php @@ -350,7 +350,7 @@ public function formatHits($hits) { if (isset($hit['highlight'])) { $highlight = ''; foreach ($hit['highlight'] as $content) { - $highlight .= implode('...', $content); + $highlight .= implode('... ', $content); } $hit['_source']['highlight'] = $highlight; } diff --git a/includes/Common/Queue.php b/includes/Common/Queue.php index e0e8057b..7a551281 100644 --- a/includes/Common/Queue.php +++ b/includes/Common/Queue.php @@ -3,6 +3,7 @@ namespace ES\Common; use DateTime; +use ES\Jobs\Job; use Exception; use DrupalQueue; @@ -140,7 +141,7 @@ public static function fixProgress(&$queue) { * Dispatch a new job. Uses the queue that has minimum items if queue name is * not provided. * - * @param \ES\Jobs $job + * @param \ES\Jobs\Job $job * @param string $queue_name * * @return boolean @@ -158,7 +159,7 @@ public static function dispatch($job, $queue_name = NULL) { /** * Execute a given job. * - * @param \ES\Jobs $job + * @param \ES\Jobs\Job $job * * @throws \Exception */ @@ -187,7 +188,7 @@ public static function run($job) { throw new Exception( 'Elasticsearch Queue: ' . get_class( $job - ) . ' is an invalid job type. Jobs must extend the ES\Jobs class' + ) . ' is an invalid job type. Jobs must extend the ES\Jobs\Job class' ); } diff --git a/includes/Jobs/DispatcherJob.php b/includes/Jobs/DispatcherJob.php index daadf28c..d4f2d10d 100644 --- a/includes/Jobs/DispatcherJob.php +++ b/includes/Jobs/DispatcherJob.php @@ -33,7 +33,7 @@ class DispatcherJob extends Job{ /** * DispatcherJob constructor. * - * @param \ES\Jobs $job The job object. + * @param \ES\Jobs\Job $job The job object. */ public function __construct($job) { $this->job = $job; @@ -72,7 +72,7 @@ public function dispatch($queue_name = NULL) { /** * Get the job. * - * @return \ES\Jobs + * @return \ES\Jobs\Job */ public function job() { return $this->job; diff --git a/includes/Jobs/Job.php b/includes/Jobs/Job.php index 41dc301e..3ce4e972 100644 --- a/includes/Jobs/Job.php +++ b/includes/Jobs/Job.php @@ -2,6 +2,8 @@ namespace ES\Jobs; +use ES\Common\Queue; + abstract class Job{ /** diff --git a/includes/tripal_elasticsearch.indices.form.inc b/includes/tripal_elasticsearch.indices.form.inc index f86b1398..c4de744f 100644 --- a/includes/tripal_elasticsearch.indices.form.inc +++ b/includes/tripal_elasticsearch.indices.form.inc @@ -20,8 +20,10 @@ function tripal_elasticsearch_indexing_form($form, &$form_state) { $indices = $es->getIndices(); } catch (\Exception $exception) { drupal_set_message($exception->getMessage(), 'warning'); - drupal_set_message("Please check your Elasticsearch Connection.", - 'warning'); + drupal_set_message( + "Please check your Elasticsearch Connection.", + 'warning' + ); return $form; } @@ -59,8 +61,10 @@ function tripal_elasticsearch_indexing_form($form, &$form_state) { $form['index_type'] = [ '#type' => 'select', '#title' => t('Index type'), - '#description' => t('If you already have a website-wide node or tripal - entities index, it must be edited or deleted in the indices list page. Website Tripal entities index is available for Tripal version 3 only.'), + '#description' => t( + 'If you already have a website-wide node or tripal + entities index, it must be edited or deleted in the indices list page. Website Tripal entities index is available for Tripal version 3 only.' + ), '#options' => $index_types, '#default_value' => isset($index_types['website']) ? 'website' : 'database', ]; @@ -69,7 +73,9 @@ function tripal_elasticsearch_indexing_form($form, &$form_state) { '#type' => 'textfield', '#title' => t('Enter a unique Elasticsearch index name'), '#field_suffix' => 'less than 28 characters', - '#description' => t('Elasticsearch index name can only contain lowercase letters, numbers and underscores. It must start with a letter.'), + '#description' => t( + 'Elasticsearch index name can only contain lowercase letters, numbers and underscores. It must start with a letter.' + ), // This field is only visible to table indexing. '#states' => [ 'visible' => [ @@ -84,7 +90,9 @@ function tripal_elasticsearch_indexing_form($form, &$form_state) { '#type' => 'fieldset', '#tree' => TRUE, '#title' => t('Elasticsearch index settings'), - '#description' => t('These settings determine how your data will be indexed and made searchable.'), + '#description' => t( + 'These settings determine how your data will be indexed and made searchable.' + ), '#collapsible' => TRUE, '#collapsed' => TRUE, '#states' => [ @@ -115,8 +123,10 @@ function tripal_elasticsearch_indexing_form($form, &$form_state) { $form['index_table'] = [ '#type' => 'select', '#title' => t('Select a table and fields to index'), - '#options' => array_merge(['' => 'Select a table'], - drupal_map_assoc(tripal_elasticsearch_get_table_list())), + '#options' => array_merge( + ['' => 'Select a table'], + drupal_map_assoc(tripal_elasticsearch_get_table_list()) + ), '#states' => [ 'visible' => [ ':input[name="index_type"]' => [ @@ -151,26 +161,34 @@ function tripal_elasticsearch_indexing_form($form, &$form_state) { $form['table_fields'][$field] = [ '#type' => 'select', '#title' => t('Field name: ' . $field), - '#description' => t('Please select a mapping type for each field. If no + '#description' => t( + 'Please select a mapping type for each field. If no mapping type is selected for a field, that field will not be indexed. This can be used to selectively index - table fields.'), - '#options' => array_merge(['' => 'Select mapping type'], - tripal_elasticsearch_get_field_mapping_types()), + table fields.' + ), + '#options' => array_merge( + ['' => 'Select mapping type'], + tripal_elasticsearch_get_field_mapping_types() + ), ]; } $form['exposed'] = [ '#type' => 'checkbox', '#title' => t('Expose Index to Cross-Site Search'), - '#description' => t("Check this box to expose your index to cross-site search. If exposed, other Tripal sites using this module will be able to search your sites and display the results on their own. Exposing an index is READ ONLY."), + '#description' => t( + "Check this box to expose your index to cross-site search. If exposed, other Tripal sites using this module will be able to search your sites and display the results on their own. Exposing an index is READ ONLY." + ), '#default_value' => FALSE, ]; $form['url'] = [ '#type' => 'textfield', '#title' => 'Results URL Path', - '#description' => t('In order for other sites to link back to your results page, you must specify a path where the form for this index can be reached.'), + '#description' => t( + 'In order for other sites to link back to your results page, you must specify a path where the form for this index can be reached.' + ), '#attributes' => [ 'placeholder' => 'E.g, elasticsearch/index-name', ], @@ -220,29 +238,47 @@ function tripal_elasticsearch_indexing_form_validate($form, &$form_state) { if ($type === "database") { // Index name validation. if (strlen($index_name) > 28) { - form_set_error('index_name', - t('String length cannot be greater than 28.')); + form_set_error( + 'index_name', + t('String length cannot be greater than 28.') + ); } if (!preg_match('/^[A-Za-z][A-Za-z0-9_]+$/', $index_name)) { - form_set_error('index_name', t('index name can only contain lowercase letters, - numbers and underscores, and must start with a letter.')); + form_set_error( + 'index_name', + t( + 'index name can only contain lowercase letters, + numbers and underscores, and must start with a letter.' + ) + ); } // 'website' is reserved for website indexing and search, use a different name // as index name for table search. if ($index_name === 'website') { - form_set_error('index_name', t('"website" is reserved for the website nodes index. Please - use a different name.')); + form_set_error( + 'index_name', + t( + '"website" is reserved for the website nodes index. Please + use a different name.' + ) + ); } // At least one table field need to be selected. - if (!is_array($form_state['values']['table_fields']) || empty($form_state['values']['table_fields'])) { - form_set_error('table_fields', - t('Please specify a mapping type for at least one field.')); + if (!is_array( + $form_state['values']['table_fields'] + ) || empty($form_state['values']['table_fields'])) { + form_set_error( + 'table_fields', + t('Please specify a mapping type for at least one field.') + ); } else { $table_fields = array_filter($form_state['values']['table_fields']); if (empty($table_fields)) { - form_set_error('table_fields', - t('Please specify a mapping type for at least one field.')); + form_set_error( + 'table_fields', + t('Please specify a mapping type for at least one field.') + ); } } } @@ -253,11 +289,17 @@ function tripal_elasticsearch_indexing_form_validate($form, &$form_state) { } else { if ($type === 'entities') { - $count = db_query('SELECT COUNT(id) FROM {tripal_elasticsearch_priority}')->fetchField(); + $count = db_query( + 'SELECT COUNT(id) FROM {tripal_elasticsearch_priority}' + )->fetchField(); if (intval($count) === 0) { - form_set_error(0, - t('Please configure and submit ') . l('the tuning form', - 'admin/tripal/extension/tripal_elasticsearch/tuning') . t(' before creating the entities index.')); + form_set_error( + 0, + t('Please configure and submit ') . l( + 'the tuning form', + 'admin/tripal/extension/tripal_elasticsearch/tuning' + ) . t(' before creating the entities index.') + ); } } } @@ -291,8 +333,10 @@ function tripal_elasticsearch_delete_indices_form($form, &$form_state) { $mappings = $client->indices()->getMapping(); } catch (\Exception $exception) { drupal_set_message($exception->getMessage(), 'warning'); - drupal_set_message("Please check your Elasticsearch Connection.", - 'warning'); + drupal_set_message( + "Please check your Elasticsearch Connection.", + 'warning' + ); return NULL; } @@ -301,7 +345,9 @@ function tripal_elasticsearch_delete_indices_form($form, &$form_state) { foreach (array_keys($mappings) as $index) { if (isset(array_keys($mappings[$index]['mappings'])[1])) { if (!in_array($index, ['entities', 'website'])) { - $existing_indices[$index] = $index . ' (indexed table: ' . array_keys($mappings[$index]['mappings'])[1] . ')'; + $existing_indices[$index] = $index . ' (indexed table: ' . array_keys( + $mappings[$index]['mappings'] + )[1] . ')'; } else { $existing_indices[$index] = $index . ' (index for site-wide search)'; @@ -315,7 +361,9 @@ function tripal_elasticsearch_delete_indices_form($form, &$form_state) { $form['indices'] = [ '#type' => 'checkboxes', '#title' => t('Existing indices'), - '#description' => t('delete indexed data from Eleasticsearch. Please make sure you know what the data is. This process can NOT be undone.'), + '#description' => t( + 'delete indexed data from Eleasticsearch. Please make sure you know what the data is. This process can NOT be undone.' + ), '#options' => $existing_indices, ]; @@ -324,8 +372,12 @@ function tripal_elasticsearch_delete_indices_form($form, &$form_state) { '#value' => t('Delete'), ]; - drupal_add_js(drupal_get_path('module', - 'tripal_elasticsearch') . '/js/indices_delete_confirm.js'); + drupal_add_js( + drupal_get_path( + 'module', + 'tripal_elasticsearch' + ) . '/js/indices_delete_confirm.js' + ); return $form; } @@ -343,10 +395,12 @@ function tripal_elasticsearch_delete_indices_form_submit($form, &$form_state) { try { $es = new \ES\Common\Instance(); $es->deleteIndex($index); - db_query('DELETE FROM {tripal_elasticsearch_indices} WHERE index_name=:index_name', + db_query( + 'DELETE FROM {tripal_elasticsearch_indices} WHERE index_name=:index_name', [ ':index_name' => $index, - ]); + ] + ); } catch (\Exception $exception) { drupal_set_message($exception->getMessage(), 'warning'); } @@ -367,8 +421,10 @@ function tripal_elasticsearch_indices_list_page() { $indices = $es->getIndices(); } catch (\Exception $exception) { drupal_set_message($exception->getMessage(), 'warning'); - drupal_set_message("Please check your Elasticsearch Connection.", - 'warning'); + drupal_set_message( + "Please check your Elasticsearch Connection.", + 'warning' + ); return ''; } @@ -405,14 +461,20 @@ function tripal_elasticsearch_indices_list_page() { $exposed_text = "public"; } - $edit = l('Edit', - 'admin/tripal/extension/tripal_elasticsearch/indices/edit/' . $index); - $delete = l('Delete', - 'admin/tripal/extension/tripal_elasticsearch/indices/delete/' . $index); + $edit = l( + 'Edit', + 'admin/tripal/extension/tripal_elasticsearch/indices/edit/' . $index + ); + $delete = l( + 'Delete', + 'admin/tripal/extension/tripal_elasticsearch/indices/delete/' . $index + ); $row = [$index, $table, $exposed_text, $edit, $delete]; if (in_array($index, ['entities', 'gene_search_index'])) { - $row[] = l('Update', - 'admin/tripal/extension/tripal_elasticsearch/indices/update/' . $index); + $row[] = l( + 'Update', + 'admin/tripal/extension/tripal_elasticsearch/indices/update/' . $index + ); } else { $row[] = t('Update not available'); @@ -421,19 +483,24 @@ function tripal_elasticsearch_indices_list_page() { } $output = '

List of Available Indices

'; - $output .= theme('table', [ - 'header' => [ - 'Index Name', - 'Indexed Table', - 'Exposed', - 'Edit', - 'Delete', - 'update', - ], - 'rows' => $rows, - ]); - $link = l('Create Index', - 'admin/tripal/extension/tripal_elasticsearch/indices/create'); + $output .= theme( + 'table', + [ + 'header' => [ + 'Index Name', + 'Indexed Table', + 'Exposed', + 'Edit', + 'Delete', + 'update', + ], + 'rows' => $rows, + ] + ); + $link = l( + 'Create Index', + 'admin/tripal/extension/tripal_elasticsearch/indices/create' + ); $output .= '

To create a new index, click the ' . $link . ' tab above.

'; return $output; @@ -491,11 +558,13 @@ function tripal_elasticsearch_index_edit_confirm($form, &$form_state, $index_nam $table_name = $table_name[0]; } - db_query("INSERT INTO {tripal_elasticsearch_indices} (index_name, table_name, exposed, url) VALUES(:index_name, :table_name, 0, '')", + db_query( + "INSERT INTO {tripal_elasticsearch_indices} (index_name, table_name, exposed, url) VALUES(:index_name, :table_name, 0, '')", [ ':index_name' => $index_name, ':table_name' => $table_name, - ]); + ] + ); $result = db_query($sql, [':index_name' => $index_name])->fetchObject(); } catch (Exception $exception) { drupal_set_message($exception->getMessage(), 'error'); @@ -542,16 +611,20 @@ function tripal_elasticsearch_index_edit_confirm($form, &$form_state, $index_nam $preset_mappings = []; if ($index_type == 'table') { - if (array_key_exists($table_name, - $index_mappings[$index_name]["mappings"])) { + if (array_key_exists( + $table_name, + $index_mappings[$index_name]["mappings"] + )) { $preset_mappings = $index_mappings[$index_name]["mappings"][$table_name]["properties"]; } } if (!$preset_mappings && $index_type == "table") { // there is no dbtable mapping settings yet. Cron still needs to run! - drupal_set_message("The index $index_name has no database mapping. Please make sure that your CRON job has finished running.", - 'warning'); + drupal_set_message( + "The index $index_name has no database mapping. Please make sure that your CRON job has finished running.", + 'warning' + ); } $index_settings = ["tokenizer" => "standard", "token_filters" => ""]; @@ -569,9 +642,12 @@ function tripal_elasticsearch_index_edit_confirm($form, &$form_state, $index_nam // to re-index need to build it the same way the options are built so they match $previous_token_filters = tripal_elasticsearch_get_token_filter_options(); - $previous_token_filters = array_map(function ($value) { - return 0; - }, $previous_token_filters); //empty the array values + $previous_token_filters = array_map( + function ($value) { + return 0; + }, + $previous_token_filters + ); //empty the array values foreach ($index_settings["token_filters"] as $setting) { $previous_token_filters[$setting] = $setting; } @@ -601,7 +677,9 @@ function tripal_elasticsearch_index_edit_confirm($form, &$form_state, $index_nam '#type' => 'fieldset', '#tree' => TRUE, '#title' => t('Elasticsearch index settings'), - '#description' => t('These settings determine how your data will be indexed and made searchable.'), + '#description' => t( + 'These settings determine how your data will be indexed and made searchable.' + ), '#collapsible' => TRUE, '#collapsed' => TRUE, ]; @@ -624,7 +702,9 @@ function tripal_elasticsearch_index_edit_confirm($form, &$form_state, $index_nam $form['exposed'] = [ '#type' => 'checkbox', '#title' => t('Expose Index to Cross-Site Search'), - '#description' => t("Check this box to expose your index to cross-site search. If exposed, other Tripal sites using this module will be able to search your sites and display the results on their own. Exposing an index is READ ONLY."), + '#description' => t( + "Check this box to expose your index to cross-site search. If exposed, other Tripal sites using this module will be able to search your sites and display the results on their own. Exposing an index is READ ONLY." + ), '#default_value' => $exposed == 1 ? TRUE : FALSE, ]; @@ -633,7 +713,9 @@ function tripal_elasticsearch_index_edit_confirm($form, &$form_state, $index_nam $form['url'] = [ '#type' => 'textfield', '#title' => 'Results URL Path', - '#description' => t('In order for other sites to link back to your results page, you must specify a path where the form for this index can be reached.'), + '#description' => t( + 'In order for other sites to link back to your results page, you must specify a path where the form for this index can be reached.' + ), '#attributes' => [ 'placeholder' => 'E.g, elasticsearch/index-name', ], @@ -656,10 +738,12 @@ function tripal_elasticsearch_index_edit_confirm($form, &$form_state, $index_nam $form['table_fields'] = [ '#type' => 'fieldset', '#title' => t('Select fields to index'), - '#description' => t('Please select a mapping type for each field. If no + '#description' => t( + 'Please select a mapping type for each field. If no mapping type is selected for a field, that field will not be indexed. This can be used to selectively index - table fields.'), + table fields.' + ), '#tree' => TRUE, '#collapsible' => TRUE, '#collapsed' => TRUE, @@ -676,16 +760,23 @@ function tripal_elasticsearch_index_edit_confirm($form, &$form_state, $index_nam '#type' => 'select', '#title' => t('Field name: ' . $field), '#default_value' => $preset, - '#options' => array_merge(['' => 'Select mapping type'], - tripal_elasticsearch_get_field_mapping_types()), + '#options' => array_merge( + ['' => 'Select mapping type'], + tripal_elasticsearch_get_field_mapping_types() + ), ]; } $description = 'Edit settings for ' . $index_name; $cancel_path = 'admin/tripal/extension/tripal_elasticsearch/indices'; - return confirm_form($form, 'Edit Index', $cancel_path, $description, - 'Submit Edits'); + return confirm_form( + $form, + 'Edit Index', + $cancel_path, + $description, + 'Submit Edits' + ); } /** @@ -706,12 +797,18 @@ function tripal_elasticsearch_index_edit_confirm_submit($form, &$form_state) { // Delete/Create for custom tables since the field settings might have changed. if ($index == $previous_values['index_name'] && $previous_index_settings['tokenizer'] && $new_index_settings['tokenizer']) { - if (empty(array_diff($previous_index_settings['token_filters'], - $new_index_settings['token_filters']))) { + if (empty( + array_diff( + $previous_index_settings['token_filters'], + $new_index_settings['token_filters'] + ) + )) { if ($type != 'table') { // No need to delete the index, just need to update the table. tripal_elasticsearch_update_index_table($form_state['values']); - drupal_set_message("Your index has been updated. Re-indexing is not necessary."); + drupal_set_message( + "Your index has been updated. Re-indexing is not necessary." + ); $form_state['redirect'] = 'admin/tripal/extension/tripal_elasticsearch/indices'; return; } @@ -744,8 +841,13 @@ function tripal_elasticsearch_index_delete_confirm($form, &$form_state, $index_n '#value' => $index_name, ]; - return confirm_form($form, 'Are you sure?', $cancel_path, $description, - 'Delete'); + return confirm_form( + $form, + 'Are you sure?', + $cancel_path, + $description, + 'Delete' + ); } /** @@ -777,10 +879,12 @@ function tripal_elasticsearch_create_index(&$values) { $index_name = 'website'; $index_table = 'node'; $tokenizer = 'standard'; - $token_filters = drupal_map_assoc([ - 'standard', - 'lowercase', - ]); + $token_filters = drupal_map_assoc( + [ + 'standard', + 'lowercase', + ] + ); $field_mapping_types = [ 'nid' => 'integer', 'type' => 'text', @@ -792,28 +896,29 @@ function tripal_elasticsearch_create_index(&$values) { $index_name = 'entities'; $index_table = 'tripal_entity'; $tokenizer = 'standard'; - $token_filters = drupal_map_assoc([ - 'standard', - 'lowercase', - ]); + $token_filters = drupal_map_assoc( + [ + 'standard', + 'lowercase', + ] + ); $field_mapping_types = [ 'entity_id' => 'integer', 'bundle_label' => 'text', 'title' => 'text', - 'content' => [ - 'dynamic' => TRUE, - 'properties' => [], - ], + 'content' => 'dynamic', ]; } elseif ($index_type === 'gene_search') { $index_name = 'gene_search_index'; $index_table = 'chado.feature'; $tokenizer = 'standard'; - $token_filters = drupal_map_assoc([ - 'standard', - 'lowercase', - ]); + $token_filters = drupal_map_assoc( + [ + 'standard', + 'lowercase', + ] + ); $field_mapping_types = [ 'node_id' => 'integer', 'entity_id' => 'integer', @@ -850,10 +955,12 @@ function tripal_elasticsearch_create_index(&$values) { $tokenizer = 'standard'; } if (!$token_filters) { - $token_filters = drupal_map_assoc([ - 'standard', - 'lowercase', - ]); + $token_filters = drupal_map_assoc( + [ + 'standard', + 'lowercase', + ] + ); } } @@ -865,14 +972,22 @@ function tripal_elasticsearch_create_index(&$values) { ':url' => $url, ]; - db_query('INSERT INTO {tripal_elasticsearch_indices} (index_name, table_name, exposed, url) VALUES (:index_name, :table_name, :exposed, :url)', - $insert); + db_query( + 'INSERT INTO {tripal_elasticsearch_indices} (index_name, table_name, exposed, url) VALUES (:index_name, :table_name, :exposed, :url)', + $insert + ); // Create the index. try { $es = new \ES\Common\Instance(); - $es->setIndexParams($index_name, 5, 0, $tokenizer, $token_filters, - $field_mapping_types)->createIndex(); + $es->setIndexParams( + $index_name, + 5, + 0, + $tokenizer, + $token_filters, + $field_mapping_types + )->createIndex(); } catch (Exception $exception) { form_set_error(0, $exception->getMessage()); watchdog('tripal_elasticsearch', $exception->getMessage()); @@ -884,34 +999,42 @@ function tripal_elasticsearch_create_index(&$values) { $job = NULL; switch ($index_type) { case 'website': - $job = new NodesIndexJob(); + $job = new \ES\Jobs\NodesIndexJob(); break; case 'gene_search': - GeneSearchIndexJob::generateDispatcherJobs(); + \ES\Jobs\GeneSearchIndexJob::generateDispatcherJobs(); break; case 'entities': - EntitiesIndexJob::generateDispatcherJobs(); + \ES\Jobs\EntitiesIndexJob::generateDispatcherJobs(); break; default: $fields = array_keys($field_mapping_types); - $job = new TableIndexJob($index_name, $index_table, $fields); + $job = new \ES\Jobs\TableIndexJob($index_name, $index_table, $fields); } // All jobs other than entities can be dispatched directly. // Tripal entities must be dispatched by bundle type. if ($job !== NULL) { - $dispatcher = new DispatcherJob($job); + $dispatcher = new \ES\Jobs\DispatcherJob($job); $dispatcher->dispatch(); } $base_url = variable_get('website_base_url'); - $cron_url = l($base_url . '/admin/config/system/cron', - 'admin/config/system/cron'); - - drupal_set_message("The indexing job for {$index_name} has been submitted to your CRON queue. You can view the status of your CRON jobs at {$cron_url}"); + $cron_url = l( + $base_url . '/admin/config/system/cron', + 'admin/config/system/cron' + ); + + drupal_set_message( + "The indexing job for {$index_name} has been submitted to your CRON queue. You can view the status of your CRON jobs at {$cron_url}" + ); if ($index_type === 'database') { - drupal_set_message("You may create a search block for this index using the " . l('form management', - 'admin/tripal/extension/tripal_elasticsearch/forms') . " section."); + drupal_set_message( + "You may create a search block for this index using the " . l( + 'form management', + 'admin/tripal/extension/tripal_elasticsearch/forms' + ) . " section." + ); } } @@ -932,48 +1055,67 @@ function tripal_elasticsearch_delete_index($index) { } // Find the index and delete it from the DB. this db is for the search blocks - db_query("DELETE FROM {tripal_elasticsearch} WHERE index_name=:index_name", - [':index_name' => $index]); + db_query( + "DELETE FROM {tripal_elasticsearch} WHERE index_name=:index_name", + [':index_name' => $index] + ); // Now delete the table info - db_query("DELETE FROM {tripal_elasticsearch_indices} WHERE index_name=:index_name", - [':index_name' => $index]); + db_query( + "DELETE FROM {tripal_elasticsearch_indices} WHERE index_name=:index_name", + [':index_name' => $index] + ); // Delete any progress indicator - db_query('DELETE FROM {tripal_elasticsearch_queues} WHERE index_name=:index_name', - [':index_name' => $index]); + db_query( + 'DELETE FROM {tripal_elasticsearch_queues} WHERE index_name=:index_name', + [':index_name' => $index] + ); } /** * Update an index. * * @param string $index index name + * + * @return void */ function tripal_elasticsearch_update_index($index) { - $result = db_query("SELECT * FROM {tripal_elasticsearch_indices} WHERE index_name=:index_name", - [':index_name' => $index])->fetchObject(); + $result = db_query( + "SELECT * FROM {tripal_elasticsearch_indices} WHERE index_name=:index_name", + [':index_name' => $index] + )->fetchObject(); if (!$result) { drupal_set_message("Index \"{$index}\" not found", 'error'); - return drupal_goto('admin/tripal/extension/tripal_elasticsearch/indices'); + drupal_goto('admin/tripal/extension/tripal_elasticsearch/indices'); + return; } if (!in_array($index, ['entities', 'gene_search_index'])) { - drupal_set_message('Update functionality is not available for this index. Please delete and recreate the index.', - 'error'); - return drupal_goto('admin/tripal/extension/tripal_elasticsearch/indices'); + drupal_set_message( + 'Update functionality is not available for this index. Please delete and recreate the index.', + 'error' + ); + drupal_goto('admin/tripal/extension/tripal_elasticsearch/indices'); + return; } switch ($index) { case 'gene_search_index': - GeneSearchIndexJob::generateDispatcherJobs(TRUE); + \ES\Jobs\GeneSearchIndexJob::generateDispatcherJobs(TRUE); break; case 'entities': - return drupal_goto('admin/tripal/extension/tripal_elasticsearch/indices/update_entities'); + drupal_goto( + 'admin/tripal/extension/tripal_elasticsearch/indices/update_entities' + ); + return; break; } - drupal_set_message("{$index} updating jobs have been dispatched successfully. Please run `drush cron` to process items in the queue."); + drupal_set_message( + "{$index} updating jobs have been dispatched successfully. Please run `drush cron` to process items in the queue." + ); drupal_goto('admin/tripal/extension/tripal_elasticsearch/indices'); } @@ -983,8 +1125,12 @@ function tripal_elasticsearch_update_index($index) { function tripal_elasticsearch_progress_page($callback = FALSE) { $content = ''; if (!$callback) { - drupal_add_js(drupal_get_path('module', - 'tripal_elasticsearch') . '/js/tripal_elasticsearch_progress_tracker.js'); + drupal_add_js( + drupal_get_path( + 'module', + 'tripal_elasticsearch' + ) . '/js/tripal_elasticsearch_progress_tracker.js' + ); $content .= '
'; } @@ -998,12 +1144,17 @@ function tripal_elasticsearch_progress_page($callback = FALSE) { $content .= '

All items have been indexed. 0 Items remaining.

'; } else { - $etr = tripal_elasticsearch_compute_ETR($data->total, $data->completed, - $data->time); - $content .= theme_progress_bar([ - 'percent' => $data->percent, - 'message' => 'Indexing ' . $data->completed . '/' . $data->total . ' items. Estimated time remaining: ' . $etr, - ]); + $etr = tripal_elasticsearch_compute_ETR( + $data->total, + $data->completed, + $data->time + ); + $content .= theme_progress_bar( + [ + 'percent' => $data->percent, + 'message' => 'Indexing ' . $data->completed . '/' . $data->total . ' items. Estimated time remaining: ' . $etr, + ] + ); } if (count($data->queues) > 1) { @@ -1014,12 +1165,17 @@ function tripal_elasticsearch_progress_page($callback = FALSE) { $content .= '

All items have been indexed. 0 Items remaining

'; } else { - $etr = tripal_elasticsearch_compute_ETR($progress->total, - $progress->completed, $progress->time); - $content .= theme_progress_bar([ - 'percent' => $progress->percent, - 'message' => $progress->remaining . ' Items remaining. Estimated time remaining: ' . $etr, - ]); + $etr = tripal_elasticsearch_compute_ETR( + $progress->total, + $progress->completed, + $progress->time + ); + $content .= theme_progress_bar( + [ + 'percent' => $progress->percent, + 'message' => $progress->remaining . ' Items remaining. Estimated time remaining: ' . $etr, + ] + ); } } } @@ -1050,11 +1206,13 @@ function tripal_elasticsearch_update_index_table(&$values) { $url = isset($values['url']) ? $values['url'] : ''; try { - db_update('tripal_elasticsearch_indices')->fields([ - 'table_name' => $index_table, - 'exposed' => $exposed, - 'url' => $url, - ])->condition('table_name', $index_table)->execute(); + db_update('tripal_elasticsearch_indices')->fields( + [ + 'table_name' => $index_table, + 'exposed' => $exposed, + 'url' => $url, + ] + )->condition('table_name', $index_table)->execute(); drupal_set_message(t('Index \'' . $index_name . '\' has been updated.')); } catch (Exception $e) { drupal_set_message("could not edit index"); @@ -1128,13 +1286,15 @@ function tripal_elasticsearch_update_entities_form_submit($form, &$form_state) { $bundle = $form_state['values']['bundle']; if (!empty($bundle)) { - EntitiesIndexJob::generateDispatcherJobs(1, FALSE, $bundle); + \ES\Jobs\EntitiesIndexJob::generateDispatcherJobs(1, FALSE, $bundle); } else { - EntitiesIndexJob::generateDispatcherJobs(1, TRUE); + \ES\Jobs\EntitiesIndexJob::generateDispatcherJobs(1, TRUE); } - drupal_set_message("Entities updating jobs have been dispatched successfully. Please run `drush cron` to process items in the queue."); + drupal_set_message( + "Entities updating jobs have been dispatched successfully. Please run `drush cron` to process items in the queue." + ); $form_state['redirect'] = 'admin/tripal/extension/tripal_elasticsearch/indices'; } diff --git a/tests/Feature/IndexTest.php b/tests/Feature/IndexTest.php new file mode 100644 index 00000000..66d77827 --- /dev/null +++ b/tests/Feature/IndexTest.php @@ -0,0 +1,32 @@ +makeInstance(); + $index = new Index($instance); + + $name = uniqid(); + + $data = $index->setIndexName($name) + ->setFields([ + 'content' => 'dynamic', + 'test' => 'text' + ]) + ->createIndex(); + + $this->assertTrue($data['acknowledged']); + $this->assertEquals($index->getIndexName(), $data['index']); + + var_dump($index->getFields(true)); + + $data = $index->deleteIndex(); + $this->assertTrue($data['acknowledged']); + } +} diff --git a/tests/TestCase.php b/tests/TestCase.php index cbe4857f..ef915b01 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -28,13 +28,21 @@ protected function tearDown() { parent::tearDown(); } + /** + * @return \ES\Common\Instance + * @throws \Exception + */ + public function makeInstance() { + return new \ES\Common\Instance(getenv('ES_HOST')); + } + /** * @param string $name * * @throws \Exception */ public function deleteIndex($name) { - $es = new \ES\Common\Instance(getenv('ES_HOST')); + $es = $this->makeInstance(); $es->deleteIndex($name); } @@ -45,7 +53,7 @@ public function deleteIndex($name) { * @throws \Exception */ public function makeIndex($name = NULL, $fields = []) { - $es = new \ES\Common\Instance(getenv('ES_HOST')); + $es = $this->makeInstance(); if (is_null($name)) { $name = uniqid(); diff --git a/tripal_elasticsearch.module b/tripal_elasticsearch.module index 95adf1f3..36975af6 100644 --- a/tripal_elasticsearch.module +++ b/tripal_elasticsearch.module @@ -1239,7 +1239,7 @@ function tripal_elasticsearch_importer_finish($importer) { ]; if (in_array($name, $supported_importers)) { - GeneSearchIndexJob::generateDispatcherJobs(TRUE); - EntitiesIndexJob::generateDispatcherJobs(1, TRUE); + \ES\Jobs\GeneSearchIndexJob::generateDispatcherJobs(TRUE); + \ES\Jobs\EntitiesIndexJob::generateDispatcherJobs(1, TRUE); } } From 3451ce468c5ed988fa7bdd44c298668d2830f56e Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Thu, 21 Feb 2019 14:04:00 -0500 Subject: [PATCH 17/21] Fix table name typo --- includes/tripal_elasticsearch.indices.form.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/includes/tripal_elasticsearch.indices.form.inc b/includes/tripal_elasticsearch.indices.form.inc index c4de744f..644c6900 100644 --- a/includes/tripal_elasticsearch.indices.form.inc +++ b/includes/tripal_elasticsearch.indices.form.inc @@ -1212,7 +1212,7 @@ function tripal_elasticsearch_update_index_table(&$values) { 'exposed' => $exposed, 'url' => $url, ] - )->condition('table_name', $index_table)->execute(); + )->condition('index_name', $index_table)->execute(); drupal_set_message(t('Index \'' . $index_name . '\' has been updated.')); } catch (Exception $e) { drupal_set_message("could not edit index"); From c5a10b863e946fc59a4092517c61d150d505c5f4 Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Fri, 22 Feb 2019 08:02:10 -0500 Subject: [PATCH 18/21] Clean up the index from empty fields and reduce complexity to a single string --- includes/Jobs/EntitiesIndexJob.php | 104 ++++++++++++------ .../tripal_elasticsearch.collections.form.inc | 81 +++++++++----- ...ipal_elasticsearch.website_search.form.inc | 78 ++++++------- tripal_elasticsearch.module | 2 +- 4 files changed, 164 insertions(+), 101 deletions(-) diff --git a/includes/Jobs/EntitiesIndexJob.php b/includes/Jobs/EntitiesIndexJob.php index b30b3440..17e610ad 100644 --- a/includes/Jobs/EntitiesIndexJob.php +++ b/includes/Jobs/EntitiesIndexJob.php @@ -1,5 +1,7 @@ loadContent($entities); foreach ($records as $record) { - $this->es->createOrUpdate($this->index, $this->index, - $record->entity_id, $record); + $this->es->createOrUpdate( + $this->index, + $this->index, + $record->entity_id, + $record + ); } - } catch (Exception $exception) { - tripal_report_error('tripal_elasticsearch', TRIPAL_ERROR, - $exception->getMessage()); + } catch (\Exception $exception) { + tripal_report_error( + 'tripal_elasticsearch', + TRIPAL_ERROR, + $exception->getMessage() + ); } } @@ -116,16 +125,23 @@ protected function loadContent($records) { $all = []; // Load entities and applicable fields - $ids = array_map(function ($record) { - return $record->entity_id; - }, $records); + $ids = array_map( + function ($record) { + return $record->entity_id; + }, + $records + ); $fields = field_info_instances('TripalEntity', $this->bundle); // Load priority list $priority = $this->getPriorityList($fields); - $entities = tripal_load_entity('TripalEntity', $ids, TRUE, - $priority['ids']); + $entities = tripal_load_entity( + 'TripalEntity', + $ids, + TRUE, + $priority['ids'] + ); foreach ($records as $record) { if (!isset($entities[$record->entity_id])) { continue; @@ -135,17 +151,17 @@ protected function loadContent($records) { $content = []; if (tripal_entity_access('view', $entity)) { foreach ($priority['names'] as $field) { - if (property_exists($entity, - $field) && isset($entity->{$field}['und'])) { + $has_prop = property_exists($entity, $field); + if ($has_prop && isset($entity->{$field}['und'])) { $content[$field] = []; + foreach ($entity->{$field}['und'] as $elements) { if (!isset($elements['value'])) { continue; } - $value = $this->extractValue($elements['value']); if (empty($value)) { @@ -155,7 +171,17 @@ protected function loadContent($records) { $content[$field][] = $value; //$content[] = $value; } + + $content[$field] = elasticsearch_recursive_implode( + ' ', + $content[$field] + ); + + if(empty($content[$field])) { + unset($content[$field]); + } } + } } @@ -230,13 +256,17 @@ protected function getAllFields($fields) { */ protected function prioritizeFields($fields) { if ($this->priority_round < 2 && $this->id === NULL) { - $results = db_query('SELECT * FROM {tripal_elasticsearch_priority} WHERE priority = :priority', + $results = db_query( + 'SELECT * FROM {tripal_elasticsearch_priority} WHERE priority = :priority', [ ':priority' => 1, - ])->fetchAll(); + ] + )->fetchAll(); } else { - $results = db_query('SELECT * FROM {tripal_elasticsearch_priority}')->fetchAll(); + $results = db_query( + 'SELECT * FROM {tripal_elasticsearch_priority}' + )->fetchAll(); } $indexed = []; @@ -325,7 +355,7 @@ protected function flatten($array, &$items) { * Get records to index. * * @return mixed - * @throws \Exception + * @throws \\Exception */ protected function get() { if ($this->id !== NULL) { @@ -333,7 +363,9 @@ protected function get() { } if ($this->limit === NULL || $this->offset === NULL) { - throw new Exception('EntitiesIndexJob: Limit and offset parameters are required if node id is not provided in the constructor.'); + throw new \Exception( + 'EntitiesIndexJob: Limit and offset parameters are required if node id is not provided in the constructor.' + ); } return $this->getMultipleEntities(); @@ -353,11 +385,14 @@ protected function getMultipleEntities() { OFFSET :offset LIMIT :limit'; - return db_query($query, [ - ':bundle' => $this->bundle, - ':limit' => $this->limit, - ':offset' => $this->offset, - ])->fetchAll(); + return db_query( + $query, + [ + ':bundle' => $this->bundle, + ':limit' => $this->limit, + ':offset' => $this->offset, + ] + )->fetchAll(); } /** @@ -391,8 +426,10 @@ public function total() { * @return int */ public function count() { - return db_query('SELECT COUNT(id) FROM {tripal_entity} WHERE status=1 AND bundle=:bundle', - [':bundle' => $this->bundle])->fetchField(); + return db_query( + 'SELECT COUNT(id) FROM {tripal_entity} WHERE status=1 AND bundle=:bundle', + [':bundle' => $this->bundle] + )->fetchField(); } /** @@ -408,13 +445,16 @@ public static function generateDispatcherJobs($round = 1, $clear_queue = FALSE, if ($clear_queue) { // Clear all entries from the queue $sql = 'SELECT item_id, data FROM queue q WHERE name LIKE :name'; - $results = db_query($sql, - [':name' => db_like('elasticsearch') . '%'])->fetchAll(); + $results = db_query( + $sql, + [':name' => db_like('elasticsearch') . '%'] + )->fetchAll(); $delete = []; foreach ($results as $result) { $class = unserialize($result->data); - if ($class instanceof DispatcherJob && $class->job() instanceof EntitiesIndexJob) { + if ($class instanceof DispatcherJob && $class->job( + ) instanceof EntitiesIndexJob) { $delete[] = $result->item_id; } elseif ($class instanceof EntitiesIndexJob) { @@ -423,8 +463,10 @@ public static function generateDispatcherJobs($round = 1, $clear_queue = FALSE, } if (!empty($delete)) { - $dsql = 'DELETE FROM queue WHERE item_id IN (' . implode(',', - $delete) . ')'; + $dsql = 'DELETE FROM queue WHERE item_id IN (' . implode( + ',', + $delete + ) . ')'; db_query($dsql)->execute(); } } diff --git a/includes/tripal_elasticsearch.collections.form.inc b/includes/tripal_elasticsearch.collections.form.inc index 72151571..f9c5858b 100644 --- a/includes/tripal_elasticsearch.collections.form.inc +++ b/includes/tripal_elasticsearch.collections.form.inc @@ -9,8 +9,10 @@ function tripal_elasticsearch_collections_form($form, &$form_state) { if (!user_is_logged_in()) { $content = '

'; - $content .= t('To create a collection, you must first ') . l('sign in', - 'user') . t(' or ') . l('register a new account', 'user/register'); + $content .= t('To create a collection, you must first ') . l( + 'sign in', + 'user' + ) . t(' or ') . l('register a new account', 'user/register'); $content .= '

'; $form['login'] = [ '#type' => 'markup', @@ -77,20 +79,27 @@ function tripal_elasticsearch_collections_form($form, &$form_state) { if ($selected_category || (isset($form_state['values']) && $form_state['values']['bundle_type'])) { $bundle = isset($form_state['values']) ? $form_state['values']['bundle_type'] : $selected_category; $field_ids = tripal_elasticsearch_get_fields_as_options($bundle); - $defaults = array_filter($field_ids, function ($field) { - return strstr(strtolower($field), - 'sequence') !== FALSE || strstr(strtolower($field), 'name') !== FALSE; - }); + $defaults = array_filter( + $field_ids, + function ($field) { + return strstr( + strtolower($field), + 'sequence' + ) !== FALSE || strstr(strtolower($field), 'name') !== FALSE; + } + ); $form['step2']['fields'] = [ '#title' => t('Select One or More Fields'), - '#description' => t('Please select the fields to include in this data + '#description' => t( + 'Please select the fields to include in this data collection. Not all of these fields appear in the search results above but they are available for this category. By default, tab-delimited and comma-separated files are generated for the collection using only the fields selected. However, some fields when selected will generate other downloadable file formats. Fields that - generate other file formats are indicated.'), + generate other file formats are indicated.' + ), '#type' => 'checkboxes', '#options' => $field_ids, '#prefix' => '
', @@ -117,7 +126,9 @@ function tripal_elasticsearch_collections_form($form, &$form_state) { '#type' => 'textarea', '#title' => t('Description'), '#required' => FALSE, - '#description' => t('Optionally, you can add a more detailed description for this collection.'), + '#description' => t( + 'Optionally, you can add a more detailed description for this collection.' + ), ]; // hidden elements @@ -143,8 +154,10 @@ function tripal_elasticsearch_collections_form_submit($form, &$form_state) { $query = $values['search_query']; // get the bundle label - $bundle_label = db_query('SELECT name FROM {tripal_bundle} WHERE name=:name', - [':name' => $bundle_name])->fetchField(); + $bundle_label = db_query( + 'SELECT name FROM {tripal_bundle} WHERE name=:name', + [':name' => $bundle_name] + )->fetchField(); $entities = []; try { @@ -153,33 +166,47 @@ function tripal_elasticsearch_collections_form_submit($form, &$form_state) { $count = $es->count(); for ($i = 0; $i < $count; $i += 500) { - $results = $es->setWebsiteSearchParams($query, $bundle_label, '', '', [ - $i, - 500, - ], TRUE)->search(); + $results = $es->setWebsiteSearchParams( + $query, + $bundle_label, + '', + '', + [ + $i, + 500, + ], + TRUE + )->search(); foreach ($results as $entity) { $entities[] = $entity['entity_id']; } } } catch (Exception $exception) { - drupal_set_message('Could not create collection due to an internal server error. Please try again later.', - 'error'); - tripal_report_error('tripal_elasticsearch', TRIPAL_ERROR, - $exception->getMessage()); + drupal_set_message( + 'Could not create collection due to an internal server error. Please try again later.', + 'error' + ); + tripal_report_error( + 'tripal_elasticsearch', + TRIPAL_ERROR, + $exception->getMessage() + ); return; } global $user; - tripal_create_collection([ - 'uid' => $user->uid, - 'collection_name' => $collection_name, - 'bundle_name' => $bundle_label, - 'ids' => $entities, - 'fields' => $fields, - 'description' => $description, - ]); + tripal_create_collection( + [ + 'uid' => $user->uid, + 'collection_name' => $collection_name, + 'bundle_name' => $bundle_label, + 'ids' => $entities, + 'fields' => $fields, + 'description' => $description, + ] + ); $form_state['redirect'] = 'user/' . $user->uid . '/files'; } diff --git a/includes/tripal_elasticsearch.website_search.form.inc b/includes/tripal_elasticsearch.website_search.form.inc index 547cf088..df8079f0 100644 --- a/includes/tripal_elasticsearch.website_search.form.inc +++ b/includes/tripal_elasticsearch.website_search.form.inc @@ -6,14 +6,14 @@ */ /** - * Render array for website_search_box_form. + * Render array for tripal_elasticsearch_site_wide_search_form. * * @param $form * @param $form_state * * @return mixed */ -function website_search_box_form($form, &$form_state) { +function tripal_elasticsearch_site_wide_search_form($form, &$form_state) { global $base_url; $default = isset($_GET['search_box']) && $_GET['search_box'] ? $_GET['search_box'] : ''; @@ -31,12 +31,6 @@ function website_search_box_form($form, &$form_state) { '#default_value' => $default, ]; - if (strstr(current_path(), 'tripal_elasticsearch/search_website') !== FALSE) { - $form['container']['search_box']['#attributes'] = [ - 'autofocus' => 'autofocus', - ]; - } - $form['container']['submit'] = [ '#type' => 'submit', '#value' => t('Search'), @@ -53,26 +47,6 @@ function website_search_box_form($form, &$form_state) { return $form; } -/** - * website_search_box_form submit. - * - * @param $form - * @param $form_state - */ -function website_search_box_form_submit($form, &$form_state) { - /* - * $keyword = $form_state['values']['search_box']; - $url = "tripal_elasticsearch/search_website/$keyword"; - // only redirect to the search results page when $keyword is not empty. - if (empty($keyword)) { - drupal_goto(current_path()); - } - else { - drupal_goto($url); - } - */ -} - /** * Form builder function for table search. * @@ -109,19 +83,28 @@ function tripal_elasticsearch_build_search_block_form($form, &$form_state, $inde preg_match_all('/\[.+?\]/', $select_options_text, $matches); $select_options_pairs = $matches[0]; - $select_options_keys = array_map(function ($string) { - $string = preg_replace('/\[/', '', $string); + $select_options_keys = array_map( + function ($string) { + $string = preg_replace('/\[/', '', $string); - return explode('|', $string)[0]; - }, $select_options_pairs); + return explode('|', $string)[0]; + }, + $select_options_pairs + ); - $select_options_values = array_map(function ($string) { - $string = preg_replace('/\]/', '', $string); + $select_options_values = array_map( + function ($string) { + $string = preg_replace('/\]/', '', $string); - return explode('|', $string)[1]; - }, $select_options_pairs); + return explode('|', $string)[1]; + }, + $select_options_pairs + ); - $select_options = array_merge(['' => 'Select'], array_combine($select_options_keys, $select_options_values)); + $select_options = array_merge( + ['' => 'Select'], + array_combine($select_options_keys, $select_options_values) + ); $default_value = ''; if (isset($_GET[$record->index_field])) { @@ -157,11 +140,16 @@ function tripal_elasticsearch_build_search_block_form($form, &$form_state, $inde //$window_range = drupal_map_assoc([1]); if (count($index_fields) > 0) { // Build search query for table search. - $query = tripal_elasticsearch_build_search_query_from_field_content_pairs($index_fields); + $query = tripal_elasticsearch_build_search_query_from_field_content_pairs( + $index_fields + ); try { $es = new \ES\Common\Instance(); - $results_count = $es->setTableSearchParams($index_name, $index_type, $query) - ->count(); + $results_count = $es->setTableSearchParams( + $index_name, + $index_type, + $query + )->count(); $count = '' . $results_count . ' results match your search.'; } catch (\Exception $e) { drupal_set_message($e->getMessage(), 'warning'); @@ -229,8 +217,14 @@ function save_to_file_ajax_callback($form, &$form_state) { */ function search_content_ajax_callback($form, &$form_state) { $commands = []; - $commands[] = ajax_command_replace('#count_ajax_wrapper', drupal_render($form['count'])); - $commands[] = ajax_command_replace("#select_window_ajax_wrapper", drupal_render($form['select_window'])); + $commands[] = ajax_command_replace( + '#count_ajax_wrapper', + drupal_render($form['count']) + ); + $commands[] = ajax_command_replace( + "#select_window_ajax_wrapper", + drupal_render($form['select_window']) + ); return ['#type' => 'ajax', '#commands' => $commands]; } diff --git a/tripal_elasticsearch.module b/tripal_elasticsearch.module index 36975af6..113ebd6f 100644 --- a/tripal_elasticsearch.module +++ b/tripal_elasticsearch.module @@ -614,7 +614,7 @@ function tripal_elasticsearch_block_view($delta = '') { switch ($delta) { case 'elasticsearch_website_search_box': $block['subject'] = ''; - $block['content'] = drupal_get_form('website_search_box_form'); + $block['content'] = drupal_get_form('tripal_elasticsearch_site_wide_search_form'); break; case 'website_search_category': $block['subject'] = ''; From 6be0434ea4b3e1ad2e44d882b75c0277888f3731 Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Fri, 22 Feb 2019 15:35:16 -0500 Subject: [PATCH 19/21] Add model tests --- includes/Common/Instance.php | 199 +++++++++++++++++++---------- includes/Indices/Index.php | 52 ++++++-- includes/Jobs/EntitiesIndexJob.php | 1 - includes/Models/Model.php | 161 ++++++++++++++++++----- includes/Models/Results.php | 10 ++ includes/Query/Clause.php | 2 +- tests/Feature/IndexTest.php | 53 ++++++-- tests/Feature/InstanceTest.php | 27 +++- tests/Feature/ModelTest.php | 33 +++++ tests/TestCase.php | 16 ++- tripal_elasticsearch.module | 2 +- 11 files changed, 421 insertions(+), 135 deletions(-) create mode 100644 includes/Models/Results.php create mode 100644 tests/Feature/ModelTest.php diff --git a/includes/Common/Instance.php b/includes/Common/Instance.php index bbb30e4f..67fd8f3d 100644 --- a/includes/Common/Instance.php +++ b/includes/Common/Instance.php @@ -12,7 +12,7 @@ * Also Provides methods for building indices, searching, * deleting and indexing. */ -class Instance{ +class Instance { /** * Elasticsearch client. @@ -43,12 +43,13 @@ class Instance{ * * @param null $host * - * @throws \Exception + * * @return void */ public function __construct($host = NULL) { + if ($host === NULL) { - $host = variable_get('elasticsearch_host'); + $host = getenv('ES_HOST') ?: variable_get('elasticsearch_host'); } if (empty($host)) { @@ -62,7 +63,7 @@ public function __construct($host = NULL) { $host = [$host]; } - // Load the elastic search library + // Load the elastic search library. libraries_load('elasticsearch-php'); $exists = class_exists(ClientBuilder::class); @@ -98,8 +99,10 @@ public function sanitizeQuery($query) { * @param string $node_type * @param string $index * @param string $index_type - * @param array $offset [int $from, int $to] - * @param bool $force_entities_only force search of entities only + * @param array $offset + * [int $from, int $to]. + * @param bool $force_entities_only + * force search of entities only. * * @return $this */ @@ -108,7 +111,7 @@ public function setWebsiteSearchParams($search_terms, $node_type = '', $index = $queries[] = [ 'query_string' => [ - //'default_field' => 'content.taxrank__genus', + // 'default_field' => 'content.taxrank__genus',. 'query' => $this->sanitizeQuery($search_terms), 'default_operator' => 'AND', ], @@ -139,7 +142,8 @@ public function setWebsiteSearchParams($search_terms, $node_type = '', $index = ) && !$force_entities_only) { $queries[1]['query_string'] = [ 'fields' => ['type', 'bundle_label'], - 'query' => '"' . $node_type . '"', // Gene or mRNA (feature,Gene) + // Gene or mRNA (feature,Gene) + 'query' => '"' . $node_type . '"', 'default_operator' => 'AND', ]; } @@ -195,11 +199,16 @@ public function hasIndex($index) { * Build table search params. * USe this method if not searching the website or entities indices. * - * @param string $index Index name - * @param string $type Index type - * @param array $query ES query array - * @param array $offset [int from, int size] - * @param boolean $highlight Whether to highlight fields + * @param string $index + * Index name. + * @param string $type + * Index type. + * @param array $query + * ES query array. + * @param array $offset + * [int from, int size]. + * @param bool $highlight + * Whether to highlight fields. * * @return $this */ @@ -208,9 +217,8 @@ public function setTableSearchParams($index, $type, $query, $offset = [], $highl $params['index'] = $index; $params['type'] = $type; - // sort the table by the first field by default - //$sort_field = array_keys($field_content_pairs)[0]; - + // Sort the table by the first field by default + // $sort_field = array_keys($field_content_pairs)[0]; $params['body'] = [ 'query' => $query, ]; @@ -251,12 +259,18 @@ public function setTableSearchParams($index, $type, $query, $offset = [], $highl /** * Build a new index parameters. * - * @param string $index_name The index name. - * @param int $shards Number of shards. - * @param int $replicas Number of replicas. - * @param string $tokenizer The type of tokenizer. - * @param array $filters The filters. - * @param array $fields The fields as ['field' => 'type', ...]. + * @param string $index_name + * The index name. + * @param int $shards + * Number of shards. + * @param int $replicas + * Number of replicas. + * @param string $tokenizer + * The type of tokenizer. + * @param array $filters + * The filters. + * @param array $fields + * The fields as ['field' => 'type', ...]. * * @return $this * The current object. @@ -281,8 +295,10 @@ public function setIndexParams($index_name, $shards = 5, $replicas = 0, $tokeniz $properties = []; foreach ($fields as $field => $mapping_type) { - if ($mapping_type === 'dynamic') { - $properties[$field] = []; + if ($mapping_type === 'object') { + $properties[$field] = [ + 'properties' => [], + ]; } else { $properties[$field] = [ @@ -312,13 +328,15 @@ public function setIndexParams($index_name, $shards = 5, $replicas = 0, $tokeniz * Perform the actual search. * Use this function after setting the search params. * - * @param bool $return_source Whether to format the results or not. + * @param bool $return_source + * Whether to format the results or not. * * @see \ES\Common\Instance::setTableSearchParams() * @see \ES\Common\Instance::setWebsiteSearchParams() * * @return array - * The search results. + * The search results. + * * @throws \Exception */ public function search($return_source = FALSE) { @@ -340,7 +358,8 @@ public function search($return_source = FALSE) { /** * Format hits. * - * @param array $hits The hits returned from the search operation. + * @param array $hits + * The hits returned from the search operation. * * @return array */ @@ -365,6 +384,7 @@ public function formatHits($hits) { * Get the number of available results for a given search query. * * @return mixed + * * @throws \Exception */ public function count() { @@ -374,10 +394,10 @@ public function count() { ); } - // Get the search query + // Get the search query. $params = $this->searchParams; - // Remove offset restrictions + // Remove offset restrictions. unset($params['from']); unset($params['size']); unset($params['body']['highlight']); @@ -392,6 +412,7 @@ public function count() { * @see \ES\Common\Instance::setIndexParams() * * @return array + * * @throws \Exception */ public function createIndex() { @@ -407,7 +428,8 @@ public function createIndex() { /** * Delete an entire index. * - * @param string $index Index name + * @param string $index + * Index name. * * @return array */ @@ -420,9 +442,12 @@ public function deleteIndex($index) { /** * Delete an entry from an index. * - * @param string $index Index name - * @param string $index_type Index type - * @param int $id Entry ID (node or entity id) + * @param string $index + * Index name. + * @param string $index_type + * Index type. + * @param int $id + * Entry ID (node or entity id) */ public function deleteEntry($index, $index_type, $id) { $params = [ @@ -437,13 +462,17 @@ public function deleteEntry($index, $index_type, $id) { /** * Create a new entry and add it to the provided index. * - * @param string $index Index name - * @param string $type Table name for table entries and index name for - * website entries - * @param int $id Entry ID (node or entity id). Set as FALSE if a table index - * entry. - * @param array $body Array of record data to index. Must match index - * structure. + * @param string $index + * Index name. + * @param string $type + * Table name for table entries and index name for + * website entries. + * @param int $id + * Entry ID (node or entity id). Set as FALSE if a table index + * entry. + * @param array $body + * Array of record data to index. Must match index + * structure. * * @return array */ @@ -464,10 +493,14 @@ public function createEntry($index, $type, $id, $body) { /** * Index multiple entries at once. * - * @param string $index Index name - * @param array $entries Array of entries - * @param string $type Index type - * @param string $id_key The object key to get the id value + * @param string $index + * Index name. + * @param array $entries + * Array of entries. + * @param string $type + * Index type. + * @param string $id_key + * The object key to get the id value. * * @return array */ @@ -476,10 +509,14 @@ public function bulkIndex($index, $entries, $type = NULL, $id_key = NULL) { } /** - * @param string $index Index name - * @param array $entries Array of entries - * @param string $type Index type - * @param string $id_key The object key to get the id value + * @param string $index + * Index name. + * @param array $entries + * Array of entries. + * @param string $type + * Index type. + * @param string $id_key + * The object key to get the id value. * * @return array */ @@ -490,13 +527,17 @@ public function bulkUpdate($index, $entries, $type = NULL, $id_key = NULL) { /** * Update a document. * - * @param string $index The index name. - * @param string $type The type name. - * @param string $id The id of the document to update. - * @param array $data The data to replace. + * @param string $index + * The index name. + * @param string $type + * The type name. + * @param string $id + * The id of the document to update. + * @param array $data + * The data to replace. * * @return array - * The returned array from the client. + * The returned array from the client. */ public function update($index, $type, $id, $data) { $params = [ @@ -514,8 +555,9 @@ public function update($index, $type, $id, $data) { /** * @param string $operation * @param string $index - * @param array $entries Array of entries of the form - * [ + * @param array $entries + * Array of entries of the form + * [ * [ // Start of entry 1 * 'field1' => 'value for field1', * 'field2' => 'value for field 2' @@ -524,7 +566,7 @@ public function update($index, $type, $id, $data) { * 'field1' => 'another value', * 'field2' => 'some other value' * ] - * ] + * ]. * @param string $type * @param string $id_key * @@ -559,6 +601,7 @@ public function bulk($operation, $index, $entries, $type = NULL, $id_key = NULL) case 'index': $params['body'][] = $entry; break; + case 'update': $params['body'][] = ['doc' => $entry]; break; @@ -574,6 +617,7 @@ public function bulk($operation, $index, $entries, $type = NULL, $id_key = NULL) * @param $per_page * * @throws \Exception + * * @return array */ public function paginate($per_page) { @@ -609,14 +653,18 @@ public function getIndices() { /** * Get all available categories. * - * @param int $version Get specific tripal version categories. - * @param boolean $get_count Add count to the list. If set to TRUE, elements - * with 0 count won't be removed. - * @param string $keyword Count keyword. + * @param int $version + * Get specific tripal version categories. + * @param bool $get_count + * Add count to the list. If set to TRUE, elements + * with 0 count won't be removed. + * @param string $keyword + * Count keyword. * * @throws \Exception + * * @return array - * If count is requested, 2 arrays will be returned. + * If count is requested, 2 arrays will be returned. * Otherwise, the structure is $array[$type_label] = $type_label */ public function getAllCategories($version = NULL, $get_count = FALSE, $keyword = '*') { @@ -709,6 +757,7 @@ public function getIndexMappings($index) { * @param string|null $category * * @throws \Exception + * * @return array */ public function searchWebIndices($terms, $size, $category = NULL) { @@ -806,7 +855,8 @@ public function getRecord($index, $type, $id) { 'id' => $id, ] ); - } catch (Exception $exception) { + } + catch (Exception $exception) { return ['found' => FALSE]; } } @@ -814,11 +864,15 @@ public function getRecord($index, $type, $id) { /** * Update field mappings of an index. * - * @param string $index_name Index name - * @param string $field_name Field name - * @param string $field_type Mapping type. E.g, text, integer, etc. + * @param string $index_name + * Index name. + * @param string $field_name + * Field name. + * @param string $field_type + * Mapping type. E.g, text, integer, etc. * * @throws \Exception + * * @return array */ public function putMapping($index_name, $field_name, $field_type, $index_type = NULL) { @@ -832,7 +886,7 @@ public function putMapping($index_name, $field_name, $field_type, $index_type = 'fields' => [ 'raw' => [ 'type' => $field_type, - //'index' => 'not_analyzed', + // 'index' => 'not_analyzed',. ], ], ], @@ -853,10 +907,14 @@ public function putMapping($index_name, $field_name, $field_type, $index_type = * Create a new element if one does not exist. Update the * element if it already exists. * - * @param string $index The index name - * @param string $index_type The index type - * @param mixed $id The document ID - * @param array $item The fields to update or create. + * @param string $index + * The index name. + * @param string $index_type + * The index type. + * @param mixed $id + * The document ID. + * @param array $item + * The fields to update or create. * * @return array */ @@ -873,4 +931,5 @@ public function createOrUpdate($index, $index_type, $id, $item) { ] ); } + } diff --git a/includes/Indices/Index.php b/includes/Indices/Index.php index 993e0f2d..456f16d8 100644 --- a/includes/Indices/Index.php +++ b/includes/Indices/Index.php @@ -6,14 +6,14 @@ use ES\Exceptions\IndexExistsException; use ES\Exceptions\IndexMissingException; use ES\Models\Model; -use mysql_xdevapi\Exception; +use Exception; /** - * Class Index + * Class Index. * * @package ES\Indices */ -class Index{ +class Index { /** * The ES instance. @@ -39,9 +39,12 @@ class Index{ /** * Index constructor. * - * @param array $data Fill the object with data. - * @param string $id The ES given id for the object. - * @param Instance $instance The ES instance. + * @param array $data + * Fill the object with data. + * @param string $id + * The ES given id for the object. + * @param \ES\Common\Instance $instance + * The ES instance. * * @throws \Exception */ @@ -52,17 +55,19 @@ public function __construct(Instance $instance = NULL) { /** * Creates an index from a given model. * - * @param \ES\Models\Model $model The model to create an index for. + * @param \ES\Models\Model $model + * The model to create an index for. * * @return array + * * @throws \Exception * @throws \ES\Exceptions\IndexExistsException */ public function createFromModel(Model $model) { - $this->setIndexName($model->getIndexName()); + $this->setName($model->getIndexName()); $this->setFields($model->getFields()); - return $this->createIndex(); + return $this->create(); } /** @@ -71,7 +76,7 @@ public function createFromModel(Model $model) { * @throws \Exception * @throws \ES\Exceptions\IndexExistsException */ - public function createIndex() { + public function create() { if (!$this->index || empty($this->index)) { throw new \Exception('Index name must be specified to create the index.'); } @@ -107,10 +112,11 @@ public function createIndex() { * Delete the index. * * @return array|bool + * * @throws \ES\Exceptions\IndexMissingException * @throws \Exception */ - public function deleteIndex() { + public function delete() { if (!$this->index || empty($this->index)) { throw new \Exception('Index name must be specified to create the index.'); } @@ -136,7 +142,7 @@ public function deleteIndex() { * * @return $this */ - public function setIndexName($index) { + public function setName($index) { $this->index = $index; return $this; @@ -147,10 +153,15 @@ public function setIndexName($index) { * * @return string */ - public function getIndexName() { + public function getName() { return $this->index; } + /** + * @param array $fields + * + * @return $this + */ public function setFields(array $fields) { $this->fields = $fields; @@ -159,6 +170,7 @@ public function setFields(array $fields) { /** * @return array + * @throws \Exception */ public function getFields($refresh = FALSE) { if (!$refresh && !empty($this->fields)) { @@ -169,6 +181,18 @@ public function getFields($refresh = FALSE) { throw new Exception('Index name must be provided before getting fields'); } - return $this->instance->getIndexMappings($this->index); + $data = $this->instance->getIndexMappings($this->index); + + $data = $data[$this->index]; + $fields = $data['mappings']['_default_']['properties'] ?? []; + + $results = []; + + foreach ($fields as $field => $props) { + $results[$field] = $props['type']; + } + + return $results; } + } diff --git a/includes/Jobs/EntitiesIndexJob.php b/includes/Jobs/EntitiesIndexJob.php index 17e610ad..f8b6de91 100644 --- a/includes/Jobs/EntitiesIndexJob.php +++ b/includes/Jobs/EntitiesIndexJob.php @@ -181,7 +181,6 @@ function ($record) { unset($content[$field]); } } - } } diff --git a/includes/Models/Model.php b/includes/Models/Model.php index 60ca4d1f..47b14448 100644 --- a/includes/Models/Model.php +++ b/includes/Models/Model.php @@ -2,6 +2,12 @@ namespace ES\Models; +use ES\Common\Instance; +use ES\Query\Builder; + +/** + * + */ class Model{ /** @@ -33,6 +39,8 @@ class Model{ /** * Fields mappings. * + * The format must be ['field' => 'mapping type'] + * * @var array */ protected $fields = []; @@ -40,9 +48,9 @@ class Model{ /** * The id of the entry. * - * @var int + * @var int|bool */ - protected $id; + protected $id = FALSE; /** * Query Builder. @@ -61,29 +69,31 @@ class Model{ /** * Index constructor. * - * @param array $data Fill the object with data. - * @param string $id The ES given id for the object. - * @param Instance $instance The ES instance. + * @param array $data + * Fill the object with data. + * @param string $id + * The ES given id for the object. + * @param \ES\Common\Instance $instance + * The ES instance. * * @throws \Exception */ - public function __construct($data = [], $id = FALSE, Instance $instance = NULL) { + public function __construct(Instance $instance = NULL) { $this->instance = $instance ?? new Instance(); - $this->fill($data); - - $this->id = $id; - $this->builder = new Builder($this->index); } /** * Add a where clause. * - * @param string $field The field. - * @param string $value The value + * @param string $field + * The field. + * @param string $value + * The value. * * @return $this * The object. + * * @see \ES\Query\Clause::where() */ public function where($field, $value = NULL) { @@ -95,11 +105,14 @@ public function where($field, $value = NULL) { /** * Add an or clause. * - * @param string $field The field. - * @param string $value The value + * @param string $field + * The field. + * @param string $value + * The value. * * @return $this * The object. + * * @see \ES\Query\Clause::orWhere() */ public function orWhere($field, $value = NULL) { @@ -121,19 +134,40 @@ public function fill(array $data = []) { } } + public function find($id) { + $record = $this->instance->getRecord( + $this->getIndexName(), + $this->getIndexType(), + $id + ); + } + /** * Save the data. * * @return $this - * The object. - * @throws \Exception + * The object. + * + * @throws \Exception */ public function save() { - if ($this->exists()) { + $exists = FALSE; + if ($this->id !== FALSE) { + $record = $this->find($this->id); + if ($record) { + $this->id = $record->id; + $exists = TRUE; + } + } + else { + $exists = $this->exists(); + } + + if ($exists) { $data = $this->instance->update( $this->getIndexName(), $this->getIndexType(), - $this->id ?? FALSE, + $this->id, $this->attributes ); @@ -142,7 +176,7 @@ public function save() { return $this; } - // There data does not exist so + // There data does not exist so. $data = $this->instance->createEntry( $this->getIndexName(), $this->getIndexType(), @@ -159,7 +193,8 @@ public function save() { * Check whether the record exists. * * @return bool - * Whether the record exists. + * Whether the record exists. + * * @throws \Exception */ public function exists() { @@ -172,7 +207,8 @@ public function exists() { * Count the number of documents in the index. * * @return int - * The number of documents in the index. + * The number of documents in the index. + * * @throws \Exception */ public function count() { @@ -185,13 +221,12 @@ public function count() { * @throws \Exception * @return \ES\Models\Model */ - public static function createOrUpdate($data, $id = FALSE) { - $record = new static($data); - - $record->setID($id); - $record->save(); + public function createOrUpdate($data, $id = FALSE) { + $this->fill($data); + $this->setID($id); + $this->save(); - return $record; + return $this; } /** @@ -207,6 +242,15 @@ public function setID($id) { return $this; } + /** + * Get the ES given id. + * + * @return bool|int + */ + public function getID() { + return $this->id; + } + /** * @param array $data * @param bool $id @@ -214,13 +258,12 @@ public function setID($id) { * @return \ES\Models\Model * @throws \Exception */ - public static function create(array $data, $id = FALSE) { - $index = new static($data); - - $index->setID($id); - $index->save(); + public function create(array $data) { + $this->fill($data); + $this->setID(FALSE); + $this->save(); - return $index; + return $this; } /** @@ -233,6 +276,9 @@ public static function create(array $data, $id = FALSE) { public function setIndexName($index) { $this->index = $index; + // Update the builder as well + $this->builder->setIndex($index); + return $this; } @@ -255,6 +301,9 @@ public function getIndexName() { public function setIndexType($type) { $this->type = $type; + // Update the builder. + $this->builder->setType($type); + return $this; } @@ -267,6 +316,17 @@ public function getIndexType() { return $this->type; } + /** + * @param array fields + * + * @return $this + */ + public function setFields($fields) { + $this->fields = $fields; + + return $this; + } + /** * Get the fields for the model. * @@ -276,10 +336,40 @@ public function getFields() { return $this->fields; } + /** + * @return array + * @throws \Exception + */ + public function search() { + $params = $this->builder->build(); + $q = $params['body']['query']['simple_query_string']['query']; + $results = $this->instance->setWebsiteSearchParams( + $q, + '', + $this->getIndexName(), + $this->getIndexType() + )->search(true); + + //$results = $this->instance->client->search($params); + + return $results; + } + + /** + * Get query paramaters. + * + * @return array + * @throws \Exception + */ + public function getQuery() { + return $this->builder->build(); + } + /** * Get an attribute. * - * @param string $name The name of attribute. + * @param string $name + * The name of attribute. * * @return mixed * The value of the attribute if it exists. @@ -299,7 +389,8 @@ public function __get($name) { /** * Check if an attribute isset. * - * @param string $name The name of the attribute. + * @param string $name + * The name of the attribute. * * @return bool * Whether the attribute has been set. diff --git a/includes/Models/Results.php b/includes/Models/Results.php new file mode 100644 index 00000000..e03ab220 --- /dev/null +++ b/includes/Models/Results.php @@ -0,0 +1,10 @@ +sanitizer->escape($field); } - $field = $this->sanitizer->escape($field); + $field = $this->sanitizer->escapeField($field); $value = $this->sanitizer->escape($value); return $field . ':' . $value; diff --git a/tests/Feature/IndexTest.php b/tests/Feature/IndexTest.php index 66d77827..50372218 100644 --- a/tests/Feature/IndexTest.php +++ b/tests/Feature/IndexTest.php @@ -3,30 +3,61 @@ namespace Tests\Feature; use ES\Indices\Index; +use ES\Models\Model; use Tests\TestCase; -class IndexTest extends TestCase{ +/** + * + */ +class IndexTest extends TestCase { - /** @test */ + /** + * @test + * @throws \Exception + */ public function testThatCreatingAndDeletingIndicesWork() { $instance = $this->makeInstance(); $index = new Index($instance); $name = uniqid(); - $data = $index->setIndexName($name) - ->setFields([ - 'content' => 'dynamic', - 'test' => 'text' - ]) - ->createIndex(); + $fields = [ + 'content' => 'object', + 'test' => 'text', + ]; + + $data = $index->setName($name)->setFields($fields)->create(); $this->assertTrue($data['acknowledged']); - $this->assertEquals($index->getIndexName(), $data['index']); + $this->assertEquals($index->getName(), $data['index']); - var_dump($index->getFields(true)); + $this->assertEquals($fields, $index->getFields(TRUE)); - $data = $index->deleteIndex(); + $data = $index->delete(); $this->assertTrue($data['acknowledged']); } + + /** + * @test + * @throws \Exception + */ + public function testCreatingFromModel() { + $instance = $this->makeInstance(); + $index = new Index($instance); + $model = new Model(); + $name = uniqid(); + $model->setIndexName($name); + $model->setFields( + [ + 'content' => 'object', + ] + ); + + $data = $index->createFromModel($model); + $this->assertTrue($data['acknowledged']); + $this->assertEquals($name, $data['index']); + + $index->delete(); + } + } diff --git a/tests/Feature/InstanceTest.php b/tests/Feature/InstanceTest.php index 1cb03eec..3a5beef0 100644 --- a/tests/Feature/InstanceTest.php +++ b/tests/Feature/InstanceTest.php @@ -10,12 +10,37 @@ class InstanceTest extends TestCase{ use DBTransaction; + protected $old_host = NULL; + + /** + * @throws \StatonLab\TripalTestSuite\Exceptions\TripalTestSuiteException + */ + public function setUp() { + parent::setUp(); + + $this->old_host = NULL; + } + + /** + * @throws \Exception + */ + public function tearDown() { + parent::tearDown(); + + if (!is_null($this->old_host)) { + putenv("ES_HOST=$this->old_host"); + } + } + /** @test */ public function testThatConnectionToAnInvalidHostFails() { variable_del('elasticsearch_host'); + $this->old_host = getenv('ES_HOST'); + putenv('ES_HOST'); $this->expectException(\Exception::class); + // Set the host to false to simulate a non-existent host new \ES\Common\Instance(); } @@ -43,7 +68,7 @@ public function testCreatingAndUpdatingDocuments() { $this->makeIndex($name, ['content' => 'text']); - $es = new Instance(); + $es = $this->makeInstance(); $data = $es->createEntry( $name, $name, diff --git a/tests/Feature/ModelTest.php b/tests/Feature/ModelTest.php new file mode 100644 index 00000000..f3612294 --- /dev/null +++ b/tests/Feature/ModelTest.php @@ -0,0 +1,33 @@ +makeInstance(); + + $name = uniqid(); + $this->makeIndex($name); + + // Insert data into the index + $this->createRecord($name, [ + 'content' => 'data' + ]); + + $model = new Model($instance); + $model->setIndexName($name); + $model->setIndexType($name); + + $data = $model->where('*')->search(); + $this->assertEquals($data['hits']['total'], 1); + } +} diff --git a/tests/TestCase.php b/tests/TestCase.php index ef915b01..fd5874a0 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -59,11 +59,25 @@ public function makeIndex($name = NULL, $fields = []) { $name = uniqid(); } - $index = $es->setIndexParams($name, 1, 0, 'standard', [], $fields) + $index = $es->setIndexParams($name, 5, 0, 'standard', [], $fields) ->createIndex(); $this->_indices[] = $index; return $index; } + + /** + * Creating records is asynchronous in ES. This methods adds the wait time + * required to make sure the record was created before making assertions. + * + * @param string $index Index name (and type!) + * @param array $data + */ + public function createRecord($index, $data) { + $instance = $this->makeInstance(); + $record = $instance->createEntry($index, $index, false, $data); + sleep(1); + return $record; + } } diff --git a/tripal_elasticsearch.module b/tripal_elasticsearch.module index 113ebd6f..7cfeeb7f 100644 --- a/tripal_elasticsearch.module +++ b/tripal_elasticsearch.module @@ -714,7 +714,7 @@ function tripal_elasticsearch_web_search_results_page_callback($node_type = '') if (count($results['results'])) { $content = tripal_elasticsearch_create_collection_button(); } - $content .= '

' . $results['total'] . ' results found Page ' . $current_page . ' out of ' . $pages . '

'; + $content .= '

' . number_format($results['total']) . ' results found Page ' . $current_page . ' out of ' . $pages . '

'; $content .= tripal_elasticsearch_get_website_search_result_table( $results['results'], FALSE From 2f8b761472d63059557a491f3abe9ed79b0556c9 Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Mon, 25 Feb 2019 10:23:41 -0500 Subject: [PATCH 20/21] Implement multi-query constructor --- includes/Common/Instance.php | 7 +-- includes/Models/Model.php | 12 ++--- includes/Query/Builder.php | 43 ++++++---------- includes/Query/BuilderContract.php | 3 -- includes/Query/Clause.php | 82 ++++++++++-------------------- includes/Query/ClauseSanitizer.php | 9 ---- tests/Feature/ClauseTest.php | 33 ++++-------- tests/Feature/ModelTest.php | 31 +++++++++-- tests/Feature/QueryBuilderTest.php | 10 +--- tests/TestCase.php | 14 ++++- 10 files changed, 96 insertions(+), 148 deletions(-) diff --git a/includes/Common/Instance.php b/includes/Common/Instance.php index 67fd8f3d..26a26303 100644 --- a/includes/Common/Instance.php +++ b/includes/Common/Instance.php @@ -12,7 +12,7 @@ * Also Provides methods for building indices, searching, * deleting and indexing. */ -class Instance { +class Instance{ /** * Elasticsearch client. @@ -142,7 +142,6 @@ public function setWebsiteSearchParams($search_terms, $node_type = '', $index = ) && !$force_entities_only) { $queries[1]['query_string'] = [ 'fields' => ['type', 'bundle_label'], - // Gene or mRNA (feature,Gene) 'query' => '"' . $node_type . '"', 'default_operator' => 'AND', ]; @@ -855,8 +854,7 @@ public function getRecord($index, $type, $id) { 'id' => $id, ] ); - } - catch (Exception $exception) { + } catch (Exception $exception) { return ['found' => FALSE]; } } @@ -931,5 +929,4 @@ public function createOrUpdate($index, $index_type, $id, $item) { ] ); } - } diff --git a/includes/Models/Model.php b/includes/Models/Model.php index 47b14448..99753c0b 100644 --- a/includes/Models/Model.php +++ b/includes/Models/Model.php @@ -116,7 +116,7 @@ public function where($field, $value = NULL) { * @see \ES\Query\Clause::orWhere() */ public function orWhere($field, $value = NULL) { - $this->builder->where($field, $value); + $this->builder->orWhere($field, $value); return $this; } @@ -342,15 +342,9 @@ public function getFields() { */ public function search() { $params = $this->builder->build(); - $q = $params['body']['query']['simple_query_string']['query']; - $results = $this->instance->setWebsiteSearchParams( - $q, - '', - $this->getIndexName(), - $this->getIndexType() - )->search(true); + $results = $this->instance->client->search($params); - //$results = $this->instance->client->search($params); + $this->builder->reset($this->index); return $results; } diff --git a/includes/Query/Builder.php b/includes/Query/Builder.php index cf4d38cc..bcc1edf6 100644 --- a/includes/Query/Builder.php +++ b/includes/Query/Builder.php @@ -95,32 +95,6 @@ public function orWhere($field, $value = NULL) { return $this; } - /** - * @param $query - * @param string $op - * - * @return $this - * - * @see \ES\Query\Clause::raw() - */ - public function raw($query, $op = 'AND') { - $this->query->raw($query, $op); - - return $this; - } - - /** - * @param $query - * - * @return $this - * @see \ES\Query\Clause::orRaw() - */ - public function orRaw($query) { - $this->query->orRaw($query); - - return $this; - } - /** * * @param string|array $fields @@ -218,8 +192,8 @@ public function build($with_range = TRUE) { 'index' => $this->index, 'body' => [ 'query' => [ - 'simple_query_string' => [ - 'query' => $this->query->build(), + 'bool' => [ + 'must' => $this->query->build(), ], ], ], @@ -252,4 +226,17 @@ public function build($with_range = TRUE) { return $params; } + + /** + * @param null $index + */ + public function reset($index = NULL) { + $this->from = NULL; + $this->query = NULL; + $this->size = NULL; + $this->type = NULL; + if (!is_null($index)) { + $this->index = $index; + } + } } diff --git a/includes/Query/BuilderContract.php b/includes/Query/BuilderContract.php index cf3aae51..b3ef3c0a 100644 --- a/includes/Query/BuilderContract.php +++ b/includes/Query/BuilderContract.php @@ -26,9 +26,6 @@ abstract public function where($field, $value = NULL); */ abstract public function orWhere($field, $value = NULL); - abstract public function raw($query, $op = 'AND'); - abstract public function orRaw($query); - /** * Add a begins with condition. * diff --git a/includes/Query/Clause.php b/includes/Query/Clause.php index 1e9f387a..845d04ed 100644 --- a/includes/Query/Clause.php +++ b/includes/Query/Clause.php @@ -7,9 +7,9 @@ class Clause extends BuilderContract{ /** * The built query. * - * @var string + * @var array */ - protected $query = ''; + protected $queries = []; /** * @var \ES\Query\ClauseSanitizer @@ -29,26 +29,35 @@ public function __construct() { * @param string $field * @param string $value * - * @return string + * @return array */ protected function makeQuery($field, $value = NULL) { if ($field instanceof \Closure) { - $query = '('; $clause = new Clause(); $field($clause); - $query .= $clause->build(); - $query .= ')'; - return $query; + return $clause->build(); } if (is_null($value)) { - return $this->sanitizer->escape($field); + return [ + 'simple_query_string' => [ + 'fields' => ['*'], + 'query' => $this->sanitizer->escape($field), + 'lenient' => TRUE, + ], + ]; } - $field = $this->sanitizer->escapeField($field); + $fields = is_array($field) ? $field : [$field]; $value = $this->sanitizer->escape($value); - return $field . ':' . $value; + return [ + 'simple_query_string' => [ + 'fields' => $fields, + 'query' => $value, + 'lenient' => TRUE, + ], + ]; } /** @@ -63,11 +72,13 @@ protected function makeQuery($field, $value = NULL) { */ public function where($field, $data = NULL) { $query = $this->makeQuery($field, $data); - if (!empty($this->query)) { - $this->query .= " AND $query"; + if (count($query) > 1) { + foreach ($query as $q) { + $this->queries[] = $q; + } } else { - $this->query = $query; + $this->queries[] = $query; } return $this; @@ -84,54 +95,15 @@ public function where($field, $data = NULL) { * The object. */ public function orWhere($field, $data = NULL) { - $query = $this->makeQuery($field, $data); - if (!empty($this->query)) { - $this->query .= " OR $query"; - } - else { - $this->query = $query; - } + $this->where($field, $data); return $this; } /** - * @return string + * @return array */ public function build() { - return $this->query; - } - - /** - * Add a raw query joined with $op. - * - * @param string $query - * @param string $op - * - * @return $this - */ - public function raw($query, $op = 'AND') { - $query = $this->sanitizer->escape($query); - - if (empty($this->query)) { - $this->query = $query; - - return $this; - } - - $this->query .= " {$op} {$query}"; - - return $this; - } - - /** - * Add a raw query joined with OR. - * - * @param string $query - * - * @return \ES\Query\Clause - */ - public function orRaw($query) { - return $this->raw($query, 'OR'); + return $this->queries; } } diff --git a/includes/Query/ClauseSanitizer.php b/includes/Query/ClauseSanitizer.php index 322ddc15..758fca96 100644 --- a/includes/Query/ClauseSanitizer.php +++ b/includes/Query/ClauseSanitizer.php @@ -12,13 +12,4 @@ class ClauseSanitizer{ public function escape($value) { return stripslashes($value); } - - /** - * @param $field - * - * @return mixed - */ - public function escapeField($field) { - return str_replace('.*', '.\\*', $field); - } } diff --git a/tests/Feature/ClauseTest.php b/tests/Feature/ClauseTest.php index 0db1c2d6..d45553f6 100644 --- a/tests/Feature/ClauseTest.php +++ b/tests/Feature/ClauseTest.php @@ -12,7 +12,7 @@ class ClauseTest extends TestCase{ public function testThatWeCanBuildWhereQueriesWithoutUsingClosures() { $clause = new Clause(); $query = $clause->where('field', 'value')->where('value')->orWhere('value'); - $this->assertEquals('field:value AND value OR value', $query->build()); + $this->assertEquals(3, count($query->build())); } /** @test */ @@ -31,7 +31,7 @@ function (BuilderContract $query) { } )->build(); - $this->assertEquals('(f:v AND f:v) OR (f:v OR v)', $query); + $this->assertEquals(4, count($query)); } /** @test */ @@ -44,18 +44,18 @@ public function testAddonBuilders() { ->fuzzy('test') ->build(); - $this->assertEquals('test* AND *test* AND *test AND test~', $query); + $this->assertEquals(4, count($query)); } /** @test */ public function testAddonBuildersThatUseOr() { $clause = new Clause(); - $query = $clause->beginsWith('test')->orBeginsWith('test')->orContains( - 'test' - )->orEndsWith('test')->orFuzzy('test')->build(); + $query = $clause->orBeginsWith('test')->orContains( + 'test' + )->orEndsWith('test')->orFuzzy('test')->build(); - $this->assertEquals('test* OR test* OR *test* OR *test OR test~', $query); + $this->assertEquals(4, count($query)); } /** @test */ @@ -68,10 +68,7 @@ public function testAddonsWithFields() { ->fuzzy('field', 'test') ->build(); - $this->assertEquals( - 'field:test* AND field:*test* AND field:*test AND field:test~', - $query - ); + $this->assertEquals(4, count($query)); } /** @test */ @@ -85,18 +82,6 @@ public function testAddonsWithFieldsUsingOr() { ->orFuzzy('field', 'test') ->build(); - $this->assertEquals( - 'field:test* OR field:test* OR field:*test* OR field:*test OR field:test~', - $query - ); - } - - /** @test */ - public function testRawQuery() { - $clause = new Clause(); - - $query = $clause->raw('test:value* AND test')->orRaw('test')->build(); - - $this->assertEquals('test:value* AND test OR test', $query); + $this->assertEquals(5, count($query)); } } diff --git a/tests/Feature/ModelTest.php b/tests/Feature/ModelTest.php index f3612294..ac3bca73 100644 --- a/tests/Feature/ModelTest.php +++ b/tests/Feature/ModelTest.php @@ -15,19 +15,40 @@ class ModelTest extends TestCase{ public function testAbilityToQueryAnIndex() { $instance = $this->makeInstance(); + $faker = $this->makeFaker(); + $name = uniqid(); - $this->makeIndex($name); + $this->makeIndex( + $name, + [ + 'content' => 'object', + ] + ); + + $name1 = $faker->name; // Insert data into the index - $this->createRecord($name, [ - 'content' => 'data' - ]); + $this->createRecord( + $name, + [ + 'content' => [ + 'field1' => $name1, + 'field2' => $faker->name, + ], + ] + ); $model = new Model($instance); + $model->setIndexName($name); $model->setIndexType($name); + $model->where($name1)->orWhere('content.*', $name1)->orWhere( + 'content.field1', + $name1 + ); + + $data = $model->search(); - $data = $model->where('*')->search(); $this->assertEquals($data['hits']['total'], 1); } } diff --git a/tests/Feature/QueryBuilderTest.php b/tests/Feature/QueryBuilderTest.php index e5bbae69..38922a4e 100644 --- a/tests/Feature/QueryBuilderTest.php +++ b/tests/Feature/QueryBuilderTest.php @@ -26,8 +26,8 @@ public function testThatParamsBuildCorrectly() { $this->assertArrayHasKey('index', $params); $this->assertArrayHasKey('query', $params['body']); $this->assertArrayHasKey( - 'query', - $params['body']['query']['simple_query_string'] + 'bool', + $params['body']['query'] ); $this->assertArrayHasKey('highlight', $params['body']); $this->assertArrayHasKey('fields', $params['body']['highlight']); @@ -36,12 +36,6 @@ public function testThatParamsBuildCorrectly() { $params['body']['highlight']['fields'] ); - // Check query value - $this->assertEquals( - 'field:value OR other:value', - $params['body']['query']['simple_query_string']['query'] - ); - // Build without range $params = $builder->build(FALSE); diff --git a/tests/TestCase.php b/tests/TestCase.php index fd5874a0..9f57df56 100644 --- a/tests/TestCase.php +++ b/tests/TestCase.php @@ -2,6 +2,7 @@ namespace Tests; +use Faker\Factory; use StatonLab\TripalTestSuite\TripalTestCase; class TestCase extends TripalTestCase{ @@ -36,6 +37,13 @@ public function makeInstance() { return new \ES\Common\Instance(getenv('ES_HOST')); } + /** + * @return \Faker\Generator + */ + public function makeFaker() { + return Factory::create(); + } + /** * @param string $name * @@ -68,15 +76,17 @@ public function makeIndex($name = NULL, $fields = []) { } /** - * Creating records is asynchronous in ES. This methods adds the wait time + * Creating records is asynchronous in ES. This method adds the wait time * required to make sure the record was created before making assertions. * * @param string $index Index name (and type!) * @param array $data + * + * @throws \Exception */ public function createRecord($index, $data) { $instance = $this->makeInstance(); - $record = $instance->createEntry($index, $index, false, $data); + $record = $instance->createEntry($index, $index, FALSE, $data); sleep(1); return $record; } From 620959d720932f641919d82461379c3cfd1e9efe Mon Sep 17 00:00:00 2001 From: Abdullah Almsaeed Date: Mon, 25 Feb 2019 10:25:14 -0500 Subject: [PATCH 21/21] Implement multi-query constructor --- includes/Models/Model.php | 10 +++++----- .../Query/{Builder.php => SimpleQueryBuilder.php} | 6 +++--- .../Query/{Clause.php => SimpleQueryClause.php} | 4 ++-- tests/Feature/ClauseTest.php | 14 +++++++------- tests/Feature/QueryBuilderTest.php | 8 ++++---- 5 files changed, 21 insertions(+), 21 deletions(-) rename includes/Query/{Builder.php => SimpleQueryBuilder.php} (96%) rename includes/Query/{Clause.php => SimpleQueryClause.php} (95%) diff --git a/includes/Models/Model.php b/includes/Models/Model.php index 99753c0b..fb6fcb1f 100644 --- a/includes/Models/Model.php +++ b/includes/Models/Model.php @@ -3,7 +3,7 @@ namespace ES\Models; use ES\Common\Instance; -use ES\Query\Builder; +use ES\Query\SimpleQueryBuilder; /** * @@ -55,7 +55,7 @@ class Model{ /** * Query Builder. * - * @var \ES\Query\Builder + * @var \ES\Query\SimpleQueryBuilder */ protected $builder; @@ -80,7 +80,7 @@ class Model{ */ public function __construct(Instance $instance = NULL) { $this->instance = $instance ?? new Instance(); - $this->builder = new Builder($this->index); + $this->builder = new SimpleQueryBuilder($this->index); } /** @@ -94,7 +94,7 @@ public function __construct(Instance $instance = NULL) { * @return $this * The object. * - * @see \ES\Query\Clause::where() + * @see \ES\Query\SimpleQueryClause::where() */ public function where($field, $value = NULL) { $this->builder->where($field, $value); @@ -113,7 +113,7 @@ public function where($field, $value = NULL) { * @return $this * The object. * - * @see \ES\Query\Clause::orWhere() + * @see \ES\Query\SimpleQueryClause::orWhere() */ public function orWhere($field, $value = NULL) { $this->builder->orWhere($field, $value); diff --git a/includes/Query/Builder.php b/includes/Query/SimpleQueryBuilder.php similarity index 96% rename from includes/Query/Builder.php rename to includes/Query/SimpleQueryBuilder.php index bcc1edf6..ab03e975 100644 --- a/includes/Query/Builder.php +++ b/includes/Query/SimpleQueryBuilder.php @@ -2,7 +2,7 @@ namespace ES\Query; -class Builder extends BuilderContract{ +class SimpleQueryBuilder extends BuilderContract{ /** * @var string @@ -27,7 +27,7 @@ class Builder extends BuilderContract{ /** * The query clause builder. * - * @var \ES\Query\Clause + * @var \ES\Query\SimpleQueryClause */ protected $query; @@ -50,7 +50,7 @@ class Builder extends BuilderContract{ */ public function __construct($index = NULL) { $this->index = $index; - $this->query = new Clause(); + $this->query = new SimpleQueryClause(); } /** diff --git a/includes/Query/Clause.php b/includes/Query/SimpleQueryClause.php similarity index 95% rename from includes/Query/Clause.php rename to includes/Query/SimpleQueryClause.php index 845d04ed..7bde5e77 100644 --- a/includes/Query/Clause.php +++ b/includes/Query/SimpleQueryClause.php @@ -2,7 +2,7 @@ namespace ES\Query; -class Clause extends BuilderContract{ +class SimpleQueryClause extends BuilderContract{ /** * The built query. @@ -33,7 +33,7 @@ public function __construct() { */ protected function makeQuery($field, $value = NULL) { if ($field instanceof \Closure) { - $clause = new Clause(); + $clause = new SimpleQueryClause(); $field($clause); return $clause->build(); } diff --git a/tests/Feature/ClauseTest.php b/tests/Feature/ClauseTest.php index d45553f6..b39d368c 100644 --- a/tests/Feature/ClauseTest.php +++ b/tests/Feature/ClauseTest.php @@ -3,21 +3,21 @@ namespace Tests\Feature; use ES\Query\BuilderContract; -use ES\Query\Clause; +use ES\Query\SimpleQueryClause; use Tests\TestCase; class ClauseTest extends TestCase{ /** @test */ public function testThatWeCanBuildWhereQueriesWithoutUsingClosures() { - $clause = new Clause(); + $clause = new SimpleQueryClause(); $query = $clause->where('field', 'value')->where('value')->orWhere('value'); $this->assertEquals(3, count($query->build())); } /** @test */ public function testThatClosuresGenerateEnclosedParameters() { - $clause = new Clause(); + $clause = new SimpleQueryClause(); $query = $clause->where( function (BuilderContract $query) { @@ -36,7 +36,7 @@ function (BuilderContract $query) { /** @test */ public function testAddonBuilders() { - $clause = new Clause(); + $clause = new SimpleQueryClause(); $query = $clause->beginsWith('test') ->contains('test') @@ -49,7 +49,7 @@ public function testAddonBuilders() { /** @test */ public function testAddonBuildersThatUseOr() { - $clause = new Clause(); + $clause = new SimpleQueryClause(); $query = $clause->orBeginsWith('test')->orContains( 'test' @@ -60,7 +60,7 @@ public function testAddonBuildersThatUseOr() { /** @test */ public function testAddonsWithFields() { - $clause = new Clause(); + $clause = new SimpleQueryClause(); $query = $clause->beginsWith('field', 'test') ->contains('field', 'test') @@ -73,7 +73,7 @@ public function testAddonsWithFields() { /** @test */ public function testAddonsWithFieldsUsingOr() { - $clause = new Clause(); + $clause = new SimpleQueryClause(); $query = $clause->beginsWith('field', 'test') ->orBeginsWith('field', 'test') diff --git a/tests/Feature/QueryBuilderTest.php b/tests/Feature/QueryBuilderTest.php index 38922a4e..860cf4be 100644 --- a/tests/Feature/QueryBuilderTest.php +++ b/tests/Feature/QueryBuilderTest.php @@ -2,14 +2,14 @@ namespace Tests\Feature; -use ES\Query\Builder; +use ES\Query\SimpleQueryBuilder; use Tests\TestCase; class QueryBuilderTest extends TestCase{ /** @test */ public function testThatParamsBuildCorrectly() { - $builder = new Builder('entities'); + $builder = new SimpleQueryBuilder('entities'); $builder->setID('some_id')->where('field', 'value')->orWhere( 'other', @@ -46,7 +46,7 @@ public function testThatParamsBuildCorrectly() { /** @test */ public function testThatAnExceptionIsThrownWhenIndexIsNotProvided() { - $builder = new Builder(); + $builder = new SimpleQueryBuilder(); $this->expectException(\Exception::class); $builder->build(); @@ -54,7 +54,7 @@ public function testThatAnExceptionIsThrownWhenIndexIsNotProvided() { /** @test */ public function testThatAnExceptionIsThrownWhenQueryIsNotProvided() { - $builder = new Builder('test'); + $builder = new SimpleQueryBuilder('test'); $this->expectException(\Exception::class); $builder->build();