Merge pull request #1 from signify-nz/signify

Necessary features and fixes required for base Signify functionality
signify-nz · Nov 1, 2024 · 5ddb9f1 · 5ddb9f1
2 parents 1ef7f24 + e0af92c
commit 5ddb9f1
Show file tree

Hide file tree

Showing 18 changed files with 247 additions and 157 deletions.
diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
diff --git a/Solr/9/templates/types.ss b/Solr/9/templates/types.ss
@@ -119,28 +119,8 @@
     NOTE: autoGeneratePhraseQueries="true" tends to not work well for non whitespace delimited languages.
     -->
 <fieldType name="text" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
-    <analyzer type="index">
-        <tokenizer class="solr.WhitespaceTokenizerFactory"/>
-        <!-- in this example, we will only use synonyms at query time
-        <filter class="solr.SynonymGraphFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
-        -->
-        <!-- Case insensitive stop word removal.
-          add enablePositionIncrements=true in both the index and query
-          analyzers to leave a 'gap' for more accurate phrase queries.
-        -->
-        <filter class="solr.LowerCaseFilterFactory"/>
-        <filter class="solr.StopFilterFactory"
-                ignoreCase="true"
-                words="stopwords.txt"
-
-        />
-        <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1"
-                catenateWords="1"
-                catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
-        <filter class="solr.FlattenGraphFilterFactory"/> <!-- required on index analyzers after graph filters -->
-    </analyzer>
     <analyzer type="query">
-        <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+        <tokenizer class="solr.ClassicTokenizerFactory"/>
         <filter class="solr.LowerCaseFilterFactory"/>
         <filter class="solr.ASCIIFoldingFilterFactory"/>
         <filter class="solr.KeywordRepeatFilterFactory"/>
@@ -158,24 +138,31 @@
         <filter class="solr.SnowballPorterFilterFactory"/>
     </analyzer>
     <analyzer type="index">
-        <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+        <tokenizer class="solr.ClassicTokenizerFactory"/>
         <filter class="solr.LowerCaseFilterFactory"/>
+        <filter class="solr.StopFilterFactory"
+                ignoreCase="true"
+                words="stopwords.txt"
+        />
         <filter class="solr.SnowballPorterFilterFactory"/>
         <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
+        <filter class="solr.WordDelimiterGraphFilterFactory" generateWordParts="1" generateNumberParts="1"
+                catenateWords="1"
+                catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
         <filter class="solr.FlattenGraphFilterFactory"/> <!-- required on index analyzers after graph filters -->
     </analyzer>
 </fieldType>
 <fieldType name="stemfield" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
     <analyzer type="index">
-        <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+        <tokenizer class="solr.ClassicTokenizerFactory"/>
         <filter class="solr.LowerCaseFilterFactory"/>
         <filter class="solr.ASCIIFoldingFilterFactory"/>
         <filter class="solr.SnowballPorterFilterFactory"/>
         <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
         <filter class="solr.FlattenGraphFilterFactory"/> <!-- required on index analyzers after graph filters -->
     </analyzer>
     <analyzer type="query">
-        <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+        <tokenizer class="solr.ClassicTokenizerFactory"/>
         <filter class="solr.LowerCaseFilterFactory"/>
         <filter class="solr.ASCIIFoldingFilterFactory"/>
         <filter class="solr.SnowballPorterFilterFactory"/>
@@ -185,7 +172,7 @@
 <!-- A copy of text that has the HTMLStripCharFilterFactory as the first index analyzer, so that html can be provided -->
 <fieldType name="htmltext" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
     <analyzer type="index">
-        <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+        <tokenizer class="solr.ClassicTokenizerFactory"/>
         <charFilter class="solr.HTMLStripCharFilterFactory"/>
         <filter class="solr.LowerCaseFilterFactory"/>
         <filter class="solr.ASCIIFoldingFilterFactory"/>
@@ -199,7 +186,7 @@
         <filter class="solr.FlattenGraphFilterFactory"/> <!-- required on index analyzers after graph filters -->
     </analyzer>
     <analyzer type="query">
-        <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+        <tokenizer class="solr.ClassicTokenizerFactory"/>
         <filter class="solr.LowerCaseFilterFactory"/>
         <filter class="solr.ASCIIFoldingFilterFactory"/>
         <filter class="solr.KeywordRepeatFilterFactory"/>
@@ -266,7 +253,7 @@
 <!-- A general unstemmed text field - good if one does not know the language of the field -->
 <fieldType name="textgen" class="solr.TextField" positionIncrementGap="100">
     <analyzer type="index">
-        <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+        <tokenizer class="solr.ClassicTokenizerFactory"/>
         <filter class="solr.LowerCaseFilterFactory"/>
         <filter class="solr.ASCIIFoldingFilterFactory"/>
         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
@@ -278,7 +265,7 @@
         <filter class="solr.FlattenGraphFilterFactory"/> <!-- required on index analyzers after graph filters -->
     </analyzer>
     <analyzer type="query">
-        <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+        <tokenizer class="solr.ClassicTokenizerFactory"/>
         <filter class="solr.LowerCaseFilterFactory"/>
         <filter class="solr.ASCIIFoldingFilterFactory"/>
         <filter class="solr.StopFilterFactory"

diff --git a/docs/03-Set-up-and-Configuration.md b/docs/03-Set-up-and-Configuration.md
@@ -77,6 +77,16 @@ SilverStripe\Core\Injector\Injector:
     class: Firesphere\SolrSearch\Stores\PostConfigStore
 ```
 
+#### Additional Solarium options
+If your Solr configuration requires use of the [additional Solarium addDocuments options](https://solarium.readthedocs.io/en/stable/queries/update-query/building-an-update-query/add-command/) for the `SolrIndexTask` to complete properly, these can also be set as config on the SolrCoreService.
+
+Example config:
+```yaml
+Firesphere\SolrSearch\Services\SolrCoreService:
+  add_docs_overwrite: true #boolean
+  add_docs_commitWithin: 1000 #int - value in milliseconds
+```
+
 ### Authentication
 
 Solr supports several ways of adding authentication to the instance.
@@ -286,6 +296,19 @@ Firesphere\SolrSearch\Services\SolrCoreService:
 
 Looking at the `tests` folder, there is a `TestIndexFour`. This index is not loaded unless explicitly asked.
 
+## Excluding unwanted classes from index
+
+To exclude unwanted subclasses from being indexed add these in a list as an `exclude_classes` config on the index. 
+
+For example, if you want to index `SilverStripe\Assets\File` but not it's subclasses `Folder` and `Image`, and you want to index `SilverStripe\CMS\Model\SiteTree` but not `RedirectorPage` the config would look like this:
+
+```yaml
+Firesphere\SolrSearch\Indexes\BaseIndex:
+  exclude_classes:
+    - SilverStripe\CMS\Model\RedirectorPage
+    - SilverStripe\Assets\Folder
+    - SilverStripe\Assets\Image
+```
 
 ----------
 <sup>1</sup> Although not required, it's highly recommended

diff --git a/readme.md b/readme.md
@@ -1,3 +1,5 @@
+**BASED ON [firesphere/solr-search](https://codeberg.org/Firesphere/silverstripe-solr)**
+
 [![Maintainability](https://api.codeclimate.com/v1/badges/55c8967ef25e37182e3d/maintainability)](https://codeclimate.com/github/Firesphere/silverstripe-solr-search/maintainability)
 [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/Firesphere/silverstripe-solr-search/badges/quality-score.png?b=primary)](https://scrutinizer-ci.com/g/Firesphere/silverstripe-solr-search/?branch=primary)
 [![Code Coverage](https://scrutinizer-ci.com/g/Firesphere/silverstripe-solr-search/badges/coverage.png?b=primary)](https://scrutinizer-ci.com/g/Firesphere/silverstripe-solr-search/?branch=primary)

diff --git a/src/Admins/SearchAdmin.php b/src/Admins/SearchAdmin.php
@@ -57,4 +57,12 @@ public function init()
 
         Requirements::css('firesphere/solr-search:client/dist/main.css');
     }
+
+    protected function getManagedModelTabs()
+    {
+        $tabs = parent::getManagedModelTabs();
+        return $tabs->filterByCallback(function ($tab) {
+            return singleton($tab->ClassName)->canView();
+        });
+    }
 }
diff --git a/src/Extensions/DataObjectExtension.php b/src/Extensions/DataObjectExtension.php
@@ -14,7 +14,6 @@
 use Firesphere\SolrSearch\Helpers\SolrLogger;
 use Firesphere\SolrSearch\Models\DirtyClass;
 use Firesphere\SolrSearch\Services\SolrCoreService;
-use Firesphere\SolrSearch\Tests\DataObjectExtensionTest;
 use Psr\Log\LoggerInterface;
 use Psr\SimpleCache\InvalidArgumentException;
 use ReflectionException;
@@ -94,15 +93,15 @@ protected function shouldPush()
      */
     protected function pushToSolr(DataObject $owner)
     {
-        $service = new SolrCoreService();
+        $service = Injector::inst()->get(SolrCoreService::class);
         if (!$service->isValidClass($owner->ClassName)) {
             return;
         }
 
         /** @var DataObject $owner */
         $record = $this->getDirtyClass(SolrCoreService::UPDATE_TYPE);
 
-        $ids = json_decode($record->IDs, 1) ?: [];
+        $ids = json_decode($record->IDs ?? '[]', 1) ?: [];
         $mode = Versioned::get_reading_mode();
         try {
             Versioned::set_reading_mode(Versioned::LIVE);
@@ -129,19 +128,21 @@ protected function pushToSolr(DataObject $owner)
      * Find or create a new DirtyClass for recording dirty IDs
      *
      * @param string $type
+     * @param string $class optional class to use. If not set uses current owner class
      * @return DirtyClass
      * @throws ValidationException
      */
-    protected function getDirtyClass(string $type)
+    protected function getDirtyClass(string $type, string $class = null)
     {
+        $params = [
+            'Class' => ($class ?? $this->owner->ClassName),
+            'Type' => $type
+        ];
         // Get the DirtyClass object for this item
         /** @var null|DirtyClass $record */
-        $record = DirtyClass::get()->filter(['Class' => $this->owner->ClassName, 'Type' => $type])->first();
+        $record = DirtyClass::get()->filter($params)->first();
         if (!$record || !$record->exists()) {
-            $record = DirtyClass::create([
-                'Class' => $this->owner->ClassName,
-                'Type'  => $type,
-            ]);
+            $record = DirtyClass::create($params);
             $record->write();
         }
 
@@ -213,6 +214,53 @@ public function doReindex()
         $this->pushToSolr($this->owner);
     }
 
+    /**
+     * Attempt to remove the item from Solr
+     *
+     * @throws ValidationException
+     * @throws HTTPException
+     */
+    private function removeItem(DataObject $item = null)
+    {
+        /** @var DataObject $owner */
+        $owner = $item ?? $this->owner;
+        /** @var DirtyClass $record */
+        $record = $this->getDirtyClass(SolrCoreService::DELETE_TYPE, $owner->ClassName);
+        $record->IDs = $record->IDs ?? '[]'; // If the record is new, or the IDs list is null, default
+        $ids = json_decode($record->IDs, 1);
+
+        try {
+            Injector::inst()->get(SolrCoreService::class)
+                ->updateItems(ArrayList::create([$owner]), SolrCoreService::DELETE_TYPE);
+            // If successful, remove it from the array
+            // Added bonus, array_flip removes duplicates
+            $this->clearIDs($owner, $ids, $record);
+            // @codeCoverageIgnoreStart
+        } catch (Exception $error) {
+            $this->registerException($ids, $record, $error);
+        }
+        // @codeCoverageIgnoreEnd
+    }
+
+    /**
+     * Clear old page type from Solr before publishing if required
+     *
+     * @throws ValidationException
+     * @throws HTTPException
+     * @throws ReflectionException
+     * @throws InvalidArgumentException
+     */
+    public function onBeforePublish()
+    {
+        // Check for changed classname and delete old record before pushing new if required.
+        if ($this->owner instanceof SiteTree) {
+            $lastPublished = Versioned::get_by_stage(SiteTree::class, Versioned::LIVE)->byID($this->owner->ID);
+            if ($lastPublished && $this->owner->ClassName !== $lastPublished->ClassName) {
+                $this->removeItem($lastPublished);
+            }
+        }
+    }
+
     /**
      * Push the item to Solr after publishing
      *
@@ -231,31 +279,14 @@ public function onAfterPublish()
     }
 
     /**
-     * Attempt to remove the item from Solr
+     * Attempt to remove the item from Solr when deleted
      *
      * @throws ValidationException
      * @throws HTTPException
      */
     public function onAfterDelete(): void
     {
-        /** @var DataObject $owner */
-        $owner = $this->owner;
-        /** @var DirtyClass $record */
-        $record = $this->getDirtyClass(SolrCoreService::DELETE_TYPE);
-        $record->IDs = $record->IDs ?? '[]'; // If the record is new, or the IDs list is null, default
-        $ids = json_decode($record->IDs, 1);
-
-        try {
-            (new SolrCoreService())
-                ->updateItems(ArrayList::create([$owner]), SolrCoreService::DELETE_TYPE);
-            // If successful, remove it from the array
-            // Added bonus, array_flip removes duplicates
-            $this->clearIDs($owner, $ids, $record);
-            // @codeCoverageIgnoreStart
-        } catch (Exception $error) {
-            $this->registerException($ids, $record, $error);
-        }
-        // @codeCoverageIgnoreEnd
+        $this->removeItem();
     }
 
     /**

diff --git a/src/Factories/DocumentFactory.php b/src/Factories/DocumentFactory.php
@@ -179,7 +179,7 @@ protected function addField($doc, $object, $options): void
         $type = $typeMap[$options['type']] ?? $typeMap['*'];
 
         foreach ($valuesForField as $value) {
-            $this->extend('onBeforeAddDoc', $options, $value);
+            $this->extend('onBeforeAddDoc', $options, $value, $object);
             $this->addToDoc($doc, $options, $type, $value);
         }
     }