Add unique index for page groups #348

classes/page_group.php

@@ -143,8 +143,9 @@ protected function update_cache() {
      *
      * @param profile $profile The profile to pull the information from.
      * @param int|null $month The month to record the profile under, or null to use the current month.
+     * @param bool $retry Whether to allow one retry for unique key errors.
      */
-    public static function record_fuzzy_counts(profile $profile, ?int $month = null) {
+    public static function record_fuzzy_counts(profile $profile, ?int $month = null, $retry = true) {
         // Do this only if both auto profiling and fuzzy counting is set.
         if (!get_config('tool_excimer', 'enable_auto') ||
             !get_config('tool_excimer', 'enable_fuzzy_count')) {
@@ -183,7 +184,17 @@ public static function record_fuzzy_counts(profile $profile, ?int $month = null)
         $pagegroup->set('fuzzydurationsum', $fuzzydurationsum);
 
         if ($existing != $pagegroup->to_record()) {
-            $pagegroup->save();
+            try {
+                $pagegroup->save();
+            } catch (\dml_exception $e) {
+                // We have a minor loss in data with concurrent updates that can cause duplicate rows.
+                // When creating new page groups we can catch unique key errors and then retry an update.
+                // Updates are harder to detect, but will only occur when fuzzycount is low, so can ignore.
+                if (!$pagegroupexisted && $retry) {
+                    // One retry should be enough to resolve unique key errors.
+                    self::record_fuzzy_counts($profile, $month, false);
+                }
+            }
         }
     }
 

db/install.xml

@@ -61,7 +61,7 @@
     <TABLE NAME="tool_excimer_page_groups" COMMENT="Metadata about groups of profiles.">
       <FIELDS>
         <FIELD NAME="id" TYPE="int" LENGTH="10" NOTNULL="true" SEQUENCE="true"/>
-        <FIELD NAME="name" TYPE="char" LENGTH="256" NOTNULL="true" SEQUENCE="false" COMMENT="Group name"/>
+        <FIELD NAME="name" TYPE="char" LENGTH="255" NOTNULL="true" SEQUENCE="false" COMMENT="Group name"/>
         <FIELD NAME="month" TYPE="int" LENGTH="6" NOTNULL="true" SEQUENCE="false" COMMENT="Month in YYYYMM format."/>
         <FIELD NAME="fuzzycount" TYPE="int" LENGTH="11" NOTNULL="true" DEFAULT="0" SEQUENCE="false" COMMENT="A approximate count of hits for this group using the approximate counting algorithm."/>
         <FIELD NAME="fuzzydurationcounts" TYPE="text" NOTNULL="false" SEQUENCE="false" COMMENT="Breakdown of fuzzy counts for durations"/>
@@ -70,6 +70,9 @@
       <KEYS>
         <KEY NAME="primary" TYPE="primary" FIELDS="id"/>
       </KEYS>
+      <INDEXES>
+        <INDEX NAME="pagegroup" UNIQUE="true" FIELDS="name, month"/>
+      </INDEXES>
     </TABLE>
   </TABLES>
 </XMLDB>

db/upgrade.php

@@ -490,5 +490,66 @@ function xmldb_tool_excimer_upgrade($oldversion) {
         upgrade_plugin_savepoint(true, 2024050700, 'tool', 'excimer');
     }
 
+    if ($oldversion < 2024082301) {
+
+        // Change precision of name to 255 so it can be used as an index.
+        // First we need to drop a few edge cases that have a length of 256.
+        $DB->delete_records_select('tool_excimer_page_groups', $DB->sql_length('name') . ' > 255');
+
+        // Changing precision of field name on table tool_excimer_page_groups to (255).
+        $table = new xmldb_table('tool_excimer_page_groups');
+        $field = new xmldb_field('name', XMLDB_TYPE_CHAR, '255', null, XMLDB_NOTNULL, null, null, 'id');
+
+        // Launch change of precision for field name.
+        $dbman->change_field_precision($table, $field);
+
+        // Find all non-unique page groups and remove duplicates.
+        $sql = "
+            SELECT pgroups.id, pgroups.name, pgroups.month, pgroups.fuzzycount
+            FROM {tool_excimer_page_groups} pgroups
+            JOIN (
+                SELECT name, month
+                FROM {tool_excimer_page_groups}
+                GROUP BY name, month
+                HAVING COUNT(*) > 1
+            ) dupl ON pgroups.name = dupl.name AND pgroups.month = dupl.month
+            ORDER BY pgroups.name, pgroups.month, pgroups.fuzzycount DESC
+        ";
+        $duplicates = $DB->get_records_sql($sql);
+
+        if (!empty($duplicates)) {
+            $previouskey = '';
+            $removeids = [];
+
+            // Duplicates are ordered, so only keep the first occurence.
+            foreach ($duplicates as $row) {
+                $key = $row->name . '_' . $row->month;
+                if ($key === $previouskey) {
+                    $removeids[] = $row->id;
+                } else {
+                    $previouskey = $key;
+                }
+            }
+
+            // Remove the duplicate rows.
+            if (!empty($removeids)) {
+                $removeids = implode(',', $removeids);
+                $DB->delete_records_select('tool_excimer_page_groups', "id IN ($removeids)");
+            }
+        }
+
+        // Define index pagegroup (unique) to be added to tool_excimer_page_groups.
+        $table = new xmldb_table('tool_excimer_page_groups');
+        $index = new xmldb_index('pagegroup', XMLDB_INDEX_UNIQUE, ['name', 'month']);
+
+        // Conditionally launch add index pagegroup.
+        if (!$dbman->index_exists($table, $index)) {
+            $dbman->add_index($table, $index);
+        }
+
+        // Excimer savepoint reached.
+        upgrade_plugin_savepoint(true, 2024082301, 'tool', 'excimer');
+    }
+
     return true;
 }

version.php

@@ -25,8 +25,8 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version = 2024052400;
-$plugin->release = 2024052400;
+$plugin->version = 2024082301;
+$plugin->release = 2024082301;
 $plugin->requires = 2017051500;    // Moodle 3.3 for Totara support.
 $plugin->supported = [35, 401];     // Supports Moodle 3.5 or later.
 // TODO $plugin->incompatible = ;  // Available as of Moodle 3.9.0 or later.