Skip to content

Commit

Permalink
Refactor progress into shared search class
Browse files Browse the repository at this point in the history
  • Loading branch information
@bwalkerl
bwalkerl committed Oct 28, 2024
1 parent 4cad4d1 commit ef77c0d
Show file tree
Hide file tree
Showing 4 changed files with 145 additions and 168 deletions.
88 changes: 0 additions & 88 deletions classes/db_search.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,6 @@ class db_search extends search {
/** @var array excludetables from config. */
protected $excludetables = null;

/**
* @var \stdClass tracking of current status */
protected $status = null;

/**
* Return the definition of the properties of this model.
*
Expand Down Expand Up @@ -173,88 +169,4 @@ public function get_min_search_length(): int {
$minsearch = empty($this->get('regex')) ? $this->get('search') : $this->get('prematch');
return strlen($minsearch);
}

/**
* Updates a progress bar using the current status.
* @param string $table table being searched
* @param string $colname column being searched
* @return void
*/
public function update_progress_bar(string $table, string $colname): void {
if (isset($this->status) && isset($this->status->progressbar)) {
$message = "Searching in $table:$colname";
$this->status->progressbar->update($this->status->rowcount, $this->status->totalrows, $message);
}
}

/**
* Updates the tracking status of a search.
* @param int $rowcount number of rows that have been searched
* @param int $matches matches found
* @throws \coding_exception
* @return void
*/
public function update_status(int $rowcount, int $matches): void {
if (!isset($this->status)) {
throw new \coding_exception('Status has not been initalised');
}

// Update row count.
$this->status->rowcount = $rowcount;

// Only save update search progress every 10 seconds or 5 percent.
$time = time();
$percent = round(100 * $rowcount / $this->status->totalrows, 2);
if ($time > $this->status->prevtime + 10 || $percent > $this->status->prevpercent + 5) {
$this->set('progress', $percent);
$this->set('matches', $matches);
$this->save();
$this->status->prevtime = $time;
$this->status->prevpercent = $percent;
}
}

/**
* Marks a search as having started and initialises tracking.
* @param int $totalrows estimate of total rows being searched
* @return void
*/
public function mark_started(int $totalrows): void {
$this->set('timestart', time());
$this->save();

// Setup tracking.
$status = new \stdClass();
$status->prevtime = time();
$status->prevpercent = 0;
$status->rowcount = 0;
$status->totalrows = $totalrows;
$status->progressbar = null;

// If called from CLI, add a progress bar.
if ($this->get('origin') === 'cli') {
$status->progressbar = new \progress_bar();
$status->progressbar->create();
}
$this->status = $status;
}

/**
* Saves the final values and marks a search as finished.
*
* @param int $matches matches found
* @param string $output
* @return void
*/
public function mark_finished(int $matches, string $output = ''): void {
// Update progress bar.
if (isset($this->status) && isset($this->status->progress)) {
$this->status->progress->update_full(100, "Finished saving searches into $output");
}

$this->set('timeend', time());
$this->set('progress', 100);
$this->set('matches', $matches);
$this->save();
}
}
61 changes: 11 additions & 50 deletions classes/file_search.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,9 +158,7 @@ public static function files(files $record, string $output = '', int $limitfrom
raise_memory_limit(MEMORY_HUGE);
$criteria = self::get_criteria($record);

$id = $record->get('id');
$shard = $record->is_shard();
$filename = $record->get_filename();
// Create temp output directory.
if (!$output) {
$tempfile = true;
Expand All @@ -177,14 +175,13 @@ public static function files(files $record, string $output = '', int $limitfrom

[$whereclause, $params] = self::make_where_clause($criteria);
$record->set('timestart', time());
$updatetime = time();
$updatepercent = 0;
$matchcount = 0;
$filecount = 0;
$totalfiles = $DB->count_records_select('files', $whereclause, $params);
if (!empty($limitnum)) {
$totalfiles = min($totalfiles, $limitnum);
}
$record->mark_started($totalfiles);
$sql = "
SELECT
f.id, f.component, f.filearea, f.contextid, f.itemid, f.filename, f.filepath, f.mimetype,
Expand All @@ -201,45 +198,21 @@ public static function files(files $record, string $output = '', int $limitfrom
";
$fileset = $DB->get_recordset_sql($sql, $params, $limitfrom, $limitnum);
foreach ($fileset as $filerecord) {
$record->update_progress_bar("Searching in $filerecord->component:$filerecord->filename");
$matchcount += self::search_file($filerecord, $criteria, $stream);
$filecount ++;
$time = time();
$percent = round(100 * $filecount / $totalfiles, 2);
if ($time > $updatetime + 10 || $percent > $updatepercent + 5) {
// Update progress bar after 5 percent or 10 seconds.
$record->set('progress', $percent);
$record->set('matches', $matchcount);
if ( ! \tool_advancedreplace\files::record_exists($id) ) {
// If record has gone, exit the job.
break;
}
$record->update();
$updatetime = $time;
$updatepercent = $percent;
// Update status. If this returns false, the record is gone so stop searching.
if (!$record->update_status($filecount, $matchcount)) {
break;
}

}
$fileset->close();
fclose($stream);

if (\tool_advancedreplace\files::record_exists($id) ) {
$record->set('timeend', time());
$record->set('progress', 100);
$record->set('matches', $matchcount);
$record->update();

// Save as pluginfile.
if (!empty($matchcount) && !$shard) {
$fs = get_file_storage();
$fileinfo = [
'contextid' => \context_system::instance()->id,
'component' => 'tool_advancedreplace',
'filearea' => 'files',
'itemid' => $id,
'filepath' => '/',
'filename' => $filename,
];
$fs->create_file_from_pathname($fileinfo, $output);
}
if ($record->search_exists()) {
$record->mark_finished($matchcount);
$record->save_pluginfile($output);
}
// Remove temp file.
if (isset($tempfile) && file_exists($output) && !$shard) {
Expand Down Expand Up @@ -270,10 +243,7 @@ public static function combine_shard_output(files $parent): void {
}

// Update parent.
$parent->set('timeend', time());
$parent->set('matches', $matches);
$parent->set('progress', 100);
$parent->update();
$parent->mark_finished($matches);

// Copy data into one csv.
$dir = make_request_directory();
Expand All @@ -297,16 +267,7 @@ public static function combine_shard_output(files $parent): void {
}

// Create new pluginfile.
$fs = get_file_storage();
$fileinfo = [
'contextid' => \context_system::instance()->id,
'component' => 'tool_advancedreplace',
'filearea' => 'files',
'itemid' => $parent->get('id'),
'filepath' => '/',
'filename' => $parent->get_filename(),
];
$fs->create_file_from_pathname($fileinfo, $output);
$parent->save_pluginfile($output);

// Remove old temp files.
foreach ($files as $file) {
Expand Down
24 changes: 6 additions & 18 deletions classes/helper.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -472,7 +472,7 @@ public static function search_db(db_search $search, string $output = ''): void {
$colstart = time();

// Show the table and column being searched.
$search->update_progress_bar($table, $colname);
$search->update_progress_bar("Searching in $table:$colname");

// Perform the search.
$results = self::search_column($search, $table, $column, $fp);
Expand All @@ -494,11 +494,10 @@ public static function search_db(db_search $search, string $output = ''): void {
];
}

if ( ! \tool_advancedreplace\db_search::record_exists($searchid) ) {
// The control row has been deleted, so we should exit.
// Update status. If this returns false, the record is gone so stop searching.
if (!$search->update_status($rowcount, $matches)) {
break 2;
}
$search->update_status($rowcount, $matches);
}
}

Expand All @@ -512,21 +511,10 @@ public static function search_db(db_search $search, string $output = ''): void {
mtrace(sprintf($format, $log->table, $log->column, $log->rows, $log->matches, $log->time));
}
}
if (\tool_advancedreplace\db_search::record_exists($searchid) ) {

if ($search->search_exists()) {
$search->mark_finished($matches, $output);
// Save as pluginfile.
if (!empty($matches)) {
$fs = get_file_storage();
$fileinfo = [
'contextid' => \context_system::instance()->id,
'component' => 'tool_advancedreplace',
'filearea' => 'search',
'itemid' => $search->get('id'),
'filepath' => '/',
'filename' => $search->get_filename(),
];
$fs->create_file_from_pathname($fileinfo, $output);
}
$search->save_pluginfile($output);
}
// Remove temp file.
if (isset($tempfile) && file_exists($output)) {
Expand Down
Loading

0 comments on commit ef77c0d

Please sign in to comment.