Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Search feature #1

Merged
merged 1 commit into from
Sep 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# .github/workflows/ci.yml
name: ci

on: [push, pull_request]

jobs:
ci:
uses: catalyst/catalyst-moodle-workflows/.github/workflows/ci.yml@main
26 changes: 25 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1 +1,25 @@
# moodle-tool_advancedreplace
# moodle-tool_advancedreplace

This is a Moodle plugin that allows administrators to search and replace strings in the Moodle database.

Administrators can search and replace strings in tables and columns of the Moodle database.
They can use simple text search or regular expressions.

## GDPR
The plugin does not store any personal data.

## Examples
- Find all occurrences of "http://example.com/" followed by any number of digits on tables:

`php admin/tool/advancedreplace/cli/find.php --regex-match="http://example.com/\d+"`
- Find all occurrences of "http://example.com/" on a table:

`php admin/tool/advancedreplace/cli/find.php --regex-match="http://example.com/" --tables=page`

- Find all occurrences of "http://example.com/" on multiple tables:

`php admin/tool/advancedreplace/cli/find.php --regex-match="http://example.com/" --tables=page,forum`

- Replace all occurrences of "http://example.com/" on different tables and columns:

`php admin/tool/advancedreplace/cli/find.php --regex-match="http://example.com/" --tables=page:content,forum:message`
193 changes: 193 additions & 0 deletions classes/helper.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.

namespace tool_advancedreplace;

use core\exception\moodle_exception;

/**
* Helper class to search and replace text throughout the whole database.
*
* @package tool_advancedreplace
* @copyright 2024 Catalyst IT Australia Pty Ltd
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class helper {

/** @var string ALL_COLUMNS Flag to indicate we search all columns in a table **/
const ALL_COLUMNS = 'all columns';

/**
* Perform a plain text search on a table and column.
*
* @param string $search The text to search for.
* @param string $table The table to search.
* @param string $column The column to search.
* @param int $limit The maximum number of results to return.
* @return array The results of the search.
*/
private static function plain_text_search(string $search, string $table,
string $column = self::ALL_COLUMNS, $limit = 0): array {
global $DB;

$results = [];

$columns = $DB->get_columns($table);

if ($column !== self::ALL_COLUMNS) {
// Only search the specified column.
$columns = array_filter($columns, function($col) use ($column) {
return $col->name == $column;
});
}

foreach ($columns as $column) {
$columnname = $DB->get_manager()->generator->getEncQuoted($column->name);

$searchsql = $DB->sql_like($columnname, '?', false);
$searchparam = '%'.$DB->sql_like_escape($search).'%';

$sql = "SELECT id, $columnname
FROM {".$table."}
WHERE $searchsql";

if ($column->meta_type === 'X' || $column->meta_type === 'C') {
$records = $DB->get_records_sql($sql, [$searchparam], 0, $limit);
if ($records) {
$results[$table][$column->name] = $records;
}
}
}

return $results;
}

/**
* Perform a regular expression search on a table and column.
* This function is only called if the database supports regular expression searches.
*
* @param string $search The regular expression to search for.
* @param string $table The table to search.
* @param string $column The column to search.
* @param $limit The maximum number of results to return.
* @return array
*/
private static function regular_expression_search(string $search, string $table,
string $column = self::ALL_COLUMNS, $limit = 0): array {
global $DB;

// Check if the database supports regular expression searches.
if (!$DB->sql_regex_supported()) {
throw new moodle_exception(get_string('errorregexnotsupported', 'tool_advancedreplace'));
}

$results = [];

$columns = $DB->get_columns($table);

if ($column !== self::ALL_COLUMNS) {
// Only search the specified column.
$columns = array_filter($columns, function($col) use ($column) {
return $col->name == $column;
});
}

foreach ($columns as $column) {
$columnname = $DB->get_manager()->generator->getEncQuoted($column->name);

$select = $columnname . ' ' . $DB->sql_regex() . ' :pattern ';
$params = ['pattern' => $search];

if ($column->meta_type === 'X' || $column->meta_type === 'C') {
$records = $DB->get_records_select($table, $select, $params, '', '*', 0, $limit);

if ($records) {
$results[$table][$column->name] = $records;
}
}
}

return $results;
}

/**
* Perform a search on a table and column.
*
* @param string $search The text to search for.
* @param bool $regex Whether to use regular expression search.
* @param string $tables A comma separated list of tables and columns to search.
* @param int $limit The maximum number of results to return.
* @return array
*/
public static function search(string $search, bool $regex = false, string $tables = '', int $limit = 0): array {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

when we scale up I'm pretty sure this will start breaking as everything is in memory

So probably fine to merge right now, but we'll need to refactor this later. We will probably want to do things in two phases, the first phase returns a list of tables we will search, and then we'll want to do a search table by tables and that second layer will return a record set we can stream through

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also thinking ahead, we talked about this script just outputting to stdout, but it might be better if it outputs to a file and then we can output a set of progress bars to stdout. Knowing how long things will take and where we are up to with ETA's is going to be fairly important. Again not critical now but I'll make issues for this

global $DB;

// Build a list of tables and columns to search.
$tablelist = explode(',', $tables);
$searchlist = [];
foreach ($tablelist as $table) {
$tableandcols = explode(':', $table);
$tablename = $tableandcols[0];
$columnname = $tableandcols[1] ?? '';

// Check if the table already exists in the list.
if (array_key_exists($tablename, $searchlist)) {
// Skip if the table has already been flagged to search all columns.
if (in_array(self::ALL_COLUMNS, $searchlist[$tablename])) {
continue;
}

// Skip if the column already exists in the list for that table.
if (!in_array($columnname, $searchlist[$tablename])) {
continue;
}
}

// Add the table to the list.
if ($columnname == '') {
// If the column is not specified, search all columns in the table.
$searchlist[$tablename][] = self::ALL_COLUMNS;
} else {
// Add the column to the list.
$searchlist[$tablename][] = $columnname;
}
}

// If no tables are specified, search all tables and columns.
if (empty($tables)) {
$tables = $DB->get_tables();
// Mark all columns in each table to be searched.
foreach ($tables as $table) {
$searchlist[$table] = [self::ALL_COLUMNS];
}
}

// Perform the search for each table and column.
$results = [];
foreach ($searchlist as $table => $columns) {
foreach ($columns as $column) {
// Perform the search on this column.
if ($regex) {
$results = array_merge($results, self::regular_expression_search($search, $table, $column, $limit));
} else {
$results = array_merge($results, self::plain_text_search($search, $table, $column, $limit));
}
}
}

return $results;
}
}
37 changes: 37 additions & 0 deletions classes/privacy/provider.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
<?php
// This file is part of Moodle - http://moodle.org/
//
// Moodle is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Moodle is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Moodle. If not, see <http://www.gnu.org/licenses/>.

namespace tool_advancedreplace\privacy;

/**
* Privacy Subsystem implementation for tool_advancedreplace.
*
* @package tool_advancedreplace
* @copyright 2024 Catalyst IT Australia Pty Ltd
* @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
*/
class provider implements \core_privacy\local\metadata\null_provider {

/**
* Get the language string identifier with the component's language
* file to explain why this plugin stores no data.
*
* @return string
*/
public static function get_reason(): string {
return 'privacy:metadata';
}
}
Loading
Loading