diff --git a/classes/form/step_form.php b/classes/form/step_form.php index 5eec7d04..7e3af96e 100644 --- a/classes/form/step_form.php +++ b/classes/form/step_form.php @@ -107,11 +107,22 @@ public function definition() { $select->setMultiple(true); $dataflow = new dataflow($dataflowid); - $varroot = $dataflow->get_variables_root(); - // List all the available fields available for configuration, in dot syntax. - $mform->addElement('static', 'fields', get_string('available_fields', 'tool_dataflows'), - $this->prepare_available_fields($varroot->get())); + try { + $varroot = $dataflow->get_variables_root(); + + // List all the available fields available for configuration, in dot syntax. + $mform->addElement( + 'static', + 'fields', + get_string('available_fields', 'tool_dataflows'), + $this->prepare_available_fields($varroot->get()) + ); + } catch (\Throwable $e) { + global $OUTPUT; + $errtext = $OUTPUT->notification($e->getMessage()); + $mform->addElement('static', 'vars_error', get_string('available_fields', 'tool_dataflows'), $errtext); + } // Check and set custom form inputs if required. Defaulting to a // textarea config input for those not yet configured. diff --git a/classes/local/execution/engine.php b/classes/local/execution/engine.php index eafeb9ec..1494178e 100644 --- a/classes/local/execution/engine.php +++ b/classes/local/execution/engine.php @@ -211,6 +211,13 @@ public function __construct(dataflow $dataflow, bool $isdryrun = false, $automat // Find the flow blocks. $this->create_flow_caps(); + + // Make the runid available to the flow. + if (!$this->isdryrun) { + $variables = $this->get_variables(); + $variables->set('run.name', $this->run->name); + $variables->set('run.id', $this->run->id); + } } /** @@ -792,7 +799,7 @@ private function setup_logging() { // Dataflow run logger. // Type: FILE_PER_RUN - // e.g. '[dataroot]/tool_dataflows/3/20060102150405-21.log' as the path. + // e.g. '[dataroot]/tool_dataflows/3/Ymd_His.uuu_21.log' as the path. if (isset($loghandlers[log_handler::FILE_PER_RUN])) { $dataflowrunlogpath = $CFG->dataroot . DIRECTORY_SEPARATOR . 'tool_dataflows' . DIRECTORY_SEPARATOR . diff --git a/classes/local/step/connector_set_multiple_variables.php b/classes/local/step/connector_set_multiple_variables.php new file mode 100644 index 00000000..b609490b --- /dev/null +++ b/classes/local/step/connector_set_multiple_variables.php @@ -0,0 +1,35 @@ +. + +namespace tool_dataflows\local\step; + +/** + * Set multiple variables connector step + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class connector_set_multiple_variables extends connector_step { + use set_multiple_variables_trait; + + /** @var int[] number of output flows (min, max). */ + protected $outputflows = [0, 1]; + + /** @var int[] number of output connectors (min, max). */ + protected $outputconnectors = [0, 1]; +} diff --git a/classes/local/step/connector_sql.php b/classes/local/step/connector_sql.php new file mode 100644 index 00000000..0cb01e5d --- /dev/null +++ b/classes/local/step/connector_sql.php @@ -0,0 +1,29 @@ +. + +namespace tool_dataflows\local\step; + +/** + * SQL connector step + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class connector_sql extends connector_step { + use sql_trait; +} diff --git a/classes/local/step/copy_file_trait.php b/classes/local/step/copy_file_trait.php index 30a081c5..05cc866b 100644 --- a/classes/local/step/copy_file_trait.php +++ b/classes/local/step/copy_file_trait.php @@ -72,7 +72,9 @@ public function execute($input = null) { $todirectory = dirname($to); if (!file_exists($todirectory)) { $this->log("Creating a directory at {$todirectory}"); - mkdir($todirectory, $CFG->directorypermissions, true); + if (!mkdir($todirectory, $CFG->directorypermissions, true)) { + throw new \moodle_exception('flow_copy_file:mkdir_failed', 'tool_dataflows', '', $todirectory); + } } // Attempt to copy the file to the destination. @@ -112,10 +114,12 @@ public function execute($input = null) { private function copy(string $from, string $to) { $this->log("Copying $from to $to"); if (!copy($from, $to)) { - throw new \moodle_exception('flow_copy_file:copy_failed', 'tool_dataflows', (object) [ - 'from' => $from, - 'to' => $to, - ]); + throw new \moodle_exception( + 'flow_copy_file:copy_failed', + 'tool_dataflows', + '', + (object) ['from' => $from, 'to' => $to] + ); } } diff --git a/classes/local/step/flow_email.php b/classes/local/step/flow_email.php index 7090dfdd..d35d1510 100644 --- a/classes/local/step/flow_email.php +++ b/classes/local/step/flow_email.php @@ -28,6 +28,12 @@ */ class flow_email extends flow_step { + /** @var int[] number of output flows (min, max). */ + protected $outputflows = [0, 1]; + + /** @var int[] number of output connectors (min, max) */ + protected $outputconnectors = [0, 1]; + /** @var bool whether or not this step type (potentially) contains a side effect or not */ protected $hassideeffect = true; diff --git a/classes/local/step/flow_set_multiple_variables.php b/classes/local/step/flow_set_multiple_variables.php new file mode 100644 index 00000000..1115240e --- /dev/null +++ b/classes/local/step/flow_set_multiple_variables.php @@ -0,0 +1,35 @@ +. + +namespace tool_dataflows\local\step; + +/** + * Set multiple variables flow step + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class flow_set_multiple_variables extends flow_step { + use set_multiple_variables_trait; + + /** @var int[] number of output flows (min, max). */ + protected $outputflows = [0, 1]; + + /** @var int[] number of output connectors (min, max). */ + protected $outputconnectors = [0, 1]; +} diff --git a/classes/local/step/flow_sql.php b/classes/local/step/flow_sql.php index 0bfff0ca..1e91ec76 100644 --- a/classes/local/step/flow_sql.php +++ b/classes/local/step/flow_sql.php @@ -27,131 +27,4 @@ class flow_sql extends flow_step { use sql_trait; - /** - * Return the definition of the fields available in this form. - * - * @return array - */ - public static function form_define_fields(): array { - return [ - 'sql' => ['type' => PARAM_TEXT, 'required' => true], - ]; - } - - /** - * Allows each step type to determine a list of optional/required form - * inputs for their configuration - * - * It's recommended you prefix the additional config related fields to avoid - * conflicts with any existing fields. - * - * @param \MoodleQuickForm $mform - */ - public function form_add_custom_inputs(\MoodleQuickForm &$mform) { - // SQL example and inputs. - $sqlexample = " - SELECT id, username - FROM {user} - WHERE id > \${{steps.xyz.var.number}} - ORDER BY id ASC - LIMIT 10"; - $sqlexamples = \html_writer::tag('pre', trim($sqlexample, " \t\r\0\x0B")); - $mform->addElement('textarea', 'config_sql', get_string('flow_sql:sql', 'tool_dataflows'), - ['max_rows' => 40, 'rows' => 5, 'style' => 'font: 87.5% monospace; width: 100%; max-width: 100%']); - $mform->addElement('static', 'config_sql_help', '', get_string('flow_sql:sql_help', 'tool_dataflows', $sqlexamples)); - } - - /** - * Allow steps to setup the form depending on current values. - * - * This method is called after definition(), data submission and set_data(). - * All form setup that is dependent on form values should go in here. - * - * @param \MoodleQuickForm $mform - * @param \stdClass $data - */ - public function form_definition_after_data(\MoodleQuickForm &$mform, \stdClass $data) { - // Validate the data. - $sqllinecount = count(explode(PHP_EOL, trim($data->config_sql))); - - // Get the element. - $element = $mform->getElement('config_sql'); - - // Update the element height based on min/max settings, but preserve - // other existing rules. - $attributes = $element->getAttributes(); - - // Set the rows at a minimum to the predefined amount in - // form_add_custom_inputs, and expand as content grows up to a maximum. - $attributes['rows'] = min( - $attributes['max_rows'], - max($attributes['rows'], $sqllinecount) - ); - $element->setAttributes($attributes); - } - - /** - * Execute configured query - * - * @param mixed $input - * @return mixed - * @throws \dml_read_exception when the SQL is not valid. - */ - public function execute($input = null) { - global $DB; - - // Construct the query. - $variables = $this->get_variables(); - $config = $variables->get_raw('config'); - [$sql, $params] = $this->evaluate_expressions($config->sql); - - // Now that we have the query, we want to get info on SQL keywords to figure out where to route the request. - // This is not used for security, just to route the request via the correct pathway for readonly databases. - $pattern = '/(SELECT|UPDATE|INSERT|DELETE)/im'; - $matches = []; - preg_match($pattern, $sql, $matches); - - // Matches[0] contains the match. Fallthrough to default on no match. - $token = $matches[0] ?? ''; - $emptydefault = new \stdClass(); - - switch(strtoupper($token)) { - case 'SELECT': - // Execute the query using get_records instead of get_record. - // This is so we can expose the number of records returned which - // can then be used by the dataflow in for e.g. a switch statement. - $records = $DB->get_records_sql($sql, $params); - - $variables->set('count', count($records)); - $invalidnum = ($records === false || count($records) !== 1); - $data = $invalidnum ? $emptydefault : array_pop($records); - $variables->set('data', $data); - break; - default: - // Default to execute. - $success = $DB->execute($sql, $params); - - // We can't really do anything with the response except check for success. - $variables->set('count', (int) $success); - $variables->set('data', $emptydefault); - break; - } - - return $input; - } - - /** - * Validate the configuration settings. - * - * @param object $config - * @return true|\lang_string[] true if valid, an array of errors otherwise - */ - public function validate_config($config) { - $errors = []; - if (empty($config->sql)) { - $errors['config_sql'] = get_string('config_field_missing', 'tool_dataflows', 'sql', true); - } - - return empty($errors) ? true : $errors; - } } diff --git a/classes/local/step/reader_csv.php b/classes/local/step/reader_csv.php index c5f8e7aa..1de5e7f0 100644 --- a/classes/local/step/reader_csv.php +++ b/classes/local/step/reader_csv.php @@ -43,6 +43,7 @@ public static function form_define_fields(): array { 'path' => ['type' => PARAM_TEXT, 'required' => true], 'headers' => ['type' => PARAM_TEXT], 'overwriteheaders' => ['type' => PARAM_BOOL], + 'continueonerror' => ['type' => PARAM_BOOL], 'delimiter' => ['type' => PARAM_TEXT], ]; } @@ -61,9 +62,11 @@ public function get_iterator(): iterator { */ public function csv_contents_generator() { $maxlinelength = 1000; - $config = $this->get_variables()->get('config'); + $variables = $this->get_variables(); + $config = $variables->get('config'); $strheaders = $config->headers; $overwriteheaders = !empty($config->overwriteheaders); + $continueonerror = !empty($config->continueonerror); $delimiter = $config->delimiter ?: self::DEFAULT_DELIMETER; $path = $this->enginestep->engine->resolve_path($config->path); @@ -92,12 +95,35 @@ public function csv_contents_generator() { // Convert header string to an actual headers array. $headers = str_getcsv($strheaders, $delimiter); $numheaders = count($headers); + $rownumber = 1; // First row is always headers. + $errors = ['header_field_count_mismatch' => 0]; while (($data = fgetcsv($handle, $maxlinelength, $delimiter)) !== false) { + $rownumber++; $numfields = count($data); if ($numfields !== $numheaders) { + // Continue on (parse) error. + if ($continueonerror) { + $errors['header_field_count_mismatch'] += 1; + $this->log->error( + get_string('reader_csv:header_field_count_mismatch', 'tool_dataflows', (object) [ + 'numfields' => $numfields, + 'numheaders' => $numheaders, + 'rownumber' => $rownumber, + ]), + [ + 'fields' => $data, + 'headers' => $headers, + ] + ); + + continue; + } + + // Throw exception on error. throw new \moodle_exception('reader_csv:header_field_count_mismatch', 'tool_dataflows', '', (object) [ 'numfields' => $numfields, 'numheaders' => $numheaders, + 'rownumber' => $rownumber, ], json_encode([ 'fields' => $data, 'headers' => $headers, @@ -109,6 +135,8 @@ public function csv_contents_generator() { } finally { fclose($handle); } + + $variables->set('errors', (object) $errors); } /** @@ -150,6 +178,10 @@ public function form_add_custom_inputs(\MoodleQuickForm &$mform) { $mform->hideIf('config_overwriteheaders', 'config_headers', 'eq', ''); $mform->disabledIf('config_overwriteheaders', 'config_headers', 'eq', ''); + // Used when we want to replace the headers (or overwrite them) using the ones we supplied instead. Defaults to off. + $mform->addElement('checkbox', 'config_continueonerror', get_string('reader_csv:continueonerror', 'tool_dataflows'), + get_string('reader_csv:continueonerror_help', 'tool_dataflows')); + // Delimiter. $mform->addElement( 'text', diff --git a/classes/local/step/set_multiple_variables_trait.php b/classes/local/step/set_multiple_variables_trait.php new file mode 100644 index 00000000..6d2ea232 --- /dev/null +++ b/classes/local/step/set_multiple_variables_trait.php @@ -0,0 +1,101 @@ +. + +namespace tool_dataflows\local\step; + +use tool_dataflows\local\variables\var_root; +use tool_dataflows\local\variables\var_object_visible; + +/** + * Set multiple variables trait + * + * Similar to the single approach, except it allows multiple to be set in a + * single step. This is great for initialising counters, and initial variables + * that need to be reset every run, but might change during. + * + * @package tool_dataflows + * @author Kevin Pham + * @copyright Catalyst IT, 2023 + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +trait set_multiple_variables_trait { + use set_variable_trait; + + /** + * Executes the step, fetching the config and actioning the step. + * + * @param mixed|null $input + * @return mixed + */ + public function execute($input = null) { + + $stepvars = $this->get_variables(); + $config = $stepvars->get('config'); + $rootvars = $this->get_variables_root(); + $this->run($rootvars, $config->field, $config->values); + + return $input; + } + + /** + * Main handler of this step, split out to make it easier to test. + * + * @param var_root $varobject + * @param string $field + * @param mixed $values + */ + public function run(var_root $varobject, string $field, $values) { + // Do nothing if the value has not changed. + $currentvalue = $varobject->get($field); + if ($currentvalue === $values) { + return; + } + + // Set the value in the variable tree. + $varobject->set($field, $values); + $this->log->info("Set '{field}' as '{values}'", ['field' => $field, 'values' => json_encode($values)]); + + // We do not persist the value if it is a dry run. + if ($this->is_dry_run()) { + return; + } + } + + /** + * Return the definition of the fields available in this form. + * + * @return array + */ + public static function form_define_fields(): array { + return [ + 'field' => ['type' => PARAM_TEXT, 'required' => true], + 'values' => ['type' => PARAM_TEXT, 'required' => true, 'yaml' => true], + ]; + } + + /** + * Custom form inputs + * + * @param \MoodleQuickForm $mform + */ + public function form_add_custom_inputs(\MoodleQuickForm &$mform) { + $mform->addElement('text', 'config_field', get_string('set_multiple_variables:field', 'tool_dataflows')); + $mform->addElement('static', 'config_field_help', '', get_string('set_multiple_variables:field_help', 'tool_dataflows')); + + $mform->addElement('textarea', 'config_values', get_string('set_multiple_variables:values', 'tool_dataflows')); + $mform->addElement('static', 'config_field_help', '', get_string('set_multiple_variables:values_help', 'tool_dataflows')); + } +} diff --git a/classes/local/step/sftp_trait.php b/classes/local/step/sftp_trait.php index 1304dea4..0c61c53b 100644 --- a/classes/local/step/sftp_trait.php +++ b/classes/local/step/sftp_trait.php @@ -35,10 +35,13 @@ trait sftp_trait { /** Shorthand sftp scheme for use in config. */ - static protected $sftpprefix = 'sftp'; + protected static $sftpprefix = 'sftp'; /** Default port to connect to. */ - static protected $defaultport = 22; + protected static $defaultport = 22; + + /** Array of SFTP objects to use for performance reasons. */ + protected $sftp = []; /** * Returns whether or not the step configured, has a side effect. @@ -93,17 +96,14 @@ public function form_add_core_inputs(\MoodleQuickForm &$mform) { $mform->addElement('text', 'config_port', get_string('connector_sftp:port', 'tool_dataflows')); $mform->setDefault('config_port', self::$defaultport); $mform->addElement('text', 'config_hostpubkey', get_string('connector_sftp:hostpubkey', 'tool_dataflows')); - $mform->addElement('static', 'config_hostpubkey_desc', '', - get_string('connector_sftp:hostpubkey_desc', 'tool_dataflows')); + $mform->addElement('static', 'config_hostpubkey_desc', '', get_string('connector_sftp:hostpubkey_desc', 'tool_dataflows')); $mform->addElement('text', 'config_username', get_string('username')); $mform->addElement('passwordunmask', 'config_password', get_string('password')); - $mform->addElement('static', 'config_password_desc', '', - get_string('connector_sftp:password_desc', 'tool_dataflows')); + $mform->addElement('static', 'config_password_desc', '', get_string('connector_sftp:password_desc', 'tool_dataflows')); $mform->addElement('text', 'config_privkeyfile', get_string('connector_sftp:privkeyfile', 'tool_dataflows')); - $mform->addElement('static', 'config_keyfile_desc', '', - get_string('connector_sftp:keyfile_desc', 'tool_dataflows')); + $mform->addElement('static', 'config_keyfile_desc', '', get_string('connector_sftp:keyfile_desc', 'tool_dataflows')); } /** @@ -118,13 +118,19 @@ public function form_add_custom_inputs(\MoodleQuickForm &$mform, $behaviour = 'c if ($behaviour === 'copy') { $mform->addElement('text', 'config_source', get_string('connector_sftp:source', 'tool_dataflows')); $mform->addElement('static', 'config_source_desc', '', get_string('connector_sftp:source_desc', 'tool_dataflows'). - \html_writer::nonempty_tag('pre', get_string('connector_sftp:path_example', 'tool_dataflows'). - get_string('path_help_examples', 'tool_dataflows'))); + \html_writer::nonempty_tag( + 'pre', + get_string('connector_sftp:path_example', 'tool_dataflows'). get_string('path_help_examples', 'tool_dataflows') + ) + ); $mform->addElement('text', 'config_target', get_string('connector_sftp:target', 'tool_dataflows')); $mform->addElement('static', 'config_target_desc', '', get_string('connector_sftp:target_desc', 'tool_dataflows'). - \html_writer::nonempty_tag('pre', get_string('connector_sftp:path_example', 'tool_dataflows'). - get_string('path_help_examples', 'tool_dataflows'))); + \html_writer::nonempty_tag( + 'pre', + get_string('connector_sftp:path_example', 'tool_dataflows'). get_string('path_help_examples', 'tool_dataflows') + ) + ); } } @@ -247,11 +253,8 @@ public function execute($input = null) { $stepvars = $this->get_variables(); $config = $stepvars->get('config'); - $this->log->debug("Connecting to {$config->host}:{$config->port}"); - // At this point we need to disconnect once we are finished. try { - // Skip if it is a dry run. if ($this->is_dry_run() && $this->has_side_effect()) { return $input; @@ -278,7 +281,7 @@ public function execute($input = null) { // Upload to remote. $this->upload($sftp, $sourcepath, $targetpath); - } finally { + } catch (\Throwable $e) { if (isset($sftp)) { $sftp->disconnect(); } @@ -287,6 +290,28 @@ public function execute($input = null) { return $input; } + /** + * Hook function that gets called when an engine step has been aborted. + */ + public function on_abort() { + if (isset($this->sftp)) { + foreach ($this->sftp as $s) { + $s->disconnect(); + } + } + } + + /** + * Hook function that gets called when an engine step has been finalised. + */ + public function on_finalise() { + if (isset($this->sftp)) { + foreach ($this->sftp as $s) { + $s->disconnect(); + } + } + } + /** * Checks and loads the appropriate key, based on config * @@ -408,6 +433,14 @@ public function resolve_path(string $pathname): string { * @return SFTP */ private function init_sftp($config): SFTP { + // Use existing cached SFTP object if available. + $cachekey = implode('|', [$config->host, $config->port, $config->username]); + if (isset($this->sftp[$cachekey])) { + return $this->sftp[$cachekey]; + } + + // Create and connect to SFTP. + $this->log->debug("Connecting to {$config->host}:{$config->port}"); $sftp = new SFTP($config->host, $config->port); $this->check_public_host_key($sftp, $config->hostpubkey); @@ -417,6 +450,10 @@ private function init_sftp($config): SFTP { } $sftp->enableDatePreservation(); + + // Cache sftp since it takes a while to attempt initial connection. + $this->sftp[$cachekey] = $sftp; + return $sftp; } } diff --git a/classes/local/step/sql_trait.php b/classes/local/step/sql_trait.php index 5b1157cc..e3c13762 100644 --- a/classes/local/step/sql_trait.php +++ b/classes/local/step/sql_trait.php @@ -75,8 +75,8 @@ private function evaluate_expressions(string $sql) { [$hasexpression] = $parser->has_expression($el); $max--; } - if (!in_array(gettype($el), ['string', 'int'])) { - throw new \moodle_exception('sql_trait:sql_param_type_not_valid', 'tool_dataflows'); + if (!in_array(gettype($el), ['string', 'int', 'integer'])) { + throw new \moodle_exception('sql_trait:sql_param_type_not_valid', 'tool_dataflows', '', gettype($el)); } return $el; }, $expressions); @@ -167,4 +167,117 @@ public function validate_config($config) { } return empty($errors) ? true : $errors; } + + /** + * Return the definition of the fields available in this form. + * + * @return array + */ + public static function form_define_fields(): array { + return [ + 'sql' => ['type' => PARAM_TEXT, 'required' => true], + ]; + } + + /** + * Allows each step type to determine a list of optional/required form + * inputs for their configuration + * + * It's recommended you prefix the additional config related fields to avoid + * conflicts with any existing fields. + * + * @param \MoodleQuickForm $mform + */ + public function form_add_custom_inputs(\MoodleQuickForm &$mform) { + // SQL example and inputs. + $sqlexample = " + SELECT id, username + FROM {user} + WHERE id > \${{steps.xyz.var.number}} + ORDER BY id ASC + LIMIT 10"; + $sqlexamples = \html_writer::tag('pre', trim($sqlexample, " \t\r\0\x0B")); + $mform->addElement('textarea', 'config_sql', get_string('flow_sql:sql', 'tool_dataflows'), + ['max_rows' => 40, 'rows' => 5, 'style' => 'font: 87.5% monospace; width: 100%; max-width: 100%']); + $mform->addElement('static', 'config_sql_help', '', get_string('flow_sql:sql_help', 'tool_dataflows', $sqlexamples)); + } + + /** + * Allow steps to setup the form depending on current values. + * + * This method is called after definition(), data submission and set_data(). + * All form setup that is dependent on form values should go in here. + * + * @param \MoodleQuickForm $mform + * @param \stdClass $data + */ + public function form_definition_after_data(\MoodleQuickForm &$mform, \stdClass $data) { + // Validate the data. + $sqllinecount = count(explode(PHP_EOL, trim($data->config_sql))); + + // Get the element. + $element = $mform->getElement('config_sql'); + + // Update the element height based on min/max settings, but preserve + // other existing rules. + $attributes = $element->getAttributes(); + + // Set the rows at a minimum to the predefined amount in + // form_add_custom_inputs, and expand as content grows up to a maximum. + $attributes['rows'] = min( + $attributes['max_rows'], + max($attributes['rows'], $sqllinecount) + ); + $element->setAttributes($attributes); + } + + /** + * Execute configured query + * + * @param mixed $input + * @return mixed + * @throws \dml_read_exception when the SQL is not valid. + */ + public function execute($input = null) { + global $DB; + + // Construct the query. + $variables = $this->get_variables(); + $config = $variables->get_raw('config'); + [$sql, $params] = $this->evaluate_expressions($config->sql); + + // Now that we have the query, we want to get info on SQL keywords to figure out where to route the request. + // This is not used for security, just to route the request via the correct pathway for readonly databases. + $pattern = '/(SELECT|UPDATE|INSERT|DELETE)/im'; + $matches = []; + preg_match($pattern, $sql, $matches); + + // Matches[0] contains the match. Fallthrough to default on no match. + $token = $matches[0] ?? ''; + $emptydefault = new \stdClass(); + + switch(strtoupper($token)) { + case 'SELECT': + // Execute the query using get_records instead of get_record. + // This is so we can expose the number of records returned which + // can then be used by the dataflow in for e.g. a switch statement. + $records = $DB->get_records_sql($sql, $params); + + $variables->set('count', count($records)); + $invalidnum = ($records === false || count($records) !== 1); + $data = $invalidnum ? $emptydefault : array_pop($records); + $variables->set('data', $data); + break; + default: + // Default to execute. + $success = $DB->execute($sql, $params); + + // We can't really do anything with the response except check for success. + $variables->set('count', (int) $success); + $variables->set('data', $emptydefault); + break; + } + + return $input; + } } diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php index ab84b7fd..ba22bc0f 100644 --- a/lang/en/tool_dataflows.php +++ b/lang/en/tool_dataflows.php @@ -145,6 +145,7 @@ $string['step_name_connector_remove_file'] = 'Remove file'; $string['step_name_connector_s3'] = 'S3 file copy'; $string['step_name_connector_set_variable'] = 'Set variable'; +$string['step_name_connector_set_multiple_variables'] = 'Set multiple variables'; $string['step_name_connector_sftp'] = 'SFTP file copy'; $string['step_name_connector_sns_notify'] = 'AWS-SNS Notification'; $string['step_name_connector_wait'] = 'Wait'; @@ -159,6 +160,7 @@ $string['step_name_flow_hash_file'] = 'Hash file'; $string['step_name_flow_remove_file'] = 'Remove file'; $string['step_name_flow_set_variable'] = 'Set variable'; +$string['step_name_flow_set_multiple_variables'] = 'Set multiple variables'; $string['step_name_flow_logic_join'] = 'Join'; $string['step_name_flow_logic_switch'] = 'Switch'; $string['step_name_flow_noop'] = 'No-op'; @@ -179,6 +181,7 @@ $string['step_name_writer_stream'] = 'Stream writer'; $string['step_name_trigger_event'] = 'Moodle event'; $string['step_name_flow_sql'] = 'SQL'; +$string['step_name_connector_sql'] = 'SQL'; // Step (type) groups. $string['stepgrouptriggers'] = 'Triggers'; @@ -350,7 +353,7 @@ $string['writer_csv:fail_to_encode'] = 'Failed to encode CSV.'; // SQL trait. -$string['sql_trait:sql_param_type_not_valid'] = 'The SQL parameter must be a valid type (string or int).'; +$string['sql_trait:sql_param_type_not_valid'] = 'The SQL parameter must be a valid type (string or int), found {$a}'; $string['sql_trait:variable_not_valid_in_position_replacement_text'] = "Invalid expression \${{ {\$a->expression} }} as `{\$a->expressionpath}` could not be resolved at line {\$a->line} character {\$a->column} in:\n{\$a->sql}"; // phpcs:disable moodle.Strings.ForbiddenStrings.Found // Reader SQL. @@ -370,7 +373,9 @@ $string['reader_csv:headers_help'] = 'If populated, then this will act as the header to map field to keys. If left blank, it will be populated automatically using the first read row.'; $string['reader_csv:overwriteheaders'] = 'Overwrite existing headers'; $string['reader_csv:overwriteheaders_help'] = 'If checked, the headers supplied above will be used instead of the ones in the file, effectively ignoring the first row.'; -$string['reader_csv:header_field_count_mismatch'] = 'Number of headers ({$a->numheaders}) should match number of fields ({$a->numfields})'; +$string['reader_csv:continueonerror'] = 'Continue on parsing errors'; +$string['reader_csv:continueonerror_help'] = 'If checked, the step will continue reading the next row of data if there are parsing errors on the current.'; +$string['reader_csv:header_field_count_mismatch'] = 'Row #{$a->rownumber}: Number of fields ({$a->numfields}) should match number of headers ({$a->numheaders})'; // Reader JSON. $string['reader_json:arrayexpression_help'] = 'Nested array to extract from JSON. For example, {$a->expression} will return the users array from the following JSON (If empty it is assumed the starting point of the JSON file is an array):{$a->jsonexample}'; @@ -546,6 +551,7 @@ $string['flow_copy_file:from'] = 'From'; $string['flow_copy_file:to'] = 'To'; $string['flow_copy_file:copy_failed'] = 'Failed to copy {$a->from} to {$a->to}'; +$string['flow_copy_file:mkdir_failed'] = 'Failed to create directory at {$a}. Please check permissions and try again.'; // Directory file count. $string['connector_directory_file_count:path'] = 'Path to directory'; @@ -588,6 +594,12 @@ $string['set_variable:value'] = 'Value'; $string['set_variable:value_help'] = 'The value could be a number, text, or an expression. For example: ${{ record.id }}.'; +// Set multiple variables step. +$string['set_multiple_variables:field'] = 'Field'; +$string['set_multiple_variables:field_help'] = 'Defines the path to the field you would like to set the value(s). For example: dataflow.vars.counter.'; +$string['set_multiple_variables:values'] = 'Values'; +$string['set_multiple_variables:values_help'] = 'A list of fields/keys and values, in YAML format.'; + // Event trigger. $string['trigger_event:policy:immediate'] = 'Run immediately'; $string['trigger_event:policy:adhoc'] = 'Run ASAP in individual tasks in parallel'; diff --git a/lib.php b/lib.php index 1853d813..f5bf130a 100644 --- a/lib.php +++ b/lib.php @@ -69,6 +69,7 @@ function tool_dataflows_step_types() { new step\connector_s3, new step\connector_set_variable, new step\connector_sftp, + new step\connector_sql, new step\connector_update_user, new step\connector_sftp_directory_file_list, new step\connector_sns_notify, @@ -106,6 +107,8 @@ function tool_dataflows_step_types() { new step\trigger_webservice, new step\writer_debugging, new step\writer_stream, + new step\connector_set_multiple_variables, + new step\flow_set_multiple_variables, ]; } diff --git a/run.php b/run.php index c1d8fae4..d3edb790 100644 --- a/run.php +++ b/run.php @@ -44,8 +44,8 @@ function tool_dataflows_mtrace_wrapper($message, $eol) { // Mark up errors.. if (preg_match('/error:/im', $message)) { $class = 'bg-danger text-white'; - } else if (preg_match('/warn:/im', $message)) { - $class = 'bg-warning'; + } else if (preg_match('/warning:/im', $message)) { + $class = 'text-warning'; } else if (preg_match('/notice:/im', $message)) { $class = 'bold text-primary'; } else if (preg_match('/info:/im', $message)) { diff --git a/version.php b/version.php index 43b109a6..f1721742 100644 --- a/version.php +++ b/version.php @@ -25,8 +25,8 @@ defined('MOODLE_INTERNAL') || die(); -$plugin->version = 2023110901; -$plugin->release = 2023110901; +$plugin->version = 2023110902; +$plugin->release = 2023110902; $plugin->requires = 2017051500; // Our lowest supported Moodle (3.3.0). $plugin->supported = [35, 401]; // Available as of Moodle 3.9.0 or later. // TODO $plugin->incompatible = ; // Available as of Moodle 3.9.0 or later.