Merge pull request #855 from catalyst/additional-goodies-csv-sftp-per…

…f-sql-and-more additional goodies csv sftp perf sql and more
catalyst · Dec 22, 2023 · 66cf26e · 66cf26e
2 parents b8819bf + f7485ef
commit 66cf26e
Show file tree

Hide file tree

Showing 16 changed files with 460 additions and 162 deletions.
diff --git a/classes/form/step_form.php b/classes/form/step_form.php
@@ -107,11 +107,22 @@ public function definition() {
         $select->setMultiple(true);
 
         $dataflow = new dataflow($dataflowid);
-        $varroot = $dataflow->get_variables_root();
 
-        // List all the available fields available for configuration, in dot syntax.
-        $mform->addElement('static', 'fields', get_string('available_fields', 'tool_dataflows'),
-            $this->prepare_available_fields($varroot->get()));
+        try {
+            $varroot = $dataflow->get_variables_root();
+
+            // List all the available fields available for configuration, in dot syntax.
+            $mform->addElement(
+                'static',
+                'fields',
+                get_string('available_fields', 'tool_dataflows'),
+                $this->prepare_available_fields($varroot->get())
+            );
+        } catch (\Throwable $e) {
+            global $OUTPUT;
+            $errtext  = $OUTPUT->notification($e->getMessage());
+            $mform->addElement('static', 'vars_error', get_string('available_fields', 'tool_dataflows'), $errtext);
+        }
 
         // Check and set custom form inputs if required. Defaulting to a
         // textarea config input for those not yet configured.

diff --git a/classes/local/execution/engine.php b/classes/local/execution/engine.php
@@ -211,6 +211,13 @@ public function __construct(dataflow $dataflow, bool $isdryrun = false, $automat
 
         // Find the flow blocks.
         $this->create_flow_caps();
+
+        // Make the runid available to the flow.
+        if (!$this->isdryrun) {
+            $variables = $this->get_variables();
+            $variables->set('run.name', $this->run->name);
+            $variables->set('run.id', $this->run->id);
+        }
     }
 
     /**
@@ -792,7 +799,7 @@ private function setup_logging() {
 
         // Dataflow run logger.
         // Type: FILE_PER_RUN
-        // e.g. '[dataroot]/tool_dataflows/3/20060102150405-21.log' as the path.
+        // e.g. '[dataroot]/tool_dataflows/3/Ymd_His.uuu_21.log' as the path.
         if (isset($loghandlers[log_handler::FILE_PER_RUN])) {
             $dataflowrunlogpath = $CFG->dataroot . DIRECTORY_SEPARATOR .
                 'tool_dataflows' . DIRECTORY_SEPARATOR .

diff --git a/classes/local/step/connector_set_multiple_variables.php b/classes/local/step/connector_set_multiple_variables.php
@@ -0,0 +1,35 @@
+<?php
+// This file is part of Moodle - https://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace tool_dataflows\local\step;
+
+/**
+ * Set multiple variables connector step
+ *
+ * @package   tool_dataflows
+ * @author    Kevin Pham <[email protected]>
+ * @copyright Catalyst IT, 2023
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class connector_set_multiple_variables extends connector_step {
+    use set_multiple_variables_trait;
+
+    /** @var int[] number of output flows (min, max). */
+    protected $outputflows = [0, 1];
+
+    /** @var int[] number of output connectors (min, max). */
+    protected $outputconnectors = [0, 1];
+}
diff --git a/classes/local/step/connector_sql.php b/classes/local/step/connector_sql.php
@@ -0,0 +1,29 @@
+<?php
+// This file is part of Moodle - https://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace tool_dataflows\local\step;
+
+/**
+ * SQL connector step
+ *
+ * @package   tool_dataflows
+ * @author    Kevin Pham <[email protected]>
+ * @copyright Catalyst IT, 2023
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class connector_sql extends connector_step {
+    use sql_trait;
+}
diff --git a/classes/local/step/copy_file_trait.php b/classes/local/step/copy_file_trait.php
@@ -72,7 +72,9 @@ public function execute($input = null) {
         $todirectory = dirname($to);
         if (!file_exists($todirectory)) {
             $this->log("Creating a directory at {$todirectory}");
-            mkdir($todirectory, $CFG->directorypermissions, true);
+            if (!mkdir($todirectory, $CFG->directorypermissions, true)) {
+                throw new \moodle_exception('flow_copy_file:mkdir_failed', 'tool_dataflows', '', $todirectory);
+            }
         }
 
         // Attempt to copy the file to the destination.
@@ -112,10 +114,12 @@ public function execute($input = null) {
     private function copy(string $from, string $to) {
         $this->log("Copying $from to $to");
         if (!copy($from, $to)) {
-            throw new \moodle_exception('flow_copy_file:copy_failed', 'tool_dataflows', (object) [
-                'from' => $from,
-                'to' => $to,
-            ]);
+            throw new \moodle_exception(
+                'flow_copy_file:copy_failed',
+                'tool_dataflows',
+                '',
+                (object) ['from' => $from, 'to' => $to]
+            );
         }
     }
 

diff --git a/classes/local/step/flow_email.php b/classes/local/step/flow_email.php
@@ -28,6 +28,12 @@
  */
 class flow_email extends flow_step {
 
+    /** @var int[] number of output flows (min, max). */
+    protected $outputflows = [0, 1];
+
+    /** @var int[] number of output connectors (min, max) */
+    protected $outputconnectors = [0, 1];
+
     /** @var bool whether or not this step type (potentially) contains a side effect or not */
     protected $hassideeffect = true;
 

diff --git a/classes/local/step/flow_set_multiple_variables.php b/classes/local/step/flow_set_multiple_variables.php
@@ -0,0 +1,35 @@
+<?php
+// This file is part of Moodle - http://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace tool_dataflows\local\step;
+
+/**
+ * Set multiple variables flow step
+ *
+ * @package   tool_dataflows
+ * @author    Kevin Pham <[email protected]>
+ * @copyright Catalyst IT, 2023
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class flow_set_multiple_variables extends flow_step {
+    use set_multiple_variables_trait;
+
+    /** @var int[] number of output flows (min, max). */
+    protected $outputflows = [0, 1];
+
+    /** @var int[] number of output connectors (min, max). */
+    protected $outputconnectors = [0, 1];
+}
diff --git a/classes/local/step/flow_sql.php b/classes/local/step/flow_sql.php
@@ -27,131 +27,4 @@
 class flow_sql extends flow_step {
     use sql_trait;
 
-    /**
-     * Return the definition of the fields available in this form.
-     *
-     * @return array
-     */
-    public static function form_define_fields(): array {
-        return [
-            'sql' => ['type' => PARAM_TEXT, 'required' => true],
-        ];
-    }
-
-    /**
-     * Allows each step type to determine a list of optional/required form
-     * inputs for their configuration
-     *
-     * It's recommended you prefix the additional config related fields to avoid
-     * conflicts with any existing fields.
-     *
-     * @param \MoodleQuickForm $mform
-     */
-    public function form_add_custom_inputs(\MoodleQuickForm &$mform) {
-        // SQL example and inputs.
-        $sqlexample = "
-        SELECT id, username
-          FROM {user}
-      WHERE id > \${{steps.xyz.var.number}}
-      ORDER BY id ASC
-         LIMIT 10";
-        $sqlexamples = \html_writer::tag('pre', trim($sqlexample, " \t\r\0\x0B"));
-        $mform->addElement('textarea', 'config_sql', get_string('flow_sql:sql', 'tool_dataflows'),
-            ['max_rows' => 40, 'rows' => 5, 'style' => 'font: 87.5% monospace; width: 100%; max-width: 100%']);
-        $mform->addElement('static', 'config_sql_help', '', get_string('flow_sql:sql_help', 'tool_dataflows', $sqlexamples));
-    }
-
-    /**
-     * Allow steps to setup the form depending on current values.
-     *
-     * This method is called after definition(), data submission and set_data().
-     * All form setup that is dependent on form values should go in here.
-     *
-     * @param \MoodleQuickForm $mform
-     * @param \stdClass $data
-     */
-    public function form_definition_after_data(\MoodleQuickForm &$mform, \stdClass $data) {
-        // Validate the data.
-        $sqllinecount = count(explode(PHP_EOL, trim($data->config_sql)));
-
-        // Get the element.
-        $element = $mform->getElement('config_sql');
-
-        // Update the element height based on min/max settings, but preserve
-        // other existing rules.
-        $attributes = $element->getAttributes();
-
-        // Set the rows at a minimum to the predefined amount in
-        // form_add_custom_inputs, and expand as content grows up to a maximum.
-        $attributes['rows'] = min(
-            $attributes['max_rows'],
-            max($attributes['rows'], $sqllinecount)
-        );
-        $element->setAttributes($attributes);
-    }
-
-    /**
-     * Execute configured query
-     *
-     * @param   mixed $input
-     * @return  mixed
-     * @throws \dml_read_exception when the SQL is not valid.
-     */
-    public function execute($input = null) {
-        global $DB;
-
-        // Construct the query.
-        $variables = $this->get_variables();
-        $config = $variables->get_raw('config');
-        [$sql, $params] = $this->evaluate_expressions($config->sql);
-
-        // Now that we have the query, we want to get info on SQL keywords to figure out where to route the request.
-        // This is not used for security, just to route the request via the correct pathway for readonly databases.
-        $pattern = '/(SELECT|UPDATE|INSERT|DELETE)/im';
-        $matches = [];
-        preg_match($pattern, $sql, $matches);
-
-        // Matches[0] contains the match. Fallthrough to default on no match.
-        $token = $matches[0] ?? '';
-        $emptydefault = new \stdClass();
-
-        switch(strtoupper($token)) {
-            case 'SELECT':
-                // Execute the query using get_records instead of get_record.
-                // This is so we can expose the number of records returned which
-                // can then be used by the dataflow in for e.g. a switch statement.
-                $records = $DB->get_records_sql($sql, $params);
-
-                $variables->set('count', count($records));
-                $invalidnum = ($records === false || count($records) !== 1);
-                $data = $invalidnum ? $emptydefault : array_pop($records);
-                $variables->set('data', $data);
-                break;
-            default:
-                // Default to execute.
-                $success = $DB->execute($sql, $params);
-
-                // We can't really do anything with the response except check for success.
-                $variables->set('count', (int) $success);
-                $variables->set('data', $emptydefault);
-                break;
-        }
-
-        return $input;
-    }
-
-    /**
-     * Validate the configuration settings.
-     *
-     * @param   object $config
-     * @return  true|\lang_string[] true if valid, an array of errors otherwise
-     */
-    public function validate_config($config) {
-        $errors = [];
-        if (empty($config->sql)) {
-            $errors['config_sql'] = get_string('config_field_missing', 'tool_dataflows', 'sql', true);
-        }
-
-        return empty($errors) ? true : $errors;
-    }
 }