Issue #158: Add a scratch directory for dataflows.

classes/helper.php

@@ -0,0 +1,110 @@
+<?php
+// This file is part of Moodle - https://moodle.org/
+//
+// Moodle is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// Moodle is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with Moodle.  If not, see <http://www.gnu.org/licenses/>.
+
+namespace tool_dataflows;
+
+/**
+ * Support functions for dataflows.
+ *
+ * @package   tool_dataflows
+ * @author    Jason den Dulk <[email protected]>
+ * @copyright 2022, Catalyst IT
+ * @license   http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later
+ */
+class helper {
+
+    /**
+     * Get the scheme for a path string. will default to 'file' if none present.
+     *
+     * @param string $path
+     * @return string
+     */
+    public static function path_get_scheme(string $path): string {
+        $splitpath = explode('://', $path, 2);
+        if (count($splitpath) !== 2) {
+            return 'file';
+        }
+        return $splitpath[0];
+    }
+
+    /**
+     * Does the path string include a scheme?
+     *
+     * @param string $path
+     * @return bool
+     */
+    public static function path_has_scheme(string $path): bool {
+        $splitpath = explode('://', $path, 2);
+        return count($splitpath) === 2;
+    }
+
+    /**
+     * Is the path a relative one?
+     * Note: Windows paths are not yet supported.
+     *
+     * @param string $path
+     * @return bool
+     */
+    public static function path_is_relative(string $path): bool {
+        if (self::path_has_scheme($path)) {
+            return false;
+        }
+
+        // Unix absolute path.
+        return substr($path, 0, 1) !== '/';
+
+        // TODO: Windows support.
+    }
+
+    /**
+     * Makes a full path from a relative one using the given base dir.
+     *
+     * @param  string $path
+     * @param  string $scratchdir
+     * @return string
+     */
+    public static function path_get_absolute(string $path, string $scratchdir): string {
+        if (self::path_is_relative($path)) {
+            return $scratchdir . '/' . $path;
+        }
+        return $path;
+    }
+
+    /**
+     * Validate a path.
+     *
+     * @param  string $path
+     * @return \lang_string|true True if the path is valid. A lang_string otherwise.
+     */
+    public static function path_validate(string $path) {
+        if (self::path_is_relative($path)) {
+            return true;
+        }
+
+        if (self::path_get_scheme($path) !== 'file') {
+            return true;
+        }
+
+        $permitteddirs = explode(PHP_EOL, trim(get_config('tool_dataflows', 'permitted_dirs')));
+        foreach ($permitteddirs as $dir) {
+            if (substr($path, 0, strlen($dir)) === $dir) {
+                return true;
+            }
+        }
+
+        return get_string('path_invalid', 'tool_dataflows', $path, true);
+    }
+}

classes/local/execution/engine.php

@@ -19,6 +19,7 @@
 use Symfony\Component\Yaml\Yaml;
 use tool_dataflows\dataflow;
 use tool_dataflows\exportable;
+use tool_dataflows\helper;
 use tool_dataflows\local\step\flow_cap;
 use tool_dataflows\run;
 
@@ -106,6 +107,9 @@ class engine {
     /** @var bool True if executing via automation. */
     protected $automated = true;
 
+    /** @var string Scratch directory for temporary files. */
+    protected $scratchdir = null;
+
     /**
      * Constructs the engine.
      *
@@ -172,6 +176,8 @@ public function initialise() {
             // Add sinks to the execution queue.
             $this->queue = $this->sinks;
             $this->set_status(self::STATUS_INITIALISED);
+
+            $this->scratchdir = make_request_directory();
         } catch (\Throwable $thrown) {
             $this->abort($thrown);
         }
@@ -339,6 +345,7 @@ public function __get($parameter) {
             case 'isdryrun':
             case 'run':
             case 'status':
+            case 'scratchdir':
                 return $this->$parameter;
             case 'name':
                 return $this->dataflow->name;
@@ -471,4 +478,20 @@ protected function status_check(int $expected) {
             );
         }
     }
+
+    /**
+     * Resolves the full path name for the givne path. If the directory does nto exist, it will create it.
+     *
+     * @param  string $pathname
+     * @param  string $mode
+     * @return false|resource
+     */
+    public function resolve_path(string $pathname) {
+        $fullpath = helper::path_get_absolute($pathname, $this->scratchdir);
+        $dir = dirname($fullpath);
+        if (!file_exists($dir)) {
+            mkdir($dir);
+        }
+        return $fullpath;
+    }
 }

classes/local/step/writer_stream.php

@@ -20,6 +20,7 @@
 use tool_dataflows\local\execution\iterators\iterator;
 use tool_dataflows\local\execution\iterators\dataflow_iterator;
 use tool_dataflows\manager;
+use tool_dataflows\helper;
 
 /**
  * Stream writer step. Writes to a PHP stream.
@@ -99,8 +100,8 @@ public function get_iterator(): iterator {
             private $writer;
 
             public function __construct(flow_engine_step $step, string $streamname, string $format, iterator $input) {
-                $this->streamname = $streamname;
-                $this->handle = fopen($streamname, 'a');
+                $this->streamname = $step->engine->resolve_path($streamname);
+                $this->handle = fopen($this->streamname, 'a');
                 if ($this->handle === false) {
                     $step->log(error_get_last()['message']);
                     throw new \moodle_exception(get_string('writer_stream:failed_to_open_stream', 'tool_dataflows', $streamname));
@@ -155,6 +156,11 @@ public function validate_config($config) {
         $errors = [];
         if (!isset($config->streamname)) {
             $errors['config_streamname'] = get_string('config_field_missing', 'tool_dataflows', 'streamname', true);
+        } else {
+            $error = helper::path_validate($config->streamname);
+            if ($error !== true) {
+                $errors['config_streamname'] = $error;
+            }
         }
         if (!isset($config->format)) {
             $errors['config_format'] = get_string('config_field_missing', 'tool_dataflows', 'format', true);

lang/en/tool_dataflows.php

@@ -34,6 +34,8 @@
 $string['pluginsettings'] = 'General settings';
 $string['enabled'] = 'Enable/disable this plugin';
 $string['enabled_help'] = '';
+$string['permitted_dirs'] = 'Permitted directories';
+$string['permitted_dirs_desc'] = 'List directories here to allow them to be read from/written to by dataflow steps.';
 
 // Manage flows / Overview.
 $string['overview'] = 'Overview';
@@ -167,6 +169,7 @@
 $string['change_state_after_concluded'] = 'Attempting to change the status of a dataflow engine after it has concluded.';
 $string['bad_status'] = 'Bad status, had "{$a->status}", expected "{$a->expected}"';
 $string['must_have_a_step_def_defined'] = 'If an engine is passed as a parameter, a step definition must alse be passed.';
+$string['path_invalid'] = 'Path "{$a}" is not permitted.';
 
 // Stream errors.
 $string['writer_stream:failed_to_open_stream'] = 'Failed to open stream "{$a}".';

settings.php

@@ -32,14 +32,27 @@
     $settings = new admin_settingpage('tool_dataflows_settings',
         get_string('pluginsettings', 'tool_dataflows'));
 
-    $settings->add(new admin_setting_configcheckbox('tool_dataflows/enabled',
-        get_string('enabled', 'tool_dataflows'),
-        get_string('enabled_help', 'tool_dataflows'), '0'));
-
     $dataflowsettings = new admin_externalpage('tool_dataflows_overview',
         get_string('pluginmanage', 'tool_dataflows'),
         new moodle_url('/admin/tool/dataflows/index.php'));
 
+    if ($ADMIN->fulltree) {
+        $settings->add(new admin_setting_configcheckbox(
+            'tool_dataflows/enabled',
+            get_string('enabled', 'tool_dataflows'),
+            get_string('enabled_help', 'tool_dataflows'),
+            '0'
+        ));
+
+        $settings->add(new admin_setting_configtextarea(
+            'tool_dataflows/permitted_dirs',
+            get_string('permitted_dirs', 'tool_dataflows'),
+            get_string('permitted_dirs_desc', 'tool_dataflows'),
+            '',
+            PARAM_RAW
+        ));
+    }
+
     $ADMIN->add('tool_dataflows', $settings);
     $ADMIN->add('tool_dataflows', $dataflowsettings);
 

version.php

@@ -25,8 +25,8 @@
 
 defined('MOODLE_INTERNAL') || die();
 
-$plugin->version = 2022062900;
-$plugin->release = 2022062900;
+$plugin->version = 2022070500;
+$plugin->release = 2022070500;
 
 $plugin->requires = 2017051500;    // Our lowest supported Moodle (3.3.0).