From a54fd41d97c53a54ea01ef4b812b504418797105 Mon Sep 17 00:00:00 2001 From: Marc-Alexandre Ghaly Date: Mon, 27 Jun 2022 17:18:19 -0400 Subject: [PATCH] Resolves Prod ready curl step #167 --- classes/local/step/connector_curl.php | 171 ++++++++++++++- lang/en/tool_dataflows.php | 12 ++ tests/tool_dataflows_curl_connector_test.php | 208 +++++++++++++++++++ 3 files changed, 389 insertions(+), 2 deletions(-) create mode 100644 tests/tool_dataflows_curl_connector_test.php diff --git a/classes/local/step/connector_curl.php b/classes/local/step/connector_curl.php index bc897f81..3885dad7 100644 --- a/classes/local/step/connector_curl.php +++ b/classes/local/step/connector_curl.php @@ -16,12 +16,14 @@ namespace tool_dataflows\local\step; use tool_dataflows\local\execution\engine_step; +use Symfony\Component\Yaml\Yaml; /** * CURL connector step type * * @package tool_dataflows * @author Kevin Pham + * @author Ghaly Marc-Alexandre * @copyright Catalyst IT, 2022 * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later */ @@ -30,15 +32,180 @@ class connector_curl extends connector_step { /** @var bool whether or not this step type (potentially) contains a side effect or not */ protected $hassideeffect = true; + /** @var int Time after which curl request is aborted */ + protected $timeout = 60; + + /** + * Return the definition of the fields available in this form. + * + * @return array + */ + protected static function form_define_fields(): array { + return [ + 'curl' => ['type' => PARAM_URL], + 'destination' => ['type' => PARAM_PATH], + 'headers' => ['type' => PARAM_RAW], + 'method' => ['type' => PARAM_TEXT], + 'rawpostdata' => ['type' => PARAM_RAW], + 'sideeffects' => ['type' => PARAM_RAW], + 'timeout' => ['type' => PARAM_INT], + ]; + } + + /** + * Allows each step type to determine a list of optional/required form + * inputs for their configuration + * + * It's recommended you prefix the additional config related fields to avoid + * conflicts with any existing fields. + * + * @param \MoodleQuickForm &$mform + */ + public function form_add_custom_inputs(\MoodleQuickForm &$mform) { + $jsonexample = [ + 'header-key' => 'header value', + 'another-key' => '1234' + ]; + $ex['json'] = \html_writer::nonempty_tag('pre', json_encode($jsonexample, JSON_PRETTY_PRINT)); + $ex['yaml'] = '
header-key: header value
another-key: 1234
'; + $mform->addElement('text', 'config_curl', get_string('connector_curl:curl', 'tool_dataflows'), + ['cols' => 50, 'rows' => 7]); + $mform->addElement('text', 'config_destination', get_string('connector_curl:destination', 'tool_dataflows')); + $mform->addHelpButton('config_destination', 'connector_curl:destination', 'tool_dataflows'); + $mform->addElement('textarea', 'config_headers', get_string('connector_curl:headers', 'tool_dataflows'), + ['cols' => 50, 'rows' => 7]); + $mform->addElement('static', 'headers_help', '', get_string('connector_curl:field_headers_help', 'tool_dataflows', $ex)); + $mform->addElement('select', 'config_method', get_string('connector_curl:method', 'tool_dataflows'), + ['get' => 'GET', 'post' => 'POST', 'put' => 'PUT']); + $mform->addElement('textarea' , 'config_rawpostdata', get_string('connector_curl:rawpostdata', 'tool_dataflows'), + ['cols' => 50, 'rows' => 7]); + $mform->addElement('checkbox', 'config_sideeffects', get_string('connector_curl:sideeffects', 'tool_dataflows'), + get_string('yes')); + $mform->hideIf('config_rawpostdata', 'config_method', 'eq', 'get'); + $mform->disabledIf('config_rawpostdata', 'config_method', 'eq', 'get'); + $mform->addElement('text', 'config_timeout', get_string('connector_curl:timeout', 'tool_dataflows')); + $mform->addHelpButton('config_timeout', 'connector_curl:timeout', 'tool_dataflows'); + } + + /** + * Validate the configuration settings. + * + * @param object $config + * @return true|\lang_string[] true if valid, an array of errors otherwise + */ + public function validate_config($config) { + $errors = []; + if (empty($config->curl)) { + $errors['config_curl'] = get_string('config_field_missing', 'tool_dataflows', 'curl', true); + } + if (empty($config->rawpostdata) && ($config->method === 'put' || $config->method === 'post')) { + $errors['config_rawpostdata'] = get_string('config_field_missing', 'tool_dataflows', 'rawpostdata', true); + } + return empty($errors) ? true : $errors; + } + /** * Executes the step * - * TODO: This will perform a curl call + * Performs a curl call according to given parameters. * * @return bool Returns true if successful, false otherwise. */ public function execute(): bool { - // TODO: Implement. + // Get variables. + $config = $this->enginestep->stepdef->config; + $isdryrun = $this->enginestep->engine->isdryrun; + $method = $config->method; + $dbgcommand = 'curl -X ' . strtoupper($method) . ' ' . $config->curl; + $result = null; + + if (!empty($config->timeout)) { + $this->timeout = (int) $config->timeout; + $dbgcommand .= ' --max-time ' . $config->timeout; + } + + $options = ['CURLOPT_TIMEOUT' => $this->timeout]; + + if ($method === 'post') { + $options['CURLOPT_POST'] = 1; + } + + if ($method === 'put') { + $options['CURLOPT_PUT'] = 1; + } + + $curl = new \curl(); + + $this->hassideeffect = !empty($config->sideeffects); + + // For put and post methods automatically overrides/has side effects. + if ($config->method != 'get') { + $this->hassideeffect = true; + } + + // Provided a header is specified add header to request. + if (!empty($config->headers)) { + $headers = $config->headers; + if (!is_array($headers)) { + $headers = json_decode($headers, true); + } + if (is_null($headers)) { + $headers = Yaml::parse($config->headers); + } + $curl->setHeader($headers); + $dbgcommand .= ' -H \'' . $config->headers . '\''; + } + + // Sets post data. + if (!empty($config->rawpostdata)) { + $options['CURLOPT_POSTFIELDS'] = $config->rawpostdata; + $dbgcommand .= ' -d \'' . $config->rawpostdata . '\''; + } + + // Download response to file provided destination is set. + if (!empty($config->destination) && !$isdryrun) { + if ($config->destination[0] === '/') { + $config->destination = ltrim($config->destination, '/'); + } + $config->destination = $this->enginestep->engine->resolve_path($config->destination); + $file = fopen($config->destination, 'w'); + $options['CURLOPT_FILE'] = $file; + } + + // Perform call. + if (!$isdryrun) { + $result = $curl->$method($config->curl, [], $options); + } + + if (!empty($file)) { + fclose($file); + } + + $info = $curl->get_info(); + // Stores response to be reusable by other steps. + // TODO : Once set_var api is refactored add response. + $response = $curl->getResponse(); + $httpcode = $info['http_code'] ?? null; + $connecttime = $info['connect_time'] ?? null; + $totaltime = $info['total_time'] ?? null; + $sizeupload = $info['size_upload'] ?? null; + $destination = !empty($config->destination) ? $config->destination : null; + $errno = $curl->get_errno(); + + if (($httpcode >= 400 || empty($response) || $errno == 28) && !$isdryrun) { + throw new \moodle_exception($httpcode . ':' . $result); + } + + if (!$isdryrun) { + $this->enginestep->stepdef->set_var('result', $result); + $this->enginestep->stepdef->set_var('httpcode', $httpcode); + $this->enginestep->stepdef->set_var('connecttime', $connecttime); + $this->enginestep->stepdef->set_var('totaltime', $totaltime); + $this->enginestep->stepdef->set_var('sizeupload', $sizeupload); + $this->enginestep->stepdef->set_var('destination', $destination); + } else { + $this->enginestep->stepdef->set_var('dbgcommand', $dbgcommand); + } return true; } } diff --git a/lang/en/tool_dataflows.php b/lang/en/tool_dataflows.php index a9d7ba3b..b8231379 100644 --- a/lang/en/tool_dataflows.php +++ b/lang/en/tool_dataflows.php @@ -282,3 +282,15 @@ $string['connector_s3:missing_s3_source_or_target'] = 'At least one source or target path must reference a location in S3.'; $string['connector_s3:source_is_a_directory'] = 'The source path is a directory but a file path is expected.'; +// Connector cURL. +$string['connector_curl:curl'] = 'Client URL'; +$string['connector_curl:destination'] = 'File / Response destination'; +$string['connector_curl:destination_help'] = 'Should this field be left blank, +response will be stored in step definition to be reused subsequently by other steps.'; +$string['connector_curl:headers'] = 'Headers'; +$string['connector_curl:method'] = 'HTTP request method'; +$string['connector_curl:rawpostdata'] = 'Raw post data'; +$string['connector_curl:sideeffects'] = 'Does this request have side effects?'; +$string['connector_curl:timeout'] = 'Timeout'; +$string['connector_curl:timeout_help'] = 'Time after which curl request will abort, default is 60 seconds.'; +$string['connector_curl:field_headers_help'] = 'Headers should be in the following JSON format: {$a->json} You can also use the following YAML format: {$a->yaml}'; diff --git a/tests/tool_dataflows_curl_connector_test.php b/tests/tool_dataflows_curl_connector_test.php new file mode 100644 index 00000000..84b70b6f --- /dev/null +++ b/tests/tool_dataflows_curl_connector_test.php @@ -0,0 +1,208 @@ +. + +namespace tool_dataflows; + +use Symfony\Component\ExpressionLanguage\ExpressionLanguage; +use Symfony\Component\Yaml\Yaml; +use tool_dataflows\local\step\connector_curl; +use tool_dataflows\local\execution\engine; +use tool_dataflows\step; +use tool_dataflows\dataflow; + +/** + * Unit test for the SQL curl connector step. + * + * @package tool_dataflows + * @author Ghaly Marc-Alexandre + * @copyright 2022, Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class tool_dataflows_curl_connector_test extends \advanced_testcase { + /** + * Set up before each test + */ + protected function setUp(): void { + parent::setUp(); + $this->resetAfterTest(); + } + + /** + * Tests the execute() function. + * + * @covers \tool_dataflows\local\step\connector_curl::execute + */ + public function test_execute() { + global $CFG; + + $connectorcurl = new connector_curl(); + $testgeturl = $this->getExternalTestFileUrl('/h5puuid.json'); + + $stepdef = new step(); + $dataflow = new dataflow(); + $dataflow->name = 'connector-step'; + $dataflow->enabled = true; + $dataflow->save(); + // Tests get method. + $stepdef->config = Yaml::dump([ + 'curl' => $testgeturl, + 'destination' => '', + 'headers' => '', + 'method' => 'get' + ]); + $stepdef->name = 'connector'; + $stepdef->type = 'tool_dataflows\local\step\connector_curl'; + $dataflow->add_step($stepdef); + ob_start(); + $engine = new engine($dataflow, false, false); + $engine->execute(); + ob_get_clean(); + $variables = $engine->get_variables()['steps']->connector->config; + // Result can be anything but for readability decoded to see vars. + $result = json_decode($variables->result, true);; + $this->assertEquals($result['uuid'], '3d188fbf-d0b7-4d4e-ae4d-4b5548df824e'); + + $this->assertEquals($variables->httpcode, 200); + $this->assertObjectHasAttribute('connecttime', $variables); + $this->assertObjectHasAttribute('totaltime', $variables); + $this->assertObjectHasAttribute('sizeupload', $variables); + + $testurl = $this->getExternalTestFileUrl('/test_post.php'); + + // Tests post method. + $stepdef->config = Yaml::dump([ + 'curl' => $testurl, + 'destination' => '', + 'headers' => '', + 'method' => 'post', + 'rawpostdata' => 'data=moodletest' + ]); + $dataflow->add_step($stepdef); + ob_start(); + $engine = new engine($dataflow, false, false); + $engine->execute(); + ob_get_clean(); + $variables = $engine->get_variables()['steps']->connector->config; + + $this->assertEmpty($variables->result); + $this->assertEquals($variables->httpcode, 200); + $this->assertObjectHasAttribute('connecttime', $variables); + $this->assertObjectHasAttribute('totaltime', $variables); + $this->assertObjectHasAttribute('sizeupload', $variables); + + // Tests put method. + $stepdef->config = Yaml::dump([ + 'curl' => $testurl, + 'destination' => '', + 'headers' => '', + 'method' => 'put', + 'rawpostdata' => 'data=moodletest' + ]); + $dataflow->add_step($stepdef); + ob_start(); + $engine = new engine($dataflow, false, false); + $engine->execute(); + ob_get_clean(); + $variables = $engine->get_variables()['steps']->connector->config; + + $this->assertEmpty($variables->result); + $this->assertEquals($variables->httpcode, 200); + $this->assertObjectHasAttribute('connecttime', $variables); + $this->assertObjectHasAttribute('totaltime', $variables); + $this->assertObjectHasAttribute('sizeupload', $variables); + + // Tests debug command when dry run. + $stepdef->config = Yaml::dump([ + 'curl' => $testurl, + 'destination' => '', + 'headers' => '', + 'method' => 'post', + 'rawpostdata' => '{ + "name": "morpheus", + "job": "leader" + }' + ]); + $dataflow->add_step($stepdef); + ob_start(); + $engine = new engine($dataflow, true, false); + $engine->execute(); + ob_get_clean(); + $variables = $engine->get_variables()['steps']->connector->config; + $this->assertObjectNotHasAttribute('result', $variables); + $this->assertEquals($variables->dbgcommand, "curl -X POST {$testurl} -d '{ + \"name\": \"morpheus\", + \"job\": \"leader\" + }'\n"); + + // Test file writting. + $tofile = "test.html"; + $stepdef->config = Yaml::dump([ + 'curl' => $testgeturl, + 'destination' => $tofile, + 'headers' => '', + 'method' => 'get' + ]); + $dataflow->add_step($stepdef); + ob_start(); + $engine = new engine($dataflow, false, false); + $engine->execute(); + ob_get_clean(); + $variables = $engine->get_variables()['steps']->connector->config; + $destination = $variables->destination; + $httpcode = $variables->httpcode; + $this->assertFileExists($destination); + unlink($destination); + + $variables = $engine->get_variables(); + $expressionlanguage = new ExpressionLanguage(); + // Checks that it can properly be referenced for future steps. + $expressedvalue = $expressionlanguage->evaluate('steps.connector.config.curl', $variables); + $this->assertEquals($testgeturl, $expressedvalue); + $expressedvalue = $expressionlanguage->evaluate('steps.connector.config.destination', $variables); + $this->assertEquals($destination, $expressedvalue); + $expressedvalue = $expressionlanguage->evaluate('steps.connector.config.httpcode', $variables); + $this->assertEquals($httpcode, $expressedvalue); + } + + /** + * Test validate_config(). + * + * @covers \tool_dataflows\local\step\connector_curl::validate_config + * @throws \coding_exception + */ + public function test_validate_config() { + // Test valid configuration. + $config = (object) [ + 'curl' => $this->getExternalTestFileUrl('/h5puuid.json'), + 'method' => 'get' + ]; + $connectorcurl = new connector_curl(); + $this->assertTrue($connectorcurl->validate_config($config)); + + // Test that whenever post/put is selected postdata exists. + $config->method = 'post'; + $config->rawpostdata = ''; + $result = $connectorcurl->validate_config($config); + $this->assertTrue(is_array($result)); + $this->assertArrayHasKey('config_rawpostdata', $result); + + // Test cURL always exists. + unset($config->curl); + $result = $connectorcurl->validate_config($config); + $this->assertTrue(is_array($result)); + $this->assertArrayHasKey('config_curl', $result); + } +}