Merge pull request #254 from stronk7/new_behat_phpunit_options

New behat phpunit options

docs/CHANGELOG.md

@@ -9,6 +9,10 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 The format of this change log follows the advice given at [Keep a CHANGELOG](http://keepachangelog.com).
 
 ## [Unreleased]
+### Added
+- Added support for the `--tags` and `--name` options to the `behat` command.
+- Added support for the `--configure`, `--testsuite` and `--filter` options to the `phpunit` command.
+
 ### Changed
 - ACTION SUGGESTED: If you are using GitHub Actions, it's recomended to use `!cancelled()` instead of `always()` for moodle-plugin-ci tests. Adding a final step that always returns failure when the workflow is cancelled will ensure that cancelled workflows are not marked as successful. For a working example, please reference the updated `gha.dist.yml` file.
 - ACTION SUGGESTED: For some (unknown) reason, Travis environments with PHP 8.2 have started to fail with error:

docs/CLI.md

@@ -241,7 +241,7 @@ Run Behat on a plugin
 
 ### Usage
 
-* `behat [-m|--moodle MOODLE] [-p|--profile PROFILE] [--suite SUITE] [--start-servers] [--auto-rerun AUTO-RERUN] [--dump] [--] <plugin>`
+* `behat [-m|--moodle MOODLE] [-p|--profile PROFILE] [--suite SUITE] [--tags TAGS] [--name NAME] [--start-servers] [--auto-rerun AUTO-RERUN] [--dump] [--] <plugin>`
 
 Run Behat on a plugin
 
@@ -269,7 +269,7 @@ Path to Moodle
 
 #### `--profile|-p`
 
-Behat profile to use
+Behat profile option to use
 
 * Accept value: yes
 * Is value required: yes
@@ -279,14 +279,34 @@ Behat profile to use
 
 #### `--suite`
 
-Behat suite to use (Moodle theme)
+Behat suite option to use (Moodle theme)
 
 * Accept value: yes
 * Is value required: yes
 * Is multiple: no
 * Is negatable: no
 * Default: `'default'`
 
+#### `--tags`
+
+Behat tags option to use. If not set, defaults to the component name
+
+* Accept value: yes
+* Is value required: yes
+* Is multiple: no
+* Is negatable: no
+* Default: `''`
+
+#### `--name`
+
+Behat name option to use
+
+* Accept value: yes
+* Is value required: yes
+* Is multiple: no
+* Is negatable: no
+* Default: `''`
+
 #### `--start-servers`
 
 Start Selenium and PHP servers
@@ -1955,7 +1975,7 @@ Run PHPUnit on a plugin
 
 ### Usage
 
-* `phpunit [-m|--moodle MOODLE] [--coverage-text] [--coverage-clover] [--coverage-pcov] [--coverage-xdebug] [--coverage-phpdbg] [--fail-on-incomplete] [--fail-on-risky] [--fail-on-skipped] [--fail-on-warning] [--testdox] [--] <plugin>`
+* `phpunit [-m|--moodle MOODLE] [-c|--configuration CONFIGURATION] [--testsuite TESTSUITE] [--filter FILTER] [--testdox] [--coverage-text] [--coverage-clover] [--coverage-pcov] [--coverage-xdebug] [--coverage-phpdbg] [--fail-on-incomplete] [--fail-on-risky] [--fail-on-skipped] [--fail-on-warning] [--] <plugin>`
 
 Run PHPUnit on a plugin
 
@@ -1981,6 +2001,46 @@ Path to Moodle
 * Is negatable: no
 * Default: `'.'`
 
+#### `--configuration|-c`
+
+PHPUnit configuration XML file (relative to plugin directory)
+
+* Accept value: yes
+* Is value required: yes
+* Is multiple: no
+* Is negatable: no
+* Default: `NULL`
+
+#### `--testsuite`
+
+PHPUnit testsuite option to use (must exist in the configuration file being used)
+
+* Accept value: yes
+* Is value required: yes
+* Is multiple: no
+* Is negatable: no
+* Default: `NULL`
+
+#### `--filter`
+
+PHPUnit filter option to use
+
+* Accept value: yes
+* Is value required: yes
+* Is multiple: no
+* Is negatable: no
+* Default: `NULL`
+
+#### `--testdox`
+
+Enable testdox formatter
+
+* Accept value: no
+* Is value required: no
+* Is multiple: no
+* Is negatable: no
+* Default: `false`
+
 #### `--coverage-text`
 
 Generate and print code coverage report in text format
@@ -2071,16 +2131,6 @@ Treat tests with warnings as failures
 * Is negatable: no
 * Default: `false`
 
-#### `--testdox`
-
-Enable testdox formatter
-
-* Accept value: no
-* Is value required: no
-* Is multiple: no
-* Is negatable: no
-* Default: `false`
-
 #### `--help|-h`
 
 Display help for the given command. When no command is given display help for the list command

src/Command/BehatCommand.php

@@ -58,8 +58,11 @@ protected function configure(): void
         parent::configure();
 
         $this->setName('behat')
-            ->addOption('profile', 'p', InputOption::VALUE_REQUIRED, 'Behat profile to use', 'default')
-            ->addOption('suite', null, InputOption::VALUE_REQUIRED, 'Behat suite to use (Moodle theme)', 'default')
+            ->addOption('profile', 'p', InputOption::VALUE_REQUIRED, 'Behat profile option to use', 'default')
+            ->addOption('suite', null, InputOption::VALUE_REQUIRED, 'Behat suite option to use (Moodle theme)', 'default')
+            ->addOption('tags', null, InputOption::VALUE_REQUIRED, 'Behat tags option to use. ' .
+                'If not set, defaults to the component name', '')
+            ->addOption('name', null, InputOption::VALUE_REQUIRED, 'Behat name option to use', '')
             ->addOption('start-servers', null, InputOption::VALUE_NONE, 'Start Selenium and PHP servers')
             ->addOption('auto-rerun', null, InputOption::VALUE_REQUIRED, 'Number of times to rerun failures', 2)
             ->addOption('dump', null, InputOption::VALUE_NONE, 'Print contents of Behat failure HTML files')
@@ -89,14 +92,19 @@ protected function execute(InputInterface $input, OutputInterface $output): int
 
         $cmd = [
             'php', 'admin/tool/behat/cli/run.php',
-            '--tags=@' . $this->plugin->getComponent(),
             '--profile=' . $input->getOption('profile'),
             '--suite=' . $input->getOption('suite'),
+            '--tags=' . ($input->getOption('tags') ?: '@' . $this->plugin->getComponent()),
             '--auto-rerun=' . $input->getOption('auto-rerun'),
             '--verbose',
             '-vvv',
         ];
 
+        $name = $input->getOption('name');
+        if (!empty($name) && is_string($name)) {
+            $cmd[] = '--name=\'' . $name . '\'';
+        }
+
         if ($output->isDecorated()) {
             $cmd[] = '--colors';
         }

src/Command/PHPUnitCommand.php

@@ -29,6 +29,20 @@ protected function configure(): void
 
         $this->setName('phpunit')
             ->setDescription('Run PHPUnit on a plugin')
+            ->addOption(
+                'configuration',
+                'c',
+                InputOption::VALUE_REQUIRED,
+                'PHPUnit configuration XML file (relative to plugin directory)'
+            )
+            ->addOption(
+                'testsuite',
+                null,
+                InputOption::VALUE_REQUIRED,
+                'PHPUnit testsuite option to use (must exist in the configuration file being used)'
+            )
+            ->addOption('filter', null, InputOption::VALUE_REQUIRED, 'PHPUnit filter option to use')
+            ->addOption('testdox', null, InputOption::VALUE_NONE, 'Enable testdox formatter')
             ->addOption('coverage-text', null, InputOption::VALUE_NONE, 'Generate and print code coverage report in text format')
             ->addOption('coverage-clover', null, InputOption::VALUE_NONE, 'Generate code coverage report in Clover XML format')
             ->addOption('coverage-pcov', null, InputOption::VALUE_NONE, 'Use the pcov extension to calculate code coverage')
@@ -37,8 +51,7 @@ protected function configure(): void
             ->addOption('fail-on-incomplete', null, InputOption::VALUE_NONE, 'Treat incomplete tests as failures')
             ->addOption('fail-on-risky', null, InputOption::VALUE_NONE, 'Treat risky tests as failures')
             ->addOption('fail-on-skipped', null, InputOption::VALUE_NONE, 'Treat skipped tests as failures')
-            ->addOption('fail-on-warning', null, InputOption::VALUE_NONE, 'Treat tests with warnings as failures')
-            ->addOption('testdox', null, InputOption::VALUE_NONE, 'Enable testdox formatter');
+            ->addOption('fail-on-warning', null, InputOption::VALUE_NONE, 'Treat tests with warnings as failures');
     }
 
     protected function initialize(InputInterface $input, OutputInterface $output): void
@@ -80,6 +93,28 @@ protected function execute(InputInterface $input, OutputInterface $output): int
     private function resolveOptions(InputInterface $input): array
     {
         $options = [];
+
+        if ($input->getOption('configuration')) {
+            $options[] = [
+                '--configuration',
+                $this->plugin->directory . '/' . $input->getOption('configuration'),
+            ];
+        }
+
+        if ($input->getOption('testsuite')) {
+            $options[] = [
+                '--testsuite',
+                $input->getOption('testsuite'),
+            ];
+        }
+
+        if ($input->getOption('filter')) {
+            $options[] = [
+                '--filter',
+                $input->getOption('filter'),
+            ];
+        }
+
         if ($this->supportsCoverage() && $input->getOption('coverage-text')) {
             $options[] = [
                 '--coverage-text',
@@ -103,16 +138,25 @@ private function resolveOptions(InputInterface $input): array
                 ];
             }
         }
-        if (is_file($this->plugin->directory . '/phpunit.xml')) {
-            $options[] = [
-                '--configuration',
-                $this->plugin->directory,
-            ];
-        } else {
-            $options[] = [
-                '--testsuite',
-                $this->plugin->getComponent(),
-            ];
+
+        // Only can set configuration or testsuite here (auto) if the former has not been set via command line option.
+        if (!$input->getOption('configuration')) {
+            // Use default configuration (phpunit.xml) only if it exists.
+            if (is_file($this->plugin->directory . '/phpunit.xml')) {
+                $options[] = [
+                    '--configuration',
+                    $this->plugin->directory . '/phpunit.xml',
+                ];
+            } else {
+                // Fallback to try to use the best testsuite potentially available.
+                // Only can set automatic testsuite if it has not been passed via command line option.
+                if (!$input->getOption('testsuite')) {
+                    $options[] = [
+                        '--testsuite',
+                        $this->plugin->getComponent() . '_testsuite', // This is our best guess.
+                    ];
+                }
+            }
         }
 
         return array_merge(...$options); // Merge all options into a single array.

tests/Command/BehatCommandTest.php

@@ -21,7 +21,7 @@
 
 class BehatCommandTest extends MoodleTestCase
 {
-    protected function executeCommand($pluginDir = null, $moodleDir = null): CommandTester
+    protected function executeCommand($pluginDir = null, $moodleDir = null, array $cmdOptions = []): CommandTester
     {
         if ($pluginDir === null) {
             $pluginDir = $this->pluginDir;
@@ -38,10 +38,15 @@ protected function executeCommand($pluginDir = null, $moodleDir = null): Command
         $application->add($command);
 
         $commandTester = new CommandTester($application->find('behat'));
-        $commandTester->execute([
-            'plugin'   => $pluginDir,
-            '--moodle' => $moodleDir,
-        ]);
+        $cmdOptions    = array_merge(
+            [
+                'plugin'   => $pluginDir,
+                '--moodle' => $moodleDir,
+            ],
+            $cmdOptions
+        );
+        $commandTester->execute($cmdOptions);
+        $this->lastCmd = $command->execute->lastCmd; // We need this for assertions against the command run.
 
         return $commandTester;
     }
@@ -50,6 +55,28 @@ public function testExecute()
     {
         $commandTester = $this->executeCommand();
         $this->assertSame(0, $commandTester->getStatusCode());
+        $this->assertMatchesRegularExpression('/php.*admin.tool.behat.cli.run/', $this->lastCmd);
+        $this->assertMatchesRegularExpression('/--profile=default.*--suite=default/', $this->lastCmd);
+        $this->assertMatchesRegularExpression('/--tags=@local_ci/', $this->lastCmd);
+        $this->assertMatchesRegularExpression('/--verbose.*-vvv/', $this->lastCmd);
+    }
+
+    public function testExecuteWithTags()
+    {
+        $commandTester = $this->executeCommand(null, null, ['--tags' => '@tag1&&@tag2']);
+        $this->assertSame(0, $commandTester->getStatusCode());
+        $this->assertMatchesRegularExpression('/--tags=@tag1&&@tag2/', $this->lastCmd);
+        $this->assertDoesNotMatchRegularExpression('/--tags=@local_ci/', $this->lastCmd);
+    }
+
+    public function testExecuteWithName()
+    {
+        $featureName = 'With "double quotes" and \'single quotes\'';
+        // Note that everything is escaped for shell execution, plus own regexp quoting.
+        $expectedName  = preg_quote(escapeshellarg("--name='$featureName'"));
+        $commandTester = $this->executeCommand(null, null, ['--name' => $featureName]);
+        $this->assertSame(0, $commandTester->getStatusCode());
+        $this->assertMatchesRegularExpression("/{$expectedName}/", $this->lastCmd);
     }
 
     public function testExecuteNoFeatures()
@@ -70,6 +97,7 @@ public function testExecuteNoPlugin()
     public function testExecuteNoMoodle()
     {
         $this->expectException(\InvalidArgumentException::class);
+        // TODO: Check what's happening here. moodleDir should be the 2nd parameter, but then the test fails.
         $this->executeCommand($this->moodleDir . '/no/moodle');
     }
 }