diff --git a/dev/src/Command/AddComponentCommand.php b/dev/src/Command/AddComponentCommand.php index c457d74f6882..f1c26f0baee5 100644 --- a/dev/src/Command/AddComponentCommand.php +++ b/dev/src/Command/AddComponentCommand.php @@ -20,18 +20,23 @@ use Google\Cloud\Dev\Composer; use Google\Cloud\Dev\Component; use Google\Cloud\Dev\NewComponent; +use Google\Cloud\Dev\RunProcess; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Helper\Table; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Question\Question; use Symfony\Component\Console\Question\ConfirmationQuestion; use Symfony\Component\Filesystem\Filesystem; +use Symfony\Component\Process\Process; +use Symfony\Component\Yaml\Yaml; use GuzzleHttp\Client; use Twig\Loader\FilesystemLoader; use Twig\Environment; use RuntimeException; +use Exception; /** * Add a Component @@ -53,17 +58,26 @@ class AddComponentCommand extends Command 'phpunit.xml.dist.twig', 'README.md.twig', ]; + private const BAZEL_VERSION = '6.0.0'; + private const OWLBOT_CLI_IMAGE = 'gcr.io/cloud-devrel-public-resources/owlbot-cli:latest'; + private const OWLBOT_PHP_IMAGE = 'gcr.io/cloud-devrel-public-resources/owlbot-php:latest'; private $input; private $output; private $rootPath; + private $httpClient; + private RunProcess $runProcess; /** * @param string $rootPath The path to the repository root directory. + * @param Client $httpClient specify the HTTP client, useful for tests. + * @param RunProcess $runProcess Instance to execute Symfony Process commands, useful for tests. */ - public function __construct($rootPath) + public function __construct($rootPath, Client $httpClient = null, RunProcess $runProcess = null) { $this->rootPath = realpath($rootPath); + $this->httpClient = $httpClient ?: new Client(); + $this->runProcess = $runProcess ?: new RunProcess(); parent::__construct(); } @@ -71,7 +85,18 @@ protected function configure() { $this->setName('add-component') ->setDescription('Add a Component') - ->addArgument('proto', InputArgument::REQUIRED, 'Path to service proto.'); + ->addArgument('proto', InputArgument::REQUIRED, 'Path to service proto.') + ->addOption( + 'googleapis-gen-path', + null, + InputOption::VALUE_REQUIRED, + 'Path to googleapis-gen repo. Option to generate the library using Owlbot:copy-code' + )->addOption( + 'bazel-path', + null, + InputOption::VALUE_REQUIRED, + 'Path to bazel (googleapis) workspace. Option to generate the library using Bazel' + ); } protected function execute(InputInterface $input, OutputInterface $output) @@ -111,16 +136,19 @@ protected function execute(InputInterface $input, OutputInterface $output) ->render(); } - $productHomepage = $this->getHelper('question')->ask( + $productDocumentation = null; + $yamlFileContent = $this->loadYamlConfigContent($new, dirname($proto)); + $productDocumentation = $yamlFileContent['publishing']['documentation_uri'] ?? null; + $productDocumentation = $productDocumentation ?: $this->getHelper('question')->ask( $input, $output, - new Question('What is the product homepage? ') + new Question('What is the product documentation URL? ') ); - - $productDocumentation = $this->getHelper('question')->ask( + $productHomePage = $this->getHomePageFromDocsUrl($productDocumentation); + $productHomePage = $productHomePage ?: $this->getHelper('question')->ask( $input, $output, - new Question('What is the product documentation URL? ') + new Question('What is the product homepage? ') ); $documentationUrl = $new->getDocumentationUrl(); @@ -155,7 +183,7 @@ protected function execute(InputInterface $input, OutputInterface $output) 'version' => $new->version, 'github_repo' => $new->githubRepo, 'documentation' => $documentationUrl, - 'product_homepage' => $productHomepage, + 'product_homepage' => $productHomePage, 'product_documentation' => $productDocumentation, ])); } @@ -184,6 +212,25 @@ protected function execute(InputInterface $input, OutputInterface $output) $composer->updateMainComposer(); $composer->createComponentComposer($new->displayName, $new->githubRepo); + if ($input->getOption('bazel-path') || $input->getOption('googleapis-gen-path')) { + $this->validateOptions($input, $output); + $this->checkDockerAvailable(); + if ($input->getOption('bazel-path')) { + $googleApisDir = realpath($input->getOption('bazel-path')); + $output->writeln("\n\nbazel build library"); + $output->writeln($this->bazelQueryAndBuildLibrary(dirname($protoFile), $googleApisDir)); + $output->writeln("\n\nCopying the library code from bazel-bin"); + $output->writeln($this->owlbotCopyBazelBin($new->componentName, $googleApisDir)); + } else { + $googleApisGenDir = realpath($input->getOption('googleapis-gen-path')); + $output->writeln("\n\nCopying the library code from googleapis-gen"); + $output->writeln($this->owlbotCopyCode($new->componentName, $googleApisGenDir)); + } + // run owlbot post-processor if a bazel-path or googleapis-gen-path was supplied + $output->writeln("\n\nOwlbot post processing"); + $output->writeln($this->owlbotPostProcessor()); + } + $output->writeln(''); $output->writeln(''); $output->writeln('Success!'); @@ -191,14 +238,166 @@ protected function execute(InputInterface $input, OutputInterface $output) return 0; } + private function validateOptions(InputInterface $input, OutputInterface $output): void + { + if ($input->getOption('bazel-path') && $input->getOption('googleapis-gen-path')) { + throw new \InvalidArgumentException( + 'The options --googleapis-gen-path and --bazel-path cannot be used together.' . + ' Please provide only one path option.' + ); + } + } + + private function loadYamlConfigContent(NewComponent $new, string $protoDir): ?array + { + $yamlFilePath = sprintf('%s/%s%s%s.yaml', + $protoDir, + strtolower($new->shortName), + $new->version ? '_' : '', + $new->version ?? '' + ); + + try { + return Yaml::parse($this->loadProtoContent($yamlFilePath)); + } catch (Exception $e) { + // Handle error gracefully. + return null; + } + } + private function loadProtoContent(string $proto): string { if (file_exists($proto)) { return file_get_contents($proto); } $protoUrl = 'https://raw.githubusercontent.com/googleapis/googleapis/master/' . $proto; - $client = new Client(); - $response = $client->get($protoUrl); + $response = $this->httpClient->get($protoUrl); return (string) $response->getBody(); } + + private function bazelQueryAndBuildLibrary(string $protoDir, string $googleApisDir): string + { + $command = ['bazel', '--version']; + $output = $this->runProcess->execute($command); + // Extract the version number from the output + $match = preg_match('/bazel\s+(?P\d+\.\d+\.\d+)/', $output, $matches); + if (!$match || $matches['version'] !== self::BAZEL_VERSION) { + throw new RuntimeException('Bazel 6.0.0 is not available'); + } + + $command = [ + 'bazel', + 'query', + 'filter("-(php)$", kind("rule", //' . $protoDir .'/...:*))' + ]; + $output = $this->runProcess->execute($command, $googleApisDir); + // Get componenets starting with //google/ and + // not ending with :(proto|grpc|gapic)-.*-php + $components = array_filter( + explode("\n", $output), + fn ($line) => $line && preg_match('/^\/\/google\/(?!:(proto|grpc|gapic)-.*-php$)/', $line) + ); + if (count($components) !== 1) { + throw new Exception( + 'expected only one bazel component, found ' . + (implode(' ', $components) ?: '0') + ); + } + + $command = ['bazel', 'build', $components[0]]; + return $this->runProcess->execute($command, $googleApisDir); + } + + private function owlbotCopyCode(string $componentName, string $googleApisGenDir): string + { + list($userId, $groupId) = $this->getUserAndGroupId(); + $command = [ + 'docker', + 'run', + '--rm', + '--user', + sprintf('%s::%s', $userId, $groupId), + '-v', + $this->rootPath . ':/repo', + '-v', + $googleApisGenDir . ':/googleapis-gen', + '-w', + '/repo', + '--env', + 'HOME=/tmp', + self::OWLBOT_CLI_IMAGE, + 'copy-code', + sprintf('--config-file=%s/.OwlBot.yaml', $componentName), + '--source-repo=/googleapis-gen' + ]; + return $this->runProcess->execute($command); + } + + private function owlbotCopyBazelBin(string $componentName, string $googleApisDir): string + { + list($userId, $groupId) = $this->getUserAndGroupId(); + $command = [ + 'docker', + 'run', + '--rm', + '--user', + sprintf('%s::%s', $userId, $groupId), + '-v', + $this->rootPath . ':/repo', + '-v', + $googleApisDir . '/bazel-bin:/bazel-bin', + self::OWLBOT_CLI_IMAGE, + 'copy-bazel-bin', + sprintf('--config-file=%s/.OwlBot.yaml', $componentName), + '--source-dir', + '/bazel-bin', + '--dest', + '/repo' + ]; + return $this->runProcess->execute($command); + } + + private function owlbotPostProcessor(): string + { + list($userId, $groupId) = $this->getUserAndGroupId(); + $command = [ + 'docker', + 'run', + '--rm', + '--user', + sprintf('%s::%s', $userId, $groupId), + '-v', + $this->rootPath . ':/repo', + '-w', + '/repo', + self::OWLBOT_PHP_IMAGE + ]; + return $this->runProcess->execute($command); + } + + private function getHomePageFromDocsUrl(?string $url): ?string + { + $productHomePage = !empty($url) ? explode('/docs', $url)[0] : null; + $response = $this->httpClient->get($productHomePage, ['http_errors' => false]); + return $response->getStatusCode() >= 400 ? null : $productHomePage; + } + + private function checkDockerAvailable(): void + { + $command = ['which', 'docker']; + $output = $this->runProcess->execute($command); + if (strlen($output) == 0) { + throw new RuntimeException( + 'Error: Docker is not available.' + ); + } + } + + private function getUserAndGroupId(): array + { + // Get the user ID and group ID + $userId = posix_getuid(); + $groupId = posix_getgid(); + return [$userId, $groupId]; + } } diff --git a/dev/src/RunProcess.php b/dev/src/RunProcess.php new file mode 100644 index 000000000000..6c879eb313e0 --- /dev/null +++ b/dev/src/RunProcess.php @@ -0,0 +1,45 @@ +mustRun(); + + return $process->getOutput() . $process->getErrorOutput(); + } +} diff --git a/dev/tests/Unit/Command/AddComponentCommandTest.php b/dev/tests/Unit/Command/AddComponentCommandTest.php index 51b70173f049..87f6ce116aa0 100644 --- a/dev/tests/Unit/Command/AddComponentCommandTest.php +++ b/dev/tests/Unit/Command/AddComponentCommandTest.php @@ -19,7 +19,10 @@ use Google\Cloud\Dev\Command\AddComponentCommand; use Google\Cloud\Dev\Composer; +use Google\Cloud\Dev\RunProcess; +use GuzzleHttp\Client; use PHPUnit\Framework\TestCase; +use Prophecy\PhpUnit\ProphecyTrait; use Symfony\Component\Console\Application; use Symfony\Component\Console\Tester\CommandTester; @@ -29,6 +32,8 @@ */ class AddComponentCommandTest extends TestCase { + use ProphecyTrait; + private static $expectedFiles = [ '.OwlBot.yaml' => '.OwlBot.yaml.test', // so OwlBot doesn't read the test file '.gitattributes' => null, @@ -59,8 +64,8 @@ public function testAddComponent() { self::$commandTester->setInputs([ 'Y', // Does this information look correct? [Y/n] - 'https://cloud.google.com/secret-mananger', // What is the product homepage? 'https://cloud.google.com/secret-manager/docs/reference/rest/', // What is the product documentation URL? + 'https://cloud.google.com/secret-manager', // What is the product homepage? ]); self::$commandTester->execute([ @@ -113,8 +118,8 @@ public function testAddComponentWithCustomOptions() 'google/cloud/custompath/(.*)', // custom value for "protoPath" 'v2', // custom value for "version" 'Y', // Does this information look correct? [Y/n] - 'https://cloud.google.com/coustom-product', // What is the product homepage? 'https://cloud.google.com/coustom-product/docs/reference/rest/', // What is the product documentation URL? + 'https://cloud.google.com/coustom-product', // What is the product homepage? ]); self::$commandTester->execute([ @@ -151,6 +156,121 @@ public function testAddComponentWithCustomOptions() $this->assertComposerJson('CustomInput'); } + public function testGoogleapisGenPath() + { + $expectedOwlbotCopyCodeCmd = sprintf( + 'docker run --rm --user %s::%s -v %s:/repo -v :/googleapis-gen -w /repo ' + . '--env HOME=/tmp gcr.io/cloud-devrel-public-resources/owlbot-cli:latest copy-code ' + . '--config-file=SecretManager/.OwlBot.yaml --source-repo=/googleapis-gen', + posix_getuid(), + posix_getgid(), + self::$tmpDir + ); + $expectedOwlbotPostProcessCmd = sprintf( + 'docker run --rm --user %s::%s -v %s:/repo -w /repo ' + . 'gcr.io/cloud-devrel-public-resources/owlbot-php:latest', + posix_getuid(), + posix_getgid(), + self::$tmpDir + ); + $runProcess = $this->prophesize(RunProcess::class); + $runProcess->execute(['which', 'docker']) + ->shouldBeCalledOnce() + ->willReturn('/path/to/docker'); + $runProcess->execute(explode(' ', $expectedOwlbotCopyCodeCmd)) + ->shouldBeCalledOnce() + ->willReturn(''); + $runProcess->execute(explode(' ', $expectedOwlbotPostProcessCmd)) + ->shouldBeCalledOnce() + ->willReturn(''); + + $application = new Application(); + $application->add(new AddComponentCommand(self::$tmpDir, null, $runProcess->reveal())); + + $commandTester = new CommandTester($application->get('add-component')); + $commandTester->setInputs([ + 'Y', // Does this information look correct? [Y/n] + 'https://cloud.google.com/secret-manager/docs/reference/rest/', // What is the product documentation URL? + 'https://cloud.google.com/secret-manager', // What is the product homepage? + ]); + + $commandTester->execute([ + 'proto' => 'google/cloud/secretmanager/v1/service.proto', + '--googleapis-gen-path' => 'path/to/bazel', + ]); + } + + public function testBazelPathAndFetchDocUri() + { + $client = new Client(); + $productHomePage = 'https://cloud.google.com/infrastructure-manager'; + $rawContentUri = 'https://raw.githubusercontent.com/googleapis/googleapis/master/'; + $proto = 'google/cloud/config/v1/config.proto'; + $yaml = 'google/cloud/config/v1/config_v1.yaml'; + $expectedOwlbotCopyBazelBinCmd = sprintf( + 'docker run --rm --user %s::%s -v %s:/repo -v /bazel-bin:/bazel-bin ' + . 'gcr.io/cloud-devrel-public-resources/owlbot-cli:latest copy-bazel-bin ' + . '--config-file=Config/.OwlBot.yaml --source-dir /bazel-bin --dest /repo', + posix_getuid(), + posix_getgid(), + self::$tmpDir + ); + $expectedOwlbotPostProcessCmd = sprintf( + 'docker run --rm --user %s::%s -v %s:/repo -w /repo ' + . 'gcr.io/cloud-devrel-public-resources/owlbot-php:latest', + posix_getuid(), + posix_getgid(), + self::$tmpDir + ); + $runProcess = $this->prophesize(RunProcess::class); + $runProcess->execute(['which', 'docker']) + ->shouldBeCalledOnce() + ->willReturn('/path/to/docker'); + $runProcess->execute(['bazel', '--version']) + ->shouldBeCalledOnce() + ->willReturn('bazel 6.0.0'); + $runProcess->execute( + ['bazel', 'query', 'filter("-(php)$", kind("rule", //google/cloud/config/v1/...:*))'], + '' + ) + ->shouldBeCalledOnce() + ->willReturn('//google/cloud/config/v1'); + $runProcess->execute(['bazel', 'build', '//' . dirname($proto)], '') + ->shouldBeCalledOnce() + ->willReturn(''); + $runProcess->execute(explode(' ', $expectedOwlbotCopyBazelBinCmd)) + ->shouldBeCalledOnce() + ->willReturn(''); + $runProcess->execute(explode(' ', $expectedOwlbotPostProcessCmd)) + ->shouldBeCalledOnce() + ->willReturn(''); + + $httpClient = $this->prophesize(Client::class); + $httpClient->get($rawContentUri . $proto) + ->shouldBeCalledOnce() + ->willReturn($client->get($rawContentUri . $proto)); + $httpClient->get($rawContentUri . $yaml) + ->shouldBeCalledOnce() + ->willReturn($client->get($rawContentUri . $yaml)); + $httpClient->get($productHomePage, ['http_errors' => false]) + ->shouldBeCalledOnce() + ->willReturn($client->get($productHomePage)); + + $application = new Application(); + $application->add(new AddComponentCommand(self::$tmpDir, $httpClient->reveal(), $runProcess->reveal())); + + $commandTester = new CommandTester($application->get('add-component')); + // No documentationPage/homePage input is required as it is fetched automatically from the yaml file. + $commandTester->setInputs([ + 'Y' // Does this information look correct? [Y/n] + ]); + + $commandTester->execute([ + 'proto' => $proto, + '--bazel-path' => '/path/to/bazel', + ]); + } + private function assertComposerJson(string $componentName) { $composerPath = sprintf('%s/../../fixtures/component/%s/composer.json', __DIR__, $componentName); diff --git a/dev/tests/fixtures/component/SecretManager/README.md b/dev/tests/fixtures/component/SecretManager/README.md index 2d02f9948e86..0608a7d425b9 100644 --- a/dev/tests/fixtures/component/SecretManager/README.md +++ b/dev/tests/fixtures/component/SecretManager/README.md @@ -1,6 +1,6 @@ # Google Cloud Secret Manager for PHP -> Idiomatic PHP client for [Google Cloud Secret Manager](https://cloud.google.com/secret-mananger). +> Idiomatic PHP client for [Google Cloud Secret Manager](https://cloud.google.com/secret-manager). [![Latest Stable Version](https://poser.pugx.org/google/cloud-secretmanager/v/stable)](https://packagist.org/packages/google/cloud-secretmanager) [![Packagist](https://img.shields.io/packagist/dm/google/cloud-secretmanager.svg)](https://packagist.org/packages/google/cloud-secretmanager)