feat(dev): add created at and gt/lt operators

dev/src/Command/ComponentInfoCommand.php

@@ -18,6 +18,8 @@
 namespace Google\Cloud\Dev\Command;
 
 use Google\Cloud\Dev\Component;
+use Google\Cloud\Dev\ComponentPackage;
+use Google\Cloud\Dev\Packagist;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Helper\Table;
 use Symfony\Component\Console\Input\InputInterface;
@@ -36,7 +38,7 @@ class ComponentInfoCommand extends Command
         'component_name' => 'Component Name',
         'package_name' => 'Package Name',
         'package_version' => 'Package Version',
-        'api_versions' => 'API Version',
+        'api_version' => 'API Version',
         'release_level' => 'Release Level',
         'migration_mode' => 'Migration Mode',
         'php_namespaces' => 'Php Namespace',
@@ -45,30 +47,33 @@ class ComponentInfoCommand extends Command
         'service_address' => 'Service Address',
         'api_shortname' => 'API Shortname',
         'description' => 'Description',
+        'created_at' => 'Created At',
         'available_api_versions' => 'Availble API Versions',
+        'downloads' => 'Downloads',
     ];
     private static $defaultFields = [
         'component_name',
         'package_name',
         'package_version',
-        'api_versions',
+        'api_version',
         'release_level',
-        'migration_mode',
         'api_shortname',
     ];
 
     private string $token;
+    private Packagist $packagist;
 
     protected function configure()
     {
         $this->setName('component-info')
+            ->setAliases(['info'])
             ->setDescription('list info of a component or the whole library')
             ->addOption('component', 'c', InputOption::VALUE_REQUIRED | InputOption::VALUE_IS_ARRAY, 'get info for a single component', [])
             ->addOption('csv', '', InputOption::VALUE_OPTIONAL, 'export findings to csv.', false)
             ->addOption('fields', 'f', InputOption::VALUE_REQUIRED, sprintf(
                 "Comma-separated list of fields, \"all\" for all fields. The following fields are available: \n - %s\n" .
-                "NOTE: \"available_api_versions\" are omited by default because they take a long time to load.\n" .
-                "Use --show-available-api-versions to include them.\n",
+                "NOTE: \"available_api_versions\", \"created_at\", and \"downloads\" are omited by default because they ".
+                "take a long time to load.\n",
                 implode("\n - ", array_keys(self::$allFields))
             ))
             ->addOption('filter', '', InputOption::VALUE_REQUIRED,
@@ -79,12 +84,6 @@ protected function configure()
                 'field to sort by (with optional ASC/DESC suffix. e.g. "component_name DESC"'
             )
             ->addOption('token', 't', InputOption::VALUE_REQUIRED, 'Github token to use for authentication', '')
-            ->addOption(
-                'show-available-api-versions',
-                '',
-                InputOption::VALUE_NONE,
-                'Show available API versions for each component. Requires an API call'
-            )
             ->addOption('expanded', '', InputOption::VALUE_NONE, 'Break down each component by packages')
         ;
     }
@@ -93,14 +92,21 @@ protected function execute(InputInterface $input, OutputInterface $output)
     {
         $fields = match($input->getOption('fields')) {
             null => self::$defaultFields,
-            'all' => array_keys(array_diff_key(self::$allFields, ['available_api_versions' => ''])),
+            'all' => array_keys(array_diff_key(
+                self::$allFields,
+                ['available_api_versions' => '', 'created_at' => '', 'downloads' => '']
+            )),
             default => explode(',', $input->getOption('fields')),
         };
 
-        if ($input->getOption('show-available-api-versions')) {
-            $fields[] = 'available_api_versions';
+        // support "+" prefix to add requested field to the default fields
+        if (0 === strpos($fields[0], '+')) {
+            $fields[0] = substr($fields[0], 1);
+            $fields = array_merge(self::$defaultFields, $fields);
         }
+
         $this->token = $input->getOption('token');
+        $this->packagist = new Packagist(new Client(), '', '');
 
         // Parse filters
         $filters = $this->parseFilters($input->getOption('filter') ?: '');
@@ -119,22 +125,10 @@ protected function execute(InputInterface $input, OutputInterface $output)
             $componentRows = $this->getComponentDetails(
                 $component,
                 $requestedFields,
+                $filters,
                 $input->getOption('expanded')
             );
 
-            foreach ($componentRows as $row) {
-                foreach ($filters as $filter) {
-                    list($field, $value, $operator) = $filter;
-                    if (!match ($operator) {
-                        '=' => ($row[$field] === $value),
-                        '!=' => ($row[$field] !== $value),
-                        '~=' => strpos($row[$field], $value) !== false,
-                        '!~=' => strpos($row[$field], $value) === false,
-                    }) {
-                        continue 3;
-                    }
-                }
-            }
             $rows = array_merge($rows, $componentRows);
         }
 
@@ -143,6 +137,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
             usort($rows, function ($a, $b) use ($field) {
                 return match ($field) {
                     'package_version' => version_compare($a[$field], $b[$field]),
+                    'downloads' => str_replace(',', '', $a[$field]) <=> str_replace(',', '', $b[$field]),
                     default => strcmp($a[$field], $b[$field]),
                 };
             });
@@ -186,61 +181,71 @@ protected function execute(InputInterface $input, OutputInterface $output)
         return 0;
     }
 
-    private function getComponentDetails(Component $component, array $requestedFields, bool $expanded): array
+    private function getComponentDetails(Component $component, array $requestedFields, array $filters, bool $expanded): array
     {
         $rows = [];
         if ($expanded) {
             foreach ($component->getComponentPackages() as $pkg) {
-                $availableApiVersions = '';
-                if (array_key_exists('available_api_versions', $requestedFields)) {
-                    $availableApiVersions = $this->getAvailableApiVersions($component);
+                if ($row = $this->getComponentDetailRow($component, $pkg, $requestedFields, $filters)) {
+                    $rows[] = $row;
                 }
-                // use "array_intersect_key" to filter out fields that were not requested.
-                // use "array_replace" to sort the fields in the order they were requested.
-                $rows[] = array_replace($requestedFields, array_intersect_key([
-                    'component_name' => $component->getName() . "\\" . $pkg->getName(),
-                    'package_name' => $component->getPackageName(),
-                    'package_version' => $component->getPackageVersion(),
-                    'api_versions' => $pkg->getName(),
-                    'release_level' => $component->getReleaseLevel(),
-                    'migration_mode' => $pkg->getMigrationStatus(),
-                    'php_namespaces' => implode("\n", array_keys($component->getNamespaces())),
-                    'github_repo' => $component->getRepoName(),
-                    'proto_path' => $pkg->getProtoPackage(),
-                    'service_address' => $pkg->getServiceAddress(),
-                    'api_shortname' => $pkg->getApiShortname(),
-                    'description' => $component->getDescription(),
-                    'available_api_versions' => $availableApiVersions,
-                ], $requestedFields));
             }
         } else {
-            // use "array_intersect_key" to filter out fields that were not requested.
-            // use "array_replace" to sort the fields in the order they were requested.
-            $details = array_replace($requestedFields, array_intersect_key([
-                'component_name' => $component->getName(),
-                'package_name' => $component->getPackageName(),
-                'package_version' => $component->getPackageVersion(),
-                'api_versions' => implode("\n", $component->getApiVersions()),
-                'release_level' => $component->getReleaseLevel(),
-                'migration_mode' => implode("\n", $component->getMigrationStatuses()),
-                'php_namespaces' => implode("\n", array_keys($component->getNamespaces())),
-                'github_repo' => $component->getRepoName(),
-                'proto_path' => implode("\n", $component->getProtoPackages()),
-                'service_address' => implode("\n", $component->getServiceAddresses()),
-                'api_shortname' => implode("\n", array_filter($component->getApiShortnames())),
-                'description' => $component->getDescription(),
-            ], $requestedFields));
-
-            if (array_key_exists('available_api_versions', $requestedFields)) {
-                $details['available_api_versions'] = $this->getAvailableApiVersions($component);
+            if ($row = $this->getComponentDetailRow($component, null, $requestedFields, $filters)) {
+                $rows[] = $row;
             }
-
-            $rows[] = $details;
         }
 
         return $rows;
     }
 
+    private function getComponentDetailRow(
+        Component $component,
+        ?ComponentPackage $package,
+        array $requestedFields,
+        array $filters,
+    ): ?array {
+        // use "array_intersect_key" to filter out fields that were not requested.
+        // use "array_replace" to sort the fields in the order they were requested.
+        $row = array_replace($requestedFields, array_intersect_key([
+            'component_name' => $component->getName() . ($package ? "/" . $package->getName() : ''),
+            'package_name' => $component->getPackageName(),
+            'package_version' => $component->getPackageVersion(),
+            'api_version' => $package ? $package->getName() : implode(",", $component->getApiVersions()),
+            'release_level' => $component->getReleaseLevel(),
+            'migration_mode' => $package ? $package->getMigrationStatus() : implode(",", $component->getMigrationStatuses()),
+            'php_namespaces' => implode(",", array_keys($component->getNamespaces())),
+            'github_repo' => $component->getRepoName(),
+            'proto_path' => $package ? $package->getProtoPackage() : implode(",", $component->getProtoPackages()),
+            'service_address' => $package ? $package->getServiceAddress() : implode(",", $component->getServiceAddresses()),
+            'api_shortname' => $package ? $package->getApiShortname() : implode(",", array_filter($component->getApiShortnames())),
+            'description' => $component->getDescription(),
+            'available_api_versions' => null,
+            'created_at' => null,
+            'downloads' => null,
+        ], $requestedFields));
+
+        // pre-filter so we don't perform excessive slow operations
+        if ($this->filterRow($row, $filters)) {
+            return null;
+        }
+        // Only add these if they've been requested (because they're slow)
+        if (array_key_exists('available_api_versions', $requestedFields)) {
+            $row['available_api_versions'] = $this->getAvailableApiVersions($component);
+        }
+        if (array_key_exists('created_at', $requestedFields)) {
+            $row['created_at'] = $component->getCreatedAt()->format('Y-m-d');
+        }
+        if (array_key_exists('downloads', $requestedFields)) {
+            $row['downloads'] = number_format($this->packagist->getDownloads($component->getPackageName()));
+        }
+        // call again in case the filters were on the slow fields
+        if ($this->filterRow($row, $filters)) {
+            return null;
+        }
+        return $row;
+    }
+
     private function getAvailableApiVersions(Component $component): string
     {
         $protos = $component->getProtoPackages();
@@ -271,7 +276,7 @@ private function parseFilters(string $filterString): array
     {
         $filters = [];
         foreach (array_filter(explode(',', $filterString)) as $filter) {
-            if (!preg_match('/^(\w+?)(!~=|~=|!=|=)(.+)$/', $filter, $matches)) {
+            if (!preg_match('/^(\w+?)(!~=|~=|!=|>=|<=|=|<|>)(.+)$/', $filter, $matches)) {
                 throw new \InvalidArgumentException(sprintf('Invalid filter: %s', $filter));
             }
             $filters[] = [$matches[1], $matches[3], $matches[2]];
@@ -284,4 +289,31 @@ private function parseFilters(string $filterString): array
         }
         return $filters;
     }
+
+    private function filterRow(array $row, array $filters): bool
+    {
+        foreach ($filters as $filter) {
+            list($field, $value, $operator) = $filter;
+            if ($row[$field] === null) {
+                // bypass filter for now - these will be added later
+                continue;
+            }
+            if (!match ($operator) {
+                '=' => ($row[$field] === $value),
+                '!=' => ($row[$field] !== $value),
+                '~=' => strpos($row[$field], $value) !== false,
+                '!~=' => strpos($row[$field], $value) === false,
+                '>','<','>=','<=' => match($field) {
+                    'downloads' => version_compare(
+                        str_replace(',' , '', $row[$field]),
+                        $value,
+                        $operator),
+                    default => version_compare($row[$field], $value, $operator),
+                },
+            }) {
+                return true; // filter out the row
+            }
+        }
+        return false;
+    }
 }

dev/src/Component.php

@@ -20,6 +20,7 @@
 use Symfony\Component\Finder\Exception\DirectoryNotFoundException;
 use Symfony\Component\Finder\Finder;
 use RuntimeException;
+use DateTime;
 
 /**
  * @internal
@@ -262,29 +263,25 @@ public function getApiVersions(): array
         return array_map(fn($pkg) => $pkg->getName(), $this->getComponentPackages());
     }
 
+    public function getCreatedAt(): DateTime
+    {
+        exec(sprintf(
+            'git log --reverse --pretty=format:"%%cd" %s/ | head -1',
+            $this->name,
+        ), $output);
+
+        return new DateTime($output[0]);
+    }
+
     private function getPackagePaths(): array
     {
         $result = (new Finder())->directories()->in($this->path . '/src/')->name(self::VERSION_REGEX);
         $paths = array_map(fn ($file) => $file->getRelativePathname(), iterator_to_array($result));
         $paths = array_reverse(array_values($paths));
-        usort($paths, [$this, 'versionCompare']);
+        usort($paths, 'version_compare');
         if (empty($paths)) {
             $paths = [''];
         }
-        return $paths;
-    }
-
-    private static function versionCompare(string $v1, string $v2)
-    {
-        // First, sort by API number (e.g. V1 vs V2)
-        $sort = substr($v1, strrpos($v1, 'V')) <=> substr($v2, strrpos($v2, 'V'));
-        if ($sort === 0) {
-            // If same API version, sort by if one is in a subdirectory
-            return strpos($v1, '/') <=> strpos($v2, '/');
-        }
-        // Else, sort by release level (e.g. beta vs alpha vs GA)
-        $v1Sort = strpos($v1, 'beta') ? 0 : (strpos($v1, 'alpha') ? -1 : 1);
-        $v2Sort = strpos($v2, 'beta') ? 0 : (strpos($v2, 'alpha') ? -1 : 1);
-        return $v2Sort <=> $v1Sort;
+        return array_reverse($paths);
     }
 }

dev/src/Packagist.php

@@ -78,6 +78,13 @@ public function getApiToken(): string
         return $this->apiToken;
     }
 
+    public function getDownloads(string $packageName): int
+    {
+        $response = $this->client->get("https://packagist.org/packages/$packageName/stats.json");
+        $data = json_decode($response->getBody()->getContents(), true);
+        return $data['downloads']['total'] ?? 0;
+    }
+
     /**
      * Log an exception
      *

dev/templates/owlbot.py.twig

@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

dev/tests/fixtures/component/CustomInput/owlbot.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

dev/tests/fixtures/component/SecretManager/owlbot.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Google LLC
+# Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.