Component split (#360)

* Reorganize namespaces and fix tests * # This is a combination of 9 commits. # This is the 1st commit message: Reorganize namespaces and fix tests Update component versions via the CLI # This is the commit message #2: Create tags in components # This is the commit message #3: Send component name and version in user agent # This is the commit message #4: Add README files # This is the commit message #5: Fix CLI # This is the commit message #6: Add split to travis # This is the commit message #7: Test datastore release # This is the commit message #8: Compile splitsh # This is the commit message #9: Reintroduce full split process * # This is a combination of 2 commits. # This is the 1st commit message: Reorganize namespaces and fix tests Update component versions via the CLI Create tags in components Send component name and version in user agent Add README files Fix CLI Add split to travis Test datastore release Compile splitsh Reintroduce full split process Release patches # This is the commit message #2: Run split in each build matrix * Reorganize namespaces and fix tests Update component versions via the CLI Create tags in components Send component name and version in user agent Add README files Fix CLI Add split to travis Test datastore release Compile splitsh Reintroduce full split process Release patches Run split in each build matrix Release datastore patch enable set -e * Work with github API directly * Fix Vision class reference * Create VERSION file for all components * Remove componentName * Fix int64 snippet tests * Update component release target * Parse documentation separately for each component * This works now * Add flag to docs to use JSON_PRETTY_PRINT for debugging * Parse correct links between components * Fix unit tests * Doc links within component should not change version * Cross-component links should go to the correct version docs * Update snippets to remove refs to ServiceBuilder * Remove unnecessary env vars * Don't write VERSION for parent * Release packages * release v0.24.1 * Add autoloaders to each component * Pass libVersion to gax config * Update push docs command * add gcsUri helper * allow gcsUri or storage object * Release 0.24.2 * Update things again * this is the worst * Link to upstream tag * tag * Update table of contents * Reset versions * Update suggests and requires * fix build docs trigger * dont throw exceptions on decode * Fix docs link * Fix various doc issues * fix namespace
googleapis · Mar 10, 2017 · 3e21b68 · 3e21b68
1 parent b73a782
commit 3e21b68
Show file tree

Hide file tree

Showing 300 changed files with 4,638 additions and 1,319 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,30 +1,31 @@
 language: php
-
 sudo: required
 dist: trusty
 
 matrix:
-    include:
-        - php: 5.5.38
-        - php: 5.6.25
-        - php: 7.0
-        - php: 7.1
-        - php: hhvm
-          group: edge
-    fast_finish: true
+  include:
+  - php: 5.5.38
+  - php: 5.6.25
+  - php: 7.0
+  - php: 7.1
+  - php: hhvm
+    group: edge
+  fast_finish: true
 
 before_script:
-    - pecl install grpc || echo 'Failed to install grpc'
-    - composer install
+  - pecl install grpc || echo 'Failed to install grpc'
+  - composer install
 
 script:
-    - ./dev/sh/tests
-    - vendor/bin/phpcs --standard=./phpcs-ruleset.xml
-    - ./dev/sh/build-docs
+  - ./dev/sh/tests
+  - vendor/bin/phpcs --standard=./phpcs-ruleset.xml
+  - ./dev/sh/build-docs
 
 after_success:
-    - bash <(curl -s https://codecov.io/bash)
-    - ./dev/sh/push-docs
+  - bash <(curl -s https://codecov.io/bash)
+  - ./dev/sh/push-docs
+  - ./dev/sh/trigger-split
+  - cat ./build/snippets-uncovered.json
 
 after_failure:
-    - echo "SNIPPET COVERAGE REPORT" && cat ./build/snippets-uncovered.json
+  - echo "SNIPPET COVERAGE REPORT" && cat ./build/snippets-uncovered.json
diff --git a/composer.json b/composer.json
@@ -76,5 +76,13 @@
     },
     "scripts": {
         "google-cloud": "dev/google-cloud"
+    },
+    "extra": {
+        "component": {
+            "id": "google-cloud",
+            "target": "[email protected]:jdpedrie-gcp/google-cloud-php.git",
+            "path": "src",
+            "entry": "ServiceBuilder.php"
+        }
     }
 }
diff --git a/dev/google-cloud b/dev/google-cloud
@@ -20,6 +20,7 @@ require __DIR__ . '/../vendor/autoload.php';
 
 use Google\Cloud\Dev\DocGenerator\Command\Docs;
 use Google\Cloud\Dev\Release\Command\Release;
+use Google\Cloud\Dev\Split\Command\Split;
 use Symfony\Component\Console\Application;
 
 if (!class_exists(Application::class)) {
@@ -33,4 +34,5 @@ if (!class_exists(Application::class)) {
 $app = new Application;
 $app->add(new Release(__DIR__));
 $app->add(new Docs(__DIR__));
+$app->add(new Split(__DIR__));
 $app->run();
diff --git a/dev/sh/build-docs b/dev/sh/build-docs
@@ -5,7 +5,13 @@ set -ev
 function buildDocs () {
   echo "doc dir before generation:"
   find docs
-  composer google-cloud docs
+
+  if [ -z "$TRAVIS_TAG" ]; then
+    ./dev/google-cloud docs
+  else
+    ./dev/google-cloud docs -r ${TRAVIS_TAG}
+  fi
+
   echo "doc dir after generation:"
   find docs
 }

diff --git a/dev/sh/compile-splitsh b/dev/sh/compile-splitsh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+mkdir $
+export GOPATH=$TRAVIS_BUILD_DIR/go
+
+go get -d github.com/libgit2/git2go
+cd $GOPATH/src/github.com/libgit2/git2go
+git checkout next
+git submodule update --init
+make install
+
+go get github.com/splitsh/lite
+go build -o $TRAVIS_BUILD_DIR/splitsh-lite github.com/splitsh/lite
diff --git a/dev/sh/push-docs b/dev/sh/push-docs
@@ -2,20 +2,11 @@
 
 set -ev
 
-function generateDocs () {
-  echo "doc dir before generation:"
-  find docs
-  composer google-cloud docs
-  echo "doc dir after generation:"
-  find docs
-}
-
 function pushDocs () {
-  git submodule add -q -f -b gh-pages https://${GH_OAUTH_TOKEN}@github.com/${GH_OWNER}/${GH_PROJECT_NAME} ghpages
-  mkdir -p ghpages/json/${1}
-  cp -R docs/json/master/* ghpages/json/${1}
-  cp docs/overview.html ghpages/json/${1}
-  cp docs/toc.json ghpages/json/${1}
+  git submodule add -q -f -b gh-pages https://${GH_OAUTH_TOKEN}@github.com/${TRAVIS_REPO_SLUG} ghpages
+
+  rsync -aP docs/json/* ghpages/json/
+
   cp docs/home.html ghpages/json
   cp docs/manifest.json ghpages
   cd ghpages
@@ -25,7 +16,7 @@ function pushDocs () {
     git config user.email "[email protected]"
     git commit -m "Updating docs for ${1}"
     git status
-    git push -q https://${GH_OAUTH_TOKEN}@github.com/${GH_OWNER}/${GH_PROJECT_NAME} HEAD:gh-pages
+    git push -q https://${GH_OAUTH_TOKEN}@github.com/${TRAVIS_REPO_SLUG} HEAD:gh-pages
   else
     echo "Nothing to commit."
   fi

diff --git a/dev/sh/split b/dev/sh/split
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+branch=$(git branch | sed -n -e 's/^\* \(.*\)/\1/p')
+
+SHA=`$TRAVIS_BUILD_DIR/splitsh-lite --prefix=$1`
+git push -q \
+    "https://${GH_OAUTH_TOKEN}@github.com/$2" \
+    $SHA:master --force
diff --git a/dev/sh/trigger-split b/dev/sh/trigger-split
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [[ "$TRAVIS_JOB_NUMBER" == *.1 && -n "$TRAVIS_TAG" ]]; then
+    $(dirname $0)/compile-splitsh
+    git fetch --unshallow
+    composer google-cloud split
+else
+    echo "Split occurs only in a tag run, and in the first matrix build"
+fi
diff --git a/dev/src/DocGenerator/Command/Docs.php b/dev/src/DocGenerator/Command/Docs.php
@@ -19,19 +19,26 @@
 
 use Google\Cloud\Dev\DocGenerator\DocGenerator;
 use Google\Cloud\Dev\DocGenerator\GuideGenerator;
+use Google\Cloud\Dev\DocGenerator\TableOfContents;
 use Google\Cloud\Dev\DocGenerator\TypeGenerator;
+use Google\Cloud\Dev\GetComponentsTrait;
 use RecursiveDirectoryIterator;
 use RecursiveIteratorIterator;
 use RecursiveRegexIterator;
 use RegexIterator;
 use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 class Docs extends Command
 {
-    const DEFAULT_OUTPUT_DIR = 'docs/json/master';
+    use GetComponentsTrait;
+
+    const DEFAULT_OUTPUT_DIR = 'docs/json';
+    const TOC_SOURCE_DIR = 'docs/contents';
+    const TOC_TEMPLATE = 'docs/toc.json';
     const DEFAULT_SOURCE_DIR = 'src';
 
     private $cliBasePath;
@@ -47,29 +54,94 @@ protected function configure()
     {
         $this->setName('docs')
             ->setDescription('Generate Documentation')
-            ->addArgument('source', InputArgument::OPTIONAL, 'The source directory to traverse and parse')
-            ->addArgument('output', InputArgument::OPTIONAL, 'The directory to output files into');
+            ->addOption('release', 'r', InputOption::VALUE_REQUIRED, 'If set, docs will be generated into tag folders' .
+                ' such as v1.0.0 rather than master.', false)
+            ->addOption('pretty', 'p', InputOption::VALUE_OPTIONAL, 'If set, json files will be written with pretty'.
+                ' formatting using PHP\'s JSON_PRETTY_PRINT flag', false);
     }
 
     protected function execute(InputInterface $input, OutputInterface $output)
     {
+        $release = ($input->getOption('release') === false && $input->getOption('release') !== 'false')
+            ? null
+            : $input->getOption('release');
+
+        $pretty = ($input->getOption('pretty') === false) ? false : true;
+
         $paths = [
-            'source' => ($input->getArgument('source'))
-                ? $this->cliBasePath .'/../'. $input->getArgument('source')
-                : $this->cliBasePath .'/../'. self::DEFAULT_SOURCE_DIR,
+            'source' => $this->cliBasePath .'/../'. self::DEFAULT_SOURCE_DIR,
+            'output' => $this->cliBasePath .'/../'. self::DEFAULT_OUTPUT_DIR,
+            'project' => $this->cliBasePath .'/../',
+            'manifest' => $this->cliBasePath .'/../docs/manifest.json',
+            'toc' => $this->cliBasePath .'/../'. self::TOC_SOURCE_DIR,
+            'tocTemplate' => $this->cliBasePath .'/../'. self::TOC_TEMPLATE
+        ];
 
-            'output' => ($input->getArgument('output'))
-                ? $this->cliBasePath .'/../'. $input->getArgument('output')
-                : $this->cliBasePath .'/../'. self::DEFAULT_OUTPUT_DIR
+        $components = $this->getComponents($paths['source']);
+        $tocTemplate = json_decode(file_get_contents($paths['tocTemplate']), true);
+
+        foreach ($components as $component) {
+            $input = $paths['project'] . $component['path'];
+            $source = $this->getFilesList($input);
+            $this->generateComponentDocumentation($output, $source, $component, $paths, $tocTemplate, $release, $pretty);
+        }
+
+        $source = [$paths['project'] .'src/ServiceBuilder.php'];
+        $component = [
+            'id' => 'google-cloud',
+            'path' => 'src/'
         ];
+        $this->generateComponentDocumentation($output, $source, $component, $paths, $tocTemplate, $release, $pretty);
+    }
+
+    private function generateComponentDocumentation(
+        OutputInterface $output,
+        array $source,
+        array $component,
+        array $paths,
+        $tocTemplate,
+        $release = false,
+        $pretty = false
+    ) {
+        $output->writeln(sprintf('Writing documentation for %s', $component['id']));
+        $output->writeln('--------------');
+
+        $version = $this->getComponentVersion($paths['manifest'], $component['id']);
+
+        $outputPath = ($release)
+            ? $paths['output'] .'/'. $component['id'] .'/'. $version
+            : $paths['output'] .'/'. $component['id'] .'/master';
+
+        $output->writeln(sprintf('Writing to %s', realpath($outputPath)));
+
+        $types = new TypeGenerator($outputPath);
+
+        $docs = new DocGenerator(
+            $types,
+            $source,
+            $outputPath,
+            $this->cliBasePath,
+            $component['id'],
+            $paths['manifest'],
+            $release
+        );
+        $docs->generate($component['path'], $pretty);
+
+        $types->write($pretty);
 
-        $types = new TypeGenerator($paths['output']);
+        $output->writeln(sprintf('Writing table of contents to %s', realpath($outputPath)));
+        $services = json_decode(file_get_contents($paths['toc'] .'/'. $component['id'] .'.json'), true);
 
-        $sourceFiles = $this->getFilesList($paths['source']);
-        $docs = new DocGenerator($types, $sourceFiles, $paths['output'], $this->cliBasePath);
-        $docs->generate();
+        $toc = new TableOfContents(
+            $tocTemplate,
+            $services,
+            $release,
+            $outputPath
+        );
+        $toc->generate($pretty);
 
-        $types->write();
+        $output->writeln(' ');
+        $output->writeln(' ');
     }
 
     private function getFilesList($source)

diff --git a/dev/src/DocGenerator/DocGenerator.php b/dev/src/DocGenerator/DocGenerator.php
@@ -32,48 +32,86 @@ class DocGenerator
     private $files;
     private $outputPath;
     private $executionPath;
+    private $componentId;
+    private $manifestPath;
+    private $release;
 
     /**
      * @param array $files
      */
-    public function __construct(TypeGenerator $types, array $files, $outputPath, $executionPath)
-    {
+    public function __construct(
+        TypeGenerator $types,
+        array $files,
+        $outputPath,
+        $executionPath,
+        $componentId,
+        $manifestPath,
+        $release
+    ) {
         $this->types = $types;
         $this->files = $files;
         $this->outputPath = $outputPath;
         $this->executionPath = $executionPath;
+        $this->componentId = $componentId;
+        $this->manifestPath = $manifestPath;
+        $this->release = $release;
     }
 
     /**
      * Generates JSON documentation from provided files.
      *
      * @return void
      */
-    public function generate()
+    public function generate($basePath, $pretty)
     {
         foreach ($this->files as $file) {
 
-            $currentFile = substr(str_replace($this->executionPath, '', $file), 3);
+            if ($basePath) {
+                $currentFileArr = explode($basePath, trim($file, '/'));
+                if (isset($currentFileArr[1])) {
+                    $currentFile = trim($currentFileArr[1], '/');
+                }
+            }
+
             $isPhp = strrpos($file, '.php') == strlen($file) - strlen('.php');
 
             if ($isPhp) {
                 $fileReflector = new FileReflector($file);
-                $parser = new CodeParser($file, $currentFile, $fileReflector);
+                $parser = new CodeParser(
+                    $file,
+                    $currentFile,
+                    $fileReflector,
+                    dirname($this->executionPath),
+                    $this->componentId,
+                    $this->manifestPath,
+                    $this->release
+                );
             } else {
                 $content = file_get_contents($file);
-                $parser = new MarkdownParser($currentFile, $content);
+                $split = explode('src/', $file);
+                $parser = new MarkdownParser($split[1], $content);
             }
 
             $document = $parser->parse();
 
-            $writer = new Writer(json_encode($document), $this->outputPath);
-            $writer->write(substr($currentFile, 4));
+            $writer = new Writer($document, $this->outputPath, $pretty);
+            $writer->write($currentFile);
 
             $this->types->addType([
                 'id' => $document['id'],
                 'title' => $document['title'],
-                'contents' => $document['id'] . '.json'
+                'contents' => $this->prune($document['id'] . '.json')
             ]);
         }
     }
+
+    private function prune($contentsFileName)
+    {
+        $explode = explode('/', $contentsFileName);
+        if (count($explode) > 1) {
+            array_shift($explode);
+        }
+
+        return implode('/', $explode);
+    }
 }