Create symbolic links in GCS to output of presubmit results. (#79)

* Create symbolic links in GCS to output of presubmit results.

test-infra/runner.py

@@ -268,13 +268,29 @@ def get_gcs_output():
               build=os.getenv("BUILD_NUMBER"))
     return output
 
+def get_symlink_output(pull_number, job_name, build_number):
+  """Return the location where the symlink should be created."""
+  # GCS layout is defined here:
+  # https://github.com/kubernetes/test-infra/tree/master/gubernator#job-artifact-gcs-layout
+  if not pull_number:
+    # Symlinks are only created for pull requests.
+    return ""
+  output = ("gs://kubernetes-jenkins/pr-logs/directory/"
+            "{job}/{build}.txt").format(
+                job=job_name,
+                build=build_number)
+  return output
+
 def create_started(gcs_client, output_dir, sha):
   """Create the started output in GCS.
 
   Args:
     gcs_client: GCS client
     output_dir: The GCS directory where the output should be written.
     sha: Sha for the mlkube.io repo
+
+  Returns:
+    blob: The created blob.
   """
   # See:
   # https://github.com/kubernetes/test-infra/tree/master/gubernator#job-artifact-gcs-layout
@@ -308,6 +324,9 @@ def create_finished(gcs_client, output_dir, success):
     gcs_client: GCS client
     output_dir: The GCS directory where the output should be written.
     success: Boolean indicating whether the test was successful.
+
+  Returns:
+    blob: The blob object that we created.
   """
   result = "FAILURE"
   if success:
@@ -328,7 +347,24 @@ def create_finished(gcs_client, output_dir, success):
   bucket = gcs_client.get_bucket(bucket)
   blob = bucket.blob(os.path.join(path, "finished.json"))
   blob.upload_from_string(json.dumps(finished))
+  return blob
+
+def create_symlink(gcs_client, symlink, output):
+  """Create a 'symlink' to the output directory.
+
+  Args:
+    gcs_client: GCS client
+    symling: GCS path of the object to server as the link
+    output: The location to point to.
+  """
+  m = GCS_REGEX.match(symlink)
+  bucket = m.group(1)
+  path = m.group(2)
 
+  bucket = gcs_client.get_bucket(bucket)
+  blob = bucket.blob(path)
+  blob.upload_from_string(output)
+  return blob
 
 def upload_outputs(gcs_client, output_dir, test_dir):
   m = GCS_REGEX.match(output_dir)
@@ -443,13 +479,18 @@ def create_latest(gcs_client, job_name, sha):
                  os.getenv("GOOGLE_APPLICATION_CREDENTIALS"))
     run(["gcloud", "auth", "activate-service-account",
          "--key-file={0}".format(os.getenv("GOOGLE_APPLICATION_CREDENTIALS"))])
-
+  job_name = os.getenv("JOB_NAME", "")
+  build_number = os.getenv("BUILD_NUMBER")
+  pull_number = os.getenv("PULL_NUMBER")
   output_dir = get_gcs_output()
-  gcs_client = storage.Client()
+  logging.info("Artifacts will be saved to: %s", output_dir)
 
+  gcs_client = storage.Client()
   create_started(gcs_client, output_dir, sha)
 
-  logging.info("Artifacts will be saved to: %s", output_dir)
+  symlink = get_symlink_output(pull_number, job_name, build_number)
+  if symlink:
+    create_symlink(gcs_client, symlink, output_dir)
 
   image = build_container(args.use_gcb, src_dir, test_dir)
   logging.info("Created image: %s", image)

test-infra/runner_test.py

@@ -6,6 +6,20 @@
 from google.cloud import storage
 
 class TestRunner(unittest.TestCase):
+  @mock.patch("runner.time.time")
+  def testCreateFinished(self, mock_time):
+    """Test create finished"""
+    mock_time.return_value = 1000
+    gcs_client = mock.MagicMock(spec=storage.Client)
+    blob = runner.create_finished(gcs_client, "gs://bucket/output", True)
+
+    expected = {
+      "timestamp": 1000,
+      "result": "SUCCESS",
+      "metadata": {},
+    }
+    blob.upload_from_string.assert_called_once_with(json.dumps(expected))
+
   @mock.patch("runner.time.time")
   def testCreateStartedPeriodic(self, mock_time):
     """Test create started for periodic job."""
@@ -22,5 +36,19 @@ def testCreateStartedPeriodic(self, mock_time):
     blob.upload_from_string.assert_called_once_with(json.dumps(expected))
 
 
+  def testGetSymlinkOutput(self):
+    location = runner.get_symlink_output("10", "mlkube-build-presubmit", "20")
+    self.assertEquals(
+      "gs://kubernetes-jenkins/pr-logs/directory/mlkube-build-presubmit/20.txt",
+      location)
+
+  def testCreateSymlinkOutput(self):
+    """Test create started for periodic job."""
+    gcs_client = mock.MagicMock(spec=storage.Client)
+    blob = runner.create_symlink(gcs_client, "gs://bucket/symlink",
+                                 "gs://bucket/output")
+
+    blob.upload_from_string.assert_called_once_with("gs://bucket/output")
+
 if __name__ == "__main__":
   unittest.main()