feat: [DataCatalogLineage] add open lineage support

DataCatalogLineage/samples/V1/LineageClient/batch_search_link_processes.php

@@ -45,8 +45,8 @@
  * have the `datalineage.events.get` permission. The project provided in the
  * URL is used for Billing and Quota.
  *
- * @param string $formattedParent The project and location you want search in the format `projects/*/locations/*`
- *                                Please see {@see LineageClient::locationName()} for help formatting this field.
+ * @param string $formattedParent The project and location where you want to search. Please see
+ *                                {@see LineageClient::locationName()} for help formatting this field.
  * @param string $linksElement    An array of links to check for their associated LineageProcesses.
  *
  *                                The maximum number of items in this array is 100.

DataCatalogLineage/samples/V1/LineageClient/create_lineage_event.php

@@ -27,6 +27,7 @@
 use Google\Cloud\DataCatalog\Lineage\V1\Client\LineageClient;
 use Google\Cloud\DataCatalog\Lineage\V1\CreateLineageEventRequest;
 use Google\Cloud\DataCatalog\Lineage\V1\LineageEvent;
+use Google\Protobuf\Timestamp;
 
 /**
  * Creates a new lineage event.
@@ -40,7 +41,9 @@ function create_lineage_event_sample(string $formattedParent): void
     $lineageClient = new LineageClient();
 
     // Prepare the request message.
-    $lineageEvent = new LineageEvent();
+    $lineageEventStartTime = new Timestamp();
+    $lineageEvent = (new LineageEvent())
+        ->setStartTime($lineageEventStartTime);
     $request = (new CreateLineageEventRequest())
         ->setParent($formattedParent)
         ->setLineageEvent($lineageEvent);

DataCatalogLineage/samples/V1/LineageClient/process_open_lineage_run_event.php

@@ -0,0 +1,77 @@
+<?php
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * GENERATED CODE WARNING
+ * This file was automatically generated - do not edit!
+ */
+
+require_once __DIR__ . '/../../../vendor/autoload.php';
+
+// [START datalineage_v1_generated_Lineage_ProcessOpenLineageRunEvent_sync]
+use Google\ApiCore\ApiException;
+use Google\Cloud\DataCatalog\Lineage\V1\Client\LineageClient;
+use Google\Cloud\DataCatalog\Lineage\V1\ProcessOpenLineageRunEventRequest;
+use Google\Cloud\DataCatalog\Lineage\V1\ProcessOpenLineageRunEventResponse;
+use Google\Protobuf\Struct;
+
+/**
+ * Creates new lineage events together with their parents: process and run.
+ * Updates the process and run if they already exist.
+ * Mapped from Open Lineage specification:
+ * https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json.
+ *
+ * @param string $parent The name of the project and its location that should own the
+ *                       process, run, and lineage event.
+ */
+function process_open_lineage_run_event_sample(string $parent): void
+{
+    // Create a client.
+    $lineageClient = new LineageClient();
+
+    // Prepare the request message.
+    $openLineage = new Struct();
+    $request = (new ProcessOpenLineageRunEventRequest())
+        ->setParent($parent)
+        ->setOpenLineage($openLineage);
+
+    // Call the API and handle any network failures.
+    try {
+        /** @var ProcessOpenLineageRunEventResponse $response */
+        $response = $lineageClient->processOpenLineageRunEvent($request);
+        printf('Response data: %s' . PHP_EOL, $response->serializeToJsonString());
+    } catch (ApiException $ex) {
+        printf('Call failed with message: %s' . PHP_EOL, $ex->getMessage());
+    }
+}
+
+/**
+ * Helper to execute the sample.
+ *
+ * This sample has been automatically generated and should be regarded as a code
+ * template only. It will require modifications to work:
+ *  - It may require correct/in-range values for request initialization.
+ *  - It may require specifying regional endpoints when creating the service client,
+ *    please see the apiEndpoint client configuration option for more details.
+ */
+function callSample(): void
+{
+    $parent = '[PARENT]';
+
+    process_open_lineage_run_event_sample($parent);
+}
+// [END datalineage_v1_generated_Lineage_ProcessOpenLineageRunEvent_sync]

DataCatalogLineage/samples/V1/LineageClient/search_links.php

@@ -40,8 +40,8 @@
  * `datalineage.events.get` permission. The project provided in the URL
  * is used for Billing and Quota.
  *
- * @param string $formattedParent The project and location you want search in the format `projects/*/locations/*`
- *                                Please see {@see LineageClient::locationName()} for help formatting this field.
+ * @param string $formattedParent The project and location you want search in. Please see
+ *                                {@see LineageClient::locationName()} for help formatting this field.
  */
 function search_links_sample(string $formattedParent): void
 {

DataCatalogLineage/src/V1/Client/LineageClient.php

@@ -50,6 +50,8 @@
 use Google\Cloud\DataCatalog\Lineage\V1\ListProcessesRequest;
 use Google\Cloud\DataCatalog\Lineage\V1\ListRunsRequest;
 use Google\Cloud\DataCatalog\Lineage\V1\Process;
+use Google\Cloud\DataCatalog\Lineage\V1\ProcessOpenLineageRunEventRequest;
+use Google\Cloud\DataCatalog\Lineage\V1\ProcessOpenLineageRunEventResponse;
 use Google\Cloud\DataCatalog\Lineage\V1\Run;
 use Google\Cloud\DataCatalog\Lineage\V1\SearchLinksRequest;
 use Google\Cloud\DataCatalog\Lineage\V1\UpdateProcessRequest;
@@ -90,6 +92,7 @@
  * @method PromiseInterface listLineageEventsAsync(ListLineageEventsRequest $request, array $optionalArgs = [])
  * @method PromiseInterface listProcessesAsync(ListProcessesRequest $request, array $optionalArgs = [])
  * @method PromiseInterface listRunsAsync(ListRunsRequest $request, array $optionalArgs = [])
+ * @method PromiseInterface processOpenLineageRunEventAsync(ProcessOpenLineageRunEventRequest $request, array $optionalArgs = [])
  * @method PromiseInterface searchLinksAsync(SearchLinksRequest $request, array $optionalArgs = [])
  * @method PromiseInterface updateProcessAsync(UpdateProcessRequest $request, array $optionalArgs = [])
  * @method PromiseInterface updateRunAsync(UpdateRunRequest $request, array $optionalArgs = [])
@@ -697,6 +700,35 @@ public function listRuns(ListRunsRequest $request, array $callOptions = []): Pag
         return $this->startApiCall('ListRuns', $request, $callOptions);
     }
 
+    /**
+     * Creates new lineage events together with their parents: process and run.
+     * Updates the process and run if they already exist.
+     * Mapped from Open Lineage specification:
+     * https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json.
+     *
+     * The async variant is {@see LineageClient::processOpenLineageRunEventAsync()} .
+     *
+     * @example samples/V1/LineageClient/process_open_lineage_run_event.php
+     *
+     * @param ProcessOpenLineageRunEventRequest $request     A request to house fields associated with the call.
+     * @param array                             $callOptions {
+     *     Optional.
+     *
+     *     @type RetrySettings|array $retrySettings
+     *           Retry settings to use for this call. Can be a {@see RetrySettings} object, or an
+     *           associative array of retry settings parameters. See the documentation on
+     *           {@see RetrySettings} for example usage.
+     * }
+     *
+     * @return ProcessOpenLineageRunEventResponse
+     *
+     * @throws ApiException Thrown if the API call fails.
+     */
+    public function processOpenLineageRunEvent(ProcessOpenLineageRunEventRequest $request, array $callOptions = []): ProcessOpenLineageRunEventResponse
+    {
+        return $this->startApiCall('ProcessOpenLineageRunEvent', $request, $callOptions)->wait();
+    }
+
     /**
      * Retrieve a list of links connected to a specific asset.
      * Links represent the data flow between **source** (upstream)

DataCatalogLineage/src/V1/Gapic/LineageGapicClient.php

@@ -55,6 +55,8 @@
 use Google\Cloud\DataCatalog\Lineage\V1\ListRunsRequest;
 use Google\Cloud\DataCatalog\Lineage\V1\ListRunsResponse;
 use Google\Cloud\DataCatalog\Lineage\V1\Process;
+use Google\Cloud\DataCatalog\Lineage\V1\ProcessOpenLineageRunEventRequest;
+use Google\Cloud\DataCatalog\Lineage\V1\ProcessOpenLineageRunEventResponse;
 use Google\Cloud\DataCatalog\Lineage\V1\Run;
 use Google\Cloud\DataCatalog\Lineage\V1\SearchLinksRequest;
 use Google\Cloud\DataCatalog\Lineage\V1\SearchLinksResponse;
@@ -63,6 +65,7 @@
 use Google\LongRunning\Operation;
 use Google\Protobuf\FieldMask;
 use Google\Protobuf\GPBEmpty;
+use Google\Protobuf\Struct;
 
 /**
  * Service Description: Lineage is used to track data flows between assets over time. You can
@@ -461,7 +464,7 @@ public function __construct(array $options = [])
      * }
      * ```
      *
-     * @param string   $parent       Required. The project and location you want search in the format `projects/*/locations/*`
+     * @param string   $parent       Required. The project and location where you want to search.
      * @param string[] $links        Required. An array of links to check for their associated LineageProcesses.
      *
      *                               The maximum number of items in this array is 100.
@@ -1170,6 +1173,61 @@ public function listRuns($parent, array $optionalArgs = [])
         return $this->getPagedListResponse('ListRuns', $optionalArgs, ListRunsResponse::class, $request);
     }
 
+    /**
+     * Creates new lineage events together with their parents: process and run.
+     * Updates the process and run if they already exist.
+     * Mapped from Open Lineage specification:
+     * https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json.
+     *
+     * Sample code:
+     * ```
+     * $lineageClient = new LineageClient();
+     * try {
+     *     $parent = 'parent';
+     *     $openLineage = new Struct();
+     *     $response = $lineageClient->processOpenLineageRunEvent($parent, $openLineage);
+     * } finally {
+     *     $lineageClient->close();
+     * }
+     * ```
+     *
+     * @param string $parent       Required. The name of the project and its location that should own the
+     *                             process, run, and lineage event.
+     * @param Struct $openLineage  Required. OpenLineage message following OpenLineage format:
+     *                             https://github.com/OpenLineage/OpenLineage/blob/main/spec/OpenLineage.json
+     * @param array  $optionalArgs {
+     *     Optional.
+     *
+     *     @type string $requestId
+     *           A unique identifier for this request. Restricted to 36 ASCII characters.
+     *           A random UUID is recommended. This request is idempotent only if a
+     *           `request_id` is provided.
+     *     @type RetrySettings|array $retrySettings
+     *           Retry settings to use for this call. Can be a {@see RetrySettings} object, or an
+     *           associative array of retry settings parameters. See the documentation on
+     *           {@see RetrySettings} for example usage.
+     * }
+     *
+     * @return \Google\Cloud\DataCatalog\Lineage\V1\ProcessOpenLineageRunEventResponse
+     *
+     * @throws ApiException if the remote call fails
+     */
+    public function processOpenLineageRunEvent($parent, $openLineage, array $optionalArgs = [])
+    {
+        $request = new ProcessOpenLineageRunEventRequest();
+        $requestParamHeaders = [];
+        $request->setParent($parent);
+        $request->setOpenLineage($openLineage);
+        $requestParamHeaders['parent'] = $parent;
+        if (isset($optionalArgs['requestId'])) {
+            $request->setRequestId($optionalArgs['requestId']);
+        }
+
+        $requestParams = new RequestParamsHeaderDescriptor($requestParamHeaders);
+        $optionalArgs['headers'] = isset($optionalArgs['headers']) ? array_merge($requestParams->getHeader(), $optionalArgs['headers']) : $requestParams->getHeader();
+        return $this->startCall('ProcessOpenLineageRunEvent', ProcessOpenLineageRunEventResponse::class, $optionalArgs, $request)->wait();
+    }
+
     /**
      * Retrieve a list of links connected to a specific asset.
      * Links represent the data flow between **source** (upstream)
@@ -1204,7 +1262,7 @@ public function listRuns($parent, array $optionalArgs = [])
      * }
      * ```
      *
-     * @param string $parent       Required. The project and location you want search in the format `projects/*/locations/*`
+     * @param string $parent       Required. The project and location you want search in.
      * @param array  $optionalArgs {
      *     Optional.
      *
@@ -1340,6 +1398,8 @@ public function updateProcess($process, array $optionalArgs = [])
      *     @type FieldMask $updateMask
      *           The list of fields to update. Currently not used. The whole message is
      *           updated.
+     *     @type bool $allowMissing
+     *           If set to true and the run is not found, the request creates it.
      *     @type RetrySettings|array $retrySettings
      *           Retry settings to use for this call. Can be a {@see RetrySettings} object, or an
      *           associative array of retry settings parameters. See the documentation on
@@ -1360,6 +1420,10 @@ public function updateRun($run, array $optionalArgs = [])
             $request->setUpdateMask($optionalArgs['updateMask']);
         }
 
+        if (isset($optionalArgs['allowMissing'])) {
+            $request->setAllowMissing($optionalArgs['allowMissing']);
+        }
+
         $requestParams = new RequestParamsHeaderDescriptor($requestParamHeaders);
         $optionalArgs['headers'] = isset($optionalArgs['headers']) ? array_merge($requestParams->getHeader(), $optionalArgs['headers']) : $requestParams->getHeader();
         return $this->startCall('UpdateRun', Run::class, $optionalArgs, $request)->wait();