feat: added RawOutputDataConfig in ExecutionSpec #patch (#329)

* feat: added RawOutputDataConfig in ExecutionSpec Signed-off-by: Kevin Su [email protected] Signed-off-by: Olga Nad <[email protected]>
flyteorg · Mar 23, 2022 · 732ff30 · 732ff30
1 parent 18db906
commit 732ff30
Show file tree

Hide file tree

Showing 14 changed files with 37 additions and 17 deletions.
diff --git a/package.json b/package.json
@@ -82,7 +82,7 @@
         "@commitlint/cli": "^8.3.5",
         "@commitlint/config-conventional": "^8.3.4",
         "@date-io/moment": "1.3.9",
-        "@flyteorg/flyteidl": "0.21.24",
+        "@flyteorg/flyteidl": "0.23.1",
         "@material-ui/core": "^4.0.0",
         "@material-ui/icons": "^4.0.0",
         "@material-ui/pickers": "^3.2.2",

diff --git a/src/components/Executions/ExecutionDetails/ExecutionMetadataExtra.tsx b/src/components/Executions/ExecutionDetails/ExecutionMetadataExtra.tsx
@@ -34,7 +34,7 @@ export const ExecutionMetadataExtra: React.FC<{
   const commonStyles = useCommonStyles();
   const styles = useStyles();
 
-  const { launchPlan: launchPlanId, maxParallelism, authRole } = execution.spec;
+  const { launchPlan: launchPlanId, maxParallelism, rawOutputDataConfig, authRole } = execution.spec;
 
   const [launchPlanSpec, setLaunchPlanSpec] = React.useState<Partial<LaunchPlanSpec>>({});
 
@@ -53,7 +53,7 @@ export const ExecutionMetadataExtra: React.FC<{
     },
     {
       label: ExecutionMetadataLabels.rawOutputPrefix,
-      value: launchPlanSpec?.rawOutputDataConfig?.outputLocationPrefix,
+      value: rawOutputDataConfig?.outputLocationPrefix || launchPlanSpec?.rawOutputDataConfig?.outputLocationPrefix,
     },
     {
       label: ExecutionMetadataLabels.parallelism,

diff --git a/src/components/Executions/ExecutionDetails/RelaunchExecutionForm.tsx b/src/components/Executions/ExecutionDetails/RelaunchExecutionForm.tsx
@@ -30,6 +30,7 @@ function useRelaunchWorkflowFormState({ execution }: RelaunchExecutionFormProps)
             launchPlan,
             disableAll,
             maxParallelism,
+            rawOutputDataConfig,
             labels,
             annotations,
             authRole,
@@ -53,6 +54,7 @@ function useRelaunchWorkflowFormState({ execution }: RelaunchExecutionFormProps)
           workflowId,
           disableAll,
           maxParallelism,
+          rawOutputDataConfig,
           labels,
           annotations,
           authRole,

diff --git a/src/components/Launch/LaunchForm/LaunchFormAdvancedInputs.tsx b/src/components/Launch/LaunchForm/LaunchFormAdvancedInputs.tsx
@@ -85,10 +85,10 @@ export const LaunchFormAdvancedInputs = React.forwardRef<
         setMaxParallelism(`${other.maxParallelism}`);
       }
       if (
-        launchPlan?.spec.rawOutputDataConfig?.outputLocationPrefix !== undefined &&
-        launchPlan?.spec.rawOutputDataConfig.outputLocationPrefix !== null
+        other?.rawOutputDataConfig?.outputLocationPrefix !== undefined &&
+          other.rawOutputDataConfig.outputLocationPrefix !== null
       ) {
-        setRawOutputDataConfig(launchPlan.spec.rawOutputDataConfig.outputLocationPrefix);
+        setRawOutputDataConfig(other.rawOutputDataConfig.outputLocationPrefix);
       }
       const newLabels = {
         ...(other.labels?.values || {}),
@@ -100,14 +100,17 @@ export const LaunchFormAdvancedInputs = React.forwardRef<
       };
       setLabelsParamData(newLabels);
       setAnnotationsParamData(newAnnotations);
-    }, [other.disableAll, other.maxParallelism, other.labels, other.annotations, launchPlan?.spec]);
+    }, [other.disableAll, other.maxParallelism, other.rawOutputDataConfig, other.labels, other.annotations, launchPlan?.spec]);
 
     React.useImperativeHandle(
       ref,
       () => ({
         getValues: () => {
           return {
             disableAll,
+            rawOutputDataConfig: {
+              outputLocationPrefix: rawOutputDataConfig
+            },
             maxParallelism: parseInt(maxParallelism || '', 10),
             labels: {
               values: labelsParamData,
@@ -121,7 +124,7 @@ export const LaunchFormAdvancedInputs = React.forwardRef<
           return true;
         },
       }),
-      [disableAll, maxParallelism, labelsParamData, annotationsParamData],
+      [disableAll, maxParallelism, rawOutputDataConfig, labelsParamData, annotationsParamData],
     );
 
     const handleDisableAllChange = React.useCallback(() => {

diff --git a/src/components/Launch/LaunchForm/launchMachine.ts b/src/components/Launch/LaunchForm/launchMachine.ts
@@ -78,6 +78,7 @@ export interface WorkflowLaunchContext extends BaseLaunchContext {
   defaultAuthRole?: Admin.IAuthRole;
   disableAll?: boolean | null;
   maxParallelism?: number | null;
+  rawOutputDataConfig?: Admin.IRawOutputDataConfig | null;
   labels?: Admin.ILabels | null;
   annotations?: Admin.IAnnotations | null;
   securityContext?: Core.ISecurityContext | null;

diff --git a/src/components/Launch/LaunchForm/types.ts b/src/components/Launch/LaunchForm/types.ts
@@ -57,6 +57,7 @@ export interface WorkflowInitialLaunchParameters extends BaseInitialLaunchParame
   securityContext?: Core.ISecurityContext;
   disableAll?: boolean | null;
   maxParallelism?: number | null;
+  rawOutputDataConfig?: Admin.IRawOutputDataConfig | null;
   labels?: Admin.ILabels | null;
   annotations?: Admin.IAnnotations | null;
 }

diff --git a/src/components/Launch/LaunchForm/useLaunchWorkflowFormState.ts b/src/components/Launch/LaunchForm/useLaunchWorkflowFormState.ts
@@ -168,7 +168,7 @@ async function submit(
 
   const { authRole, securityContext } = roleInputRef.current?.getValue() as LaunchRoles;
   const literals = formInputsRef.current.getValues();
-  const { disableAll, labels, annotations, maxParallelism } =
+  const { disableAll, labels, annotations, maxParallelism, rawOutputDataConfig } =
     advancedOptionsRef.current?.getValues() || {};
   const launchPlanId = launchPlan.id;
   const { domain, project } = workflowVersion;
@@ -179,6 +179,7 @@ async function submit(
     securityContext,
     disableAll,
     maxParallelism,
+    rawOutputDataConfig,
     domain,
     labels,
     launchPlanId,
@@ -241,6 +242,7 @@ export function useLaunchWorkflowFormState({
     values: defaultInputValues,
     disableAll,
     maxParallelism,
+    rawOutputDataConfig,
     labels,
     annotations,
     securityContext,
@@ -273,6 +275,7 @@ export function useLaunchWorkflowFormState({
       sourceId,
       disableAll,
       maxParallelism,
+      rawOutputDataConfig,
       labels,
       annotations,
     },

diff --git a/src/components/Literals/__stories__/Collection.stories.tsx b/src/components/Literals/__stories__/Collection.stories.tsx
@@ -16,7 +16,7 @@ const stories = storiesOf('Literals/Collection', module);
 stories.addDecorator(CardDecorator);
 
 function renderCollection(label: string, collection: LiteralCollection) {
-  return <LiteralValue label={label} literal={{ collection, value: 'collection' }} />;
+  return <LiteralValue label={label} literal={{ collection, value: 'collection', hash: '' }} />;
 }
 
 stories.add('Binary', () =>

diff --git a/src/components/Literals/__stories__/Map.stories.tsx b/src/components/Literals/__stories__/Map.stories.tsx
@@ -16,7 +16,7 @@ const stories = storiesOf('Literals/Map', module);
 stories.addDecorator(CardDecorator);
 
 function renderMap(label: string, map: LiteralMap) {
-  return <LiteralValue label={label} literal={{ map, value: 'map' }} />;
+  return <LiteralValue label={label} literal={{ map, value: 'map', hash: '' }} />;
 }
 
 stories.add('Binary', () =>

diff --git a/src/components/Literals/__stories__/ProtobufStruct.stories.tsx b/src/components/Literals/__stories__/ProtobufStruct.stories.tsx
@@ -15,6 +15,7 @@ function renderStruct(label: string, struct: ProtobufStruct) {
       literal={{
         scalar: { value: 'generic', generic: struct },
         value: 'scalar',
+        hash: '',
       }}
     />
   );

diff --git a/src/components/Literals/__stories__/literalValues.ts b/src/components/Literals/__stories__/literalValues.ts
@@ -14,10 +14,11 @@ import {
  * each value in a `Literal` and setting the appropriate `value` to allow
  * lookup of whichever field is populated.
  */
-function toLiterals<T>(type: keyof Core.ILiteral, values: Dictionary<T>): Dictionary<Literal> {
+function toLiterals<T>(type: 'scalar' | 'collection' | 'map', values: Dictionary<T>): Dictionary<Literal> {
   return mapValues(values, (value: T) => ({
     value: type,
     [type]: value,
+    hash: '',
   }));
 }
 
@@ -30,4 +31,5 @@ export const schemaLiterals = toLiterals('scalar', schemaScalars);
 export const noneTypeLiteral: Literal = {
   value: 'scalar',
   scalar: noneTypeScalar,
+  hash: ''
 };
diff --git a/src/models/Common/types.ts b/src/models/Common/types.ts
@@ -63,10 +63,11 @@ export type UrlBlob = Admin.IUrlBlob;
 export type Error = RequiredNonNullable<Core.IError>;
 
 export interface Literal extends Core.Literal {
-  value: keyof Core.ILiteral;
+  value: 'scalar' | 'collection' | 'map';
   collection?: Core.ILiteralCollection;
   map?: Core.ILiteralMap;
   scalar?: Scalar;
+  hash: string;
 }
 
 /** A Core.ILiteral guaranteed to have all subproperties necessary to specify

diff --git a/src/models/Execution/api.ts b/src/models/Execution/api.ts
@@ -84,6 +84,7 @@ export interface CreateWorkflowExecutionArguments {
   disableAll?: boolean | null;
   labels?: Admin.ILabels | null;
   maxParallelism?: number | null;
+  rawOutputDataConfig?: Admin.IRawOutputDataConfig | null;
   inputs: Core.ILiteralMap;
   launchPlanId: Identifier;
   project: string;
@@ -102,6 +103,7 @@ export const createWorkflowExecution = (
     disableAll,
     labels,
     maxParallelism,
+    rawOutputDataConfig,
     inputs,
     launchPlanId: launchPlan,
     project,
@@ -136,6 +138,10 @@ export const createWorkflowExecution = (
     spec.maxParallelism = maxParallelism;
   }
 
+  if (rawOutputDataConfig?.outputLocationPrefix) {
+    spec.rawOutputDataConfig = rawOutputDataConfig;
+  }
+
   return postAdminEntity<Admin.IExecutionCreateRequest, Admin.ExecutionCreateResponse>(
     {
       data: {

diff --git a/yarn.lock b/yarn.lock
@@ -1731,10 +1731,10 @@
     minimatch "^3.0.4"
     strip-json-comments "^3.1.1"
 
-"@flyteorg/flyteidl@0.21.24":
-  version "0.21.24"
-  resolved "https://registry.yarnpkg.com/@flyteorg/flyteidl/-/flyteidl-0.21.24.tgz#08e2c53a56d09ddcf2d6a2c3c183ce4d8231c23e"
-  integrity sha512-1gcbmTKS3sppPqE61zgYhk93DQh6vf36qHftjQ6bdMMeozM6JRwbYk/cKbyrc3DCk+PGoxf46jjSXSK1NJhASQ==
+"@flyteorg/flyteidl@0.23.1":
+  version "0.23.1"
+  resolved "https://registry.yarnpkg.com/@flyteorg/flyteidl/-/flyteidl-0.23.1.tgz#da88166e1c0bd404a4f00db425fe07afaaa76a8e"
+  integrity sha512-3M6o3EObbE35cFoY88/DNPcKEp+g2mOuFc2DNlCFntKH1lsB+EEl9k+chuWy8F8iWe7DHkCKJvM5AeEjSseahg==
 
 "@gar/promisify@^1.0.1":
   version "1.1.3"