Merge branch 'master' of github.com:flyteorg/flyte into add-webhook-p…

…riorityClassName

.github/workflows/tests.yml

@@ -27,20 +27,35 @@ jobs:
   docs:
     runs-on: ubuntu-latest
     steps:
-      - name: Fetch the code
+      - name: Fetch flyte code
+        uses: actions/checkout@v4
+        with:
+          path: "${{ github.workspace }}/flyte"
+      - name: Fetch flytekit code
         uses: actions/checkout@v4
+        with:
+          repository: flyteorg/flytekit
+          path: "${{ github.workspace }}/flytekit"
       - uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
           python-version: 3.9
       - shell: bash -el {0}
+        working-directory: ${{ github.workspace }}/flyte
         run: |
           conda install -c conda-forge conda-lock
           conda-lock install -n monodocs-env monodocs-environment.lock.yaml
       - shell: bash -el {0}
+        working-directory: ${{ github.workspace }}/flyte
         run: |
           conda activate monodocs-env
+          export SETUPTOOLS_SCM_PRETEND_VERSION="2.0.0"
           pip install -e ./flyteidl
+      - shell: bash -el {0}
+        working-directory: ${{ github.workspace }}/flytekit
+        run: |
+          conda activate monodocs-env
+          pip install -e .
           conda info
           conda list
           conda config --show-sources
@@ -49,6 +64,7 @@ jobs:
       - name: Setup Graphviz
         uses: ts-graphviz/setup-graphviz@v1
       - name: Build the documentation
+        working-directory: ${{ github.workspace }}/flyte
         shell: bash -el {0}
         run: |
           conda activate monodocs-env

docs/community/troubleshoot.rst

@@ -14,7 +14,7 @@ Before getting started, collect the following information from the underlying in
 
 .. prompt:: bash $
 
- $ kubectl describe pod <PodName> -n <namespace>
+ kubectl describe pod <PodName> -n <namespace>
 
 Where <PodName> will typically correspond to the node execution string that you can find in the UI.
 
@@ -23,7 +23,7 @@ Where <PodName> will typically correspond to the node execution string that you
 
 .. prompt:: bash $
 
- $ kubectl logs pods -n <namespace>
+ kubectl logs pods -n <namespace>
 
 Where <namespace> will typically correspond to the Flyte <project>-<domain>, e.g. flytesnacks-development.
 
@@ -70,7 +70,7 @@ This issue is more common on MacOS devices. Make sure that your Docker daemon ha
 
 .. prompt:: bash $
 
- $ flytectl demo start --env HTTP_PROXY=<your-proxy-IP>
+ flytectl demo start --env HTTP_PROXY=<your-proxy-IP>
 
 - If you're building a custom Docker image, make sure to use a tag other than ``latest``. Otherwise, the Kubernetes default pull policy will be changed from ``IfNotPresent`` to ``Always``, forcing an image pull with every Pod deployment.
 
@@ -85,14 +85,14 @@ Issues running workloads
 
 .. prompt:: bash $
 
- $ export FLYTECTL_CONFIG=~/.flyte/config-sandbox.yaml
+ export FLYTECTL_CONFIG=~/.flyte/config-sandbox.yaml
 
 ``ModuleNotFoundError``
 ^^^^^^^^^^^^^^^^^^^^^^^
 
 - If you're using a custom container image and using Docker, make sure your ``Dockerfile`` is located at the same level of the ``flyte`` directory and that there is an empty ``__init__.py`` file in your project's folder :
 
-.. prompt:: bash $
+.. prompt::
 
  myflyteapp
  ├── Dockerfile
@@ -111,11 +111,11 @@ Issues running workloads
 
 .. prompt:: bash $
 
- $ kubectl describe sa <my-flyte-sa> -n <flyte-namespace>
+ kubectl describe sa <my-flyte-sa> -n <flyte-namespace>
 
 Example output:
 
-.. prompt:: bash $
+.. prompt::
 
  Name:                <my-flyte-sa>
  Namespace:           flyte
@@ -130,7 +130,7 @@ Example output:
 
 .. prompt:: bash $
 
- $ kubectl annotate serviceaccount -n <flyte-namespace> <http://eks.amazonaws.com/role-arn=arn:aws:iam::xxxx:role/<flyte-iam-role>eks.amazonaws.com/role-arn=arn:aws:iam::xxxx:role/<flyte-iam-role>
+ kubectl annotate serviceaccount -n <flyte-namespace> <http://eks.amazonaws.com/role-arn=arn:aws:iam::xxxx:role/<flyte-iam-role>eks.amazonaws.com/role-arn=arn:aws:iam::xxxx:role/<flyte-iam-role>
 
 - Refer to this community-maintained `guides <https://github.com/davidmirror-ops/flyte-the-hard-way/blob/main/docs/03-roles-service-accounts.md>`_ for further information about Flyte deployment on EKS
 

docs/deployment/deployment/multicluster.rst

@@ -398,11 +398,14 @@ label has to be 1.
 
 11. Verify that all Pods in the ``flyte`` namespace are ``Running``: 
 
+.. prompt:: bash $
+
+   kubectl get pods -n flyte
+
 Example output:
 
-.. prompt:: bash $
+.. prompt::
 
-   kubectl get pods -n flyte                                                                                                                  
    NAME                             READY   STATUS    RESTARTS   AGE
    datacatalog-86f6b9bf64-bp2cj     1/1     Running   0          23h
    datacatalog-86f6b9bf64-fjzcp     1/1     Running   0          23h
@@ -658,4 +661,4 @@ The process can be repeated for additional clusters.
         
   15. A successful execution should be visible on the UI, confirming it ran in the new cluster:
 
-     .. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/common/multicluster-execution.png
+     .. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/common/multicluster-execution.png

flyteplugins/go/tasks/logs/logging_utils.go

@@ -52,6 +52,7 @@ func GetLogsForContainerInPod(ctx context.Context, logPlugin tasklog.Plugin, tas
 			TaskExecutionID:      taskExecID,
 			ExtraTemplateVars:    extraLogTemplateVars,
 			TaskTemplate:         taskTemplate,
+			HostName:             pod.Spec.Hostname,
 		},
 	)
 

flyteplugins/go/tasks/logs/logging_utils_test.go

@@ -210,6 +210,40 @@ func TestGetLogsForContainerInPod_All(t *testing.T) {
 	assert.Len(t, logs, 2)
 }
 
+func TestGetLogsForContainerInPod_HostName(t *testing.T) {
+	logPlugin, err := InitializeLogPlugins(&LogConfig{
+		IsKubernetesEnabled: true,
+		KubernetesURL:       "k8s.com",
+		IsCloudwatchEnabled: true,
+		CloudwatchRegion:    "us-east-1",
+		CloudwatchLogGroup:  "/kubernetes/flyte-production",
+	})
+	assert.NoError(t, err)
+
+	pod := &v1.Pod{
+		Spec: v1.PodSpec{
+			Containers: []v1.Container{
+				{
+					Name: "ContainerName",
+				},
+			},
+			Hostname: "my-hostname",
+		},
+		Status: v1.PodStatus{
+			ContainerStatuses: []v1.ContainerStatus{
+				{
+					ContainerID: "ContainerID",
+				},
+			},
+		},
+	}
+	pod.Name = podName
+
+	logs, err := GetLogsForContainerInPod(context.TODO(), logPlugin, dummyTaskExecID(), pod, 0, " Suffix", nil, nil)
+	assert.Nil(t, err)
+	assert.Len(t, logs, 2)
+}
+
 func TestGetLogsForContainerInPod_Stackdriver(t *testing.T) {
 	logPlugin, err := InitializeLogPlugins(&LogConfig{
 		IsStackDriverEnabled:       true,
@@ -268,7 +302,7 @@ func TestGetLogsForContainerInPod_LegacyTemplate(t *testing.T) {
 				MessageFormat: core.TaskLog_JSON,
 				Name:          "Stackdriver Logs my-Suffix",
 			},
-		})
+		}, "")
 	})
 
 	t.Run("StackDriver", func(t *testing.T) {
@@ -281,11 +315,11 @@ func TestGetLogsForContainerInPod_LegacyTemplate(t *testing.T) {
 				MessageFormat: core.TaskLog_JSON,
 				Name:          "Stackdriver Logs my-Suffix",
 			},
-		})
+		}, "")
 	})
 }
 
-func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.TaskTemplate, expectedTaskLogs []*core.TaskLog) {
+func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.TaskTemplate, expectedTaskLogs []*core.TaskLog, hostname string) {
 	logPlugin, err := InitializeLogPlugins(config)
 	assert.NoError(tb, err)
 
@@ -300,6 +334,7 @@ func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.Ta
 					Name: "ContainerName",
 				},
 			},
+			Hostname: hostname,
 		},
 		Status: v1.PodStatus{
 			ContainerStatuses: []v1.ContainerStatus{
@@ -347,7 +382,27 @@ func TestGetLogsForContainerInPod_Templates(t *testing.T) {
 			MessageFormat: core.TaskLog_JSON,
 			Name:          "Internal my-Suffix",
 		},
-	})
+	}, "")
+}
+
+func TestGetLogsForContainerInPodTemplates_Hostname(t *testing.T) {
+	assertTestSucceeded(t, &LogConfig{
+		Templates: []tasklog.TemplateLogPlugin{
+			{
+				DisplayName: "StackDriver",
+				TemplateURIs: []string{
+					"{{ .hostname }}/{{ .namespace }}/{{ .podName }}/{{ .containerName }}/{{ .containerId }}",
+				},
+				MessageFormat: core.TaskLog_JSON,
+			},
+		},
+	}, nil, []*core.TaskLog{
+		{
+			Uri:           "my-hostname/my-namespace/my-pod/ContainerName/ContainerID",
+			MessageFormat: core.TaskLog_JSON,
+			Name:          "StackDriver my-Suffix",
+		},
+	}, "my-hostname")
 }
 
 func TestGetLogsForContainerInPod_Flyteinteractive(t *testing.T) {
@@ -507,7 +562,7 @@ func TestGetLogsForContainerInPod_Flyteinteractive(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			assertTestSucceeded(t, tt.config, tt.template, tt.expectedTaskLogs)
+			assertTestSucceeded(t, tt.config, tt.template, tt.expectedTaskLogs, "")
 		})
 	}
 }