From a946070093f30b3cbed04fd7b006adcbe7279be1 Mon Sep 17 00:00:00 2001 From: Chi-Sheng Liu Date: Fri, 1 Mar 2024 17:06:50 +0800 Subject: [PATCH 1/3] docs: Remove redundant dollar signs (#4964) Signed-off-by: Chi-Sheng Liu --- docs/community/troubleshoot.rst | 16 ++++++++-------- docs/deployment/deployment/multicluster.rst | 9 ++++++--- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/docs/community/troubleshoot.rst b/docs/community/troubleshoot.rst index 1228b5f5a0f..41bc6624c39 100644 --- a/docs/community/troubleshoot.rst +++ b/docs/community/troubleshoot.rst @@ -14,7 +14,7 @@ Before getting started, collect the following information from the underlying in .. prompt:: bash $ - $ kubectl describe pod -n + kubectl describe pod -n Where will typically correspond to the node execution string that you can find in the UI. @@ -23,7 +23,7 @@ Where will typically correspond to the node execution string that you .. prompt:: bash $ - $ kubectl logs pods -n + kubectl logs pods -n Where will typically correspond to the Flyte -, e.g. flytesnacks-development. @@ -70,7 +70,7 @@ This issue is more common on MacOS devices. Make sure that your Docker daemon ha .. prompt:: bash $ - $ flytectl demo start --env HTTP_PROXY= + flytectl demo start --env HTTP_PROXY= - If you're building a custom Docker image, make sure to use a tag other than ``latest``. Otherwise, the Kubernetes default pull policy will be changed from ``IfNotPresent`` to ``Always``, forcing an image pull with every Pod deployment. @@ -85,14 +85,14 @@ Issues running workloads .. prompt:: bash $ - $ export FLYTECTL_CONFIG=~/.flyte/config-sandbox.yaml + export FLYTECTL_CONFIG=~/.flyte/config-sandbox.yaml ``ModuleNotFoundError`` ^^^^^^^^^^^^^^^^^^^^^^^ - If you're using a custom container image and using Docker, make sure your ``Dockerfile`` is located at the same level of the ``flyte`` directory and that there is an empty ``__init__.py`` file in your project's folder : -.. prompt:: bash $ +.. prompt:: myflyteapp ├── Dockerfile @@ -111,11 +111,11 @@ Issues running workloads .. prompt:: bash $ - $ kubectl describe sa -n + kubectl describe sa -n Example output: -.. prompt:: bash $ +.. prompt:: Name: Namespace: flyte @@ -130,7 +130,7 @@ Example output: .. prompt:: bash $ - $ kubectl annotate serviceaccount -n eks.amazonaws.com/role-arn=arn:aws:iam::xxxx:role/ + kubectl annotate serviceaccount -n eks.amazonaws.com/role-arn=arn:aws:iam::xxxx:role/ - Refer to this community-maintained `guides `_ for further information about Flyte deployment on EKS diff --git a/docs/deployment/deployment/multicluster.rst b/docs/deployment/deployment/multicluster.rst index e8c6b84f130..52e0378955f 100644 --- a/docs/deployment/deployment/multicluster.rst +++ b/docs/deployment/deployment/multicluster.rst @@ -398,11 +398,14 @@ label has to be 1. 11. Verify that all Pods in the ``flyte`` namespace are ``Running``: +.. prompt:: bash $ + + kubectl get pods -n flyte + Example output: -.. prompt:: bash $ +.. prompt:: - kubectl get pods -n flyte NAME READY STATUS RESTARTS AGE datacatalog-86f6b9bf64-bp2cj 1/1 Running 0 23h datacatalog-86f6b9bf64-fjzcp 1/1 Running 0 23h @@ -658,4 +661,4 @@ The process can be repeated for additional clusters. 15. A successful execution should be visible on the UI, confirming it ran in the new cluster: - .. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/common/multicluster-execution.png \ No newline at end of file + .. image:: https://raw.githubusercontent.com/flyteorg/static-resources/main/common/multicluster-execution.png From cf21877017c2bc615fd13a0b21311e2a340085ea Mon Sep 17 00:00:00 2001 From: novahow <58504997+novahow@users.noreply.github.com> Date: Mon, 4 Mar 2024 16:27:35 +0800 Subject: [PATCH 2/3] add hostname to input (#4939) Signed-off-by: novahow --- flyteplugins/go/tasks/logs/logging_utils.go | 1 + .../go/tasks/logs/logging_utils_test.go | 65 +++++++++++++++++-- 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/flyteplugins/go/tasks/logs/logging_utils.go b/flyteplugins/go/tasks/logs/logging_utils.go index 04d23f4ec87..45d12624dea 100644 --- a/flyteplugins/go/tasks/logs/logging_utils.go +++ b/flyteplugins/go/tasks/logs/logging_utils.go @@ -52,6 +52,7 @@ func GetLogsForContainerInPod(ctx context.Context, logPlugin tasklog.Plugin, tas TaskExecutionID: taskExecID, ExtraTemplateVars: extraLogTemplateVars, TaskTemplate: taskTemplate, + HostName: pod.Spec.Hostname, }, ) diff --git a/flyteplugins/go/tasks/logs/logging_utils_test.go b/flyteplugins/go/tasks/logs/logging_utils_test.go index 946d069d50f..e8c716e3410 100644 --- a/flyteplugins/go/tasks/logs/logging_utils_test.go +++ b/flyteplugins/go/tasks/logs/logging_utils_test.go @@ -210,6 +210,40 @@ func TestGetLogsForContainerInPod_All(t *testing.T) { assert.Len(t, logs, 2) } +func TestGetLogsForContainerInPod_HostName(t *testing.T) { + logPlugin, err := InitializeLogPlugins(&LogConfig{ + IsKubernetesEnabled: true, + KubernetesURL: "k8s.com", + IsCloudwatchEnabled: true, + CloudwatchRegion: "us-east-1", + CloudwatchLogGroup: "/kubernetes/flyte-production", + }) + assert.NoError(t, err) + + pod := &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "ContainerName", + }, + }, + Hostname: "my-hostname", + }, + Status: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{ + { + ContainerID: "ContainerID", + }, + }, + }, + } + pod.Name = podName + + logs, err := GetLogsForContainerInPod(context.TODO(), logPlugin, dummyTaskExecID(), pod, 0, " Suffix", nil, nil) + assert.Nil(t, err) + assert.Len(t, logs, 2) +} + func TestGetLogsForContainerInPod_Stackdriver(t *testing.T) { logPlugin, err := InitializeLogPlugins(&LogConfig{ IsStackDriverEnabled: true, @@ -268,7 +302,7 @@ func TestGetLogsForContainerInPod_LegacyTemplate(t *testing.T) { MessageFormat: core.TaskLog_JSON, Name: "Stackdriver Logs my-Suffix", }, - }) + }, "") }) t.Run("StackDriver", func(t *testing.T) { @@ -281,11 +315,11 @@ func TestGetLogsForContainerInPod_LegacyTemplate(t *testing.T) { MessageFormat: core.TaskLog_JSON, Name: "Stackdriver Logs my-Suffix", }, - }) + }, "") }) } -func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.TaskTemplate, expectedTaskLogs []*core.TaskLog) { +func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.TaskTemplate, expectedTaskLogs []*core.TaskLog, hostname string) { logPlugin, err := InitializeLogPlugins(config) assert.NoError(tb, err) @@ -300,6 +334,7 @@ func assertTestSucceeded(tb testing.TB, config *LogConfig, taskTemplate *core.Ta Name: "ContainerName", }, }, + Hostname: hostname, }, Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ @@ -347,7 +382,27 @@ func TestGetLogsForContainerInPod_Templates(t *testing.T) { MessageFormat: core.TaskLog_JSON, Name: "Internal my-Suffix", }, - }) + }, "") +} + +func TestGetLogsForContainerInPodTemplates_Hostname(t *testing.T) { + assertTestSucceeded(t, &LogConfig{ + Templates: []tasklog.TemplateLogPlugin{ + { + DisplayName: "StackDriver", + TemplateURIs: []string{ + "{{ .hostname }}/{{ .namespace }}/{{ .podName }}/{{ .containerName }}/{{ .containerId }}", + }, + MessageFormat: core.TaskLog_JSON, + }, + }, + }, nil, []*core.TaskLog{ + { + Uri: "my-hostname/my-namespace/my-pod/ContainerName/ContainerID", + MessageFormat: core.TaskLog_JSON, + Name: "StackDriver my-Suffix", + }, + }, "my-hostname") } func TestGetLogsForContainerInPod_Flyteinteractive(t *testing.T) { @@ -507,7 +562,7 @@ func TestGetLogsForContainerInPod_Flyteinteractive(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assertTestSucceeded(t, tt.config, tt.template, tt.expectedTaskLogs) + assertTestSucceeded(t, tt.config, tt.template, tt.expectedTaskLogs, "") }) } } From 4a95abffbe3d2782e01ddcda6ad1ab15b962b8cb Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Mon, 4 Mar 2024 14:16:47 -0800 Subject: [PATCH 3/3] Fix Monodocs build (#5000) Signed-off-by: Kevin Su --- .github/workflows/tests.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index aea9e0fabbc..831beda79d9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,20 +27,35 @@ jobs: docs: runs-on: ubuntu-latest steps: - - name: Fetch the code + - name: Fetch flyte code + uses: actions/checkout@v4 + with: + path: "${{ github.workspace }}/flyte" + - name: Fetch flytekit code uses: actions/checkout@v4 + with: + repository: flyteorg/flytekit + path: "${{ github.workspace }}/flytekit" - uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true python-version: 3.9 - shell: bash -el {0} + working-directory: ${{ github.workspace }}/flyte run: | conda install -c conda-forge conda-lock conda-lock install -n monodocs-env monodocs-environment.lock.yaml - shell: bash -el {0} + working-directory: ${{ github.workspace }}/flyte run: | conda activate monodocs-env + export SETUPTOOLS_SCM_PRETEND_VERSION="2.0.0" pip install -e ./flyteidl + - shell: bash -el {0} + working-directory: ${{ github.workspace }}/flytekit + run: | + conda activate monodocs-env + pip install -e . conda info conda list conda config --show-sources @@ -49,6 +64,7 @@ jobs: - name: Setup Graphviz uses: ts-graphviz/setup-graphviz@v1 - name: Build the documentation + working-directory: ${{ github.workspace }}/flyte shell: bash -el {0} run: | conda activate monodocs-env