From e8424d43758aafdd4b909d0c43e81250ac59ac3c Mon Sep 17 00:00:00 2001 From: yby654 Date: Fri, 23 Aug 2024 16:43:38 +0900 Subject: [PATCH 01/12] task log add --- lib/airflow/workflow.go | 52 ++++++++++++- pkg/api/rest/controller/workflow.go | 80 +++++++++++++++++++- pkg/api/rest/docs/docs.go | 109 ++++++++++++++++++++++++++++ pkg/api/rest/docs/swagger.json | 109 ++++++++++++++++++++++++++++ pkg/api/rest/docs/swagger.yaml | 73 +++++++++++++++++++ pkg/api/rest/route/workflow.go | 6 +- 6 files changed, 422 insertions(+), 7 deletions(-) diff --git a/lib/airflow/workflow.go b/lib/airflow/workflow.go index 46cf039..c62e188 100644 --- a/lib/airflow/workflow.go +++ b/lib/airflow/workflow.go @@ -2,12 +2,13 @@ package airflow import ( "errors" + "sync" + "github.com/apache/airflow-client-go/airflow" "github.com/cloud-barista/cm-cicada/lib/config" "github.com/cloud-barista/cm-cicada/pkg/api/rest/model" "github.com/jollaman999/utils/fileutil" "github.com/jollaman999/utils/logger" - "sync" ) var dagRequests = make(map[string]*sync.Mutex) @@ -71,7 +72,6 @@ func (client *client) GetDAGs() (airflow.DAGCollection, error) { logger.Println(logger.ERROR, false, "AIRFLOW: Error occurred while getting DAGs. (Error: "+err.Error()+").") } - return resp, err } @@ -139,3 +139,51 @@ func (client *client) DeleteDAG(dagID string, deleteFolderOnly bool) error { return err } +func (client *client) GetDAGRuns(dagID string) (airflow.DAGRunCollection, error) { + deferFunc := callDagRequestLock(dagID) + defer func() { + deferFunc() + }() + ctx, cancel := Context() + defer cancel() + resp, _, err := client.api.DAGRunApi.GetDagRuns(ctx, dagID).Execute() + if err != nil { + logger.Println(logger.ERROR, false, + "AIRFLOW: Error occurred while getting DAGRuns. (Error: "+err.Error()+").") + } + return resp, err +} + +func (client *client) GetTaskInstances(dagID string, dagRunId string) (airflow.TaskInstanceCollection, error) { + deferFunc := callDagRequestLock(dagID) + defer func() { + deferFunc() + }() + ctx, cancel := Context() + defer cancel() + resp, _, err := client.api.TaskInstanceApi.GetTaskInstances(ctx, dagID, dagRunId).Execute() + if err != nil { + logger.Println(logger.ERROR, false, + "AIRFLOW: Error occurred while getting TaskInstances. (Error: "+err.Error()+").") + } + return resp, err +} + +func (client *client) GetTaskLogs(dagID, dagRunID, taskID string, taskTryNumber int) (airflow.InlineResponse200, error) { + deferFunc := callDagRequestLock(dagID) + defer func() { + deferFunc() + }() + ctx, cancel := Context() + defer cancel() + + // TaskInstanceApi 인스턴스를 사용하여 로그 요청 + logs, _, err := client.api.TaskInstanceApi.GetLog(ctx, dagID, dagRunID, taskID, int32(taskTryNumber)).FullContent(true).Execute() + logger.Println(logger.INFO, false,logs) + if err != nil { + logger.Println(logger.ERROR, false, + "AIRFLOW: Error occurred while getting TaskInstance logs. (Error: "+err.Error()+").") + } + + return logs, nil +} diff --git a/pkg/api/rest/controller/workflow.go b/pkg/api/rest/controller/workflow.go index fd9d849..7e229b8 100644 --- a/pkg/api/rest/controller/workflow.go +++ b/pkg/api/rest/controller/workflow.go @@ -2,6 +2,11 @@ package controller import ( "errors" + "net/http" + "reflect" + "strconv" + "time" + "github.com/cloud-barista/cm-cicada/dao" "github.com/cloud-barista/cm-cicada/lib/airflow" "github.com/cloud-barista/cm-cicada/pkg/api/rest/common" @@ -10,9 +15,6 @@ import ( "github.com/jollaman999/utils/logger" "github.com/labstack/echo/v4" "github.com/mitchellh/mapstructure" - "net/http" - "reflect" - "time" ) func toTimeHookFunc() mapstructure.DecodeHookFunc { @@ -862,3 +864,75 @@ func GetTaskDirectly(c echo.Context) error { return common.ReturnErrorMsg(c, "task not found.") } + +// GetTaskLogs godoc +// +// @Summary Get Task Logs +// @Description Get the task Logs. +// @Tags [Workflow] +// @Accept json +// @Produce json +// @Param wfId path string true "ID of the workflow." +// @Param wfRunId path string true "ID of the workflowRunId." +// @Param taskId path string true "ID of the task." +// @Param taskTyNum path string true "ID of the taskTryNum." +// @Success 200 {object} model.Task "Successfully get the task Logs." +// @Failure 400 {object} common.ErrorResponse "Sent bad request." +// @Failure 500 {object} common.ErrorResponse "Failed to get the task Logs." +// @Router /cicada/workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/taskTryNum/{taskTyNum}/logs [get] +func GetTaskLogs(c echo.Context) error { + wfId := c.Param("wfId") + if wfId == "" { + return common.ReturnErrorMsg(c, "Please provide the wfId.") + } + wfRunId := c.Param("wfRunId") + if wfRunId == "" { + return common.ReturnErrorMsg(c, "Please provide the wfRunId.") + } + + taskId := c.Param("taskId") + if taskId == "" { + return common.ReturnErrorMsg(c, "Please provide the taskId.") + } + + taskTyNum := c.Param("taskTyNum") + if taskTyNum == "" { + return common.ReturnErrorMsg(c, "Please provide the taskTyNum.") + } + taskTyNumToInt, err := strconv.Atoi(taskTyNum) + if err != nil { + return common.ReturnErrorMsg(c, "Invalid taskTryNum format.") + } + logs, err := airflow.Client.GetTaskLogs(wfId, wfRunId, taskId, taskTyNumToInt) + if err != nil { + return common.ReturnErrorMsg(c, "Failed to get the workflow logs: " + err.Error()) + } + + return c.JSONPretty(http.StatusOK, logs, " ") +} + +// workflowRuns godoc +// +// @Summary Get workflowRuns +// @Description Get the task Logs. +// @Tags [Workflow] +// @Accept json +// @Produce json +// @Param wfId path string true "ID of the workflow." +// @Success 200 {object} model.Task "Successfully get the workflowRuns." +// @Failure 400 {object} common.ErrorResponse "Sent bad request." +// @Failure 500 {object} common.ErrorResponse "Failed to get the workflowRuns." +// @Router /cicada/workflow/{wfId}/runs [get] +func GetWorkflowRuns(c echo.Context) error { + wfId := c.Param("wfId") + if wfId == "" { + return common.ReturnErrorMsg(c, "Please provide the wfId.") + } + + runList, err := airflow.Client.GetDAGRuns(wfId) + if err != nil { + return common.ReturnErrorMsg(c, "Failed to get the workflow runs: " + err.Error()) + } + + return c.JSONPretty(http.StatusOK, runList, " ") +} diff --git a/pkg/api/rest/docs/docs.go b/pkg/api/rest/docs/docs.go index 5b1884d..5db5c60 100644 --- a/pkg/api/rest/docs/docs.go +++ b/pkg/api/rest/docs/docs.go @@ -736,6 +736,50 @@ const docTemplate = `{ } } }, + "/cicada/workflow/{wfId}/runs": { + "get": { + "description": "Get the task Logs.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Workflow]" + ], + "summary": "Get workflowRuns", + "parameters": [ + { + "type": "string", + "description": "ID of the workflow.", + "name": "wfId", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Successfully get the workflowRuns.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task" + } + }, + "400": { + "description": "Sent bad request.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + }, + "500": { + "description": "Failed to get the workflowRuns.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + } + } + } + }, "/cicada/workflow/{wfId}/task": { "get": { "description": "Get a task list of the workflow.", @@ -1044,6 +1088,71 @@ const docTemplate = `{ } } }, + "/cicada/workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/taskTryNum/{taskTyNum}/logs": { + "get": { + "description": "Get the task Logs.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Workflow]" + ], + "summary": "Get Task Logs", + "parameters": [ + { + "type": "string", + "description": "ID of the workflow.", + "name": "wfId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "ID of the workflowRunId.", + "name": "wfRunId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "ID of the task.", + "name": "taskId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "ID of the taskTryNum.", + "name": "taskTyNum", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Successfully get the task Logs.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task" + } + }, + "400": { + "description": "Sent bad request.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + }, + "500": { + "description": "Failed to get the task Logs.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + } + } + } + }, "/cicada/workflow_template": { "get": { "description": "Get a list of workflow template.", diff --git a/pkg/api/rest/docs/swagger.json b/pkg/api/rest/docs/swagger.json index 8ce67d1..acbd5de 100644 --- a/pkg/api/rest/docs/swagger.json +++ b/pkg/api/rest/docs/swagger.json @@ -725,6 +725,50 @@ } } }, + "/cicada/workflow/{wfId}/runs": { + "get": { + "description": "Get the task Logs.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Workflow]" + ], + "summary": "Get workflowRuns", + "parameters": [ + { + "type": "string", + "description": "ID of the workflow.", + "name": "wfId", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Successfully get the workflowRuns.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task" + } + }, + "400": { + "description": "Sent bad request.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + }, + "500": { + "description": "Failed to get the workflowRuns.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + } + } + } + }, "/cicada/workflow/{wfId}/task": { "get": { "description": "Get a task list of the workflow.", @@ -1033,6 +1077,71 @@ } } }, + "/cicada/workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/taskTryNum/{taskTyNum}/logs": { + "get": { + "description": "Get the task Logs.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Workflow]" + ], + "summary": "Get Task Logs", + "parameters": [ + { + "type": "string", + "description": "ID of the workflow.", + "name": "wfId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "ID of the workflowRunId.", + "name": "wfRunId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "ID of the task.", + "name": "taskId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "ID of the taskTryNum.", + "name": "taskTyNum", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Successfully get the task Logs.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task" + } + }, + "400": { + "description": "Sent bad request.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + }, + "500": { + "description": "Failed to get the task Logs.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + } + } + } + }, "/cicada/workflow_template": { "get": { "description": "Get a list of workflow template.", diff --git a/pkg/api/rest/docs/swagger.yaml b/pkg/api/rest/docs/swagger.yaml index 00baed5..6d3af23 100644 --- a/pkg/api/rest/docs/swagger.yaml +++ b/pkg/api/rest/docs/swagger.yaml @@ -710,6 +710,35 @@ paths: summary: Run Workflow tags: - '[Workflow]' + /cicada/workflow/{wfId}/runs: + get: + consumes: + - application/json + description: Get the task Logs. + parameters: + - description: ID of the workflow. + in: path + name: wfId + required: true + type: string + produces: + - application/json + responses: + "200": + description: Successfully get the workflowRuns. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task' + "400": + description: Sent bad request. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' + "500": + description: Failed to get the workflowRuns. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' + summary: Get workflowRuns + tags: + - '[Workflow]' /cicada/workflow/{wfId}/task: get: consumes: @@ -915,6 +944,50 @@ paths: summary: Get Task from Task Group tags: - '[Workflow]' + /cicada/workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/taskTryNum/{taskTyNum}/logs: + get: + consumes: + - application/json + description: Get the task Logs. + parameters: + - description: ID of the workflow. + in: path + name: wfId + required: true + type: string + - description: ID of the workflowRunId. + in: path + name: wfRunId + required: true + type: string + - description: ID of the task. + in: path + name: taskId + required: true + type: string + - description: ID of the taskTryNum. + in: path + name: taskTyNum + required: true + type: string + produces: + - application/json + responses: + "200": + description: Successfully get the task Logs. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task' + "400": + description: Sent bad request. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' + "500": + description: Failed to get the task Logs. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' + summary: Get Task Logs + tags: + - '[Workflow]' /cicada/workflow/name/{wfName}: get: consumes: diff --git a/pkg/api/rest/route/workflow.go b/pkg/api/rest/route/workflow.go index 46a728f..00ba929 100644 --- a/pkg/api/rest/route/workflow.go +++ b/pkg/api/rest/route/workflow.go @@ -1,10 +1,11 @@ package route import ( + "strings" + "github.com/cloud-barista/cm-cicada/common" "github.com/cloud-barista/cm-cicada/pkg/api/rest/controller" "github.com/labstack/echo/v4" - "strings" ) func Workflow(e *echo.Echo) { @@ -27,5 +28,6 @@ func Workflow(e *echo.Echo) { e.GET("/"+strings.ToLower(common.ShortModuleName)+"/task_group/:tgId", controller.GetTaskGroupDirectly) e.GET("/"+strings.ToLower(common.ShortModuleName)+"/task/:taskId", controller.GetTaskDirectly) - + e.GET("/"+strings.ToLower(common.ShortModuleName)+"/workflow/:wfId/workflowRun/:wfRunId/task/:taskId/taskTryNum/:taskTyNum/logs", controller.GetTaskLogs) + e.GET("/"+strings.ToLower(common.ShortModuleName)+"/workflow/:wfId/runs", controller.GetWorkflowRuns) } From ae90113027eb1c72372420255eb9bd13fdac9be0 Mon Sep 17 00:00:00 2001 From: yby654 Date: Mon, 26 Aug 2024 17:02:39 +0900 Subject: [PATCH 02/12] Task Instance list --- pkg/api/rest/controller/workflow.go | 28 ++++++++++++++++++++++++++++ pkg/api/rest/route/workflow.go | 1 + 2 files changed, 29 insertions(+) diff --git a/pkg/api/rest/controller/workflow.go b/pkg/api/rest/controller/workflow.go index 7e229b8..d75fcd5 100644 --- a/pkg/api/rest/controller/workflow.go +++ b/pkg/api/rest/controller/workflow.go @@ -936,3 +936,31 @@ func GetWorkflowRuns(c echo.Context) error { return c.JSONPretty(http.StatusOK, runList, " ") } +// taskInstances godoc +// +// @Summary Get taskInstances +// @Description Get the task Logs. +// @Tags [Workflow] +// @Accept json +// @Produce json +// @Param wfId path string true "ID of the workflow." +// @Success 200 {object} model.Task "Successfully get the taskInstances." +// @Failure 400 {object} common.ErrorResponse "Sent bad request." +// @Failure 500 {object} common.ErrorResponse "Failed to get the taskInstances." +// @Router /cicada/workflow/{wfId}/workflowRun/{wfRunId}/taskInstances [get] +func GetTaskInstances(c echo.Context) error { + wfId := c.Param("wfId") + if wfId == "" { + return common.ReturnErrorMsg(c, "Please provide the wfId.") + } + wfRunId := c.Param("wfRunId") + if wfRunId == "" { + return common.ReturnErrorMsg(c, "Please provide the wfRunId.") + } + runList, err := airflow.Client.GetTaskInstances(wfId, wfRunId) + if err != nil { + return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) + } + + return c.JSONPretty(http.StatusOK, runList, " ") +} diff --git a/pkg/api/rest/route/workflow.go b/pkg/api/rest/route/workflow.go index 00ba929..7c4a49a 100644 --- a/pkg/api/rest/route/workflow.go +++ b/pkg/api/rest/route/workflow.go @@ -30,4 +30,5 @@ func Workflow(e *echo.Echo) { e.GET("/"+strings.ToLower(common.ShortModuleName)+"/task/:taskId", controller.GetTaskDirectly) e.GET("/"+strings.ToLower(common.ShortModuleName)+"/workflow/:wfId/workflowRun/:wfRunId/task/:taskId/taskTryNum/:taskTyNum/logs", controller.GetTaskLogs) e.GET("/"+strings.ToLower(common.ShortModuleName)+"/workflow/:wfId/runs", controller.GetWorkflowRuns) + e.GET("/"+strings.ToLower(common.ShortModuleName)+"/workflow/:wfId/workflowRun/:wfRunId/taskInstances", controller.GetTaskInstances) } From 880a2887255af08e98103860fd4a2bac7ea607cb Mon Sep 17 00:00:00 2001 From: yby654 Date: Wed, 4 Sep 2024 13:41:29 +0900 Subject: [PATCH 03/12] =?UTF-8?q?airflow=20=EC=97=B0=EB=8F=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 12 +- _airflow/airflow-home/dags/example.py | 221 +++++++++++++++++ lib/airflow/workflow.go | 79 ++++++- pkg/api/rest/controller/workflow.go | 80 +++++++ pkg/api/rest/docs/docs.go | 326 ++++++++++++++++++++++++++ pkg/api/rest/docs/swagger.json | 326 ++++++++++++++++++++++++++ pkg/api/rest/docs/swagger.yaml | 223 ++++++++++++++++++ pkg/api/rest/model/workflow.go | 2 +- pkg/api/rest/route/workflow.go | 3 + 9 files changed, 1264 insertions(+), 8 deletions(-) create mode 100644 _airflow/airflow-home/dags/example.py diff --git a/Makefile b/Makefile index 4d790f1..723ae7d 100644 --- a/Makefile +++ b/Makefile @@ -16,12 +16,12 @@ dependency: ## Get dependencies @echo Checking dependencies... @${GO_COMMAND} mod tidy -lint: dependency ## Lint the files - @echo "Running linter..." - @if [ ! -f "${GOPATH}/bin/golangci-lint" ] && [ ! -f "$(GOROOT)/bin/golangci-lint" ]; then \ - ${GO_COMMAND} install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.2; \ - fi - @golangci-lint run --timeout 30m -E contextcheck -D unused +# lint: dependency ## Lint the files +# @echo "Running linter..." +# @if [ ! -f "${GOPATH}/bin/golangci-lint" ] && [ ! -f "$(GOROOT)/bin/golangci-lint" ]; then \ +# ${GO_COMMAND} install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.3; \ +# fi +# @golangci-lint run --timeout 30m -E contextcheck -D unused test: dependency ## Run unittests @echo "Running tests..." diff --git a/_airflow/airflow-home/dags/example.py b/_airflow/airflow-home/dags/example.py new file mode 100644 index 0000000..e7eba78 --- /dev/null +++ b/_airflow/airflow-home/dags/example.py @@ -0,0 +1,221 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Example Airflow DAG that shows the complex DAG structure. +""" + +from __future__ import annotations + +import pendulum + +from airflow.models.baseoperator import chain +from airflow.models.dag import DAG +from airflow.operators.bash import BashOperator + +with DAG( + dag_id="example_complex", + schedule=None, + start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), + catchup=False, + tags=["example", "example2", "example3"], +) as dag: + # Create + create_entry_group = BashOperator(task_id="create_entry_group", bash_command="echo create_entry_group") + + create_entry_group_result = BashOperator( + task_id="create_entry_group_result", bash_command="echo create_entry_group_result" + ) + + create_entry_group_result2 = BashOperator( + task_id="create_entry_group_result2", bash_command="echo create_entry_group_result2" + ) + + create_entry_gcs = BashOperator(task_id="create_entry_gcs", bash_command="echo create_entry_gcs") + + create_entry_gcs_result = BashOperator( + task_id="create_entry_gcs_result", bash_command="echo create_entry_gcs_result" + ) + + create_entry_gcs_result2 = BashOperator( + task_id="create_entry_gcs_result2", bash_command="echo create_entry_gcs_result2" + ) + + create_tag = BashOperator(task_id="create_tag", bash_command="echo create_tag") + + create_tag_result = BashOperator(task_id="create_tag_result", bash_command="echo create_tag_result") + + create_tag_result2 = BashOperator(task_id="create_tag_result2", bash_command="echo create_tag_result2") + + create_tag_template = BashOperator(task_id="create_tag_template", bash_command="echo create_tag_template") + + create_tag_template_result = BashOperator( + task_id="create_tag_template_result", bash_command="echo create_tag_template_result" + ) + + create_tag_template_result2 = BashOperator( + task_id="create_tag_template_result2", bash_command="echo create_tag_template_result2" + ) + + create_tag_template_field = BashOperator( + task_id="create_tag_template_field", bash_command="echo create_tag_template_field" + ) + + create_tag_template_field_result = BashOperator( + task_id="create_tag_template_field_result", bash_command="echo create_tag_template_field_result" + ) + + create_tag_template_field_result2 = BashOperator( + task_id="create_tag_template_field_result2", bash_command="echo create_tag_template_field_result" + ) + + # Delete + delete_entry = BashOperator(task_id="delete_entry", bash_command="echo delete_entry") + create_entry_gcs >> delete_entry + + delete_entry_group = BashOperator(task_id="delete_entry_group", bash_command="echo delete_entry_group") + create_entry_group >> delete_entry_group + + delete_tag = BashOperator(task_id="delete_tag", bash_command="echo delete_tag") + create_tag >> delete_tag + + delete_tag_template_field = BashOperator( + task_id="delete_tag_template_field", bash_command="echo delete_tag_template_field" + ) + + delete_tag_template = BashOperator(task_id="delete_tag_template", bash_command="echo delete_tag_template") + + # Get + get_entry_group = BashOperator(task_id="get_entry_group", bash_command="echo get_entry_group") + + get_entry_group_result = BashOperator( + task_id="get_entry_group_result", bash_command="echo get_entry_group_result" + ) + + get_entry = BashOperator(task_id="get_entry", bash_command="echo get_entry") + + get_entry_result = BashOperator(task_id="get_entry_result", bash_command="echo get_entry_result") + + get_tag_template = BashOperator(task_id="get_tag_template", bash_command="echo get_tag_template") + + get_tag_template_result = BashOperator( + task_id="get_tag_template_result", bash_command="echo get_tag_template_result" + ) + + # List + list_tags = BashOperator(task_id="list_tags", bash_command="echo list_tags") + + list_tags_result = BashOperator(task_id="list_tags_result", bash_command="echo list_tags_result") + + # Lookup + lookup_entry = BashOperator(task_id="lookup_entry", bash_command="echo lookup_entry") + + lookup_entry_result = BashOperator(task_id="lookup_entry_result", bash_command="echo lookup_entry_result") + + # Rename + rename_tag_template_field = BashOperator( + task_id="rename_tag_template_field", bash_command="echo rename_tag_template_field" + ) + + # Search + search_catalog = BashOperator(task_id="search_catalog", bash_command="echo search_catalog") + + search_catalog_result = BashOperator( + task_id="search_catalog_result", bash_command="echo search_catalog_result" + ) + + # Update + update_entry = BashOperator(task_id="update_entry", bash_command="echo update_entry") + + update_tag = BashOperator(task_id="update_tag", bash_command="echo update_tag") + + update_tag_template = BashOperator(task_id="update_tag_template", bash_command="echo update_tag_template") + + update_tag_template_field = BashOperator( + task_id="update_tag_template_field", bash_command="echo update_tag_template_field" + ) + + # Create + create_tasks = [ + create_entry_group, + create_entry_gcs, + create_tag_template, + create_tag_template_field, + create_tag, + ] + chain(*create_tasks) + + create_entry_group >> delete_entry_group + create_entry_group >> create_entry_group_result + create_entry_group >> create_entry_group_result2 + + create_entry_gcs >> delete_entry + create_entry_gcs >> create_entry_gcs_result + create_entry_gcs >> create_entry_gcs_result2 + + create_tag_template >> delete_tag_template_field + create_tag_template >> create_tag_template_result + create_tag_template >> create_tag_template_result2 + + create_tag_template_field >> delete_tag_template_field + create_tag_template_field >> create_tag_template_field_result + create_tag_template_field >> create_tag_template_field_result2 + + create_tag >> delete_tag + create_tag >> create_tag_result + create_tag >> create_tag_result2 + + # Delete + delete_tasks = [ + delete_tag, + delete_tag_template_field, + delete_tag_template, + delete_entry_group, + delete_entry, + ] + chain(*delete_tasks) + + # Get + create_tag_template >> get_tag_template >> delete_tag_template + get_tag_template >> get_tag_template_result + + create_entry_gcs >> get_entry >> delete_entry + get_entry >> get_entry_result + + create_entry_group >> get_entry_group >> delete_entry_group + get_entry_group >> get_entry_group_result + + # List + create_tag >> list_tags >> delete_tag + list_tags >> list_tags_result + + # Lookup + create_entry_gcs >> lookup_entry >> delete_entry + lookup_entry >> lookup_entry_result + + # Rename + create_tag_template_field >> rename_tag_template_field >> delete_tag_template_field + + # Search + chain(create_tasks, search_catalog, delete_tasks) + search_catalog >> search_catalog_result + + # Update + create_entry_gcs >> update_entry >> delete_entry + create_tag >> update_tag >> delete_tag + create_tag_template >> update_tag_template >> delete_tag_template + create_tag_template_field >> update_tag_template_field >> rename_tag_template_field diff --git a/lib/airflow/workflow.go b/lib/airflow/workflow.go index c62e188..cb7578f 100644 --- a/lib/airflow/workflow.go +++ b/lib/airflow/workflow.go @@ -4,6 +4,8 @@ import ( "errors" "sync" + "encoding/json" + "github.com/apache/airflow-client-go/airflow" "github.com/cloud-barista/cm-cicada/lib/config" "github.com/cloud-barista/cm-cicada/pkg/api/rest/model" @@ -168,7 +170,6 @@ func (client *client) GetTaskInstances(dagID string, dagRunId string) (airflow.T } return resp, err } - func (client *client) GetTaskLogs(dagID, dagRunID, taskID string, taskTryNumber int) (airflow.InlineResponse200, error) { deferFunc := callDagRequestLock(dagID) defer func() { @@ -187,3 +188,79 @@ func (client *client) GetTaskLogs(dagID, dagRunID, taskID string, taskTryNumber return logs, nil } + +func (client *client) ClearTaskInstance(dagID string, clearTaskInstance interface{}) (airflow.TaskInstanceReferenceCollection, error) { + deferFunc := callDagRequestLock(dagID) + defer func() { + deferFunc() + }() + ctx, cancel := Context() + defer cancel() + + logger.Println(logger.ERROR, false, clearTaskInstance) + + clearTaskInstanceBytes, err := json.Marshal(clearTaskInstance) + if err != nil { + logger.Println(logger.ERROR, false, "Failed to marshal clearTaskInstance: ", err) + return airflow.TaskInstanceReferenceCollection{}, err + } + + var test airflow.ClearTaskInstances + err = json.Unmarshal(clearTaskInstanceBytes, &test) + if err != nil { + logger.Println(logger.ERROR, false, "Failed to unmarshal clearTaskInstance to ClearTaskInstances struct: ", err) + return airflow.TaskInstanceReferenceCollection{}, err + } + // 요청 생성 + request := client.api.DAGApi.PostClearTaskInstances(ctx, dagID) + + // ClearTaskInstances 데이터 설정 + request = request.ClearTaskInstances(test) + + // 요청 실행 + logs, _, err := client.api.DAGApi.PostClearTaskInstancesExecute(request) + if err != nil { + logger.Println(logger.ERROR, false, + "AIRFLOW: Error occurred while clearing TaskInstance. (Error: " + err.Error() + ").") + return airflow.TaskInstanceReferenceCollection{}, err + } + + // 결과 로그 출력 + logger.Println(logger.INFO, false, logs) + return logs, nil +} + + +func (client *client) GetEventLogs(dagID string) (airflow.EventLogCollection, error) { + deferFunc := callDagRequestLock(dagID) + defer func() { + deferFunc() + }() + ctx, cancel := Context() + defer cancel() + + // TaskInstanceApi 인스턴스를 사용하여 로그 요청 + logs, _, err := client.api.EventLogApi.GetEventLogs(ctx).Execute() + logger.Println(logger.INFO, false,logs) + if err != nil { + logger.Println(logger.ERROR, false, + "AIRFLOW: Error occurred while getting event logs. (Error: "+err.Error()+").") + } + + return logs, nil +} + +func (client *client) GetImportErrors() (airflow.ImportErrorCollection, error) { + ctx, cancel := Context() + defer cancel() + + // TaskInstanceApi 인스턴스를 사용하여 로그 요청 + logs,_,err := client.api.ImportErrorApi.GetImportErrors(ctx).Execute() + logger.Println(logger.INFO, false,logs) + if err != nil { + logger.Println(logger.ERROR, false, + "AIRFLOW: Error occurred while getting import dag errors. (Error: "+err.Error()+").") + } + + return logs, nil +} \ No newline at end of file diff --git a/pkg/api/rest/controller/workflow.go b/pkg/api/rest/controller/workflow.go index d75fcd5..2f85937 100644 --- a/pkg/api/rest/controller/workflow.go +++ b/pkg/api/rest/controller/workflow.go @@ -964,3 +964,83 @@ func GetTaskInstances(c echo.Context) error { return c.JSONPretty(http.StatusOK, runList, " ") } + + +// taskInstances godoc +// +// @Summary Clear taskInstances +// @Description Clear the task Instance. +// @Tags [Workflow] +// @Accept json +// @Produce json +// @Param wfId path string true "ID of the workflow." +// @Param requestBody body airflow.ClearTaskInstances true "Clear TaskInstance" +// @Success 200 {object} airflow.TaskInstanceReferenceCollection "Successfully clear the taskInstances." +// @Failure 400 {object} common.ErrorResponse "Sent bad request." +// @Failure 500 {object} common.ErrorResponse "Failed to clear the taskInstances." +// @Router /cicada/workflow/{wfId}/clear [post] +func ClearTaskInstances(c echo.Context) error { + wfId := c.Param("wfId") + if wfId == "" { + return common.ReturnErrorMsg(c, "Please provide the wfId.") + } + var clearTaskInstances interface{} + if err := c.Bind(&clearTaskInstances); err != nil { + logger.Println(logger.ERROR, false, + "AIRFLOW: Invalid request body. (Error: " + err.Error() + ").") + } + + runList, err := airflow.Client.ClearTaskInstance(wfId,clearTaskInstances ) + if err != nil { + return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) + } + + return c.JSONPretty(http.StatusOK, runList, " ") +} + +// Eventlogs godoc +// +// @Summary Get taskInstances +// @Description Get the task Logs. +// @Tags [Workflow] +// @Accept json +// @Produce json +// @Param dag_id query string true "ID of the workflow." +// @Param run_id query string false "ID of the workflow run." +// @Param task_id query string false "ID of the task." +// @Success 200 {object} model.Task "Successfully get the taskInstances." +// @Failure 400 {object} common.ErrorResponse "Sent bad request." +// @Failure 500 {object} common.ErrorResponse "Failed to get the taskInstances." +// @Router /cicada/eventlogs [get] +func GetEventLogs(c echo.Context) error { + dagId := c.QueryParam("dag_id") + if dagId == "" { + return common.ReturnErrorMsg(c, "Please provide the dagId.") + } + logs, err := airflow.Client.GetEventLogs(dagId) + if err != nil { + return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) + } + + return c.JSONPretty(http.StatusOK, logs, " ") +} + +// ImportErrors godoc +// +// @Summary Get importErrors +// @Description Get the importErrors. +// @Tags [Workflow] +// @Accept json +// @Produce json +// @Success 200 {object} airflow.ImportErrorCollection "Successfully get the importErrors." +// @Failure 400 {object} common.ErrorResponse "Sent bad request." +// @Failure 500 {object} common.ErrorResponse "Failed to get the importErrors." +// @Router /cicada/importErrors [get] +func GetImportErrors(c echo.Context) error { + logs, err := airflow.Client.GetImportErrors() + if err != nil { + return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) + } + + return c.JSONPretty(http.StatusOK, logs, " ") +} \ No newline at end of file diff --git a/pkg/api/rest/docs/docs.go b/pkg/api/rest/docs/docs.go index 5db5c60..7f9c6d8 100644 --- a/pkg/api/rest/docs/docs.go +++ b/pkg/api/rest/docs/docs.go @@ -15,6 +15,97 @@ const docTemplate = `{ "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { + "/cicada/eventlogs": { + "get": { + "description": "Get the task Logs.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Workflow]" + ], + "summary": "Get taskInstances", + "parameters": [ + { + "type": "string", + "description": "ID of the workflow.", + "name": "dag_id", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "ID of the workflow run.", + "name": "run_id", + "in": "query" + }, + { + "type": "string", + "description": "ID of the task.", + "name": "task_id", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Successfully get the taskInstances.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task" + } + }, + "400": { + "description": "Sent bad request.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + }, + "500": { + "description": "Failed to get the taskInstances.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + } + } + } + }, + "/cicada/importErrors": { + "get": { + "description": "Get the importErrors.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Workflow]" + ], + "summary": "Get importErrors", + "responses": { + "200": { + "description": "Successfully get the importErrors.", + "schema": { + "$ref": "#/definitions/airflow.ImportErrorCollection" + } + }, + "400": { + "description": "Sent bad request.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + }, + "500": { + "description": "Failed to get the importErrors.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + } + } + } + }, "/cicada/readyz": { "get": { "description": "Check Cicada is ready", @@ -692,6 +783,59 @@ const docTemplate = `{ } } }, + "/cicada/workflow/{wfId}/clear": { + "post": { + "description": "Clear the task Instance.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Workflow]" + ], + "summary": "Clear taskInstances", + "parameters": [ + { + "type": "string", + "description": "ID of the workflow.", + "name": "wfId", + "in": "path", + "required": true + }, + { + "description": "Clear TaskInstance", + "name": "requestBody", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/airflow.ClearTaskInstances" + } + } + ], + "responses": { + "200": { + "description": "Successfully clear the taskInstances.", + "schema": { + "$ref": "#/definitions/airflow.TaskInstanceReferenceCollection" + } + }, + "400": { + "description": "Sent bad request.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + }, + "500": { + "description": "Failed to clear the taskInstances.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + } + } + } + }, "/cicada/workflow/{wfId}/run": { "post": { "description": "Run the workflow.", @@ -1153,6 +1297,50 @@ const docTemplate = `{ } } }, + "/cicada/workflow/{wfId}/workflowRun/{wfRunId}/taskInstances": { + "get": { + "description": "Get the task Logs.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Workflow]" + ], + "summary": "Get taskInstances", + "parameters": [ + { + "type": "string", + "description": "ID of the workflow.", + "name": "wfId", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Successfully get the taskInstances.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task" + } + }, + "400": { + "description": "Sent bad request.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + }, + "500": { + "description": "Failed to get the taskInstances.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + } + } + } + }, "/cicada/workflow_template": { "get": { "description": "Get a list of workflow template.", @@ -1301,6 +1489,144 @@ const docTemplate = `{ } }, "definitions": { + "airflow.ClearTaskInstances": { + "type": "object", + "properties": { + "dag_run_id": { + "description": "The DagRun ID for this task instance", + "allOf": [ + { + "$ref": "#/definitions/airflow.NullableString" + } + ] + }, + "dry_run": { + "description": "If set, don't actually run this operation. The response will contain a list of task instances planned to be cleaned, but not modified in any way.", + "type": "boolean" + }, + "end_date": { + "description": "The maximum execution date to clear.", + "type": "string" + }, + "include_downstream": { + "description": "If set to true, downstream tasks are also affected.", + "type": "boolean" + }, + "include_future": { + "description": "If set to True, also tasks from future DAG Runs are affected.", + "type": "boolean" + }, + "include_parentdag": { + "description": "Clear tasks in the parent dag of the subdag.", + "type": "boolean" + }, + "include_past": { + "description": "If set to True, also tasks from past DAG Runs are affected.", + "type": "boolean" + }, + "include_subdags": { + "description": "Clear tasks in subdags and clear external tasks indicated by ExternalTaskMarker.", + "type": "boolean" + }, + "include_upstream": { + "description": "If set to true, upstream tasks are also affected.", + "type": "boolean" + }, + "only_failed": { + "description": "Only clear failed tasks.", + "type": "boolean" + }, + "only_running": { + "description": "Only clear running tasks.", + "type": "boolean" + }, + "reset_dag_runs": { + "description": "Set state of DAG runs to RUNNING.", + "type": "boolean" + }, + "start_date": { + "description": "The minimum execution date to clear.", + "type": "string" + }, + "task_ids": { + "description": "A list of task ids to clear. *New in version 2.1.0*", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "airflow.ImportError": { + "type": "object", + "properties": { + "filename": { + "description": "The filename", + "type": "string" + }, + "import_error_id": { + "description": "The import error ID.", + "type": "integer" + }, + "stack_trace": { + "description": "The full stackstrace..", + "type": "string" + }, + "timestamp": { + "description": "The time when this error was created.", + "type": "string" + } + } + }, + "airflow.ImportErrorCollection": { + "type": "object", + "properties": { + "import_errors": { + "type": "array", + "items": { + "$ref": "#/definitions/airflow.ImportError" + } + }, + "total_entries": { + "description": "Count of objects in the current result set.", + "type": "integer" + } + } + }, + "airflow.NullableString": { + "type": "object" + }, + "airflow.TaskInstanceReference": { + "type": "object", + "properties": { + "dag_id": { + "description": "The DAG ID.", + "type": "string" + }, + "dag_run_id": { + "description": "The DAG run ID.", + "type": "string" + }, + "execution_date": { + "type": "string" + }, + "task_id": { + "description": "The task ID.", + "type": "string" + } + } + }, + "airflow.TaskInstanceReferenceCollection": { + "type": "object", + "properties": { + "task_instances": { + "type": "array", + "items": { + "$ref": "#/definitions/airflow.TaskInstanceReference" + } + } + } + }, "github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse": { "type": "object", "properties": { diff --git a/pkg/api/rest/docs/swagger.json b/pkg/api/rest/docs/swagger.json index acbd5de..85ca90b 100644 --- a/pkg/api/rest/docs/swagger.json +++ b/pkg/api/rest/docs/swagger.json @@ -4,6 +4,97 @@ "contact": {} }, "paths": { + "/cicada/eventlogs": { + "get": { + "description": "Get the task Logs.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Workflow]" + ], + "summary": "Get taskInstances", + "parameters": [ + { + "type": "string", + "description": "ID of the workflow.", + "name": "dag_id", + "in": "query", + "required": true + }, + { + "type": "string", + "description": "ID of the workflow run.", + "name": "run_id", + "in": "query" + }, + { + "type": "string", + "description": "ID of the task.", + "name": "task_id", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Successfully get the taskInstances.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task" + } + }, + "400": { + "description": "Sent bad request.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + }, + "500": { + "description": "Failed to get the taskInstances.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + } + } + } + }, + "/cicada/importErrors": { + "get": { + "description": "Get the importErrors.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Workflow]" + ], + "summary": "Get importErrors", + "responses": { + "200": { + "description": "Successfully get the importErrors.", + "schema": { + "$ref": "#/definitions/airflow.ImportErrorCollection" + } + }, + "400": { + "description": "Sent bad request.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + }, + "500": { + "description": "Failed to get the importErrors.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + } + } + } + }, "/cicada/readyz": { "get": { "description": "Check Cicada is ready", @@ -681,6 +772,59 @@ } } }, + "/cicada/workflow/{wfId}/clear": { + "post": { + "description": "Clear the task Instance.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Workflow]" + ], + "summary": "Clear taskInstances", + "parameters": [ + { + "type": "string", + "description": "ID of the workflow.", + "name": "wfId", + "in": "path", + "required": true + }, + { + "description": "Clear TaskInstance", + "name": "requestBody", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/airflow.ClearTaskInstances" + } + } + ], + "responses": { + "200": { + "description": "Successfully clear the taskInstances.", + "schema": { + "$ref": "#/definitions/airflow.TaskInstanceReferenceCollection" + } + }, + "400": { + "description": "Sent bad request.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + }, + "500": { + "description": "Failed to clear the taskInstances.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + } + } + } + }, "/cicada/workflow/{wfId}/run": { "post": { "description": "Run the workflow.", @@ -1142,6 +1286,50 @@ } } }, + "/cicada/workflow/{wfId}/workflowRun/{wfRunId}/taskInstances": { + "get": { + "description": "Get the task Logs.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Workflow]" + ], + "summary": "Get taskInstances", + "parameters": [ + { + "type": "string", + "description": "ID of the workflow.", + "name": "wfId", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Successfully get the taskInstances.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task" + } + }, + "400": { + "description": "Sent bad request.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + }, + "500": { + "description": "Failed to get the taskInstances.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + } + } + } + }, "/cicada/workflow_template": { "get": { "description": "Get a list of workflow template.", @@ -1290,6 +1478,144 @@ } }, "definitions": { + "airflow.ClearTaskInstances": { + "type": "object", + "properties": { + "dag_run_id": { + "description": "The DagRun ID for this task instance", + "allOf": [ + { + "$ref": "#/definitions/airflow.NullableString" + } + ] + }, + "dry_run": { + "description": "If set, don't actually run this operation. The response will contain a list of task instances planned to be cleaned, but not modified in any way.", + "type": "boolean" + }, + "end_date": { + "description": "The maximum execution date to clear.", + "type": "string" + }, + "include_downstream": { + "description": "If set to true, downstream tasks are also affected.", + "type": "boolean" + }, + "include_future": { + "description": "If set to True, also tasks from future DAG Runs are affected.", + "type": "boolean" + }, + "include_parentdag": { + "description": "Clear tasks in the parent dag of the subdag.", + "type": "boolean" + }, + "include_past": { + "description": "If set to True, also tasks from past DAG Runs are affected.", + "type": "boolean" + }, + "include_subdags": { + "description": "Clear tasks in subdags and clear external tasks indicated by ExternalTaskMarker.", + "type": "boolean" + }, + "include_upstream": { + "description": "If set to true, upstream tasks are also affected.", + "type": "boolean" + }, + "only_failed": { + "description": "Only clear failed tasks.", + "type": "boolean" + }, + "only_running": { + "description": "Only clear running tasks.", + "type": "boolean" + }, + "reset_dag_runs": { + "description": "Set state of DAG runs to RUNNING.", + "type": "boolean" + }, + "start_date": { + "description": "The minimum execution date to clear.", + "type": "string" + }, + "task_ids": { + "description": "A list of task ids to clear. *New in version 2.1.0*", + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "airflow.ImportError": { + "type": "object", + "properties": { + "filename": { + "description": "The filename", + "type": "string" + }, + "import_error_id": { + "description": "The import error ID.", + "type": "integer" + }, + "stack_trace": { + "description": "The full stackstrace..", + "type": "string" + }, + "timestamp": { + "description": "The time when this error was created.", + "type": "string" + } + } + }, + "airflow.ImportErrorCollection": { + "type": "object", + "properties": { + "import_errors": { + "type": "array", + "items": { + "$ref": "#/definitions/airflow.ImportError" + } + }, + "total_entries": { + "description": "Count of objects in the current result set.", + "type": "integer" + } + } + }, + "airflow.NullableString": { + "type": "object" + }, + "airflow.TaskInstanceReference": { + "type": "object", + "properties": { + "dag_id": { + "description": "The DAG ID.", + "type": "string" + }, + "dag_run_id": { + "description": "The DAG run ID.", + "type": "string" + }, + "execution_date": { + "type": "string" + }, + "task_id": { + "description": "The task ID.", + "type": "string" + } + } + }, + "airflow.TaskInstanceReferenceCollection": { + "type": "object", + "properties": { + "task_instances": { + "type": "array", + "items": { + "$ref": "#/definitions/airflow.TaskInstanceReference" + } + } + } + }, "github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse": { "type": "object", "properties": { diff --git a/pkg/api/rest/docs/swagger.yaml b/pkg/api/rest/docs/swagger.yaml index 6d3af23..ea18ffb 100644 --- a/pkg/api/rest/docs/swagger.yaml +++ b/pkg/api/rest/docs/swagger.yaml @@ -1,4 +1,103 @@ definitions: + airflow.ClearTaskInstances: + properties: + dag_run_id: + allOf: + - $ref: '#/definitions/airflow.NullableString' + description: The DagRun ID for this task instance + dry_run: + description: If set, don't actually run this operation. The response will + contain a list of task instances planned to be cleaned, but not modified + in any way. + type: boolean + end_date: + description: The maximum execution date to clear. + type: string + include_downstream: + description: If set to true, downstream tasks are also affected. + type: boolean + include_future: + description: If set to True, also tasks from future DAG Runs are affected. + type: boolean + include_parentdag: + description: Clear tasks in the parent dag of the subdag. + type: boolean + include_past: + description: If set to True, also tasks from past DAG Runs are affected. + type: boolean + include_subdags: + description: Clear tasks in subdags and clear external tasks indicated by + ExternalTaskMarker. + type: boolean + include_upstream: + description: If set to true, upstream tasks are also affected. + type: boolean + only_failed: + description: Only clear failed tasks. + type: boolean + only_running: + description: Only clear running tasks. + type: boolean + reset_dag_runs: + description: Set state of DAG runs to RUNNING. + type: boolean + start_date: + description: The minimum execution date to clear. + type: string + task_ids: + description: A list of task ids to clear. *New in version 2.1.0* + items: + type: string + type: array + type: object + airflow.ImportError: + properties: + filename: + description: The filename + type: string + import_error_id: + description: The import error ID. + type: integer + stack_trace: + description: The full stackstrace.. + type: string + timestamp: + description: The time when this error was created. + type: string + type: object + airflow.ImportErrorCollection: + properties: + import_errors: + items: + $ref: '#/definitions/airflow.ImportError' + type: array + total_entries: + description: Count of objects in the current result set. + type: integer + type: object + airflow.NullableString: + type: object + airflow.TaskInstanceReference: + properties: + dag_id: + description: The DAG ID. + type: string + dag_run_id: + description: The DAG run ID. + type: string + execution_date: + type: string + task_id: + description: The task ID. + type: string + type: object + airflow.TaskInstanceReferenceCollection: + properties: + task_instances: + items: + $ref: '#/definitions/airflow.TaskInstanceReference' + type: array + type: object github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse: properties: error: @@ -263,6 +362,66 @@ definitions: info: contact: {} paths: + /cicada/eventlogs: + get: + consumes: + - application/json + description: Get the task Logs. + parameters: + - description: ID of the workflow. + in: query + name: dag_id + required: true + type: string + - description: ID of the workflow run. + in: query + name: run_id + type: string + - description: ID of the task. + in: query + name: task_id + type: string + produces: + - application/json + responses: + "200": + description: Successfully get the taskInstances. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task' + "400": + description: Sent bad request. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' + "500": + description: Failed to get the taskInstances. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' + summary: Get taskInstances + tags: + - '[Workflow]' + /cicada/importErrors: + get: + consumes: + - application/json + description: Get the importErrors. + produces: + - application/json + responses: + "200": + description: Successfully get the importErrors. + schema: + $ref: '#/definitions/airflow.ImportErrorCollection' + "400": + description: Sent bad request. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' + "500": + description: Failed to get the importErrors. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' + summary: Get importErrors + tags: + - '[Workflow]' /cicada/readyz: get: consumes: @@ -681,6 +840,41 @@ paths: summary: Update Workflow tags: - '[Workflow]' + /cicada/workflow/{wfId}/clear: + post: + consumes: + - application/json + description: Clear the task Instance. + parameters: + - description: ID of the workflow. + in: path + name: wfId + required: true + type: string + - description: Clear TaskInstance + in: body + name: requestBody + required: true + schema: + $ref: '#/definitions/airflow.ClearTaskInstances' + produces: + - application/json + responses: + "200": + description: Successfully clear the taskInstances. + schema: + $ref: '#/definitions/airflow.TaskInstanceReferenceCollection' + "400": + description: Sent bad request. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' + "500": + description: Failed to clear the taskInstances. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' + summary: Clear taskInstances + tags: + - '[Workflow]' /cicada/workflow/{wfId}/run: post: consumes: @@ -988,6 +1182,35 @@ paths: summary: Get Task Logs tags: - '[Workflow]' + /cicada/workflow/{wfId}/workflowRun/{wfRunId}/taskInstances: + get: + consumes: + - application/json + description: Get the task Logs. + parameters: + - description: ID of the workflow. + in: path + name: wfId + required: true + type: string + produces: + - application/json + responses: + "200": + description: Successfully get the taskInstances. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task' + "400": + description: Sent bad request. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' + "500": + description: Failed to get the taskInstances. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' + summary: Get taskInstances + tags: + - '[Workflow]' /cicada/workflow/name/{wfName}: get: consumes: diff --git a/pkg/api/rest/model/workflow.go b/pkg/api/rest/model/workflow.go index d135e3b..c5b2e1f 100644 --- a/pkg/api/rest/model/workflow.go +++ b/pkg/api/rest/model/workflow.go @@ -120,4 +120,4 @@ func (d *CreateDataReq) Scan(value interface{}) error { return errors.New("Invalid type for Data") } return json.Unmarshal(bytes, d) -} +} \ No newline at end of file diff --git a/pkg/api/rest/route/workflow.go b/pkg/api/rest/route/workflow.go index 7c4a49a..77b52ea 100644 --- a/pkg/api/rest/route/workflow.go +++ b/pkg/api/rest/route/workflow.go @@ -31,4 +31,7 @@ func Workflow(e *echo.Echo) { e.GET("/"+strings.ToLower(common.ShortModuleName)+"/workflow/:wfId/workflowRun/:wfRunId/task/:taskId/taskTryNum/:taskTyNum/logs", controller.GetTaskLogs) e.GET("/"+strings.ToLower(common.ShortModuleName)+"/workflow/:wfId/runs", controller.GetWorkflowRuns) e.GET("/"+strings.ToLower(common.ShortModuleName)+"/workflow/:wfId/workflowRun/:wfRunId/taskInstances", controller.GetTaskInstances) + e.POST("/"+strings.ToLower(common.ShortModuleName)+"/workflow/:wfId/clear", controller.ClearTaskInstances) + e.GET("/"+strings.ToLower(common.ShortModuleName)+"/eventlogs", controller.GetEventLogs) + e.GET("/"+strings.ToLower(common.ShortModuleName)+"/importErrors", controller.GetImportErrors) } From e3cb8ddd1175a6cc0018f3f03d49b7f61bdcf724 Mon Sep 17 00:00:00 2001 From: yby654 Date: Tue, 10 Sep 2024 09:48:15 +0900 Subject: [PATCH 04/12] Workflow run modify --- pkg/api/rest/controller/workflow.go | 31 +++++++++++++---- pkg/api/rest/docs/docs.go | 53 ++++++++++++++++++++++++++++- pkg/api/rest/docs/swagger.json | 53 ++++++++++++++++++++++++++++- pkg/api/rest/docs/swagger.yaml | 36 +++++++++++++++++++- pkg/api/rest/model/workflow.go | 28 +++++++++++++++ 5 files changed, 192 insertions(+), 9 deletions(-) diff --git a/pkg/api/rest/controller/workflow.go b/pkg/api/rest/controller/workflow.go index 2f85937..847d616 100644 --- a/pkg/api/rest/controller/workflow.go +++ b/pkg/api/rest/controller/workflow.go @@ -919,22 +919,41 @@ func GetTaskLogs(c echo.Context) error { // @Accept json // @Produce json // @Param wfId path string true "ID of the workflow." -// @Success 200 {object} model.Task "Successfully get the workflowRuns." +// @Success 200 {object} model.WorkflowRun "Successfully get the workflowRuns." // @Failure 400 {object} common.ErrorResponse "Sent bad request." // @Failure 500 {object} common.ErrorResponse "Failed to get the workflowRuns." // @Router /cicada/workflow/{wfId}/runs [get] func GetWorkflowRuns(c echo.Context) error { wfId := c.Param("wfId") if wfId == "" { - return common.ReturnErrorMsg(c, "Please provide the wfId.") + return common.ReturnErrorMsg(c, "Please provide the wfId.") } - + runList, err := airflow.Client.GetDAGRuns(wfId) if err != nil { - return common.ReturnErrorMsg(c, "Failed to get the workflow runs: " + err.Error()) - } + return common.ReturnErrorMsg(c, "Failed to get the workflow runs: " + err.Error()) + } + + var transformedRuns []model.WorkflowRun; + + for _, dagRun := range *runList.DagRuns { + transformedRun := model.WorkflowRun { + WorkflowID: dagRun.DagId, + WorkflowRunID: dagRun.GetDagRunId(), + DataIntervalStart: dagRun.GetDataIntervalStart(), + DataIntervalEnd: dagRun.GetDataIntervalEnd(), + State: string(dagRun.GetState()), + ExecutionDate: dagRun.GetExecutionDate(), + StartDate: dagRun.GetStartDate(), + EndDate: dagRun.GetEndDate(), + RunType: dagRun.GetRunType(), + LastSchedulingDecision: dagRun.GetLastSchedulingDecision(), + DurationDate: (dagRun.GetEndDate().Sub(dagRun.GetStartDate()).Seconds()), + } + transformedRuns = append(transformedRuns,transformedRun ) +} - return c.JSONPretty(http.StatusOK, runList, " ") + return c.JSONPretty(http.StatusOK, transformedRuns, " ") } // taskInstances godoc // diff --git a/pkg/api/rest/docs/docs.go b/pkg/api/rest/docs/docs.go index 7f9c6d8..02c5d7d 100644 --- a/pkg/api/rest/docs/docs.go +++ b/pkg/api/rest/docs/docs.go @@ -906,7 +906,7 @@ const docTemplate = `{ "200": { "description": "Successfully get the workflowRuns.", "schema": { - "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task" + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.WorkflowRun" } }, "400": { @@ -1995,6 +1995,57 @@ const docTemplate = `{ } } }, + "github_com_cloud-barista_cm-cicada_pkg_api_rest_model.WorkflowRun": { + "type": "object", + "properties": { + "conf": { + "type": "object", + "additionalProperties": true + }, + "data_interval_end": { + "type": "string" + }, + "data_interval_start": { + "type": "string" + }, + "duration_date": { + "type": "number" + }, + "end_date": { + "type": "string" + }, + "execution_date": { + "type": "string" + }, + "external_trigger": { + "type": "boolean" + }, + "last_scheduling_decision": { + "type": "string" + }, + "logical_date": { + "type": "string" + }, + "note": { + "type": "string" + }, + "run_type": { + "type": "string" + }, + "start_date": { + "type": "string" + }, + "state": { + "type": "string" + }, + "workflow_id": { + "type": "string" + }, + "workflow_run_id": { + "type": "string" + } + } + }, "github_com_cloud-barista_cm-cicada_pkg_api_rest_model.WorkflowTemplate": { "type": "object", "required": [ diff --git a/pkg/api/rest/docs/swagger.json b/pkg/api/rest/docs/swagger.json index 85ca90b..d49f197 100644 --- a/pkg/api/rest/docs/swagger.json +++ b/pkg/api/rest/docs/swagger.json @@ -895,7 +895,7 @@ "200": { "description": "Successfully get the workflowRuns.", "schema": { - "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task" + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.WorkflowRun" } }, "400": { @@ -1984,6 +1984,57 @@ } } }, + "github_com_cloud-barista_cm-cicada_pkg_api_rest_model.WorkflowRun": { + "type": "object", + "properties": { + "conf": { + "type": "object", + "additionalProperties": true + }, + "data_interval_end": { + "type": "string" + }, + "data_interval_start": { + "type": "string" + }, + "duration_date": { + "type": "number" + }, + "end_date": { + "type": "string" + }, + "execution_date": { + "type": "string" + }, + "external_trigger": { + "type": "boolean" + }, + "last_scheduling_decision": { + "type": "string" + }, + "logical_date": { + "type": "string" + }, + "note": { + "type": "string" + }, + "run_type": { + "type": "string" + }, + "start_date": { + "type": "string" + }, + "state": { + "type": "string" + }, + "workflow_id": { + "type": "string" + }, + "workflow_run_id": { + "type": "string" + } + } + }, "github_com_cloud-barista_cm-cicada_pkg_api_rest_model.WorkflowTemplate": { "type": "object", "required": [ diff --git a/pkg/api/rest/docs/swagger.yaml b/pkg/api/rest/docs/swagger.yaml index ea18ffb..7fb7cc2 100644 --- a/pkg/api/rest/docs/swagger.yaml +++ b/pkg/api/rest/docs/swagger.yaml @@ -346,6 +346,40 @@ definitions: - id - name type: object + github_com_cloud-barista_cm-cicada_pkg_api_rest_model.WorkflowRun: + properties: + conf: + additionalProperties: true + type: object + data_interval_end: + type: string + data_interval_start: + type: string + duration_date: + type: number + end_date: + type: string + execution_date: + type: string + external_trigger: + type: boolean + last_scheduling_decision: + type: string + logical_date: + type: string + note: + type: string + run_type: + type: string + start_date: + type: string + state: + type: string + workflow_id: + type: string + workflow_run_id: + type: string + type: object github_com_cloud-barista_cm-cicada_pkg_api_rest_model.WorkflowTemplate: properties: data: @@ -921,7 +955,7 @@ paths: "200": description: Successfully get the workflowRuns. schema: - $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task' + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.WorkflowRun' "400": description: Sent bad request. schema: diff --git a/pkg/api/rest/model/workflow.go b/pkg/api/rest/model/workflow.go index c5b2e1f..5cb440f 100644 --- a/pkg/api/rest/model/workflow.go +++ b/pkg/api/rest/model/workflow.go @@ -92,6 +92,34 @@ type CreateWorkflowReq struct { Data CreateDataReq `gorm:"column:data" json:"data" mapstructure:"data" validate:"required"` } +type Monit struct { + WorkflowID string + WorkflowVersion string + Status string + startTime time.Time + endTime time.Time + Duration time.Time + WorkflowInput string + WorkflowResult string +} + +type WorkflowRun struct { + WorkflowRunID string `json:"workflow_run_id,omitempty"` + WorkflowID *string `json:"workflow_id,omitempty"` + LogicalDate string `json:"logical_date,omitempty"` + ExecutionDate time.Time `json:"execution_date,omitempty"` + StartDate time.Time `json:"start_date,omitempty"` + EndDate time.Time `json:"end_date,omitempty"` + DurationDate float64 `json:"duration_date,omitempty"` + DataIntervalStart time.Time `json:"data_interval_start,omitempty"` + DataIntervalEnd time.Time `json:"data_interval_end,omitempty"` + LastSchedulingDecision time.Time `json:"last_scheduling_decision,omitempty"` + RunType string `json:"run_type,omitempty"` + State string `json:"state,omitempty"` + ExternalTrigger *bool `json:"external_trigger,omitempty"` + Conf map[string]interface{} `json:"conf,omitempty"` + Note string `json:"note,omitempty"` +} func (d Data) Value() (driver.Value, error) { return json.Marshal(d) } From f45e8b350dba661d7387e940ed75407dca186a1b Mon Sep 17 00:00:00 2001 From: yby654 Date: Tue, 10 Sep 2024 15:21:49 +0900 Subject: [PATCH 05/12] eventlogs --- lib/airflow/workflow.go | 10 +-- pkg/api/rest/controller/workflow.go | 58 +++++++++------ pkg/api/rest/docs/docs.go | 108 ++++++++++++++++++---------- pkg/api/rest/docs/swagger.json | 38 +++++----- pkg/api/rest/docs/swagger.yaml | 38 +++++----- pkg/api/rest/model/workflow.go | 1 + 6 files changed, 145 insertions(+), 108 deletions(-) diff --git a/lib/airflow/workflow.go b/lib/airflow/workflow.go index cb7578f..1696005 100644 --- a/lib/airflow/workflow.go +++ b/lib/airflow/workflow.go @@ -1,11 +1,10 @@ package airflow import ( + "encoding/json" "errors" "sync" - "encoding/json" - "github.com/apache/airflow-client-go/airflow" "github.com/cloud-barista/cm-cicada/lib/config" "github.com/cloud-barista/cm-cicada/pkg/api/rest/model" @@ -238,16 +237,13 @@ func (client *client) GetEventLogs(dagID string) (airflow.EventLogCollection, er }() ctx, cancel := Context() defer cancel() - - // TaskInstanceApi 인스턴스를 사용하여 로그 요청 - logs, _, err := client.api.EventLogApi.GetEventLogs(ctx).Execute() - logger.Println(logger.INFO, false,logs) + req, _, err := client.api.EventLogApi.GetEventLogs(ctx).Execute() if err != nil { logger.Println(logger.ERROR, false, "AIRFLOW: Error occurred while getting event logs. (Error: "+err.Error()+").") } - return logs, nil + return req, nil } func (client *client) GetImportErrors() (airflow.ImportErrorCollection, error) { diff --git a/pkg/api/rest/controller/workflow.go b/pkg/api/rest/controller/workflow.go index 06e8738..f7d3534 100644 --- a/pkg/api/rest/controller/workflow.go +++ b/pkg/api/rest/controller/workflow.go @@ -894,7 +894,7 @@ func GetTaskDirectly(c echo.Context) error { // @Success 200 {object} model.Task "Successfully get the task Logs." // @Failure 400 {object} common.ErrorResponse "Sent bad request." // @Failure 500 {object} common.ErrorResponse "Failed to get the task Logs." -// @Router /cicada/workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/taskTryNum/{taskTyNum}/logs [get] +// @Router /workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/taskTryNum/{taskTyNum}/logs [get] func GetTaskLogs(c echo.Context) error { wfId := c.Param("wfId") if wfId == "" { @@ -937,7 +937,7 @@ func GetTaskLogs(c echo.Context) error { // @Success 200 {object} model.WorkflowRun "Successfully get the workflowRuns." // @Failure 400 {object} common.ErrorResponse "Sent bad request." // @Failure 500 {object} common.ErrorResponse "Failed to get the workflowRuns." -// @Router /cicada/workflow/{wfId}/runs [get] +// @Router /workflow/{wfId}/runs [get] func GetWorkflowRuns(c echo.Context) error { wfId := c.Param("wfId") if wfId == "" { @@ -981,7 +981,7 @@ func GetWorkflowRuns(c echo.Context) error { // @Success 200 {object} model.Task "Successfully get the taskInstances." // @Failure 400 {object} common.ErrorResponse "Sent bad request." // @Failure 500 {object} common.ErrorResponse "Failed to get the taskInstances." -// @Router /cicada/workflow/{wfId}/workflowRun/{wfRunId}/taskInstances [get] +// @Router /workflow/{wfId}/workflowRun/{wfRunId}/taskInstances [get] func GetTaskInstances(c echo.Context) error { wfId := c.Param("wfId") if wfId == "" { @@ -1012,7 +1012,7 @@ func GetTaskInstances(c echo.Context) error { // @Success 200 {object} airflow.TaskInstanceReferenceCollection "Successfully clear the taskInstances." // @Failure 400 {object} common.ErrorResponse "Sent bad request." // @Failure 500 {object} common.ErrorResponse "Failed to clear the taskInstances." -// @Router /cicada/workflow/{wfId}/clear [post] +// @Router /workflow/{wfId}/clear [post] func ClearTaskInstances(c echo.Context) error { wfId := c.Param("wfId") if wfId == "" { @@ -1034,28 +1034,42 @@ func ClearTaskInstances(c echo.Context) error { // Eventlogs godoc // -// @Summary Get taskInstances -// @Description Get the task Logs. -// @Tags [Workflow] -// @Accept json -// @Produce json -// @Param dag_id query string true "ID of the workflow." -// @Param run_id query string false "ID of the workflow run." -// @Param task_id query string false "ID of the task." -// @Success 200 {object} model.Task "Successfully get the taskInstances." -// @Failure 400 {object} common.ErrorResponse "Sent bad request." -// @Failure 500 {object} common.ErrorResponse "Failed to get the taskInstances." -// @Router /cicada/eventlogs [get] +// @ID get-Eventlog +// @Summary Get Eventlog +// @Description Get Eventlog. +// @Tags [Workflow] +// @Accept json +// @Produce json +// @Param dag_id query string true "ID of the workflow." +// @Param wfRunId query string false "ID of the workflow run." +// @Param taskId query string false "ID of the task." +// @Success 200 {object} model.Workflow "Successfully get the workflow." +// @Failure 400 {object} common.ErrorResponse "Sent bad request." +// @Failure 500 {object} common.ErrorResponse "Failed to get the workflow." +// @Router /eventlogs [get] func GetEventLogs(c echo.Context) error { - dagId := c.QueryParam("dag_id") - if dagId == "" { - return common.ReturnErrorMsg(c, "Please provide the dagId.") - } - logs, err := airflow.Client.GetEventLogs(dagId) + wfId := c.QueryParam("dag_id") + // if wfId == "" { + // return common.ReturnErrorMsg(c, "Please provide the dagId.") + // } + logs, err := airflow.Client.GetEventLogs(wfId) if err != nil { return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) } + wfRunId := c.QueryParam("wfRunId") + if wfRunId != ""{ + + } + taskId := c.QueryParam("taskId") + if taskId != ""{ + + } + // logs, err := airflow.Client.GetEventLogs(wfId) + // if err != nil { + // return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) + // } + return c.JSONPretty(http.StatusOK, logs, " ") } @@ -1069,7 +1083,7 @@ func GetEventLogs(c echo.Context) error { // @Success 200 {object} airflow.ImportErrorCollection "Successfully get the importErrors." // @Failure 400 {object} common.ErrorResponse "Sent bad request." // @Failure 500 {object} common.ErrorResponse "Failed to get the importErrors." -// @Router /cicada/importErrors [get] +// @Router /importErrors [get] func GetImportErrors(c echo.Context) error { logs, err := airflow.Client.GetImportErrors() if err != nil { diff --git a/pkg/api/rest/docs/docs.go b/pkg/api/rest/docs/docs.go index 02c5d7d..629836f 100644 --- a/pkg/api/rest/docs/docs.go +++ b/pkg/api/rest/docs/docs.go @@ -10,14 +10,18 @@ const docTemplate = `{ "description": "{{escape .Description}}", "title": "{{.Title}}", "contact": {}, + "license": { + "name": "Apache 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.html" + }, "version": "{{.Version}}" }, "host": "{{.Host}}", "basePath": "{{.BasePath}}", "paths": { - "/cicada/eventlogs": { + "/eventlogs": { "get": { - "description": "Get the task Logs.", + "description": "Get Eventlog.", "consumes": [ "application/json" ], @@ -27,7 +31,8 @@ const docTemplate = `{ "tags": [ "[Workflow]" ], - "summary": "Get taskInstances", + "summary": "Get Eventlog", + "operationId": "get-Eventlog", "parameters": [ { "type": "string", @@ -39,21 +44,21 @@ const docTemplate = `{ { "type": "string", "description": "ID of the workflow run.", - "name": "run_id", + "name": "wfRunId", "in": "query" }, { "type": "string", "description": "ID of the task.", - "name": "task_id", + "name": "taskId", "in": "query" } ], "responses": { "200": { - "description": "Successfully get the taskInstances.", + "description": "Successfully get the workflow.", "schema": { - "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task" + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Workflow" } }, "400": { @@ -63,7 +68,7 @@ const docTemplate = `{ } }, "500": { - "description": "Failed to get the taskInstances.", + "description": "Failed to get the workflow.", "schema": { "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" } @@ -71,7 +76,7 @@ const docTemplate = `{ } } }, - "/cicada/importErrors": { + "/importErrors": { "get": { "description": "Get the importErrors.", "consumes": [ @@ -106,7 +111,7 @@ const docTemplate = `{ } } }, - "/cicada/readyz": { + "/readyz": { "get": { "description": "Check Cicada is ready", "consumes": [ @@ -116,9 +121,10 @@ const docTemplate = `{ "application/json" ], "tags": [ - "[Admin] System management" + "[Admin]\tSystem management" ], "summary": "Check Ready", + "operationId": "health-check-readyz", "responses": { "200": { "description": "Successfully get ready state.", @@ -135,7 +141,7 @@ const docTemplate = `{ } } }, - "/cicada/task/{taskId}": { + "/task/{taskId}": { "get": { "description": "Get the task directly.", "consumes": [ @@ -148,6 +154,7 @@ const docTemplate = `{ "[Workflow]" ], "summary": "Get Task Directly", + "operationId": "get-task-directly", "parameters": [ { "type": "string", @@ -179,7 +186,7 @@ const docTemplate = `{ } } }, - "/cicada/task_component": { + "/task_component": { "get": { "description": "Get a list of task component.", "consumes": [ @@ -192,6 +199,7 @@ const docTemplate = `{ "[Task Component]" ], "summary": "List TaskComponent", + "operationId": "list-task-component", "parameters": [ { "type": "string", @@ -242,6 +250,7 @@ const docTemplate = `{ "[Task Component]" ], "summary": "Create TaskComponent", + "operationId": "create-task-component", "parameters": [ { "description": "task component of the node.", @@ -275,7 +284,7 @@ const docTemplate = `{ } } }, - "/cicada/task_component/name/{tcName}": { + "/task_component/name/{tcName}": { "get": { "description": "Get the task component by name.", "consumes": [ @@ -288,6 +297,7 @@ const docTemplate = `{ "[Task Component]" ], "summary": "Get TaskComponent by Name", + "operationId": "get-task-component-by-name", "parameters": [ { "type": "string", @@ -319,7 +329,7 @@ const docTemplate = `{ } } }, - "/cicada/task_component/{tcId}": { + "/task_component/{tcId}": { "get": { "description": "Get the task component.", "consumes": [ @@ -332,6 +342,7 @@ const docTemplate = `{ "[Task Component]" ], "summary": "Get TaskComponent", + "operationId": "get-task-component", "parameters": [ { "type": "string", @@ -374,6 +385,7 @@ const docTemplate = `{ "[Task Component]" ], "summary": "Update TaskComponent", + "operationId": "update-task-component", "parameters": [ { "type": "string", @@ -425,6 +437,7 @@ const docTemplate = `{ "[Task Component]" ], "summary": "Delete TaskComponent", + "operationId": "delete-task-component", "parameters": [ { "type": "string", @@ -456,7 +469,7 @@ const docTemplate = `{ } } }, - "/cicada/task_group/{tgId}": { + "/task_group/{tgId}": { "get": { "description": "Get the task group directly.", "consumes": [ @@ -469,6 +482,7 @@ const docTemplate = `{ "[Workflow]" ], "summary": "Get TaskGroup Directly", + "operationId": "get-task-group-directly", "parameters": [ { "type": "string", @@ -500,7 +514,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow": { + "/workflow": { "get": { "description": "Get a workflow list.", "consumes": [ @@ -513,12 +527,13 @@ const docTemplate = `{ "[Workflow]" ], "summary": "List Workflow", + "operationId": "list-workflow", "parameters": [ { "type": "string", "description": "Name of the workflow", "name": "name", - "in": "path" + "in": "query" }, { "type": "string", @@ -569,6 +584,7 @@ const docTemplate = `{ "[Workflow]" ], "summary": "Create Workflow", + "operationId": "create-workflow", "parameters": [ { "description": "Workflow content", @@ -602,7 +618,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow/name/{wfName}": { + "/workflow/name/{wfName}": { "get": { "description": "Get the workflow by name.", "consumes": [ @@ -615,6 +631,7 @@ const docTemplate = `{ "[Workflow]" ], "summary": "Get Workflow by Name", + "operationId": "get-workflow-by-name", "parameters": [ { "type": "string", @@ -646,7 +663,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow/{wfId}": { + "/workflow/{wfId}": { "get": { "description": "Get the workflow.", "consumes": [ @@ -659,6 +676,7 @@ const docTemplate = `{ "[Workflow]" ], "summary": "Get Workflow", + "operationId": "get-workflow", "parameters": [ { "type": "string", @@ -701,6 +719,7 @@ const docTemplate = `{ "[Workflow]" ], "summary": "Update Workflow", + "operationId": "update-workflow", "parameters": [ { "type": "string", @@ -752,6 +771,7 @@ const docTemplate = `{ "[Workflow]" ], "summary": "Delete Workflow", + "operationId": "delete-workflow", "parameters": [ { "type": "string", @@ -783,7 +803,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow/{wfId}/clear": { + "/workflow/{wfId}/clear": { "post": { "description": "Clear the task Instance.", "consumes": [ @@ -836,7 +856,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow/{wfId}/run": { + "/workflow/{wfId}/run": { "post": { "description": "Run the workflow.", "consumes": [ @@ -849,6 +869,7 @@ const docTemplate = `{ "[Workflow]" ], "summary": "Run Workflow", + "operationId": "run-workflow", "parameters": [ { "type": "string", @@ -880,7 +901,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow/{wfId}/runs": { + "/workflow/{wfId}/runs": { "get": { "description": "Get the task Logs.", "consumes": [ @@ -924,7 +945,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow/{wfId}/task": { + "/workflow/{wfId}/task": { "get": { "description": "Get a task list of the workflow.", "consumes": [ @@ -937,6 +958,7 @@ const docTemplate = `{ "[Workflow]" ], "summary": "List Task", + "operationId": "list-task", "parameters": [ { "type": "string", @@ -971,7 +993,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow/{wfId}/task/{taskId}": { + "/workflow/{wfId}/task/{taskId}": { "get": { "description": "Get the task.", "consumes": [ @@ -984,6 +1006,7 @@ const docTemplate = `{ "[Workflow]" ], "summary": "Get Task", + "operationId": "get-task", "parameters": [ { "type": "string", @@ -1022,7 +1045,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow/{wfId}/task_group": { + "/workflow/{wfId}/task_group": { "get": { "description": "Get a task group list of the workflow.", "consumes": [ @@ -1035,6 +1058,7 @@ const docTemplate = `{ "[Workflow]" ], "summary": "List TaskGroup", + "operationId": "list-task-group", "parameters": [ { "type": "string", @@ -1069,7 +1093,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow/{wfId}/task_group/{tgId}": { + "/workflow/{wfId}/task_group/{tgId}": { "get": { "description": "Get the task group.", "consumes": [ @@ -1082,6 +1106,7 @@ const docTemplate = `{ "[Workflow]" ], "summary": "Get TaskGroup", + "operationId": "get-task-group", "parameters": [ { "type": "string", @@ -1120,7 +1145,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow/{wfId}/task_group/{tgId}/task": { + "/workflow/{wfId}/task_group/{tgId}/task": { "get": { "description": "Get a task list from the task group.", "consumes": [ @@ -1133,6 +1158,7 @@ const docTemplate = `{ "[Workflow]" ], "summary": "List Task from Task Group", + "operationId": "list-task-from-task-group", "parameters": [ { "type": "string", @@ -1174,7 +1200,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow/{wfId}/task_group/{tgId}/task/{taskId}": { + "/workflow/{wfId}/task_group/{tgId}/task/{taskId}": { "get": { "description": "Get the task from the task group.", "consumes": [ @@ -1187,6 +1213,7 @@ const docTemplate = `{ "[Workflow]" ], "summary": "Get Task from Task Group", + "operationId": "get-task-from-task-group", "parameters": [ { "type": "string", @@ -1232,7 +1259,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/taskTryNum/{taskTyNum}/logs": { + "/workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/taskTryNum/{taskTyNum}/logs": { "get": { "description": "Get the task Logs.", "consumes": [ @@ -1297,7 +1324,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow/{wfId}/workflowRun/{wfRunId}/taskInstances": { + "/workflow/{wfId}/workflowRun/{wfRunId}/taskInstances": { "get": { "description": "Get the task Logs.", "consumes": [ @@ -1341,7 +1368,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow_template": { + "/workflow_template": { "get": { "description": "Get a list of workflow template.", "consumes": [ @@ -1354,12 +1381,13 @@ const docTemplate = `{ "[Workflow Template]" ], "summary": "List WorkflowTemplate", + "operationId": "list-workflow-template", "parameters": [ { "type": "string", "description": "Name of the workflow template", "name": "name", - "in": "path" + "in": "query" }, { "type": "string", @@ -1399,7 +1427,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow_template/name/{wfName}": { + "/workflow_template/name/{wfName}": { "get": { "description": "Get the workflow template by name.", "consumes": [ @@ -1412,6 +1440,7 @@ const docTemplate = `{ "[Workflow Template]" ], "summary": "Get WorkflowTemplate by Name", + "operationId": "get-workflow-template-by-name", "parameters": [ { "type": "string", @@ -1443,7 +1472,7 @@ const docTemplate = `{ } } }, - "/cicada/workflow_template/{wftId}": { + "/workflow_template/{wftId}": { "get": { "description": "Get the workflow template.", "consumes": [ @@ -1456,6 +1485,7 @@ const docTemplate = `{ "[Workflow Template]" ], "summary": "Get WorkflowTemplate", + "operationId": "get-workflow-template", "parameters": [ { "type": "string", @@ -2070,12 +2100,12 @@ const docTemplate = `{ // SwaggerInfo holds exported Swagger Info so clients can modify it var SwaggerInfo = &swag.Spec{ - Version: "", + Version: "latest", Host: "", - BasePath: "", + BasePath: "/cicada", Schemes: []string{}, - Title: "", - Description: "", + Title: "CM-Cicada REST API", + Description: "Workflow management module", InfoInstanceName: "swagger", SwaggerTemplate: docTemplate, LeftDelim: "{{", diff --git a/pkg/api/rest/docs/swagger.json b/pkg/api/rest/docs/swagger.json index 03e6449..54a23c5 100644 --- a/pkg/api/rest/docs/swagger.json +++ b/pkg/api/rest/docs/swagger.json @@ -12,9 +12,9 @@ }, "basePath": "/cicada", "paths": { - "/cicada/eventlogs": { + "/eventlogs": { "get": { - "description": "Get the task Logs.", + "description": "Get Eventlog.", "consumes": [ "application/json" ], @@ -24,7 +24,8 @@ "tags": [ "[Workflow]" ], - "summary": "Get taskInstances", + "summary": "Get Eventlog", + "operationId": "get-Eventlog", "parameters": [ { "type": "string", @@ -36,21 +37,21 @@ { "type": "string", "description": "ID of the workflow run.", - "name": "run_id", + "name": "wfRunId", "in": "query" }, { "type": "string", "description": "ID of the task.", - "name": "task_id", + "name": "taskId", "in": "query" } ], "responses": { "200": { - "description": "Successfully get the taskInstances.", + "description": "Successfully get the workflow.", "schema": { - "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task" + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Workflow" } }, "400": { @@ -60,7 +61,7 @@ } }, "500": { - "description": "Failed to get the taskInstances.", + "description": "Failed to get the workflow.", "schema": { "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" } @@ -68,7 +69,7 @@ } } }, - "/cicada/importErrors": { + "/importErrors": { "get": { "description": "Get the importErrors.", "consumes": [ @@ -103,7 +104,7 @@ } } }, - "/cicada/readyz": { + "/readyz": { "get": { "description": "Check Cicada is ready", "consumes": [ @@ -795,8 +796,7 @@ } } }, - "/workflow/{wfId}/run": { - "/cicada/workflow/{wfId}/clear": { + "/workflow/{wfId}/clear": { "post": { "description": "Clear the task Instance.", "consumes": [ @@ -849,7 +849,7 @@ } } }, - "/cicada/workflow/{wfId}/run": { + "/workflow/{wfId}/run": { "post": { "description": "Run the workflow.", "consumes": [ @@ -894,8 +894,7 @@ } } }, - "/workflow/{wfId}/task": { - "/cicada/workflow/{wfId}/runs": { + "/workflow/{wfId}/runs": { "get": { "description": "Get the task Logs.", "consumes": [ @@ -939,7 +938,7 @@ } } }, - "/cicada/workflow/{wfId}/task": { + "/workflow/{wfId}/task": { "get": { "description": "Get a task list of the workflow.", "consumes": [ @@ -1253,8 +1252,7 @@ } } }, - "/workflow_template": { - "/cicada/workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/taskTryNum/{taskTyNum}/logs": { + "/workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/taskTryNum/{taskTyNum}/logs": { "get": { "description": "Get the task Logs.", "consumes": [ @@ -1319,7 +1317,7 @@ } } }, - "/cicada/workflow/{wfId}/workflowRun/{wfRunId}/taskInstances": { + "/workflow/{wfId}/workflowRun/{wfRunId}/taskInstances": { "get": { "description": "Get the task Logs.", "consumes": [ @@ -1363,7 +1361,7 @@ } } }, - "/cicada/workflow_template": { + "/workflow_template": { "get": { "description": "Get a list of workflow template.", "consumes": [ diff --git a/pkg/api/rest/docs/swagger.yaml b/pkg/api/rest/docs/swagger.yaml index c0f3b04..9d3ce7d 100644 --- a/pkg/api/rest/docs/swagger.yaml +++ b/pkg/api/rest/docs/swagger.yaml @@ -403,11 +403,12 @@ info: title: CM-Cicada REST API version: latest paths: - /cicada/eventlogs: + /eventlogs: get: consumes: - application/json - description: Get the task Logs. + description: Get Eventlog. + operationId: get-Eventlog parameters: - description: ID of the workflow. in: query @@ -416,31 +417,31 @@ paths: type: string - description: ID of the workflow run. in: query - name: run_id + name: wfRunId type: string - description: ID of the task. in: query - name: task_id + name: taskId type: string produces: - application/json responses: "200": - description: Successfully get the taskInstances. + description: Successfully get the workflow. schema: - $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task' + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Workflow' "400": description: Sent bad request. schema: $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' "500": - description: Failed to get the taskInstances. + description: Failed to get the workflow. schema: $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' - summary: Get taskInstances + summary: Get Eventlog tags: - '[Workflow]' - /cicada/importErrors: + /importErrors: get: consumes: - application/json @@ -463,7 +464,7 @@ paths: summary: Get importErrors tags: - '[Workflow]' - /cicada/readyz: + /readyz: get: consumes: - application/json @@ -895,8 +896,7 @@ paths: summary: Update Workflow tags: - '[Workflow]' - /workflow/{wfId}/run: - /cicada/workflow/{wfId}/clear: + /workflow/{wfId}/clear: post: consumes: - application/json @@ -931,7 +931,7 @@ paths: summary: Clear taskInstances tags: - '[Workflow]' - /cicada/workflow/{wfId}/run: + /workflow/{wfId}/run: post: consumes: - application/json @@ -961,8 +961,7 @@ paths: summary: Run Workflow tags: - '[Workflow]' - /workflow/{wfId}/task: - /cicada/workflow/{wfId}/runs: + /workflow/{wfId}/runs: get: consumes: - application/json @@ -991,7 +990,7 @@ paths: summary: Get workflowRuns tags: - '[Workflow]' - /cicada/workflow/{wfId}/task: + /workflow/{wfId}/task: get: consumes: - application/json @@ -1202,8 +1201,7 @@ paths: summary: Get Task from Task Group tags: - '[Workflow]' - /workflow/name/{wfName}: - /cicada/workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/taskTryNum/{taskTyNum}/logs: + /workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/taskTryNum/{taskTyNum}/logs: get: consumes: - application/json @@ -1247,7 +1245,7 @@ paths: summary: Get Task Logs tags: - '[Workflow]' - /cicada/workflow/{wfId}/workflowRun/{wfRunId}/taskInstances: + /workflow/{wfId}/workflowRun/{wfRunId}/taskInstances: get: consumes: - application/json @@ -1276,7 +1274,7 @@ paths: summary: Get taskInstances tags: - '[Workflow]' - /cicada/workflow/name/{wfName}: + /workflow/name/{wfName}: get: consumes: - application/json diff --git a/pkg/api/rest/model/workflow.go b/pkg/api/rest/model/workflow.go index 5cb440f..a921cf8 100644 --- a/pkg/api/rest/model/workflow.go +++ b/pkg/api/rest/model/workflow.go @@ -120,6 +120,7 @@ type WorkflowRun struct { Conf map[string]interface{} `json:"conf,omitempty"` Note string `json:"note,omitempty"` } + func (d Data) Value() (driver.Value, error) { return json.Marshal(d) } From fba76a445ee90acab1fcbaf334d7ae91bdcecbae Mon Sep 17 00:00:00 2001 From: yby654 Date: Thu, 12 Sep 2024 15:44:36 +0900 Subject: [PATCH 06/12] =?UTF-8?q?taskinstance=20=EC=A1=B0=ED=9A=8C,=20task?= =?UTF-8?q?clear=20=EA=B8=B0=EB=8A=A5=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _airflow/airflow-home/dags/test.py | 107 +++++++++++++ conf/cm-cicada.yaml | 1 + lib/airflow/workflow.go | 69 ++++++--- pkg/api/rest/common/url.go | 22 +++ pkg/api/rest/controller/workflow.go | 78 ++++++++-- pkg/api/rest/docs/docs.go | 226 +++++++++++++--------------- pkg/api/rest/docs/swagger.json | 226 +++++++++++++--------------- pkg/api/rest/docs/swagger.yaml | 157 +++++++++---------- pkg/api/rest/model/workflow.go | 13 ++ pkg/api/rest/route/workflow.go | 2 +- 10 files changed, 529 insertions(+), 372 deletions(-) create mode 100644 _airflow/airflow-home/dags/test.py create mode 100644 pkg/api/rest/common/url.go diff --git a/_airflow/airflow-home/dags/test.py b/_airflow/airflow-home/dags/test.py new file mode 100644 index 0000000..79a5dfa --- /dev/null +++ b/_airflow/airflow-home/dags/test.py @@ -0,0 +1,107 @@ + + +from datetime import datetime, timedelta +from textwrap import dedent + +from airflow import DAG + +# Operators; we need this to operate! +from airflow.operators.bash import BashOperator +from airflow.operators.dummy import DummyOperator +from airflow.operators.python import BranchPythonOperator +from airflow.operators.python_operator import PythonOperator +from airflow.utils.trigger_rule import TriggerRule + +default_args = { + 'owner': 'owner-name', + 'depends_on_past': False, + 'email': ['your-email@g.com'], + 'email_on_failure': False, + 'email_on_retry': False, + 'retries': 1, + 'retry_delay': timedelta(minutes=15), +} + +dag_args = dict( + dag_id="tutorial-python-op", + default_args=default_args, + description='tutorial DAG python', + schedule_interval=timedelta(minutes=50), + start_date=datetime(2022, 2, 1), + tags=['example-sj'], +) + +def random_branch_path(): + # 필요 라이브러리는 여기서 import + # https://airflow.apache.org/docs/apache-airflow/stable/best-practices.html#writing-a-dag 참고 + from random import randint + + return "cal_a_id" if randint(1, 2) == 1 else "cal_m_id" + +def calc_add(x, y, **kwargs): + result = x + y + print("x + y : ", result) + kwargs['task_instance'].xcom_push(key='calc_result', value=result) + return "calc add" + +def calc_mul(x, y, **kwargs): + result = x * y + print("x * y : ", result) + kwargs['task_instance'].xcom_push(key='calc_result', value=result) + return "calc mul" + +def print_result(**kwargs): + r = kwargs["task_instance"].xcom_pull(key='calc_result') + print("message : ", r) + print("*"*100) + print(kwargs) + +def end_seq(): + print("end") + +with DAG( **dag_args ) as dag: + start = BashOperator( + task_id='start', + bash_command='echo "start!"', + ) + + branch = BranchPythonOperator( + task_id='branch', + python_callable=random_branch_path, + ) + + calc_add = PythonOperator( + task_id='cal_a_id', + python_callable=calc_add, + op_kwargs={'x': 10, 'y':4} + ) + + calc_mul = PythonOperator( + task_id='cal_m_id', + python_callable=calc_mul, + op_kwargs={'x': 10, 'y':4} + ) + + msg = PythonOperator( + task_id='msg', + python_callable=print_result, + trigger_rule=TriggerRule.NONE_FAILED + ) + + complete_py = PythonOperator( + task_id='complete_py', + python_callable=end_seq, + trigger_rule=TriggerRule.NONE_FAILED + ) + + complete = BashOperator( + task_id='complete_bash', + depends_on_past=False, + bash_command='echo "complete~!"', + trigger_rule=TriggerRule.NONE_FAILED + ) + + + start >> branch >> calc_add >> msg >> complete_py >> complete + start >> branch >> calc_mul >> msg >> complete + diff --git a/conf/cm-cicada.yaml b/conf/cm-cicada.yaml index aa170e5..3a96a32 100644 --- a/conf/cm-cicada.yaml +++ b/conf/cm-cicada.yaml @@ -6,6 +6,7 @@ cm-cicada: templates_directory: "./lib/airflow/example/workflow_template/" airflow-server: address: 127.0.0.1:8080 + # address: 192.168.160.139:8080 use_tls: false # skip_tls_verify: true init_retry: 5 diff --git a/lib/airflow/workflow.go b/lib/airflow/workflow.go index 1696005..457dd3e 100644 --- a/lib/airflow/workflow.go +++ b/lib/airflow/workflow.go @@ -1,8 +1,8 @@ package airflow import ( - "encoding/json" "errors" + "fmt" "sync" "github.com/apache/airflow-client-go/airflow" @@ -162,13 +162,19 @@ func (client *client) GetTaskInstances(dagID string, dagRunId string) (airflow.T }() ctx, cancel := Context() defer cancel() - resp, _, err := client.api.TaskInstanceApi.GetTaskInstances(ctx, dagID, dagRunId).Execute() + // dagID = "a7192143-6aa7-4dee-856c-366caa91d126" + // dagRunId = "manual__2024-09-10T07:20:26.072377+00:00" + // req := client.api.TaskInstanceApi.GetTaskInstances(ctx, dagID, dagRunId) + // resp, _, err := client.api.TaskInstanceApi.GetTaskInstancesExecute(req) + resp, http, err := client.api.TaskInstanceApi.GetTaskInstances(ctx, dagID, dagRunId).Execute() + fmt.Println("test : ", http) if err != nil { logger.Println(logger.ERROR, false, "AIRFLOW: Error occurred while getting TaskInstances. (Error: "+err.Error()+").") } return resp, err } + func (client *client) GetTaskLogs(dagID, dagRunID, taskID string, taskTryNumber int) (airflow.InlineResponse200, error) { deferFunc := callDagRequestLock(dagID) defer func() { @@ -188,33 +194,46 @@ func (client *client) GetTaskLogs(dagID, dagRunID, taskID string, taskTryNumber return logs, nil } -func (client *client) ClearTaskInstance(dagID string, clearTaskInstance interface{}) (airflow.TaskInstanceReferenceCollection, error) { +func (client *client) ClearTaskInstance(dagID string, dagRunID string, taskID string) (airflow.TaskInstanceReferenceCollection, error) { deferFunc := callDagRequestLock(dagID) defer func() { deferFunc() }() ctx, cancel := Context() defer cancel() - - logger.Println(logger.ERROR, false, clearTaskInstance) - - clearTaskInstanceBytes, err := json.Marshal(clearTaskInstance) - if err != nil { - logger.Println(logger.ERROR, false, "Failed to marshal clearTaskInstance: ", err) - return airflow.TaskInstanceReferenceCollection{}, err + + dryRun := false + taskIds := []string{taskID} + includeDownstream := true + includeFuture := false + includeParentdag := false + includePast := false + includeSubdags := true + includeUpstream := false + onlyFailed := false + onlyRunning := false + resetDagRuns := true + + clearTask := airflow.ClearTaskInstances{ + DryRun: &dryRun, + TaskIds: &taskIds, + IncludeSubdags: &includeSubdags, + IncludeParentdag: &includeParentdag, + IncludeUpstream: &includeUpstream, + IncludeDownstream: &includeDownstream, + IncludeFuture: &includeFuture, + IncludePast: &includePast, + OnlyFailed: &onlyFailed, + OnlyRunning: &onlyRunning, + ResetDagRuns: &resetDagRuns, + DagRunId: *airflow.NewNullableString(&dagRunID), } - var test airflow.ClearTaskInstances - err = json.Unmarshal(clearTaskInstanceBytes, &test) - if err != nil { - logger.Println(logger.ERROR, false, "Failed to unmarshal clearTaskInstance to ClearTaskInstances struct: ", err) - return airflow.TaskInstanceReferenceCollection{}, err - } // 요청 생성 request := client.api.DAGApi.PostClearTaskInstances(ctx, dagID) // ClearTaskInstances 데이터 설정 - request = request.ClearTaskInstances(test) + request = request.ClearTaskInstances(clearTask) // 요청 실행 logs, _, err := client.api.DAGApi.PostClearTaskInstancesExecute(request) @@ -258,5 +277,21 @@ func (client *client) GetImportErrors() (airflow.ImportErrorCollection, error) { "AIRFLOW: Error occurred while getting import dag errors. (Error: "+err.Error()+").") } + return logs, nil +} + + +func (client *client) PatchDag(dagID string, dagBody airflow.DAG) (airflow.DAG, error){ + ctx, cancel := Context() + defer cancel() + + // TaskInstanceApi 인스턴스를 사용하여 로그 요청 + logs,_,err := client.api.DAGApi.PatchDag(ctx, dagID).DAG(dagBody).Execute() + logger.Println(logger.INFO, false,logs) + if err != nil { + logger.Println(logger.ERROR, false, + "AIRFLOW: Error occurred while getting import dag errors. (Error: "+err.Error()+").") + } + return logs, nil } \ No newline at end of file diff --git a/pkg/api/rest/common/url.go b/pkg/api/rest/common/url.go new file mode 100644 index 0000000..65f730b --- /dev/null +++ b/pkg/api/rest/common/url.go @@ -0,0 +1,22 @@ +package common + +import ( + "fmt" + "net/url" +) + +func UrlDecode(text string) (string) { + decodedStr, err := url.QueryUnescape(text) + if err != nil { + fmt.Println("Error decoding URL:", err) + return err.Error() + } else { + return decodedStr + } +} + +func UrlEncode(text string) (string) { + encodedStr := url.QueryEscape(text) + + return encodedStr +} \ No newline at end of file diff --git a/pkg/api/rest/controller/workflow.go b/pkg/api/rest/controller/workflow.go index f7d3534..ca4349b 100644 --- a/pkg/api/rest/controller/workflow.go +++ b/pkg/api/rest/controller/workflow.go @@ -2,6 +2,7 @@ package controller import ( "errors" + "fmt" "net/http" "reflect" "strconv" @@ -475,10 +476,10 @@ func UpdateWorkflow(c echo.Context) error { return common.ReturnErrorMsg(c, err.Error()) } - err = airflow.Client.DeleteDAG(oldWorkflow.ID, true) - if err != nil { - return common.ReturnErrorMsg(c, "Failed to update the workflow. (Error:"+err.Error()+")") - } + // err = airflow.Client.DeleteDAG(oldWorkflow.ID, true) + // if err != nil { + // return common.ReturnErrorMsg(c, "Failed to update the workflow. (Error:"+err.Error()+")") + // } err = airflow.Client.CreateDAG(oldWorkflow) if err != nil { @@ -978,7 +979,8 @@ func GetWorkflowRuns(c echo.Context) error { // @Accept json // @Produce json // @Param wfId path string true "ID of the workflow." -// @Success 200 {object} model.Task "Successfully get the taskInstances." +// @Param wfRunId path string true "ID of the workflow." +// @Success 200 {object} model.TaskInstance "Successfully get the taskInstances." // @Failure 400 {object} common.ErrorResponse "Sent bad request." // @Failure 500 {object} common.ErrorResponse "Failed to get the taskInstances." // @Router /workflow/{wfId}/workflowRun/{wfRunId}/taskInstances [get] @@ -991,14 +993,50 @@ func GetTaskInstances(c echo.Context) error { if wfRunId == "" { return common.ReturnErrorMsg(c, "Please provide the wfRunId.") } - runList, err := airflow.Client.GetTaskInstances(wfId, wfRunId) + runList, err := airflow.Client.GetTaskInstances(common.UrlDecode(wfId),common.UrlDecode(wfRunId) ) if err != nil { return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) } + var taskInstances []model.TaskInstance; + layout := time.RFC3339Nano - return c.JSONPretty(http.StatusOK, runList, " ") + for _, taskInstance := range *runList.TaskInstances { + taskDBInfo, err := dao.TaskGetByWorkflowIDAndName(taskInstance.GetDagId(),taskInstance.GetTaskId()) + if err != nil { + return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) + } + taskId := &taskDBInfo.ID + executionDate, err:= time.Parse(layout, taskInstance.GetExecutionDate()) + if err != nil { + fmt.Println("Error parsing execution date:", err) + continue + } + startDate, err:= time.Parse(layout, taskInstance.GetExecutionDate()) + if err != nil { + fmt.Println("Error parsing execution date:", err) + continue + } + endDate, err:= time.Parse(layout, taskInstance.GetExecutionDate()) + if err != nil { + fmt.Println("Error parsing execution date:", err) + continue + } + taskInfo := model.TaskInstance { + WorkflowID: taskInstance.DagId, + WorkflowRunID: taskInstance.GetDagRunId(), + TaskID: *taskId, + TaskName: taskInstance.GetTaskId(), + State: string(taskInstance.GetState()), + ExecutionDate: executionDate, + StartDate: startDate, + EndDate: endDate, + DurationDate: float64(taskInstance.GetDuration()), + TryNumber: int(taskInstance.GetTryNumber()), + } + taskInstances = append(taskInstances,taskInfo ) +} + return c.JSONPretty(http.StatusOK, taskInstances, " ") } - // taskInstances godoc // @@ -1008,23 +1046,33 @@ func GetTaskInstances(c echo.Context) error { // @Accept json // @Produce json // @Param wfId path string true "ID of the workflow." -// @Param requestBody body airflow.ClearTaskInstances true "Clear TaskInstance" +// @Param wfRunId path string true "ID of the wfRunId." +// @Param taskId path string true "ID of the taskId." // @Success 200 {object} airflow.TaskInstanceReferenceCollection "Successfully clear the taskInstances." // @Failure 400 {object} common.ErrorResponse "Sent bad request." // @Failure 500 {object} common.ErrorResponse "Failed to clear the taskInstances." -// @Router /workflow/{wfId}/clear [post] +// @Router /workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/clear [post] func ClearTaskInstances(c echo.Context) error { wfId := c.Param("wfId") if wfId == "" { return common.ReturnErrorMsg(c, "Please provide the wfId.") } - var clearTaskInstances interface{} - if err := c.Bind(&clearTaskInstances); err != nil { - logger.Println(logger.ERROR, false, - "AIRFLOW: Invalid request body. (Error: " + err.Error() + ").") + wfRunId := c.Param("wfRunId") + if wfRunId == "" { + return common.ReturnErrorMsg(c, "Please provide the wfRunId.") + } + taskId := c.Param("taskId") + if taskId == "" { + return common.ReturnErrorMsg(c, "Please provide the taskId.") + } else { + taskDBInfo, err := dao.TaskGet(taskId) + if err != nil { + return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) + } + taskId = taskDBInfo.Name } - runList, err := airflow.Client.ClearTaskInstance(wfId,clearTaskInstances ) + runList, err := airflow.Client.ClearTaskInstance(wfId, common.UrlDecode(wfRunId),taskId ) if err != nil { return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) } diff --git a/pkg/api/rest/docs/docs.go b/pkg/api/rest/docs/docs.go index 629836f..2b3b49d 100644 --- a/pkg/api/rest/docs/docs.go +++ b/pkg/api/rest/docs/docs.go @@ -803,59 +803,6 @@ const docTemplate = `{ } } }, - "/workflow/{wfId}/clear": { - "post": { - "description": "Clear the task Instance.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "[Workflow]" - ], - "summary": "Clear taskInstances", - "parameters": [ - { - "type": "string", - "description": "ID of the workflow.", - "name": "wfId", - "in": "path", - "required": true - }, - { - "description": "Clear TaskInstance", - "name": "requestBody", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/airflow.ClearTaskInstances" - } - } - ], - "responses": { - "200": { - "description": "Successfully clear the taskInstances.", - "schema": { - "$ref": "#/definitions/airflow.TaskInstanceReferenceCollection" - } - }, - "400": { - "description": "Sent bad request.", - "schema": { - "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" - } - }, - "500": { - "description": "Failed to clear the taskInstances.", - "schema": { - "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" - } - } - } - } - }, "/workflow/{wfId}/run": { "post": { "description": "Run the workflow.", @@ -1259,6 +1206,64 @@ const docTemplate = `{ } } }, + "/workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/clear": { + "post": { + "description": "Clear the task Instance.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Workflow]" + ], + "summary": "Clear taskInstances", + "parameters": [ + { + "type": "string", + "description": "ID of the workflow.", + "name": "wfId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "ID of the wfRunId.", + "name": "wfRunId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "ID of the taskId.", + "name": "taskId", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Successfully clear the taskInstances.", + "schema": { + "$ref": "#/definitions/airflow.TaskInstanceReferenceCollection" + } + }, + "400": { + "description": "Sent bad request.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + }, + "500": { + "description": "Failed to clear the taskInstances.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + } + } + } + }, "/workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/taskTryNum/{taskTyNum}/logs": { "get": { "description": "Get the task Logs.", @@ -1344,13 +1349,20 @@ const docTemplate = `{ "name": "wfId", "in": "path", "required": true + }, + { + "type": "string", + "description": "ID of the workflow.", + "name": "wfRunId", + "in": "path", + "required": true } ], "responses": { "200": { "description": "Successfully get the taskInstances.", "schema": { - "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task" + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.TaskInstance" } }, "400": { @@ -1519,74 +1531,6 @@ const docTemplate = `{ } }, "definitions": { - "airflow.ClearTaskInstances": { - "type": "object", - "properties": { - "dag_run_id": { - "description": "The DagRun ID for this task instance", - "allOf": [ - { - "$ref": "#/definitions/airflow.NullableString" - } - ] - }, - "dry_run": { - "description": "If set, don't actually run this operation. The response will contain a list of task instances planned to be cleaned, but not modified in any way.", - "type": "boolean" - }, - "end_date": { - "description": "The maximum execution date to clear.", - "type": "string" - }, - "include_downstream": { - "description": "If set to true, downstream tasks are also affected.", - "type": "boolean" - }, - "include_future": { - "description": "If set to True, also tasks from future DAG Runs are affected.", - "type": "boolean" - }, - "include_parentdag": { - "description": "Clear tasks in the parent dag of the subdag.", - "type": "boolean" - }, - "include_past": { - "description": "If set to True, also tasks from past DAG Runs are affected.", - "type": "boolean" - }, - "include_subdags": { - "description": "Clear tasks in subdags and clear external tasks indicated by ExternalTaskMarker.", - "type": "boolean" - }, - "include_upstream": { - "description": "If set to true, upstream tasks are also affected.", - "type": "boolean" - }, - "only_failed": { - "description": "Only clear failed tasks.", - "type": "boolean" - }, - "only_running": { - "description": "Only clear running tasks.", - "type": "boolean" - }, - "reset_dag_runs": { - "description": "Set state of DAG runs to RUNNING.", - "type": "boolean" - }, - "start_date": { - "description": "The minimum execution date to clear.", - "type": "string" - }, - "task_ids": { - "description": "A list of task ids to clear. *New in version 2.1.0*", - "type": "array", - "items": { - "type": "string" - } - } - } - }, "airflow.ImportError": { "type": "object", "properties": { @@ -1623,9 +1567,6 @@ const docTemplate = `{ } } }, - "airflow.NullableString": { - "type": "object" - }, "airflow.TaskInstanceReference": { "type": "object", "properties": { @@ -2000,6 +1941,41 @@ const docTemplate = `{ } } }, + "github_com_cloud-barista_cm-cicada_pkg_api_rest_model.TaskInstance": { + "type": "object", + "properties": { + "duration_date": { + "type": "number" + }, + "end_date": { + "type": "string" + }, + "execution_date": { + "type": "string" + }, + "start_date": { + "type": "string" + }, + "state": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "task_name": { + "type": "string" + }, + "try_number": { + "type": "integer" + }, + "workflow_id": { + "type": "string" + }, + "workflow_run_id": { + "type": "string" + } + } + }, "github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Workflow": { "type": "object", "required": [ diff --git a/pkg/api/rest/docs/swagger.json b/pkg/api/rest/docs/swagger.json index 54a23c5..e959967 100644 --- a/pkg/api/rest/docs/swagger.json +++ b/pkg/api/rest/docs/swagger.json @@ -796,59 +796,6 @@ } } }, - "/workflow/{wfId}/clear": { - "post": { - "description": "Clear the task Instance.", - "consumes": [ - "application/json" - ], - "produces": [ - "application/json" - ], - "tags": [ - "[Workflow]" - ], - "summary": "Clear taskInstances", - "parameters": [ - { - "type": "string", - "description": "ID of the workflow.", - "name": "wfId", - "in": "path", - "required": true - }, - { - "description": "Clear TaskInstance", - "name": "requestBody", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/airflow.ClearTaskInstances" - } - } - ], - "responses": { - "200": { - "description": "Successfully clear the taskInstances.", - "schema": { - "$ref": "#/definitions/airflow.TaskInstanceReferenceCollection" - } - }, - "400": { - "description": "Sent bad request.", - "schema": { - "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" - } - }, - "500": { - "description": "Failed to clear the taskInstances.", - "schema": { - "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" - } - } - } - } - }, "/workflow/{wfId}/run": { "post": { "description": "Run the workflow.", @@ -1252,6 +1199,64 @@ } } }, + "/workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/clear": { + "post": { + "description": "Clear the task Instance.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "[Workflow]" + ], + "summary": "Clear taskInstances", + "parameters": [ + { + "type": "string", + "description": "ID of the workflow.", + "name": "wfId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "ID of the wfRunId.", + "name": "wfRunId", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "ID of the taskId.", + "name": "taskId", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "Successfully clear the taskInstances.", + "schema": { + "$ref": "#/definitions/airflow.TaskInstanceReferenceCollection" + } + }, + "400": { + "description": "Sent bad request.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + }, + "500": { + "description": "Failed to clear the taskInstances.", + "schema": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse" + } + } + } + } + }, "/workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/taskTryNum/{taskTyNum}/logs": { "get": { "description": "Get the task Logs.", @@ -1337,13 +1342,20 @@ "name": "wfId", "in": "path", "required": true + }, + { + "type": "string", + "description": "ID of the workflow.", + "name": "wfRunId", + "in": "path", + "required": true } ], "responses": { "200": { "description": "Successfully get the taskInstances.", "schema": { - "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task" + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.TaskInstance" } }, "400": { @@ -1512,74 +1524,6 @@ } }, "definitions": { - "airflow.ClearTaskInstances": { - "type": "object", - "properties": { - "dag_run_id": { - "description": "The DagRun ID for this task instance", - "allOf": [ - { - "$ref": "#/definitions/airflow.NullableString" - } - ] - }, - "dry_run": { - "description": "If set, don't actually run this operation. The response will contain a list of task instances planned to be cleaned, but not modified in any way.", - "type": "boolean" - }, - "end_date": { - "description": "The maximum execution date to clear.", - "type": "string" - }, - "include_downstream": { - "description": "If set to true, downstream tasks are also affected.", - "type": "boolean" - }, - "include_future": { - "description": "If set to True, also tasks from future DAG Runs are affected.", - "type": "boolean" - }, - "include_parentdag": { - "description": "Clear tasks in the parent dag of the subdag.", - "type": "boolean" - }, - "include_past": { - "description": "If set to True, also tasks from past DAG Runs are affected.", - "type": "boolean" - }, - "include_subdags": { - "description": "Clear tasks in subdags and clear external tasks indicated by ExternalTaskMarker.", - "type": "boolean" - }, - "include_upstream": { - "description": "If set to true, upstream tasks are also affected.", - "type": "boolean" - }, - "only_failed": { - "description": "Only clear failed tasks.", - "type": "boolean" - }, - "only_running": { - "description": "Only clear running tasks.", - "type": "boolean" - }, - "reset_dag_runs": { - "description": "Set state of DAG runs to RUNNING.", - "type": "boolean" - }, - "start_date": { - "description": "The minimum execution date to clear.", - "type": "string" - }, - "task_ids": { - "description": "A list of task ids to clear. *New in version 2.1.0*", - "type": "array", - "items": { - "type": "string" - } - } - } - }, "airflow.ImportError": { "type": "object", "properties": { @@ -1616,9 +1560,6 @@ } } }, - "airflow.NullableString": { - "type": "object" - }, "airflow.TaskInstanceReference": { "type": "object", "properties": { @@ -1993,6 +1934,41 @@ } } }, + "github_com_cloud-barista_cm-cicada_pkg_api_rest_model.TaskInstance": { + "type": "object", + "properties": { + "duration_date": { + "type": "number" + }, + "end_date": { + "type": "string" + }, + "execution_date": { + "type": "string" + }, + "start_date": { + "type": "string" + }, + "state": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "task_name": { + "type": "string" + }, + "try_number": { + "type": "integer" + }, + "workflow_id": { + "type": "string" + }, + "workflow_run_id": { + "type": "string" + } + } + }, "github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Workflow": { "type": "object", "required": [ diff --git a/pkg/api/rest/docs/swagger.yaml b/pkg/api/rest/docs/swagger.yaml index 9d3ce7d..b91c665 100644 --- a/pkg/api/rest/docs/swagger.yaml +++ b/pkg/api/rest/docs/swagger.yaml @@ -1,56 +1,5 @@ basePath: /cicada definitions: - airflow.ClearTaskInstances: - properties: - dag_run_id: - allOf: - - $ref: '#/definitions/airflow.NullableString' - description: The DagRun ID for this task instance - dry_run: - description: If set, don't actually run this operation. The response will - contain a list of task instances planned to be cleaned, but not modified - in any way. - type: boolean - end_date: - description: The maximum execution date to clear. - type: string - include_downstream: - description: If set to true, downstream tasks are also affected. - type: boolean - include_future: - description: If set to True, also tasks from future DAG Runs are affected. - type: boolean - include_parentdag: - description: Clear tasks in the parent dag of the subdag. - type: boolean - include_past: - description: If set to True, also tasks from past DAG Runs are affected. - type: boolean - include_subdags: - description: Clear tasks in subdags and clear external tasks indicated by - ExternalTaskMarker. - type: boolean - include_upstream: - description: If set to true, upstream tasks are also affected. - type: boolean - only_failed: - description: Only clear failed tasks. - type: boolean - only_running: - description: Only clear running tasks. - type: boolean - reset_dag_runs: - description: Set state of DAG runs to RUNNING. - type: boolean - start_date: - description: The minimum execution date to clear. - type: string - task_ids: - description: A list of task ids to clear. *New in version 2.1.0* - items: - type: string - type: array - type: object airflow.ImportError: properties: filename: @@ -76,8 +25,6 @@ definitions: description: Count of objects in the current result set. type: integer type: object - airflow.NullableString: - type: object airflow.TaskInstanceReference: properties: dag_id: @@ -330,6 +277,29 @@ definitions: - name - tasks type: object + github_com_cloud-barista_cm-cicada_pkg_api_rest_model.TaskInstance: + properties: + duration_date: + type: number + end_date: + type: string + execution_date: + type: string + start_date: + type: string + state: + type: string + task_id: + type: string + task_name: + type: string + try_number: + type: integer + workflow_id: + type: string + workflow_run_id: + type: string + type: object github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Workflow: properties: created_at: @@ -896,41 +866,6 @@ paths: summary: Update Workflow tags: - '[Workflow]' - /workflow/{wfId}/clear: - post: - consumes: - - application/json - description: Clear the task Instance. - parameters: - - description: ID of the workflow. - in: path - name: wfId - required: true - type: string - - description: Clear TaskInstance - in: body - name: requestBody - required: true - schema: - $ref: '#/definitions/airflow.ClearTaskInstances' - produces: - - application/json - responses: - "200": - description: Successfully clear the taskInstances. - schema: - $ref: '#/definitions/airflow.TaskInstanceReferenceCollection' - "400": - description: Sent bad request. - schema: - $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' - "500": - description: Failed to clear the taskInstances. - schema: - $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' - summary: Clear taskInstances - tags: - - '[Workflow]' /workflow/{wfId}/run: post: consumes: @@ -1201,6 +1136,45 @@ paths: summary: Get Task from Task Group tags: - '[Workflow]' + /workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/clear: + post: + consumes: + - application/json + description: Clear the task Instance. + parameters: + - description: ID of the workflow. + in: path + name: wfId + required: true + type: string + - description: ID of the wfRunId. + in: path + name: wfRunId + required: true + type: string + - description: ID of the taskId. + in: path + name: taskId + required: true + type: string + produces: + - application/json + responses: + "200": + description: Successfully clear the taskInstances. + schema: + $ref: '#/definitions/airflow.TaskInstanceReferenceCollection' + "400": + description: Sent bad request. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' + "500": + description: Failed to clear the taskInstances. + schema: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse' + summary: Clear taskInstances + tags: + - '[Workflow]' /workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/taskTryNum/{taskTyNum}/logs: get: consumes: @@ -1256,13 +1230,18 @@ paths: name: wfId required: true type: string + - description: ID of the workflow. + in: path + name: wfRunId + required: true + type: string produces: - application/json responses: "200": description: Successfully get the taskInstances. schema: - $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task' + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.TaskInstance' "400": description: Sent bad request. schema: diff --git a/pkg/api/rest/model/workflow.go b/pkg/api/rest/model/workflow.go index a921cf8..255716c 100644 --- a/pkg/api/rest/model/workflow.go +++ b/pkg/api/rest/model/workflow.go @@ -121,6 +121,19 @@ type WorkflowRun struct { Note string `json:"note,omitempty"` } +type TaskInstance struct { + WorkflowRunID string `json:"workflow_run_id,omitempty"` + WorkflowID *string `json:"workflow_id,omitempty"` + TaskID string `json:"task_id,omitempty"` + TaskName string `json:"task_name,omitempty"` + State string `json:"state,omitempty"` + StartDate time.Time `json:"start_date,omitempty"` + EndDate time.Time `json:"end_date,omitempty"` + DurationDate float64 `json:"duration_date"` + ExecutionDate time.Time `json:"execution_date,omitempty"` + TryNumber int `json:"try_number"` +} + func (d Data) Value() (driver.Value, error) { return json.Marshal(d) } diff --git a/pkg/api/rest/route/workflow.go b/pkg/api/rest/route/workflow.go index 77b52ea..dffbcc4 100644 --- a/pkg/api/rest/route/workflow.go +++ b/pkg/api/rest/route/workflow.go @@ -31,7 +31,7 @@ func Workflow(e *echo.Echo) { e.GET("/"+strings.ToLower(common.ShortModuleName)+"/workflow/:wfId/workflowRun/:wfRunId/task/:taskId/taskTryNum/:taskTyNum/logs", controller.GetTaskLogs) e.GET("/"+strings.ToLower(common.ShortModuleName)+"/workflow/:wfId/runs", controller.GetWorkflowRuns) e.GET("/"+strings.ToLower(common.ShortModuleName)+"/workflow/:wfId/workflowRun/:wfRunId/taskInstances", controller.GetTaskInstances) - e.POST("/"+strings.ToLower(common.ShortModuleName)+"/workflow/:wfId/clear", controller.ClearTaskInstances) + e.POST("/"+strings.ToLower(common.ShortModuleName)+"/workflow/:wfId/workflowRun/:wfRunId/task/:taskId/clear", controller.ClearTaskInstances) e.GET("/"+strings.ToLower(common.ShortModuleName)+"/eventlogs", controller.GetEventLogs) e.GET("/"+strings.ToLower(common.ShortModuleName)+"/importErrors", controller.GetImportErrors) } From 522e6292457b66e21ae5aa98b30752a79d09d044 Mon Sep 17 00:00:00 2001 From: yby654 Date: Fri, 13 Sep 2024 11:48:14 +0900 Subject: [PATCH 07/12] =?UTF-8?q?get=20task=20log=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/airflow/workflow.go | 4 --- pkg/api/rest/controller/workflow.go | 33 ++++++++++++++---- pkg/api/rest/docs/docs.go | 53 +++++++++++++++-------------- pkg/api/rest/docs/swagger.json | 53 +++++++++++++++-------------- pkg/api/rest/docs/swagger.yaml | 40 +++++++++++----------- pkg/api/rest/model/workflow.go | 11 ++++++ 6 files changed, 115 insertions(+), 79 deletions(-) diff --git a/lib/airflow/workflow.go b/lib/airflow/workflow.go index 457dd3e..5e472d2 100644 --- a/lib/airflow/workflow.go +++ b/lib/airflow/workflow.go @@ -162,10 +162,6 @@ func (client *client) GetTaskInstances(dagID string, dagRunId string) (airflow.T }() ctx, cancel := Context() defer cancel() - // dagID = "a7192143-6aa7-4dee-856c-366caa91d126" - // dagRunId = "manual__2024-09-10T07:20:26.072377+00:00" - // req := client.api.TaskInstanceApi.GetTaskInstances(ctx, dagID, dagRunId) - // resp, _, err := client.api.TaskInstanceApi.GetTaskInstancesExecute(req) resp, http, err := client.api.TaskInstanceApi.GetTaskInstances(ctx, dagID, dagRunId).Execute() fmt.Println("test : ", http) if err != nil { diff --git a/pkg/api/rest/controller/workflow.go b/pkg/api/rest/controller/workflow.go index ca4349b..da6101d 100644 --- a/pkg/api/rest/controller/workflow.go +++ b/pkg/api/rest/controller/workflow.go @@ -892,7 +892,7 @@ func GetTaskDirectly(c echo.Context) error { // @Param wfRunId path string true "ID of the workflowRunId." // @Param taskId path string true "ID of the task." // @Param taskTyNum path string true "ID of the taskTryNum." -// @Success 200 {object} model.Task "Successfully get the task Logs." +// @Success 200 {object} airflow.InlineResponse200 "Successfully get the task Logs." // @Failure 400 {object} common.ErrorResponse "Sent bad request." // @Failure 500 {object} common.ErrorResponse "Failed to get the task Logs." // @Router /workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/taskTryNum/{taskTyNum}/logs [get] @@ -910,6 +910,10 @@ func GetTaskLogs(c echo.Context) error { if taskId == "" { return common.ReturnErrorMsg(c, "Please provide the taskId.") } + taskInfo, err := dao.TaskGet(taskId) + if err != nil { + return common.ReturnErrorMsg(c, "Invalid get tasK from taskId.") + } taskTyNum := c.Param("taskTyNum") if taskTyNum == "" { @@ -919,7 +923,7 @@ func GetTaskLogs(c echo.Context) error { if err != nil { return common.ReturnErrorMsg(c, "Invalid taskTryNum format.") } - logs, err := airflow.Client.GetTaskLogs(wfId, wfRunId, taskId, taskTyNumToInt) + logs, err := airflow.Client.GetTaskLogs(wfId, common.UrlDecode(wfRunId), taskInfo.Name, taskTyNumToInt) if err != nil { return common.ReturnErrorMsg(c, "Failed to get the workflow logs: " + err.Error()) } @@ -1048,7 +1052,7 @@ func GetTaskInstances(c echo.Context) error { // @Param wfId path string true "ID of the workflow." // @Param wfRunId path string true "ID of the wfRunId." // @Param taskId path string true "ID of the taskId." -// @Success 200 {object} airflow.TaskInstanceReferenceCollection "Successfully clear the taskInstances." +// @Success 200 {object} model.TaskInstanceReference "Successfully clear the taskInstances." // @Failure 400 {object} common.ErrorResponse "Sent bad request." // @Failure 500 {object} common.ErrorResponse "Failed to clear the taskInstances." // @Router /workflow/{wfId}/workflowRun/{wfRunId}/task/{taskId}/clear [post] @@ -1071,13 +1075,30 @@ func ClearTaskInstances(c echo.Context) error { } taskId = taskDBInfo.Name } - - runList, err := airflow.Client.ClearTaskInstance(wfId, common.UrlDecode(wfRunId),taskId ) + var TaskInstanceReferences []model.TaskInstanceReference; + clearList, err := airflow.Client.ClearTaskInstance(wfId, common.UrlDecode(wfRunId),taskId ) if err != nil { return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) } - return c.JSONPretty(http.StatusOK, runList, " ") + for _, taskInstance := range *clearList.TaskInstances { + taskDBInfo, err := dao.TaskGetByWorkflowIDAndName(taskInstance.GetDagId(),taskInstance.GetTaskId()) + if err != nil { + return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) + } + taskId := &taskDBInfo.ID + taskInfo := model.TaskInstanceReference { + WorkflowID: taskInstance.DagId, + WorkflowRunID: taskInstance.DagRunId, + TaskId: taskId, + TaskName: taskInstance.GetTaskId(), + ExecutionDate: taskInstance.ExecutionDate, + } + TaskInstanceReferences = append(TaskInstanceReferences,taskInfo ) + } + + + return c.JSONPretty(http.StatusOK, TaskInstanceReferences, " ") } // Eventlogs godoc diff --git a/pkg/api/rest/docs/docs.go b/pkg/api/rest/docs/docs.go index 2b3b49d..de177ae 100644 --- a/pkg/api/rest/docs/docs.go +++ b/pkg/api/rest/docs/docs.go @@ -1246,7 +1246,7 @@ const docTemplate = `{ "200": { "description": "Successfully clear the taskInstances.", "schema": { - "$ref": "#/definitions/airflow.TaskInstanceReferenceCollection" + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.TaskInstanceReference" } }, "400": { @@ -1311,7 +1311,7 @@ const docTemplate = `{ "200": { "description": "Successfully get the task Logs.", "schema": { - "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task" + "$ref": "#/definitions/airflow.InlineResponse200" } }, "400": { @@ -1567,37 +1567,17 @@ const docTemplate = `{ } } }, - "airflow.TaskInstanceReference": { + "airflow.InlineResponse200": { "type": "object", "properties": { - "dag_id": { - "description": "The DAG ID.", - "type": "string" - }, - "dag_run_id": { - "description": "The DAG run ID.", + "content": { "type": "string" }, - "execution_date": { - "type": "string" - }, - "task_id": { - "description": "The task ID.", + "continuation_token": { "type": "string" } } }, - "airflow.TaskInstanceReferenceCollection": { - "type": "object", - "properties": { - "task_instances": { - "type": "array", - "items": { - "$ref": "#/definitions/airflow.TaskInstanceReference" - } - } - } - }, "github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse": { "type": "object", "properties": { @@ -1976,6 +1956,29 @@ const docTemplate = `{ } } }, + "github_com_cloud-barista_cm-cicada_pkg_api_rest_model.TaskInstanceReference": { + "type": "object", + "properties": { + "execution_date": { + "type": "string" + }, + "task_id": { + "description": "The task ID.", + "type": "string" + }, + "task_name": { + "type": "string" + }, + "workflow_id": { + "description": "The DAG ID.", + "type": "string" + }, + "workflow_run_id": { + "description": "The DAG run ID.", + "type": "string" + } + } + }, "github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Workflow": { "type": "object", "required": [ diff --git a/pkg/api/rest/docs/swagger.json b/pkg/api/rest/docs/swagger.json index e959967..e7a38a9 100644 --- a/pkg/api/rest/docs/swagger.json +++ b/pkg/api/rest/docs/swagger.json @@ -1239,7 +1239,7 @@ "200": { "description": "Successfully clear the taskInstances.", "schema": { - "$ref": "#/definitions/airflow.TaskInstanceReferenceCollection" + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.TaskInstanceReference" } }, "400": { @@ -1304,7 +1304,7 @@ "200": { "description": "Successfully get the task Logs.", "schema": { - "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task" + "$ref": "#/definitions/airflow.InlineResponse200" } }, "400": { @@ -1560,37 +1560,17 @@ } } }, - "airflow.TaskInstanceReference": { + "airflow.InlineResponse200": { "type": "object", "properties": { - "dag_id": { - "description": "The DAG ID.", - "type": "string" - }, - "dag_run_id": { - "description": "The DAG run ID.", + "content": { "type": "string" }, - "execution_date": { - "type": "string" - }, - "task_id": { - "description": "The task ID.", + "continuation_token": { "type": "string" } } }, - "airflow.TaskInstanceReferenceCollection": { - "type": "object", - "properties": { - "task_instances": { - "type": "array", - "items": { - "$ref": "#/definitions/airflow.TaskInstanceReference" - } - } - } - }, "github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse": { "type": "object", "properties": { @@ -1969,6 +1949,29 @@ } } }, + "github_com_cloud-barista_cm-cicada_pkg_api_rest_model.TaskInstanceReference": { + "type": "object", + "properties": { + "execution_date": { + "type": "string" + }, + "task_id": { + "description": "The task ID.", + "type": "string" + }, + "task_name": { + "type": "string" + }, + "workflow_id": { + "description": "The DAG ID.", + "type": "string" + }, + "workflow_run_id": { + "description": "The DAG run ID.", + "type": "string" + } + } + }, "github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Workflow": { "type": "object", "required": [ diff --git a/pkg/api/rest/docs/swagger.yaml b/pkg/api/rest/docs/swagger.yaml index b91c665..f755989 100644 --- a/pkg/api/rest/docs/swagger.yaml +++ b/pkg/api/rest/docs/swagger.yaml @@ -25,27 +25,13 @@ definitions: description: Count of objects in the current result set. type: integer type: object - airflow.TaskInstanceReference: + airflow.InlineResponse200: properties: - dag_id: - description: The DAG ID. - type: string - dag_run_id: - description: The DAG run ID. + content: type: string - execution_date: - type: string - task_id: - description: The task ID. + continuation_token: type: string type: object - airflow.TaskInstanceReferenceCollection: - properties: - task_instances: - items: - $ref: '#/definitions/airflow.TaskInstanceReference' - type: array - type: object github_com_cloud-barista_cm-cicada_pkg_api_rest_common.ErrorResponse: properties: error: @@ -300,6 +286,22 @@ definitions: workflow_run_id: type: string type: object + github_com_cloud-barista_cm-cicada_pkg_api_rest_model.TaskInstanceReference: + properties: + execution_date: + type: string + task_id: + description: The task ID. + type: string + task_name: + type: string + workflow_id: + description: The DAG ID. + type: string + workflow_run_id: + description: The DAG run ID. + type: string + type: object github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Workflow: properties: created_at: @@ -1163,7 +1165,7 @@ paths: "200": description: Successfully clear the taskInstances. schema: - $ref: '#/definitions/airflow.TaskInstanceReferenceCollection' + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.TaskInstanceReference' "400": description: Sent bad request. schema: @@ -1207,7 +1209,7 @@ paths: "200": description: Successfully get the task Logs. schema: - $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Task' + $ref: '#/definitions/airflow.InlineResponse200' "400": description: Sent bad request. schema: diff --git a/pkg/api/rest/model/workflow.go b/pkg/api/rest/model/workflow.go index 255716c..13a4974 100644 --- a/pkg/api/rest/model/workflow.go +++ b/pkg/api/rest/model/workflow.go @@ -134,6 +134,17 @@ type TaskInstance struct { TryNumber int `json:"try_number"` } +type TaskInstanceReference struct { + // The task ID. + TaskId *string `json:"task_id,omitempty"` + TaskName string `json:"task_name,omitempty"` + // The DAG ID. + WorkflowID *string `json:"workflow_id,omitempty"` + // The DAG run ID. + WorkflowRunID *string `json:"workflow_run_id,omitempty"` + ExecutionDate *string `json:"execution_date,omitempty"` +} + func (d Data) Value() (driver.Value, error) { return json.Marshal(d) } From fd3e5a7a8945ca221c6800aa404bd1a242bdf0f2 Mon Sep 17 00:00:00 2001 From: yby654 Date: Wed, 25 Sep 2024 12:08:06 +0900 Subject: [PATCH 08/12] =?UTF-8?q?workflow=20=EA=B8=B0=EB=8A=A5=20=EC=B6=94?= =?UTF-8?q?=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _airflow/airflow-home/dags/test.py | 206 ++++++++++++++--------------- lib/airflow/workflow.go | 15 ++- 2 files changed, 114 insertions(+), 107 deletions(-) diff --git a/_airflow/airflow-home/dags/test.py b/_airflow/airflow-home/dags/test.py index 79a5dfa..b20cee5 100644 --- a/_airflow/airflow-home/dags/test.py +++ b/_airflow/airflow-home/dags/test.py @@ -1,107 +1,107 @@ -from datetime import datetime, timedelta -from textwrap import dedent - -from airflow import DAG - -# Operators; we need this to operate! -from airflow.operators.bash import BashOperator -from airflow.operators.dummy import DummyOperator -from airflow.operators.python import BranchPythonOperator -from airflow.operators.python_operator import PythonOperator -from airflow.utils.trigger_rule import TriggerRule - -default_args = { - 'owner': 'owner-name', - 'depends_on_past': False, - 'email': ['your-email@g.com'], - 'email_on_failure': False, - 'email_on_retry': False, - 'retries': 1, - 'retry_delay': timedelta(minutes=15), -} - -dag_args = dict( - dag_id="tutorial-python-op", - default_args=default_args, - description='tutorial DAG python', - schedule_interval=timedelta(minutes=50), - start_date=datetime(2022, 2, 1), - tags=['example-sj'], -) - -def random_branch_path(): - # 필요 라이브러리는 여기서 import - # https://airflow.apache.org/docs/apache-airflow/stable/best-practices.html#writing-a-dag 참고 - from random import randint - - return "cal_a_id" if randint(1, 2) == 1 else "cal_m_id" - -def calc_add(x, y, **kwargs): - result = x + y - print("x + y : ", result) - kwargs['task_instance'].xcom_push(key='calc_result', value=result) - return "calc add" - -def calc_mul(x, y, **kwargs): - result = x * y - print("x * y : ", result) - kwargs['task_instance'].xcom_push(key='calc_result', value=result) - return "calc mul" - -def print_result(**kwargs): - r = kwargs["task_instance"].xcom_pull(key='calc_result') - print("message : ", r) - print("*"*100) - print(kwargs) - -def end_seq(): - print("end") - -with DAG( **dag_args ) as dag: - start = BashOperator( - task_id='start', - bash_command='echo "start!"', - ) - - branch = BranchPythonOperator( - task_id='branch', - python_callable=random_branch_path, - ) +# from datetime import datetime, timedelta +# from textwrap import dedent + +# from airflow import DAG + +# # Operators; we need this to operate! +# from airflow.operators.bash import BashOperator +# from airflow.operators.dummy import DummyOperator +# from airflow.operators.python import BranchPythonOperator +# from airflow.operators.python_operator import PythonOperator +# from airflow.utils.trigger_rule import TriggerRule + +# default_args = { +# 'owner': 'owner-name', +# 'depends_on_past': False, +# 'email': ['your-email@g.com'], +# 'email_on_failure': False, +# 'email_on_retry': False, +# 'retries': 1, +# 'retry_delay': timedelta(minutes=15), +# } + +# dag_args = dict( +# dag_id="tutorial-python-op", +# default_args=default_args, +# description='tutorial DAG python', +# schedule_interval=timedelta(minutes=50), +# start_date=datetime(2022, 2, 1), +# tags=['example-sj'], +# ) + +# def random_branch_path(): +# # 필요 라이브러리는 여기서 import +# # https://airflow.apache.org/docs/apache-airflow/stable/best-practices.html#writing-a-dag 참고 +# from random import randint + +# return "cal_a_id" if randint(1, 2) == 1 else "cal_m_id" + +# def calc_add(x, y, **kwargs): +# result = x + y +# print("x + y : ", result) +# kwargs['task_instance'].xcom_push(key='calc_result', value=result) +# return "calc add" + +# def calc_mul(x, y, **kwargs): +# result = x * y +# print("x * y : ", result) +# kwargs['task_instance'].xcom_push(key='calc_result', value=result) +# return "calc mul" + +# def print_result(**kwargs): +# r = kwargs["task_instance"].xcom_pull(key='calc_result') +# print("message : ", r) +# print("*"*100) +# print(kwargs) + +# def end_seq(): +# print("end") + +# with DAG( **dag_args ) as dag: +# start = BashOperator( +# task_id='start', +# bash_command='echo "start!"', +# ) + +# branch = BranchPythonOperator( +# task_id='branch', +# python_callable=random_branch_path, +# ) - calc_add = PythonOperator( - task_id='cal_a_id', - python_callable=calc_add, - op_kwargs={'x': 10, 'y':4} - ) - - calc_mul = PythonOperator( - task_id='cal_m_id', - python_callable=calc_mul, - op_kwargs={'x': 10, 'y':4} - ) - - msg = PythonOperator( - task_id='msg', - python_callable=print_result, - trigger_rule=TriggerRule.NONE_FAILED - ) - - complete_py = PythonOperator( - task_id='complete_py', - python_callable=end_seq, - trigger_rule=TriggerRule.NONE_FAILED - ) - - complete = BashOperator( - task_id='complete_bash', - depends_on_past=False, - bash_command='echo "complete~!"', - trigger_rule=TriggerRule.NONE_FAILED - ) - - - start >> branch >> calc_add >> msg >> complete_py >> complete - start >> branch >> calc_mul >> msg >> complete +# calc_add = PythonOperator( +# task_id='cal_a_id', +# python_callable=calc_add, +# op_kwargs={'x': 10, 'y':4} +# ) + +# calc_mul = PythonOperator( +# task_id='cal_m_id', +# python_callable=calc_mul, +# op_kwargs={'x': 10, 'y':4} +# ) + +# msg = PythonOperator( +# task_id='msg', +# python_callable=print_result, +# trigger_rule=TriggerRule.NONE_FAILED +# ) + +# complete_py = PythonOperator( +# task_id='complete_py', +# python_callable=end_seq, +# trigger_rule=TriggerRule.NONE_FAILED +# ) + +# complete = BashOperator( +# task_id='complete_bash', +# depends_on_past=False, +# bash_command='echo "complete~!"', +# trigger_rule=TriggerRule.NONE_FAILED +# ) + + +# start >> branch >> calc_add >> msg >> complete_py >> complete +# start >> branch >> calc_mul >> msg >> complete diff --git a/lib/airflow/workflow.go b/lib/airflow/workflow.go index 5e472d2..0ee5321 100644 --- a/lib/airflow/workflow.go +++ b/lib/airflow/workflow.go @@ -239,8 +239,6 @@ func (client *client) ClearTaskInstance(dagID string, dagRunID string, taskID st return airflow.TaskInstanceReferenceCollection{}, err } - // 결과 로그 출력 - logger.Println(logger.INFO, false, logs) return logs, nil } @@ -252,13 +250,22 @@ func (client *client) GetEventLogs(dagID string) (airflow.EventLogCollection, er }() ctx, cancel := Context() defer cancel() - req, _, err := client.api.EventLogApi.GetEventLogs(ctx).Execute() + + eventLogs, _, err := client.api.EventLogApi.GetEventLogs(ctx).Execute() if err != nil { logger.Println(logger.ERROR, false, "AIRFLOW: Error occurred while getting event logs. (Error: "+err.Error()+").") } + var filteredLogs []airflow.EventLog + for _, log := range eventLogs.GetEventLogs() { + // 로그 필터링 + if log.GetDagId() == dagID { + filteredLogs = append(filteredLogs, log) + fmt.Println("Filtered Event Log:", log) + } + } - return req, nil + return eventLogs, err } func (client *client) GetImportErrors() (airflow.ImportErrorCollection, error) { From 0f1dedf44b3e8a916fa015f93f5afb0f5a0ba05d Mon Sep 17 00:00:00 2001 From: yby654 Date: Thu, 26 Sep 2024 14:45:51 +0900 Subject: [PATCH 09/12] =?UTF-8?q?task=20log=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _airflow/airflow-home/dags/example.py | 3 +-- pkg/api/rest/controller/workflow.go | 8 ++++++-- pkg/api/rest/docs/docs.go | 5 ++++- pkg/api/rest/docs/swagger.json | 5 ++++- pkg/api/rest/docs/swagger.yaml | 4 +++- pkg/api/rest/model/workflow.go | 3 +++ 6 files changed, 21 insertions(+), 7 deletions(-) diff --git a/_airflow/airflow-home/dags/example.py b/_airflow/airflow-home/dags/example.py index e7eba78..a91eb20 100644 --- a/_airflow/airflow-home/dags/example.py +++ b/_airflow/airflow-home/dags/example.py @@ -147,8 +147,7 @@ update_tag_template_field = BashOperator( task_id="update_tag_template_field", bash_command="echo update_tag_template_field" - ) - + # Create create_tasks = [ create_entry_group, diff --git a/pkg/api/rest/controller/workflow.go b/pkg/api/rest/controller/workflow.go index da6101d..7024050 100644 --- a/pkg/api/rest/controller/workflow.go +++ b/pkg/api/rest/controller/workflow.go @@ -928,7 +928,11 @@ func GetTaskLogs(c echo.Context) error { return common.ReturnErrorMsg(c, "Failed to get the workflow logs: " + err.Error()) } - return c.JSONPretty(http.StatusOK, logs, " ") + taskLog := model.TaskLog { + Content: *logs.Content, + } + + return c.JSONPretty(http.StatusOK, taskLog, " ") } // workflowRuns godoc @@ -939,7 +943,7 @@ func GetTaskLogs(c echo.Context) error { // @Accept json // @Produce json // @Param wfId path string true "ID of the workflow." -// @Success 200 {object} model.WorkflowRun "Successfully get the workflowRuns." +// @Success 200 {object} []model.WorkflowRun "Successfully get the workflowRuns." // @Failure 400 {object} common.ErrorResponse "Sent bad request." // @Failure 500 {object} common.ErrorResponse "Failed to get the workflowRuns." // @Router /workflow/{wfId}/runs [get] diff --git a/pkg/api/rest/docs/docs.go b/pkg/api/rest/docs/docs.go index de177ae..061ed36 100644 --- a/pkg/api/rest/docs/docs.go +++ b/pkg/api/rest/docs/docs.go @@ -874,7 +874,10 @@ const docTemplate = `{ "200": { "description": "Successfully get the workflowRuns.", "schema": { - "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.WorkflowRun" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.WorkflowRun" + } } }, "400": { diff --git a/pkg/api/rest/docs/swagger.json b/pkg/api/rest/docs/swagger.json index e7a38a9..e7a2e4c 100644 --- a/pkg/api/rest/docs/swagger.json +++ b/pkg/api/rest/docs/swagger.json @@ -867,7 +867,10 @@ "200": { "description": "Successfully get the workflowRuns.", "schema": { - "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.WorkflowRun" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.WorkflowRun" + } } }, "400": { diff --git a/pkg/api/rest/docs/swagger.yaml b/pkg/api/rest/docs/swagger.yaml index f755989..cae055d 100644 --- a/pkg/api/rest/docs/swagger.yaml +++ b/pkg/api/rest/docs/swagger.yaml @@ -915,7 +915,9 @@ paths: "200": description: Successfully get the workflowRuns. schema: - $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.WorkflowRun' + items: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.WorkflowRun' + type: array "400": description: Sent bad request. schema: diff --git a/pkg/api/rest/model/workflow.go b/pkg/api/rest/model/workflow.go index 682c8fe..1ab2411 100644 --- a/pkg/api/rest/model/workflow.go +++ b/pkg/api/rest/model/workflow.go @@ -145,6 +145,9 @@ type TaskInstanceReference struct { ExecutionDate *string `json:"execution_date,omitempty"` } +type TaskLog struct { + Content string `json:"content,omitempty"` +} func (d Data) Value() (driver.Value, error) { return json.Marshal(d) } From 2d766c16c29ee7aae3190c5c62747709ca7df1b3 Mon Sep 17 00:00:00 2001 From: yby654 Date: Mon, 7 Oct 2024 15:14:26 +0900 Subject: [PATCH 10/12] =?UTF-8?q?event=20log=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _airflow/airflow-home/dags/example.py | 2 +- lib/airflow/workflow.go | 68 +++++++++++++++++++++------ pkg/api/rest/controller/workflow.go | 60 ++++++++++++++++------- pkg/api/rest/docs/docs.go | 33 ++++++++++++- pkg/api/rest/docs/swagger.json | 33 ++++++++++++- pkg/api/rest/docs/swagger.yaml | 23 ++++++++- pkg/api/rest/model/workflow.go | 11 +++++ 7 files changed, 191 insertions(+), 39 deletions(-) diff --git a/_airflow/airflow-home/dags/example.py b/_airflow/airflow-home/dags/example.py index a91eb20..0b7248e 100644 --- a/_airflow/airflow-home/dags/example.py +++ b/_airflow/airflow-home/dags/example.py @@ -147,7 +147,7 @@ update_tag_template_field = BashOperator( task_id="update_tag_template_field", bash_command="echo update_tag_template_field" - + ) # Create create_tasks = [ create_entry_group, diff --git a/lib/airflow/workflow.go b/lib/airflow/workflow.go index b18ff2a..b759aa1 100644 --- a/lib/airflow/workflow.go +++ b/lib/airflow/workflow.go @@ -1,8 +1,13 @@ package airflow import ( + "encoding/base64" + "encoding/json" "errors" "fmt" + "io/ioutil" + "net/http" + "net/url" "sync" "github.com/apache/airflow-client-go/airflow" @@ -244,30 +249,59 @@ func (client *client) ClearTaskInstance(dagID string, dagRunID string, taskID st return logs, nil } - -func (client *client) GetEventLogs(dagID string) (airflow.EventLogCollection, error) { +func (client *client) GetEventLogs(dagID string, dagRunId string, taskId string) (airflow.EventLogCollection, error) { deferFunc := callDagRequestLock(dagID) defer func() { deferFunc() }() ctx, cancel := Context() defer cancel() - - eventLogs, _, err := client.api.EventLogApi.GetEventLogs(ctx).Execute() + + localBasePath, err:=client.api.GetConfig().ServerURLWithContext(ctx, "EventLogApiService.GetEventLog") + baseURL := "http://"+client.api.GetConfig().Host+localBasePath + "/eventLogs" + queryParams := map[string]string{ + "offset": "0", + "limit": "100", + "dag_id": dagID, + "run_id": dagRunId, + "task_id": taskId, + "order_by": "-when", + "excluded_events": "gantt,landing_times,tries,duration,calendar,graph,grid,tree,tree_data", + } + query := url.Values{} + for key, value := range queryParams { + query.Add(key, value) + } + queryString := query.Encode() + fullURL := fmt.Sprintf("%s?%s", baseURL, queryString) + httpclient := client.api.GetConfig().HTTPClient + // fmt.Println(&httpclient.) + + // 요청 생성 + req, err := http.NewRequest("GET", fullURL, nil) if err != nil { - logger.Println(logger.ERROR, false, - "AIRFLOW: Error occurred while getting event logs. (Error: "+err.Error()+").") + fmt.Println("Error creating request:", err) } - var filteredLogs []airflow.EventLog - for _, log := range eventLogs.GetEventLogs() { - // 로그 필터링 - if log.GetDagId() == dagID { - filteredLogs = append(filteredLogs, log) - fmt.Println("Filtered Event Log:", log) - } + cred := ctx.Value(airflow.ContextBasicAuth).(airflow.BasicAuth) + addBasicAuth(req, cred.UserName, cred.Password) + res, err := httpclient.Do(req) + if err != nil { + fmt.Println("Error sending request:", err) + } + defer res.Body.Close() + body, err := ioutil.ReadAll(res.Body) + if err != nil { + fmt.Println("Error reading response body:", err) } + + var eventlogs airflow.EventLogCollection + err = json.Unmarshal(body, &eventlogs) + if err != nil { + fmt.Println("Error unmarshal response body:", err) + } + - return eventLogs, err + return eventlogs, err } func (client *client) GetImportErrors() (airflow.ImportErrorCollection, error) { @@ -299,4 +333,10 @@ func (client *client) PatchDag(dagID string, dagBody airflow.DAG) (airflow.DAG, } return logs, nil +} + +func addBasicAuth(req *http.Request, username, password string) { + auth := username + ":" + password + encodedAuth := base64.StdEncoding.EncodeToString([]byte(auth)) + req.Header.Add("Authorization", "Basic "+encodedAuth) } \ No newline at end of file diff --git a/pkg/api/rest/controller/workflow.go b/pkg/api/rest/controller/workflow.go index 7024050..aeadc60 100644 --- a/pkg/api/rest/controller/workflow.go +++ b/pkg/api/rest/controller/workflow.go @@ -476,10 +476,10 @@ func UpdateWorkflow(c echo.Context) error { return common.ReturnErrorMsg(c, err.Error()) } - // err = airflow.Client.DeleteDAG(oldWorkflow.ID, true) - // if err != nil { - // return common.ReturnErrorMsg(c, "Failed to update the workflow. (Error:"+err.Error()+")") - // } + err = airflow.Client.DeleteDAG(oldWorkflow.ID, true) + if err != nil { + return common.ReturnErrorMsg(c, "Failed to update the workflow. (Error:"+err.Error()+")") + } err = airflow.Client.CreateDAG(oldWorkflow) if err != nil { @@ -1113,37 +1113,61 @@ func ClearTaskInstances(c echo.Context) error { // @Tags [Workflow] // @Accept json // @Produce json -// @Param dag_id query string true "ID of the workflow." +// @Param wfId query string true "ID of the workflow." // @Param wfRunId query string false "ID of the workflow run." // @Param taskId query string false "ID of the task." -// @Success 200 {object} model.Workflow "Successfully get the workflow." +// @Success 200 {object} []model.EventLog "Successfully get the workflow." // @Failure 400 {object} common.ErrorResponse "Sent bad request." // @Failure 500 {object} common.ErrorResponse "Failed to get the workflow." // @Router /eventlogs [get] func GetEventLogs(c echo.Context) error { wfId := c.QueryParam("dag_id") - // if wfId == "" { - // return common.ReturnErrorMsg(c, "Please provide the dagId.") - // } - logs, err := airflow.Client.GetEventLogs(wfId) - if err != nil { - return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) + if wfId == "" { + return common.ReturnErrorMsg(c, "Please provide the dagId.") } - wfRunId := c.QueryParam("wfRunId") - if wfRunId != ""{ - + var wfRunId,taskId, taskName string + + if c.QueryParam("wfRunId") != "" { + wfRunId = c.QueryParam("wfRunId") + } + if c.QueryParam("taskId") != "" { + taskId = c.QueryParam("taskId") + taskDBInfo, err := dao.TaskGet(taskId) + if err != nil { + return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) + } + taskName = taskDBInfo.Name + } + var eventLogs []model.EventLog; + logs, err := airflow.Client.GetEventLogs(wfId,wfRunId,taskName) + if err != nil { + return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) } - taskId := c.QueryParam("taskId") - if taskId != ""{ + for _, log := range *logs.EventLogs { + taskDBInfo, err := dao.TaskGetByWorkflowIDAndName(wfId,log.GetTaskId()) + if err != nil { + return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) + } + taskId := &taskDBInfo.ID + eventlog := model.EventLog { + WorkflowID: log.GetDagId(), + WorkflowRunID: wfRunId, + TaskID: *taskId, + TaskName: log.GetTaskId(), + Extra: log.GetExtra(), + Event: log.GetEvent(), + When: log.GetWhen(), + } + eventLogs = append(eventLogs,eventlog) } // logs, err := airflow.Client.GetEventLogs(wfId) // if err != nil { // return common.ReturnErrorMsg(c, "Failed to get the taskInstances: " + err.Error()) // } - return c.JSONPretty(http.StatusOK, logs, " ") + return c.JSONPretty(http.StatusOK, eventLogs, " ") } // ImportErrors godoc diff --git a/pkg/api/rest/docs/docs.go b/pkg/api/rest/docs/docs.go index 061ed36..f5ea375 100644 --- a/pkg/api/rest/docs/docs.go +++ b/pkg/api/rest/docs/docs.go @@ -37,7 +37,7 @@ const docTemplate = `{ { "type": "string", "description": "ID of the workflow.", - "name": "dag_id", + "name": "wfId", "in": "query", "required": true }, @@ -58,7 +58,10 @@ const docTemplate = `{ "200": { "description": "Successfully get the workflow.", "schema": { - "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Workflow" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.EventLog" + } } }, "400": { @@ -1705,6 +1708,32 @@ const docTemplate = `{ } } }, + "github_com_cloud-barista_cm-cicada_pkg_api_rest_model.EventLog": { + "type": "object", + "properties": { + "event": { + "type": "string" + }, + "extra": { + "type": "string" + }, + "start_date": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "task_name": { + "type": "string" + }, + "workflow_id": { + "type": "string" + }, + "workflow_run_id": { + "type": "string" + } + } + }, "github_com_cloud-barista_cm-cicada_pkg_api_rest_model.GetWorkflowTemplate": { "type": "object", "required": [ diff --git a/pkg/api/rest/docs/swagger.json b/pkg/api/rest/docs/swagger.json index e7a2e4c..1be5554 100644 --- a/pkg/api/rest/docs/swagger.json +++ b/pkg/api/rest/docs/swagger.json @@ -30,7 +30,7 @@ { "type": "string", "description": "ID of the workflow.", - "name": "dag_id", + "name": "wfId", "in": "query", "required": true }, @@ -51,7 +51,10 @@ "200": { "description": "Successfully get the workflow.", "schema": { - "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Workflow" + "type": "array", + "items": { + "$ref": "#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.EventLog" + } } }, "400": { @@ -1698,6 +1701,32 @@ } } }, + "github_com_cloud-barista_cm-cicada_pkg_api_rest_model.EventLog": { + "type": "object", + "properties": { + "event": { + "type": "string" + }, + "extra": { + "type": "string" + }, + "start_date": { + "type": "string" + }, + "task_id": { + "type": "string" + }, + "task_name": { + "type": "string" + }, + "workflow_id": { + "type": "string" + }, + "workflow_run_id": { + "type": "string" + } + } + }, "github_com_cloud-barista_cm-cicada_pkg_api_rest_model.GetWorkflowTemplate": { "type": "object", "required": [ diff --git a/pkg/api/rest/docs/swagger.yaml b/pkg/api/rest/docs/swagger.yaml index cae055d..17658a1 100644 --- a/pkg/api/rest/docs/swagger.yaml +++ b/pkg/api/rest/docs/swagger.yaml @@ -114,6 +114,23 @@ definitions: required: - task_groups type: object + github_com_cloud-barista_cm-cicada_pkg_api_rest_model.EventLog: + properties: + event: + type: string + extra: + type: string + start_date: + type: string + task_id: + type: string + task_name: + type: string + workflow_id: + type: string + workflow_run_id: + type: string + type: object github_com_cloud-barista_cm-cicada_pkg_api_rest_model.GetWorkflowTemplate: properties: data: @@ -384,7 +401,7 @@ paths: parameters: - description: ID of the workflow. in: query - name: dag_id + name: wfId required: true type: string - description: ID of the workflow run. @@ -401,7 +418,9 @@ paths: "200": description: Successfully get the workflow. schema: - $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.Workflow' + items: + $ref: '#/definitions/github_com_cloud-barista_cm-cicada_pkg_api_rest_model.EventLog' + type: array "400": description: Sent bad request. schema: diff --git a/pkg/api/rest/model/workflow.go b/pkg/api/rest/model/workflow.go index 1ab2411..b7ae2f0 100644 --- a/pkg/api/rest/model/workflow.go +++ b/pkg/api/rest/model/workflow.go @@ -148,6 +148,17 @@ type TaskInstanceReference struct { type TaskLog struct { Content string `json:"content,omitempty"` } + +type EventLog struct { + WorkflowRunID string `json:"workflow_run_id,omitempty"` + WorkflowID string `json:"workflow_id"` + TaskID string `json:"task_id"` + TaskName string `json:"task_name"` + Event string `json:"event,omitempty"` + When time.Time `json:"start_date,omitempty"` + Extra string `json:"extra,omitempty"` +} + func (d Data) Value() (driver.Value, error) { return json.Marshal(d) } From cb155c8519d4f739d14ca59f8869badafc59d61b Mon Sep 17 00:00:00 2001 From: yby654 Date: Mon, 7 Oct 2024 15:20:39 +0900 Subject: [PATCH 11/12] =?UTF-8?q?test=20data=20=EC=82=AD=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _airflow/airflow-home/dags/example.py | 220 -------------------------- _airflow/airflow-home/dags/test.py | 107 ------------- 2 files changed, 327 deletions(-) delete mode 100644 _airflow/airflow-home/dags/example.py delete mode 100644 _airflow/airflow-home/dags/test.py diff --git a/_airflow/airflow-home/dags/example.py b/_airflow/airflow-home/dags/example.py deleted file mode 100644 index 0b7248e..0000000 --- a/_airflow/airflow-home/dags/example.py +++ /dev/null @@ -1,220 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -Example Airflow DAG that shows the complex DAG structure. -""" - -from __future__ import annotations - -import pendulum - -from airflow.models.baseoperator import chain -from airflow.models.dag import DAG -from airflow.operators.bash import BashOperator - -with DAG( - dag_id="example_complex", - schedule=None, - start_date=pendulum.datetime(2021, 1, 1, tz="UTC"), - catchup=False, - tags=["example", "example2", "example3"], -) as dag: - # Create - create_entry_group = BashOperator(task_id="create_entry_group", bash_command="echo create_entry_group") - - create_entry_group_result = BashOperator( - task_id="create_entry_group_result", bash_command="echo create_entry_group_result" - ) - - create_entry_group_result2 = BashOperator( - task_id="create_entry_group_result2", bash_command="echo create_entry_group_result2" - ) - - create_entry_gcs = BashOperator(task_id="create_entry_gcs", bash_command="echo create_entry_gcs") - - create_entry_gcs_result = BashOperator( - task_id="create_entry_gcs_result", bash_command="echo create_entry_gcs_result" - ) - - create_entry_gcs_result2 = BashOperator( - task_id="create_entry_gcs_result2", bash_command="echo create_entry_gcs_result2" - ) - - create_tag = BashOperator(task_id="create_tag", bash_command="echo create_tag") - - create_tag_result = BashOperator(task_id="create_tag_result", bash_command="echo create_tag_result") - - create_tag_result2 = BashOperator(task_id="create_tag_result2", bash_command="echo create_tag_result2") - - create_tag_template = BashOperator(task_id="create_tag_template", bash_command="echo create_tag_template") - - create_tag_template_result = BashOperator( - task_id="create_tag_template_result", bash_command="echo create_tag_template_result" - ) - - create_tag_template_result2 = BashOperator( - task_id="create_tag_template_result2", bash_command="echo create_tag_template_result2" - ) - - create_tag_template_field = BashOperator( - task_id="create_tag_template_field", bash_command="echo create_tag_template_field" - ) - - create_tag_template_field_result = BashOperator( - task_id="create_tag_template_field_result", bash_command="echo create_tag_template_field_result" - ) - - create_tag_template_field_result2 = BashOperator( - task_id="create_tag_template_field_result2", bash_command="echo create_tag_template_field_result" - ) - - # Delete - delete_entry = BashOperator(task_id="delete_entry", bash_command="echo delete_entry") - create_entry_gcs >> delete_entry - - delete_entry_group = BashOperator(task_id="delete_entry_group", bash_command="echo delete_entry_group") - create_entry_group >> delete_entry_group - - delete_tag = BashOperator(task_id="delete_tag", bash_command="echo delete_tag") - create_tag >> delete_tag - - delete_tag_template_field = BashOperator( - task_id="delete_tag_template_field", bash_command="echo delete_tag_template_field" - ) - - delete_tag_template = BashOperator(task_id="delete_tag_template", bash_command="echo delete_tag_template") - - # Get - get_entry_group = BashOperator(task_id="get_entry_group", bash_command="echo get_entry_group") - - get_entry_group_result = BashOperator( - task_id="get_entry_group_result", bash_command="echo get_entry_group_result" - ) - - get_entry = BashOperator(task_id="get_entry", bash_command="echo get_entry") - - get_entry_result = BashOperator(task_id="get_entry_result", bash_command="echo get_entry_result") - - get_tag_template = BashOperator(task_id="get_tag_template", bash_command="echo get_tag_template") - - get_tag_template_result = BashOperator( - task_id="get_tag_template_result", bash_command="echo get_tag_template_result" - ) - - # List - list_tags = BashOperator(task_id="list_tags", bash_command="echo list_tags") - - list_tags_result = BashOperator(task_id="list_tags_result", bash_command="echo list_tags_result") - - # Lookup - lookup_entry = BashOperator(task_id="lookup_entry", bash_command="echo lookup_entry") - - lookup_entry_result = BashOperator(task_id="lookup_entry_result", bash_command="echo lookup_entry_result") - - # Rename - rename_tag_template_field = BashOperator( - task_id="rename_tag_template_field", bash_command="echo rename_tag_template_field" - ) - - # Search - search_catalog = BashOperator(task_id="search_catalog", bash_command="echo search_catalog") - - search_catalog_result = BashOperator( - task_id="search_catalog_result", bash_command="echo search_catalog_result" - ) - - # Update - update_entry = BashOperator(task_id="update_entry", bash_command="echo update_entry") - - update_tag = BashOperator(task_id="update_tag", bash_command="echo update_tag") - - update_tag_template = BashOperator(task_id="update_tag_template", bash_command="echo update_tag_template") - - update_tag_template_field = BashOperator( - task_id="update_tag_template_field", bash_command="echo update_tag_template_field" - ) - # Create - create_tasks = [ - create_entry_group, - create_entry_gcs, - create_tag_template, - create_tag_template_field, - create_tag, - ] - chain(*create_tasks) - - create_entry_group >> delete_entry_group - create_entry_group >> create_entry_group_result - create_entry_group >> create_entry_group_result2 - - create_entry_gcs >> delete_entry - create_entry_gcs >> create_entry_gcs_result - create_entry_gcs >> create_entry_gcs_result2 - - create_tag_template >> delete_tag_template_field - create_tag_template >> create_tag_template_result - create_tag_template >> create_tag_template_result2 - - create_tag_template_field >> delete_tag_template_field - create_tag_template_field >> create_tag_template_field_result - create_tag_template_field >> create_tag_template_field_result2 - - create_tag >> delete_tag - create_tag >> create_tag_result - create_tag >> create_tag_result2 - - # Delete - delete_tasks = [ - delete_tag, - delete_tag_template_field, - delete_tag_template, - delete_entry_group, - delete_entry, - ] - chain(*delete_tasks) - - # Get - create_tag_template >> get_tag_template >> delete_tag_template - get_tag_template >> get_tag_template_result - - create_entry_gcs >> get_entry >> delete_entry - get_entry >> get_entry_result - - create_entry_group >> get_entry_group >> delete_entry_group - get_entry_group >> get_entry_group_result - - # List - create_tag >> list_tags >> delete_tag - list_tags >> list_tags_result - - # Lookup - create_entry_gcs >> lookup_entry >> delete_entry - lookup_entry >> lookup_entry_result - - # Rename - create_tag_template_field >> rename_tag_template_field >> delete_tag_template_field - - # Search - chain(create_tasks, search_catalog, delete_tasks) - search_catalog >> search_catalog_result - - # Update - create_entry_gcs >> update_entry >> delete_entry - create_tag >> update_tag >> delete_tag - create_tag_template >> update_tag_template >> delete_tag_template - create_tag_template_field >> update_tag_template_field >> rename_tag_template_field diff --git a/_airflow/airflow-home/dags/test.py b/_airflow/airflow-home/dags/test.py deleted file mode 100644 index b20cee5..0000000 --- a/_airflow/airflow-home/dags/test.py +++ /dev/null @@ -1,107 +0,0 @@ - - -# from datetime import datetime, timedelta -# from textwrap import dedent - -# from airflow import DAG - -# # Operators; we need this to operate! -# from airflow.operators.bash import BashOperator -# from airflow.operators.dummy import DummyOperator -# from airflow.operators.python import BranchPythonOperator -# from airflow.operators.python_operator import PythonOperator -# from airflow.utils.trigger_rule import TriggerRule - -# default_args = { -# 'owner': 'owner-name', -# 'depends_on_past': False, -# 'email': ['your-email@g.com'], -# 'email_on_failure': False, -# 'email_on_retry': False, -# 'retries': 1, -# 'retry_delay': timedelta(minutes=15), -# } - -# dag_args = dict( -# dag_id="tutorial-python-op", -# default_args=default_args, -# description='tutorial DAG python', -# schedule_interval=timedelta(minutes=50), -# start_date=datetime(2022, 2, 1), -# tags=['example-sj'], -# ) - -# def random_branch_path(): -# # 필요 라이브러리는 여기서 import -# # https://airflow.apache.org/docs/apache-airflow/stable/best-practices.html#writing-a-dag 참고 -# from random import randint - -# return "cal_a_id" if randint(1, 2) == 1 else "cal_m_id" - -# def calc_add(x, y, **kwargs): -# result = x + y -# print("x + y : ", result) -# kwargs['task_instance'].xcom_push(key='calc_result', value=result) -# return "calc add" - -# def calc_mul(x, y, **kwargs): -# result = x * y -# print("x * y : ", result) -# kwargs['task_instance'].xcom_push(key='calc_result', value=result) -# return "calc mul" - -# def print_result(**kwargs): -# r = kwargs["task_instance"].xcom_pull(key='calc_result') -# print("message : ", r) -# print("*"*100) -# print(kwargs) - -# def end_seq(): -# print("end") - -# with DAG( **dag_args ) as dag: -# start = BashOperator( -# task_id='start', -# bash_command='echo "start!"', -# ) - -# branch = BranchPythonOperator( -# task_id='branch', -# python_callable=random_branch_path, -# ) - -# calc_add = PythonOperator( -# task_id='cal_a_id', -# python_callable=calc_add, -# op_kwargs={'x': 10, 'y':4} -# ) - -# calc_mul = PythonOperator( -# task_id='cal_m_id', -# python_callable=calc_mul, -# op_kwargs={'x': 10, 'y':4} -# ) - -# msg = PythonOperator( -# task_id='msg', -# python_callable=print_result, -# trigger_rule=TriggerRule.NONE_FAILED -# ) - -# complete_py = PythonOperator( -# task_id='complete_py', -# python_callable=end_seq, -# trigger_rule=TriggerRule.NONE_FAILED -# ) - -# complete = BashOperator( -# task_id='complete_bash', -# depends_on_past=False, -# bash_command='echo "complete~!"', -# trigger_rule=TriggerRule.NONE_FAILED -# ) - - -# start >> branch >> calc_add >> msg >> complete_py >> complete -# start >> branch >> calc_mul >> msg >> complete - From efc761cdef625afb936e85242a6a181423c168d5 Mon Sep 17 00:00:00 2001 From: yby654 Date: Mon, 7 Oct 2024 15:28:36 +0900 Subject: [PATCH 12/12] =?UTF-8?q?conf,=20makefile=20=EC=A3=BC=EC=84=9D=20?= =?UTF-8?q?=EC=A0=9C=EA=B1=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 12 ++++++------ conf/cm-cicada.yaml | 1 - 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index c865947..a9507c7 100644 --- a/Makefile +++ b/Makefile @@ -16,12 +16,12 @@ dependency: ## Get dependencies @echo Checking dependencies... @${GO_COMMAND} mod tidy -# lint: dependency ## Lint the files -# @echo "Running linter..." -# @if [ ! -f "${GOPATH}/bin/golangci-lint" ] && [ ! -f "$(GOROOT)/bin/golangci-lint" ]; then \ -# ${GO_COMMAND} install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.3; \ -# fi -# @golangci-lint run --timeout 30m -E contextcheck -D unused +lint: dependency ## Lint the files + @echo "Running linter..." + @if [ ! -f "${GOPATH}/bin/golangci-lint" ] && [ ! -f "$(GOROOT)/bin/golangci-lint" ]; then \ + ${GO_COMMAND} install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.60.3; \ + fi + @golangci-lint run --timeout 30m -E contextcheck -D unused test: dependency ## Run unittests @echo "Running tests..." diff --git a/conf/cm-cicada.yaml b/conf/cm-cicada.yaml index 3a96a32..aa170e5 100644 --- a/conf/cm-cicada.yaml +++ b/conf/cm-cicada.yaml @@ -6,7 +6,6 @@ cm-cicada: templates_directory: "./lib/airflow/example/workflow_template/" airflow-server: address: 127.0.0.1:8080 - # address: 192.168.160.139:8080 use_tls: false # skip_tls_verify: true init_retry: 5