From d1274a91df001a37bb6e03008a280662bd4fb047 Mon Sep 17 00:00:00 2001 From: mansonliwh <61920998+mansonliwh@users.noreply.github.com> Date: Mon, 14 Oct 2024 19:18:22 +0800 Subject: [PATCH 1/6] [AMORO-3038] Unify the API set for dashboard and open API (#3244) --- .../server/dashboard/DashboardServer.java | 16 +++---- .../controller/CatalogController.java | 2 +- .../PlatformFileInfoController.java | 2 +- amoro-web/mock/modules/catalogs.js | 26 ++++++------ amoro-web/mock/modules/common.js | 10 ++--- amoro-web/mock/modules/optimize.js | 22 +++++----- amoro-web/mock/modules/overview.js | 10 ++--- amoro-web/mock/modules/settings.js | 4 +- amoro-web/mock/modules/table.js | 22 +++++----- amoro-web/mock/modules/terminal.js | 30 ++++++------- amoro-web/src/services/global.service.ts | 2 +- amoro-web/src/services/login.service.ts | 6 +-- amoro-web/src/services/optimize.service.ts | 28 ++++++------- amoro-web/src/services/overview.service.ts | 14 +++---- amoro-web/src/services/setting.services.ts | 16 +++---- amoro-web/src/services/table.service.ts | 42 +++++++++---------- amoro-web/src/services/terminal.service.ts | 14 +++---- amoro-web/src/utils/request.ts | 1 + amoro-web/src/views/catalogs/Detail.vue | 2 +- charts/amoro/templates/amoro-deployment.yaml | 4 +- 20 files changed, 138 insertions(+), 135 deletions(-) diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java index 4567024637..f858e8e810 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/DashboardServer.java @@ -74,6 +74,8 @@ public class DashboardServer { public static final Logger LOG = LoggerFactory.getLogger(DashboardServer.class); private static final String AUTH_TYPE_BASIC = "basic"; + private static final String X_REQUEST_SOURCE_HEADER = "X-Request-Source"; + private static final String X_REQUEST_SOURCE_WEB = "Web"; private final CatalogController catalogController; private final HealthCheckController healthCheckController; @@ -189,14 +191,13 @@ public EndpointGroup endpoints() { // for dashboard api path( - "/ams/v1", + "/api/ams/v1", () -> { // login controller get("/login/current", loginController::getCurrent); post("/login", loginController::login); post("/logout", loginController::logout); }); - path("ams/v1", apiGroup()); // for open api path("/api/ams/v1", apiGroup()); @@ -356,7 +357,8 @@ private EndpointGroup apiGroup() { public void preHandleRequest(Context ctx) { String uriPath = ctx.path(); - if (needApiKeyCheck(uriPath)) { + String requestSource = ctx.header(X_REQUEST_SOURCE_HEADER); + if (needApiKeyCheck(uriPath) && !X_REQUEST_SOURCE_WEB.equalsIgnoreCase(requestSource)) { if (AUTH_TYPE_BASIC.equalsIgnoreCase(authType)) { BasicAuthCredentials cred = ctx.basicAuthCredentials(); if (!(basicAuthUser.equals(cred.component1()) @@ -392,10 +394,10 @@ public void handleException(Exception e, Context ctx) { } private static final String[] urlWhiteList = { - "/ams/v1/versionInfo", - "/ams/v1/login", - "/ams/v1/health/status", - "/ams/v1/login/current", + "/api/ams/v1/versionInfo", + "/api/ams/v1/login", + "/api/ams/v1/health/status", + "/api/ams/v1/login/current", "/", "/overview", "/introduce", diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java index 78d10823bd..c85111a806 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/CatalogController.java @@ -610,7 +610,7 @@ public void deleteCatalog(Context ctx) { /** Construct a url */ private String constructCatalogConfigFileUrl(String catalogName, String type, String key) { return String.format( - "/ams/v1/catalogs/%s/config/%s/%s", catalogName, type, key.replaceAll("\\.", "-")); + "/api/ams/v1/catalogs/%s/config/%s/%s", catalogName, type, key.replaceAll("\\.", "-")); } /** Get the config file content uri("/catalogs/{catalogName}/config/{type}/{key} */ diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/PlatformFileInfoController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/PlatformFileInfoController.java index fe1018ebf4..b3e4689901 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/PlatformFileInfoController.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/PlatformFileInfoController.java @@ -64,7 +64,7 @@ public void uploadFile(Context ctx) throws IOException { Integer fid = platformFileInfoService.addFile(name, content); Map result = new HashMap<>(); result.put("id", String.valueOf(fid)); - result.put("url", "/ams/v1/files/" + fid); + result.put("url", "/api/ams/v1/files/" + fid); ctx.json(OkResponse.of(result)); } diff --git a/amoro-web/mock/modules/catalogs.js b/amoro-web/mock/modules/catalogs.js index 15623acf26..79195ae47b 100644 --- a/amoro-web/mock/modules/catalogs.js +++ b/amoro-web/mock/modules/catalogs.js @@ -18,7 +18,7 @@ export default [ { - url: '/mock/ams/v1/catalogs', + url: '/mock/api/ams/v1/catalogs', method: 'get', response: () => ({ "message": "success", @@ -56,27 +56,27 @@ export default [ }), }, { - url: '/mock/ams/v1/catalogs', + url: '/mock/api/ams/v1/catalogs', method: 'post', response: () => ({"message":"success","code":200,"result":""}), }, { - url: '/mock/ams/v1/catalogs/test_catalog/databases', + url: '/mock/api/ams/v1/catalogs/test_catalog/databases', method: 'get', response: () => { return { "message": "success", "code": 200, "result": ["db", "test", "acc"] } }, }, { - url: '/mock/ams/v1/catalogs/test_catalog/databases/db/tables', + url: '/mock/api/ams/v1/catalogs/test_catalog/databases/db/tables', method: 'get', response: () => { return { "message": "success", "code": 200, "result": [{ "name": "user", "type": "ICEBERG" },{ "name": "wf", "type": "ICEBERG" }, { "name": "xcvz", "type": "ICEBERG" }] }; }, }, { - url: '/mock/ams/v1/catalogs/:id', + url: '/mock/api/ams/v1/catalogs/:id', method: 'get', response: () => ({ "message": "success", @@ -92,15 +92,15 @@ export default [ "storage.type": "Hadoop", "hive.site": { "fileName": "hive-site.xml", - "fileUrl": "/ams/v1/catalogs/test_catalog/config/storage-config/hive-site" + "fileUrl": "/api/ams/v1/catalogs/test_catalog/config/storage-config/hive-site" }, "hadoop.core.site": { "fileName": "core-site.xml", - "fileUrl": "/ams/v1/catalogs/test_catalog/config/storage-config/hadoop-core-site" + "fileUrl": "/api/ams/v1/catalogs/test_catalog/config/storage-config/hadoop-core-site" }, "hadoop.hdfs.site": { "fileName": "hdfs-site.xml", - "fileUrl": "/ams/v1/catalogs/test_catalog/config/storage-config/hadoop-hdfs-site" + "fileUrl": "/api/ams/v1/catalogs/test_catalog/config/storage-config/hadoop-hdfs-site" } }, "authConfig": { @@ -116,22 +116,22 @@ export default [ }), }, { - url: '/mock/ams/v1/catalogs/:id', + url: '/mock/api/ams/v1/catalogs/:id', method: 'put', response: () => ({ "message": "success", "code": 200, "result": null }), }, { - url: '/mock/ams/v1/catalogs/:id', + url: '/mock/api/ams/v1/catalogs/:id', method: 'delete', response: () => ({ "message": "success", "code": 200, "result": true }), }, { - url: '/mock/ams/v1/catalogs/:id/delete/check', + url: '/mock/api/ams/v1/catalogs/:id/delete/check', method: 'get', response: () => ({ "message": "success", "code": 200, "result": true }), }, { - url: '/mock/ams/v1/catalogs/metastore/types', + url: '/mock/api/ams/v1/catalogs/metastore/types', method: 'get', response: () => ({ "message": "success", @@ -161,7 +161,7 @@ export default [ }), }, { - url: '/mock/ams/v1/tables/catalogs/:catalog/dbs/:dbId/tables/:tableName/optimizing-processes/:processesId/tasks', + url: '/mock/api/ams/v1/tables/catalogs/:catalog/dbs/:dbId/tables/:tableName/optimizing-processes/:processesId/tasks', method: 'get', response: () => ({ "message": "success", diff --git a/amoro-web/mock/modules/common.js b/amoro-web/mock/modules/common.js index afc71a272b..089dea2448 100644 --- a/amoro-web/mock/modules/common.js +++ b/amoro-web/mock/modules/common.js @@ -18,7 +18,7 @@ export default [ { - url: '/mock/ams/v1/login/current', + url: '/mock/api/ams/v1/login/current', method: 'get', response: () => ({ code: 200, @@ -30,7 +30,7 @@ export default [ }), }, { - url: '/mock/ams/v1/login', + url: '/mock/api/ams/v1/login', method: 'post', response: () => ({ code: 200, @@ -39,7 +39,7 @@ export default [ }), }, { - url: '/mock/ams/v1/versionInfo', + url: '/mock/api/ams/v1/versionInfo', method: 'get', response: () => ({ code: 200, @@ -51,7 +51,7 @@ export default [ }), }, { - url: '/mock/ams/v1/upgrade/properties', + url: '/mock/api/ams/v1/upgrade/properties', method: 'get', response: () => ({ code: 200, @@ -63,7 +63,7 @@ export default [ }) }, { - url: '/mock/ams/v1/logout', + url: '/mock/api/ams/v1/logout', method: 'post', response: () => ({ code: 200, diff --git a/amoro-web/mock/modules/optimize.js b/amoro-web/mock/modules/optimize.js index d8f275e50e..588baf9589 100644 --- a/amoro-web/mock/modules/optimize.js +++ b/amoro-web/mock/modules/optimize.js @@ -18,7 +18,7 @@ export default [ { - url: '/mock/ams/v1/optimize/resourceGroups/get', + url: '/mock/api/ams/v1/optimize/resourceGroups/get', method: 'get', response: () => ({ code: 200, @@ -29,7 +29,7 @@ export default [ }), }, { - url: '/mock/ams/v1/optimize/containers/get', + url: '/mock/api/ams/v1/optimize/containers/get', method: 'get', response: () => ({ code: 200, @@ -38,7 +38,7 @@ export default [ }), }, { - url: '/mock/ams/v1/optimize/actions', + url: '/mock/api/ams/v1/optimize/actions', method: 'get', response: () => ({ code: 200, @@ -47,7 +47,7 @@ export default [ }), }, { - url: '/mock/ams/v1/optimize/optimizerGroups/:groups/tables', + url: '/mock/api/ams/v1/optimize/optimizerGroups/:groups/tables', method: 'get', response: () => ({ "message": "success", @@ -85,12 +85,12 @@ export default [ }), }, { - url: '/mock/ams/v1/optimize/optimizerGroups/local/optimizers', + url: '/mock/api/ams/v1/optimize/optimizerGroups/local/optimizers', method: 'post', response: () => ({ "message": "success", "code": 200, "result": "success to scaleOut optimizer" }), }, { - url: '/mock/ams/v1/optimize/resourceGroups', + url: '/mock/api/ams/v1/optimize/resourceGroups', method: 'get', response: () => ({ "message": "success", @@ -133,7 +133,7 @@ export default [ }), }, { - url: '/mock/ams/v1/optimize/resourceGroups', + url: '/mock/api/ams/v1/optimize/resourceGroups', method: 'put', response: () => ({ "message": "success", @@ -142,7 +142,7 @@ export default [ }), }, { - url: '/mock/ams/v1/optimize/resourceGroups', + url: '/mock/api/ams/v1/optimize/resourceGroups', method: 'post', response: () => ({ "message": "success", @@ -151,12 +151,12 @@ export default [ }), }, { - url: '/mock/ams/v1/optimize/resourceGroups/:id/delete/check', + url: '/mock/api/ams/v1/optimize/resourceGroups/:id/delete/check', method: 'get', response: () => ({ "message": "success", "code": 200, "result": true }), }, { - url: '/mock/ams/v1/optimize/resourceGroups/:id', + url: '/mock/api/ams/v1/optimize/resourceGroups/:id', method: 'delete', response: () => ({ "message": "success", @@ -165,7 +165,7 @@ export default [ }), }, { - url: '/mock/ams/v1/optimize/optimizerGroups/all/optimizers', + url: '/mock/api/ams/v1/optimize/optimizerGroups/all/optimizers', method: 'get', response: () => ({ "message": "success", diff --git a/amoro-web/mock/modules/overview.js b/amoro-web/mock/modules/overview.js index 1e5ac44755..4946e68dfd 100644 --- a/amoro-web/mock/modules/overview.js +++ b/amoro-web/mock/modules/overview.js @@ -18,7 +18,7 @@ export default [ { - url: '/mock/ams/v1/overview/summary', + url: '/mock/api/ams/v1/overview/summary', method: 'get', response: () => ({ code: 200, @@ -33,7 +33,7 @@ export default [ }), }, { - url: '/mock/ams/v1/overview/optimizing', + url: '/mock/api/ams/v1/overview/optimizing', method: 'get', response: () => ({ code: 200, @@ -48,7 +48,7 @@ export default [ }), }, { - url: '/mock/ams/v1/overview/top', + url: '/mock/api/ams/v1/overview/top', method: 'get', response: () => ({ code: 200, @@ -68,7 +68,7 @@ export default [ }), }, { - url: '/mock/ams/v1/overview/resource', + url: '/mock/api/ams/v1/overview/resource', method: 'get', response: () => ({ code: 200, @@ -97,7 +97,7 @@ export default [ }), }, { - url: '/mock/ams/v1/overview/dataSize', + url: '/mock/api/ams/v1/overview/dataSize', method: 'get', response: () => ({ code: 200, diff --git a/amoro-web/mock/modules/settings.js b/amoro-web/mock/modules/settings.js index 3412937860..f8fc1f7e48 100644 --- a/amoro-web/mock/modules/settings.js +++ b/amoro-web/mock/modules/settings.js @@ -18,7 +18,7 @@ export default [ { - url: '/mock/ams/v1/settings/system', + url: '/mock/api/ams/v1/settings/system', method: 'get', response: () => ({ code: 200, @@ -30,7 +30,7 @@ export default [ }), }, { - url: '/mock/ams/v1/settings/containers', + url: '/mock/api/ams/v1/settings/containers', method: 'get', response: () => ({ code: 200, diff --git a/amoro-web/mock/modules/table.js b/amoro-web/mock/modules/table.js index fc654349fe..372e81fd97 100644 --- a/amoro-web/mock/modules/table.js +++ b/amoro-web/mock/modules/table.js @@ -18,7 +18,7 @@ export default [ { - url: '/mock/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/details', + url: '/mock/api/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/details', method: 'get', response: () => ({ "message": "success", @@ -91,7 +91,7 @@ export default [ }), }, { - url: '/mock/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/partitions', + url: '/mock/api/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/partitions', method: 'get', response: () => ({ "message": "success", @@ -120,7 +120,7 @@ export default [ }), }, { - url: '/mock/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/branches', + url: '/mock/api/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/branches', method: 'get', response: () => ({ "message": "success", @@ -141,12 +141,12 @@ export default [ }), }, { - url: '/mock/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/tags', + url: '/mock/api/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/tags', method: 'get', response: () => ({ "message": "success", "code": 200, "result": { "list": [], "total": 0 } }), }, { - url: '/mock/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/snapshots', + url: '/mock/api/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/snapshots', method: 'get', response: () => ({ "message": "success", @@ -193,12 +193,12 @@ export default [ }), }, { - url: '/mock/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/operations', + url: '/mock/api/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/operations', method: 'get', response: () => ({ "message": "success", "code": 200, "result": { "list": [], "total": 0 } }), }, { - url: '/mock/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/partitions/:filter/files', + url: '/mock/api/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/partitions/:filter/files', method: 'get', response: () => ({ "message": "success", @@ -223,7 +223,7 @@ export default [ }), }, { - url: '/mock/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/snapshots/:snapshotId/detail', + url: '/mock/api/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/snapshots/:snapshotId/detail', method: 'get', response: () => ({ "message": "success", @@ -260,7 +260,7 @@ export default [ }), }, { - url: '/mock/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/optimizing-processes', + url: '/mock/api/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/optimizing-processes', method: 'get', response: () => ({ "message": "success", @@ -307,7 +307,7 @@ export default [ }), }, { - url: '/mock/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/optimizing-types', + url: '/mock/api/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/optimizing-types', method: 'get', response: () => ({ "message": "success", @@ -320,7 +320,7 @@ export default [ }), }, { - url: '/mock/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/operations', + url: '/mock/api/ams/v1/tables/catalogs/test_catalog/dbs/db/tables/user/operations', method: 'get', response: () => ({ "message": "success", diff --git a/amoro-web/mock/modules/terminal.js b/amoro-web/mock/modules/terminal.js index 1e5eacd87a..bd6125dac1 100644 --- a/amoro-web/mock/modules/terminal.js +++ b/amoro-web/mock/modules/terminal.js @@ -18,7 +18,7 @@ export default [ { - url: '/mock/ams/v1/terminal/examples', + url: '/mock/api/ams/v1/terminal/examples', method: 'get', response: () => ({ "message": "success", @@ -36,12 +36,12 @@ export default [ }), }, { - url: '/mock/ams/v1/terminal/latestInfos', + url: '/mock/api/ams/v1/terminal/latestInfos', method: 'get', response: () => ({ "message": "success", "code": 200, "result": { "sessionId": "", "sql": "" } }), }, { - url: '/mock/ams/v1/terminal/catalogs/:catalogName/execute', + url: '/mock/api/ams/v1/terminal/catalogs/:catalogName/execute', method: 'post', response: () => ({ "message": "success", @@ -52,7 +52,7 @@ export default [ }), }, { - url: '/mock/ams/v1/terminal/:sessionId/logs', + url: '/mock/api/ams/v1/terminal/:sessionId/logs', method: 'get', response: () => ({ "message": "success", @@ -96,23 +96,23 @@ export default [ }), }, { - url: '/mock/ams/v1/terminal/:sessionId/result', + url: '/mock/api/ams/v1/terminal/:sessionId/result', method: 'get', response: () => ({ "message": "success", "code": 200, "result": [] }), }, { - url: '/mock/ams/v1/terminal/:sessionId/result', + url: '/mock/api/ams/v1/terminal/:sessionId/result', method: 'get', response: () => ({ "message": "success", "code": 200, "result": [] }), }, { - url: '/mock/ams/v1/terminal/:sessionId/stop', + url: '/mock/api/ams/v1/terminal/:sessionId/stop', method: 'get', response: () => ({ "message": "success", "code": 200, "result": null }), }, { - url: '/mock/ams/v1/terminal/examples/CreateTable', + url: '/mock/api/ams/v1/terminal/examples/CreateTable', method: 'get', response: () => ({ "message": "success", @@ -121,12 +121,12 @@ export default [ }), }, { - url: '/mock/ams/v1/terminal/examples/DeleteTable', + url: '/mock/api/ams/v1/terminal/examples/DeleteTable', method: 'get', response: () => ({ "message": "success", "code": 200, "result": "drop table db_name.table_name;" }), }, { - url: '/mock/ams/v1/terminal/examples/EditTable', + url: '/mock/api/ams/v1/terminal/examples/EditTable', method: 'get', response: () => ({ "message": "success", @@ -135,7 +135,7 @@ export default [ }), }, { - url: '/mock/ams/v1/terminal/examples/SetProperties', + url: '/mock/api/ams/v1/terminal/examples/SetProperties', method: 'get', response: () => ({ "message": "success", @@ -144,7 +144,7 @@ export default [ }), }, { - url: '/mock/ams/v1/terminal/examples/UnsetProperties', + url: '/mock/api/ams/v1/terminal/examples/UnsetProperties', method: 'get', response: () => ({ "message": "success", @@ -153,17 +153,17 @@ export default [ }), }, { - url: '/mock/ams/v1/terminal/examples/ShowDatabases', + url: '/mock/api/ams/v1/terminal/examples/ShowDatabases', method: 'get', response: () => ({ "message": "success", "code": 200, "result": "show databases;" }), }, { - url: '/mock/ams/v1/terminal/examples/ShowTables', + url: '/mock/api/ams/v1/terminal/examples/ShowTables', method: 'get', response: () => ({ "message": "success", "code": 200, "result": "show tables;" }), }, { - url: '/mock/ams/v1/terminal/examples/Describe', + url: '/mock/api/ams/v1/terminal/examples/Describe', method: 'get', response: () => ({"message":"success","code":200,"result":"desc db_name.table_name;"}), }, diff --git a/amoro-web/src/services/global.service.ts b/amoro-web/src/services/global.service.ts index d8408cfa63..23405e46db 100644 --- a/amoro-web/src/services/global.service.ts +++ b/amoro-web/src/services/global.service.ts @@ -19,5 +19,5 @@ import request from '@/utils/request' export function getVersionInfo() { - return request.get('ams/v1/versionInfo') + return request.get('api/ams/v1/versionInfo') } diff --git a/amoro-web/src/services/login.service.ts b/amoro-web/src/services/login.service.ts index e8a216392c..8dc6cee154 100644 --- a/amoro-web/src/services/login.service.ts +++ b/amoro-web/src/services/login.service.ts @@ -26,18 +26,18 @@ export class LoginService { user: string password: string }) { - return request.post('ams/v1/login', params, { returnCode: true }) + return request.post('api/ams/v1/login', params, { returnCode: true }) } public logout() { - return request.post('ams/v1/logout', { handleError: false, returnCode: true }) + return request.post('api/ams/v1/logout', { handleError: false, returnCode: true }) } /** * Check the login interface */ public getCurUserInfo(token: string) { - return request.get('ams/v1/login/current', { handleError: false, params: { token } }).then((res) => { + return request.get('api/ams/v1/login/current', { handleError: false, params: { token } }).then((res) => { // if (res) { // store.updateUserInfo({ // userName: res.userName diff --git a/amoro-web/src/services/optimize.service.ts b/amoro-web/src/services/optimize.service.ts index ac82b7281b..fb067cfb3e 100644 --- a/amoro-web/src/services/optimize.service.ts +++ b/amoro-web/src/services/optimize.service.ts @@ -19,11 +19,11 @@ import request from '@/utils/request' export function getOptimizerGroups() { - return request.get('ams/v1/optimize/optimizerGroups') + return request.get('api/ams/v1/optimize/optimizerGroups') } export function getOptimizerAction() { - return request.get('/ams/v1/optimize/actions') + return request.get('api/ams/v1/optimize/actions') } export function getOptimizerTableList( @@ -37,7 +37,7 @@ export function getOptimizerTableList( }, ) { const { optimizerGroup, dbSearchInput, tableSearchInput, page, pageSize, actions } = params - return request.get(`ams/v1/optimize/optimizerGroups/${optimizerGroup}/tables`, { params: { dbSearchInput, tableSearchInput, page, pageSize, actions } }) + return request.get(`api/ams/v1/optimize/optimizerGroups/${optimizerGroup}/tables`, { params: { dbSearchInput, tableSearchInput, page, pageSize, actions } }) } export function getOptimizerResourceList( @@ -48,13 +48,13 @@ export function getOptimizerResourceList( }, ) { const { optimizerGroup, page, pageSize } = params - return request.get(`ams/v1/optimize/optimizerGroups/${optimizerGroup}/optimizers`, { params: { page, pageSize } }) + return request.get(`api/ams/v1/optimize/optimizerGroups/${optimizerGroup}/optimizers`, { params: { page, pageSize } }) } export function getQueueResourceInfo( optimizerGroup: string, ) { - return request.get(`ams/v1/optimize/optimizerGroups/${optimizerGroup}/info`) + return request.get(`api/ams/v1/optimize/optimizerGroups/${optimizerGroup}/info`) } export function scaleoutResource( @@ -64,7 +64,7 @@ export function scaleoutResource( }, ) { const { optimizerGroup, parallelism } = params - return request.post(`ams/v1/optimize/optimizerGroups/${optimizerGroup}/optimizers`, { parallelism }) + return request.post(`api/ams/v1/optimize/optimizerGroups/${optimizerGroup}/optimizers`, { parallelism }) } export function createOptimizerResource( @@ -74,7 +74,7 @@ export function createOptimizerResource( }, ) { const { optimizerGroup, parallelism } = params - return request.post(`ams/v1/optimize/optimizers`, { optimizerGroup, parallelism }) + return request.post(`api/ams/v1/optimize/optimizers`, { optimizerGroup, parallelism }) } export function releaseResource( @@ -84,31 +84,31 @@ export function releaseResource( }, ) { const { jobId } = params - return request.delete(`ams/v1/optimize/optimizers/${jobId}`) + return request.delete(`api/ams/v1/optimize/optimizers/${jobId}`) } export async function getResourceGroupsListAPI() { - const result = await request.get('ams/v1/optimize/resourceGroups') + const result = await request.get('api/ams/v1/optimize/resourceGroups') return result } export async function getGroupContainerListAPI() { - const result = await request.get('ams/v1/optimize/containers/get') + const result = await request.get('api/ams/v1/optimize/containers/get') return result } export function addResourceGroupsAPI(params: { name: string, container: string, properties: { [prop: string]: string } }) { - return request.post('ams/v1/optimize/resourceGroups', params) + return request.post('api/ams/v1/optimize/resourceGroups', params) } export function updateResourceGroupsAPI(params: { name: string, container: string, properties: { [prop: string]: string } }) { - return request.put('ams/v1/optimize/resourceGroups', params) + return request.put('api/ams/v1/optimize/resourceGroups', params) } export function groupDeleteCheckAPI(params: { name: string }) { - return request.get(`/ams/v1/optimize/resourceGroups/${params.name}/delete/check`) + return request.get(`/api/ams/v1/optimize/resourceGroups/${params.name}/delete/check`) } export function groupDeleteAPI(params: { name: string }) { - return request.delete(`/ams/v1/optimize/resourceGroups/${params.name}`) + return request.delete(`/api/ams/v1/optimize/resourceGroups/${params.name}`) } diff --git a/amoro-web/src/services/overview.service.ts b/amoro-web/src/services/overview.service.ts index f6d1e22be3..71e95721cf 100644 --- a/amoro-web/src/services/overview.service.ts +++ b/amoro-web/src/services/overview.service.ts @@ -19,19 +19,19 @@ import request from '@/utils/request' export function getOverviewSummary() { - return request.get('ams/v1/overview/summary') + return request.get('api/ams/v1/overview/summary') } export function getOverviewFormat() { - return request.get('ams/v1/overview/format') + return request.get('api/ams/v1/overview/format') } export function getOverviewOptimizingStatus() { - return request.get('ams/v1/overview/optimizing') + return request.get('api/ams/v1/overview/optimizing') } export function getOverviewOperations() { - return request.get('ams/v1/overview/operations') + return request.get('api/ams/v1/overview/operations') } export function getTop10TableList(params: { @@ -40,7 +40,7 @@ export function getTop10TableList(params: { count?: number }) { const { order, orderBy, count } = params - return request.get(`ams/v1/overview/top`, { + return request.get(`api/ams/v1/overview/top`, { params: { order: order || 'asc', orderBy: orderBy || 'healthScore', @@ -50,9 +50,9 @@ export function getTop10TableList(params: { } export function getResourceUsageList(startTime: number) { - return request.get(`ams/v1/overview/resource`, { params: { startTime } }) + return request.get(`api/ams/v1/overview/resource`, { params: { startTime } }) } export function getDataSizeList(startTime: number) { - return request.get(`ams/v1/overview/dataSize`, { params: { startTime } }) + return request.get(`api/ams/v1/overview/dataSize`, { params: { startTime } }) } diff --git a/amoro-web/src/services/setting.services.ts b/amoro-web/src/services/setting.services.ts index 73d13f11c0..e7ce6652d6 100644 --- a/amoro-web/src/services/setting.services.ts +++ b/amoro-web/src/services/setting.services.ts @@ -20,16 +20,16 @@ import type { IMap } from '@/types/common.type' import request from '@/utils/request' export function getCatalogsTypes() { - return request.get('ams/v1/catalogs/metastore/types') + return request.get('api/ams/v1/catalogs/metastore/types') } export function getCatalogsSetting(catalogName: string) { - return request.get(`ams/v1/catalogs/${catalogName}`) + return request.get(`api/ams/v1/catalogs/${catalogName}`) } export function delCatalog(catalogName: string) { - return request.delete(`ams/v1/catalogs/${catalogName}`) + return request.delete(`api/ams/v1/catalogs/${catalogName}`) } export function checkCatalogStatus(catalogName: string) { - return request.get(`ams/v1/catalogs/${catalogName}/delete/check`) + return request.get(`api/ams/v1/catalogs/${catalogName}/delete/check`) } export function saveCatalogsSetting(params: { name: string @@ -43,13 +43,13 @@ export function saveCatalogsSetting(params: { const { isCreate, name } = params delete params.isCreate if (isCreate) { - return request.post('ams/v1/catalogs', { ...params }) + return request.post('api/ams/v1/catalogs', { ...params }) } - return request.put(`ams/v1/catalogs/${name}`, { ...params }) + return request.put(`api/ams/v1/catalogs/${name}`, { ...params }) } export function getSystemSetting() { - return request.get('ams/v1/settings/system') + return request.get('api/ams/v1/settings/system') } export function getContainersSetting() { - return request.get('ams/v1/settings/containers') + return request.get('api/ams/v1/settings/containers') } diff --git a/amoro-web/src/services/table.service.ts b/amoro-web/src/services/table.service.ts index 03ccc88d1d..8b83d71d6b 100644 --- a/amoro-web/src/services/table.service.ts +++ b/amoro-web/src/services/table.service.ts @@ -21,14 +21,14 @@ import type { ICatalogItem, IMap } from '@/types/common.type' import request from '@/utils/request' export function getCatalogList(): Promise { - return request.get('ams/v1/catalogs') + return request.get('api/ams/v1/catalogs') } export function getDatabaseList(params: { catalog: string keywords: string }): Promise { const { catalog, keywords } = params - return request.get(`ams/v1/catalogs/${catalog}/databases`, { params: { keywords } }) + return request.get(`api/ams/v1/catalogs/${catalog}/databases`, { params: { keywords } }) } export function getTableList(params: { @@ -37,26 +37,26 @@ export function getTableList(params: { keywords: string }) { const { catalog, db, keywords } = params - return request.get(`ams/v1/catalogs/${catalog}/databases/${db}/tables`, { params: { keywords } }) + return request.get(`api/ams/v1/catalogs/${catalog}/databases/${db}/tables`, { params: { keywords } }) } // get tables detail export function getTableDetail( { catalog = '' as string, db = '' as string, table = '' as string, token = '' as string }, ) { - return request.get(`ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/details`, { params: { token } }) + return request.get(`api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/details`, { params: { token } }) } export function getHiveTableDetail( { catalog = '' as string, db = '' as string, table = '' as string }, ) { - return request.get(`ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/hive/details`) + return request.get(`api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/hive/details`) } export function getUpgradeStatus( { catalog = '' as string, db = '' as string, table = '' as string }, ) { - return request.get(`ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/upgrade/status`) + return request.get(`api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/upgrade/status`) } // get partions table export function getPartitionTable( @@ -71,7 +71,7 @@ export function getPartitionTable( }, ) { const { catalog, db, table, filter, page, pageSize, token } = params - return request.get(`ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/partitions`, { params: { filter, page, pageSize, token } }) + return request.get(`api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/partitions`, { params: { filter, page, pageSize, token } }) } // get partions @@ -86,7 +86,7 @@ export function getPartitions( }, ) { const { catalog, db, table, page, pageSize, token } = params - return request.get(`ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/partitions`, { params: { page, pageSize, token } }) + return request.get(`api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/partitions`, { params: { page, pageSize, token } }) } // get partions-files export function getPartitionFiles( @@ -102,7 +102,7 @@ export function getPartitionFiles( }, ) { const { catalog, db, table, partition, specId, page, pageSize, token } = params - return request.get(`ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/partitions/${partition}/files`, { params: { specId, page, pageSize, token } }) + return request.get(`api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/partitions/${partition}/files`, { params: { specId, page, pageSize, token } }) } // get snapshots export function getSnapshots( @@ -118,7 +118,7 @@ export function getSnapshots( }, ) { const { catalog, db, table, page, pageSize, token, ref, operation } = params - return request.get(`ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/snapshots`, { params: { page, pageSize, token, ref, operation } }) + return request.get(`api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/snapshots`, { params: { page, pageSize, token, ref, operation } }) } // get Snapshot detail @@ -134,7 +134,7 @@ export function getDetailBySnapshotId( }, ) { const { catalog, db, table, snapshotId, page, pageSize, token } = params - return request.get(`ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/snapshots/${snapshotId}/detail`, { params: { page, pageSize, token } }) + return request.get(`api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/snapshots/${snapshotId}/detail`, { params: { page, pageSize, token } }) } // get operations export function getOperations( @@ -148,7 +148,7 @@ export function getOperations( }, ) { const { catalog, db, table, page, pageSize, token } = params - return request.get(`ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/operations`, { params: { page, pageSize, token } }) + return request.get(`api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/operations`, { params: { page, pageSize, token } }) } // get optimizing processes export function getOptimizingProcesses( @@ -164,7 +164,7 @@ export function getOptimizingProcesses( }, ) { const { catalog, db, table, type, status, page, pageSize, token } = params - return request.get(`ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/optimizing-processes`, { params: { page, pageSize, token, type, status } }) + return request.get(`api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/optimizing-processes`, { params: { page, pageSize, token, type, status } }) } // get optimizing process types @@ -177,7 +177,7 @@ export function getTableOptimizingTypes( }, ) { const { catalog, db, table, token } = params - return request.get(`ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/optimizing-types`, { params: { token } }) + return request.get(`api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/optimizing-types`, { params: { token } }) } // get optimizing taskes @@ -193,38 +193,38 @@ export function getTasksByOptimizingProcessId( }, ) { const { catalog, db, table, processId, page, pageSize, token } = params - return request.get(`ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/optimizing-processes/${processId}/tasks`, { params: { page, pageSize, token } }) + return request.get(`api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/optimizing-processes/${processId}/tasks`, { params: { page, pageSize, token } }) } export function upgradeHiveTable( { catalog = '' as string, db = '' as string, table = '' as string, properties = {} as IMap, pkList = [] as IMap[] }, ) { - return request.post(`ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/upgrade`, { + return request.post(`api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/upgrade`, { properties, pkList, }) } export function getUpgradeProperties() { - return request.get('ams/v1/upgrade/properties') + return request.get('api/ams/v1/upgrade/properties') } export function cancelOptimizingProcess( { catalog = '' as string, db = '' as string, table = '' as string, processId = '' as string }, ) { - return request.post(`ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/optimizing-processes/${processId}/cancel`) + return request.post(`api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/optimizing-processes/${processId}/cancel`) } export function getBranches(params: { catalog: string, db: string, table: string }) { const { catalog, db, table } = params - return request.get(`/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/branches`) + return request.get(`/api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/branches`) } export function getTags(params: { catalog: string, db: string, table: string }) { const { catalog, db, table } = params - return request.get(`/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/tags`) + return request.get(`/api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/tags`) } export function getConsumers(params: { catalog: string, db: string, table: string }) { const { catalog, db, table } = params - return request.get(`/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/consumers`) + return request.get(`/api/ams/v1/tables/catalogs/${catalog}/dbs/${db}/tables/${table}/consumers`) } diff --git a/amoro-web/src/services/terminal.service.ts b/amoro-web/src/services/terminal.service.ts index 288a7a4c55..3be9423233 100644 --- a/amoro-web/src/services/terminal.service.ts +++ b/amoro-web/src/services/terminal.service.ts @@ -19,15 +19,15 @@ import request from '@/utils/request' export function getJobDebugResult(sessionId: string) { - return request.get(`ams/v1/terminal/${sessionId}/result`) + return request.get(`api/ams/v1/terminal/${sessionId}/result`) } export function getShortcutsList() { - return request.get('ams/v1/terminal/examples') + return request.get('api/ams/v1/terminal/examples') } export function getExampleSqlCode(exampleName: string) { - return request.get(`ams/v1/terminal/examples/${exampleName}`) + return request.get(`api/ams/v1/terminal/examples/${exampleName}`) } export function executeSql(params: { @@ -35,18 +35,18 @@ export function executeSql(params: { sql: string }) { const { catalog, sql } = params - return request.post(`ams/v1/terminal/catalogs/${catalog}/execute`, { sql }) + return request.post(`api/ams/v1/terminal/catalogs/${catalog}/execute`, { sql }) } export function stopSql(sessionId: string) { - return request.put(`ams/v1/terminal/${sessionId}/stop`) + return request.put(`api/ams/v1/terminal/${sessionId}/stop`) } export function getLogsResult(sessionId: string) { - return request.get(`ams/v1/terminal/${sessionId}/logs`) + return request.get(`api/ams/v1/terminal/${sessionId}/logs`) } // get the last executed sql and sessionId export function getLastDebugInfo() { - return request.get('ams/v1/terminal/latestInfos') + return request.get('api/ams/v1/terminal/latestInfos') } diff --git a/amoro-web/src/utils/request.ts b/amoro-web/src/utils/request.ts index c9d28ff577..bd6dcf57de 100644 --- a/amoro-web/src/utils/request.ts +++ b/amoro-web/src/utils/request.ts @@ -64,6 +64,7 @@ const DEFAULT_CONFIG = { timeout: 45000, headers: { 'Content-Type': 'application/json', + 'X-Request-Source' : 'Web' }, } diff --git a/amoro-web/src/views/catalogs/Detail.vue b/amoro-web/src/views/catalogs/Detail.vue index 0dc7f70a6c..707a6a0514 100644 --- a/amoro-web/src/views/catalogs/Detail.vue +++ b/amoro-web/src/views/catalogs/Detail.vue @@ -91,7 +91,7 @@ const isEdit = computed(() => { return props.isEdit }) const uploadUrl = computed(() => { - return '/ams/v1/files' + return '/api/ams/v1/files' }) const isNewCatalog = computed(() => { const catalog = (route.query?.catalogname || '').toString() diff --git a/charts/amoro/templates/amoro-deployment.yaml b/charts/amoro/templates/amoro-deployment.yaml index bce9dc1bb0..05d84d2e84 100644 --- a/charts/amoro/templates/amoro-deployment.yaml +++ b/charts/amoro/templates/amoro-deployment.yaml @@ -83,7 +83,7 @@ spec: {{/* TODO If Support Replica can be use more than 1 */}} {{- if .Values.livenessProbe.enabled }} livenessProbe: httpGet: - path: /ams/v1/health/status + path: /api/ams/v1/health/status port: rest initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.livenessProbe.periodSeconds }} @@ -94,7 +94,7 @@ spec: {{/* TODO If Support Replica can be use more than 1 */}} {{- if .Values.readinessProbe.enabled }} readinessProbe: httpGet: - path: ams/v1/versionInfo + path: api/ams/v1/versionInfo port: rest initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.readinessProbe.periodSeconds }} From 39a5aa1823e68dd3cbb5b55c74c2882f86de7534 Mon Sep 17 00:00:00 2001 From: Congxian Qiu Date: Tue, 15 Oct 2024 11:55:29 +0800 Subject: [PATCH 2/6] [AMORO-3253] Optijmizing getOPtimizingProcessTasks and getOptimizingTypes rest efficiency (#3254) Before the change we'll check whether the table exists by loading the table, the loading step maybe too expensive, and it's redundant because the table exists will be checked in the folllowing step when loading table to retrieve the info. --- .../amoro/server/dashboard/controller/TableController.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java index 1fd1f596fa..9db6f074df 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/dashboard/controller/TableController.java @@ -338,8 +338,6 @@ public void getOptimizingTypes(Context ctx) { String db = ctx.pathParam("db"); String table = ctx.pathParam("table"); TableIdentifier tableIdentifier = TableIdentifier.of(catalog, db, table); - ServerCatalog serverCatalog = tableService.getServerCatalog(catalog); - Preconditions.checkState(serverCatalog.tableExists(db, table), "no such table"); Map values = tableDescriptor.getTableOptimizingTypes(tableIdentifier.buildTableIdentifier()); @@ -361,10 +359,8 @@ public void getOptimizingProcessTasks(Context ctx) { int offset = (page - 1) * pageSize; int limit = pageSize; - ServerCatalog serverCatalog = tableService.getServerCatalog(catalog); Preconditions.checkArgument(offset >= 0, "offset[%s] must >= 0", offset); Preconditions.checkArgument(limit >= 0, "limit[%s] must >= 0", limit); - Preconditions.checkState(serverCatalog.tableExists(db, table), "no such table"); TableIdentifier tableIdentifier = TableIdentifier.of(catalog, db, table); List optimizingTaskInfos = From 4524764f1083c7144e97370346d217914deed48d Mon Sep 17 00:00:00 2001 From: Congxian Qiu Date: Wed, 16 Oct 2024 18:39:01 +0800 Subject: [PATCH 3/6] [AMORO-3065] Optimizing rest api efficiency for table detail (#3260) For table detail we'll retrive detail info from underlying table, before the change there will be two call to the catalog when loading the table: 1) check whether the database exists, 2) load the table for the detail info. After the change, the first call to catalog has been removed, and use the second call to check if the table exists. --- .../src/main/java/org/apache/amoro/CommonUnifiedCatalog.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java b/amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java index 79b6e55be8..b49d860fbb 100644 --- a/amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java +++ b/amoro-common/src/main/java/org/apache/amoro/CommonUnifiedCatalog.java @@ -122,10 +122,6 @@ public void dropDatabase(String database) { @Override public AmoroTable loadTable(String database, String table) { - if (!databaseExists(database)) { - throw new NoSuchDatabaseException("Database: " + database + " does not exist."); - } - return formatCatalogAsOrder( TableFormat.MIXED_HIVE, TableFormat.MIXED_ICEBERG, From 8954349d975c7b8a7e924b9d70455aad173bf36f Mon Sep 17 00:00:00 2001 From: Darcy Date: Wed, 16 Oct 2024 18:44:29 +0800 Subject: [PATCH 4/6] Fix: The issue of startup failure caused by upgrading the iceberg version. (#3262) fix: The issue of startup failure caused by upgrading the iceberg version. --- .../amoro/server/optimizing/OptimizingQueue.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizingQueue.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizingQueue.java index 2dcfcaca86..fb844a41a5 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizingQueue.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/OptimizingQueue.java @@ -666,12 +666,13 @@ private void cancelTasks() { } private void loadTaskRuntimes(OptimizingProcess optimizingProcess) { - List> taskRuntimes = - getAs( - OptimizingMapper.class, - mapper -> - mapper.selectTaskRuntimes(tableRuntime.getTableIdentifier().getId(), processId)); try { + List> taskRuntimes = + getAs( + OptimizingMapper.class, + mapper -> + mapper.selectTaskRuntimes( + tableRuntime.getTableIdentifier().getId(), processId)); Map inputs = TaskFilesPersistence.loadTaskInputs(processId); taskRuntimes.forEach( taskRuntime -> { From cc29688004ecd0b24b5bbafaa2f8326cb19a087e Mon Sep 17 00:00:00 2001 From: 7hong Date: Wed, 16 Oct 2024 20:31:40 +0800 Subject: [PATCH 5/6] [AMORO-3239] Fix stack overflow caused by reading too many partitions in the filter (#3240) * [AMORO-3239] Fix stack overflow caused by reading too many partitions in the filter * [AMORO-3239] Add the "ignore-filter-partition-count" parameter * move parameter "optimizer.ignore-filter-partition-count" to "self-optimizing.skip-filter-partition-count" * move parameter "self-optimizing.skip-filter-partition-count" to "refresh-tables.max-pending-partition-count" --- .../java/org/apache/amoro/server/AmoroManagementConf.java | 6 ++++++ .../amoro/server/optimizing/plan/OptimizingEvaluator.java | 6 +++++- .../amoro/server/optimizing/plan/OptimizingPlanner.java | 2 +- .../amoro/server/table/executor/AsyncTableExecutors.java | 3 ++- .../table/executor/TableRuntimeRefreshExecutor.java | 8 ++++++-- .../apache/amoro/server/TestDefaultOptimizingService.java | 2 +- .../apache/amoro/server/dashboard/TestOverviewCache.java | 2 +- .../server/optimizing/plan/TestOptimizingEvaluator.java | 2 +- .../amoro/server/table/TestTableSummaryMetrics.java | 2 +- dist/src/main/amoro-bin/conf/config.yaml | 1 + 10 files changed, 25 insertions(+), 9 deletions(-) diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java index 415995d815..f52a0113f2 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/AmoroManagementConf.java @@ -162,6 +162,12 @@ public class AmoroManagementConf { .defaultValue(60000L) .withDescription("Interval for refreshing table metadata."); + public static final ConfigOption REFRESH_MAX_PENDING_PARTITIONS = + ConfigOptions.key("refresh-tables.max-pending-partition-count") + .intType() + .defaultValue(100) + .withDescription("Filters will not be used beyond that number of partitions"); + public static final ConfigOption BLOCKER_TIMEOUT = ConfigOptions.key("blocker.timeout") .longType() diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/OptimizingEvaluator.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/OptimizingEvaluator.java index 9be5be7974..22ebd838b1 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/OptimizingEvaluator.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/OptimizingEvaluator.java @@ -62,15 +62,18 @@ public class OptimizingEvaluator { protected final MixedTable mixedTable; protected final TableRuntime tableRuntime; protected final TableSnapshot currentSnapshot; + protected final int maxPendingPartitions; protected boolean isInitialized = false; protected Map needOptimizingPlanMap = Maps.newHashMap(); protected Map partitionPlanMap = Maps.newHashMap(); - public OptimizingEvaluator(TableRuntime tableRuntime, MixedTable table) { + public OptimizingEvaluator( + TableRuntime tableRuntime, MixedTable table, int maxPendingPartitions) { this.tableRuntime = tableRuntime; this.mixedTable = table; this.currentSnapshot = IcebergTableUtil.getSnapshot(table, tableRuntime); + this.maxPendingPartitions = maxPendingPartitions; } public TableRuntime getTableRuntime() { @@ -137,6 +140,7 @@ private void initPartitionPlans(TableFileScanHelper tableFileScanHelper) { needOptimizingPlanMap.putAll( partitionPlanMap.entrySet().stream() .filter(entry -> entry.getValue().isNecessary()) + .limit(maxPendingPartitions) .collect(Collectors.toMap(entry -> entry.getKey(), entry -> entry.getValue()))); } diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/OptimizingPlanner.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/OptimizingPlanner.java index 3e93129c84..c87a66c8a3 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/OptimizingPlanner.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/OptimizingPlanner.java @@ -65,7 +65,7 @@ public OptimizingPlanner( MixedTable table, double availableCore, long maxInputSizePerThread) { - super(tableRuntime, table); + super(tableRuntime, table, Integer.MAX_VALUE); this.partitionFilter = tableRuntime.getPendingInput() == null ? Expressions.alwaysTrue() diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/AsyncTableExecutors.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/AsyncTableExecutors.java index a45385026e..1185232550 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/AsyncTableExecutors.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/AsyncTableExecutors.java @@ -77,7 +77,8 @@ public void setup(TableManager tableManager, Configurations conf) { new TableRuntimeRefreshExecutor( tableManager, conf.getInteger(AmoroManagementConf.REFRESH_TABLES_THREAD_COUNT), - conf.getLong(AmoroManagementConf.REFRESH_TABLES_INTERVAL)); + conf.getLong(AmoroManagementConf.REFRESH_TABLES_INTERVAL), + conf.getInteger(AmoroManagementConf.REFRESH_MAX_PENDING_PARTITIONS)); if (conf.getBoolean(AmoroManagementConf.AUTO_CREATE_TAGS_ENABLED)) { this.tagsAutoCreatingExecutor = new TagsAutoCreatingExecutor( diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/TableRuntimeRefreshExecutor.java b/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/TableRuntimeRefreshExecutor.java index 4fa3598a85..e613027f03 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/TableRuntimeRefreshExecutor.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/table/executor/TableRuntimeRefreshExecutor.java @@ -31,10 +31,13 @@ public class TableRuntimeRefreshExecutor extends BaseTableExecutor { // 1 minutes private final long interval; + private final int maxPendingPartitions; - public TableRuntimeRefreshExecutor(TableManager tableRuntimes, int poolSize, long interval) { + public TableRuntimeRefreshExecutor( + TableManager tableRuntimes, int poolSize, long interval, int maxPendingPartitions) { super(tableRuntimes, poolSize); this.interval = interval; + this.maxPendingPartitions = maxPendingPartitions; } @Override @@ -48,7 +51,8 @@ protected long getNextExecutingTime(TableRuntime tableRuntime) { private void tryEvaluatingPendingInput(TableRuntime tableRuntime, MixedTable table) { if (tableRuntime.isOptimizingEnabled() && !tableRuntime.getOptimizingStatus().isProcessing()) { - OptimizingEvaluator evaluator = new OptimizingEvaluator(tableRuntime, table); + OptimizingEvaluator evaluator = + new OptimizingEvaluator(tableRuntime, table, maxPendingPartitions); if (evaluator.isNecessary()) { OptimizingEvaluator.PendingInput pendingInput = evaluator.getOptimizingPendingInput(); logger.debug( diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java b/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java index 5a237700d7..7945dd8eea 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/TestDefaultOptimizingService.java @@ -414,7 +414,7 @@ protected void reboot() throws InterruptedException { private class TableRuntimeRefresher extends TableRuntimeRefreshExecutor { public TableRuntimeRefresher() { - super(tableService(), 1, Integer.MAX_VALUE); + super(tableService(), 1, Integer.MAX_VALUE, Integer.MAX_VALUE); } void refreshPending() { diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestOverviewCache.java b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestOverviewCache.java index 1b59308b9d..5ac3c85c3d 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestOverviewCache.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/dashboard/TestOverviewCache.java @@ -109,7 +109,7 @@ private void appendData(UnkeyedTable table, int id) { void refreshPending() { TableRuntimeRefreshExecutor refresher = - new TableRuntimeRefreshExecutor(tableService(), 1, Integer.MAX_VALUE); + new TableRuntimeRefreshExecutor(tableService(), 1, Integer.MAX_VALUE, Integer.MAX_VALUE); refresher.execute(tableService().getRuntime(serverTableIdentifier().getId())); refresher.dispose(); } diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/plan/TestOptimizingEvaluator.java b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/plan/TestOptimizingEvaluator.java index 63a853fe14..395aa653db 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/plan/TestOptimizingEvaluator.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/optimizing/plan/TestOptimizingEvaluator.java @@ -110,7 +110,7 @@ public void testFragmentFiles() { } protected OptimizingEvaluator buildOptimizingEvaluator() { - return new OptimizingEvaluator(getTableRuntime(), getMixedTable()); + return new OptimizingEvaluator(getTableRuntime(), getMixedTable(), 100); } protected void assertEmptyInput(OptimizingEvaluator.PendingInput input) { diff --git a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableSummaryMetrics.java b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableSummaryMetrics.java index d5d42186c9..bd2436ccb4 100644 --- a/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableSummaryMetrics.java +++ b/amoro-ams/src/test/java/org/apache/amoro/server/table/TestTableSummaryMetrics.java @@ -142,7 +142,7 @@ private void appendPosDelete(UnkeyedTable table) { void refreshPending() { TableRuntimeRefreshExecutor refresher = - new TableRuntimeRefreshExecutor(tableService(), 1, Integer.MAX_VALUE); + new TableRuntimeRefreshExecutor(tableService(), 1, Integer.MAX_VALUE, Integer.MAX_VALUE); refresher.execute(tableService().getRuntime(serverTableIdentifier().getId())); refresher.dispose(); } diff --git a/dist/src/main/amoro-bin/conf/config.yaml b/dist/src/main/amoro-bin/conf/config.yaml index ae1de24e48..d887239bb1 100644 --- a/dist/src/main/amoro-bin/conf/config.yaml +++ b/dist/src/main/amoro-bin/conf/config.yaml @@ -43,6 +43,7 @@ ams: refresh-tables: thread-count: 10 interval: 60000 # 1min + max-pending-partition-count: 100 # default 100 self-optimizing: commit-thread-count: 10 From bd2485a4be5ef89a818fe96e60d74b758450c6e0 Mon Sep 17 00:00:00 2001 From: 7hong Date: Thu, 17 Oct 2024 14:19:01 +0800 Subject: [PATCH 6/6] [AMORO-3269] Use actualPartitionPlans to determine the optimizingType (#3270) Use actualPartitionPlans to determine the optimizingType --- .../amoro/server/optimizing/plan/OptimizingPlanner.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/OptimizingPlanner.java b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/OptimizingPlanner.java index c87a66c8a3..cf6fc9e713 100644 --- a/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/OptimizingPlanner.java +++ b/amoro-ams/src/main/java/org/apache/amoro/server/optimizing/plan/OptimizingPlanner.java @@ -175,11 +175,11 @@ public List planTasks() { tasks.addAll(partitionPlan.splitTasks((int) (actualInputSize / avgThreadCost))); } if (!tasks.isEmpty()) { - if (evaluators.stream() - .anyMatch(evaluator -> evaluator.getOptimizingType() == OptimizingType.FULL)) { + if (actualPartitionPlans.stream() + .anyMatch(plan -> plan.getOptimizingType() == OptimizingType.FULL)) { optimizingType = OptimizingType.FULL; - } else if (evaluators.stream() - .anyMatch(evaluator -> evaluator.getOptimizingType() == OptimizingType.MAJOR)) { + } else if (actualPartitionPlans.stream() + .anyMatch(plan -> plan.getOptimizingType() == OptimizingType.MAJOR)) { optimizingType = OptimizingType.MAJOR; } else { optimizingType = OptimizingType.MINOR;