diff --git a/models/interfaces.ts b/models/interfaces.ts index fe67645e..c8e4860a 100644 --- a/models/interfaces.ts +++ b/models/interfaces.ts @@ -73,6 +73,7 @@ export interface Policy { description: string; default_state: string; states: State[]; + ism_template: any; } export interface State { diff --git a/public/pages/ChangePolicy/components/NewPolicy/NewPolicy.tsx b/public/pages/ChangePolicy/components/NewPolicy/NewPolicy.tsx index 274e3b9a..ce376941 100644 --- a/public/pages/ChangePolicy/components/NewPolicy/NewPolicy.tsx +++ b/public/pages/ChangePolicy/components/NewPolicy/NewPolicy.tsx @@ -19,7 +19,7 @@ import { EuiSpacer, EuiText, EuiRadioGroup, EuiFormRow, EuiSelect, EuiComboBox, import { ContentPanel } from "../../../../components/ContentPanel"; import { IndexService } from "../../../../services"; import { Radio } from "../../containers/ChangePolicy/ChangePolicy"; -import { Policy } from "../../../../../models/interfaces"; +import { DocumentPolicy, Policy } from "../../../../../models/interfaces"; import { PolicyOption } from "../../models/interfaces"; import { getErrorMessage } from "../../../../utils/helpers"; import { DOCUMENTATION_URL } from "../../../../utils/constants"; @@ -60,9 +60,9 @@ export default class NewPolicy extends React.Component ({ - label: hit._id, - value: hit._source.policy, + const policies = searchPoliciesResponse.response.policies.map((p: DocumentPolicy) => ({ + label: p.id, + value: p.policy, })); this.setState({ policies }); } else { diff --git a/public/pages/EditRollup/containers/EditRollup.tsx b/public/pages/EditRollup/containers/EditRollup.tsx index a4591df7..e3a359c9 100644 --- a/public/pages/EditRollup/containers/EditRollup.tsx +++ b/public/pages/EditRollup/containers/EditRollup.tsx @@ -257,6 +257,7 @@ export default class EditRollup extends Component spec", () => { }); it("successfully calls search policies on mount", async () => { - httpClientMock.post = jest.fn().mockResolvedValue({ ok: true, resp: { hits: { hits: [{ _id: "test_index" }] } } }); + httpClientMock.get = jest.fn().mockResolvedValue({ ok: true, resp: { hits: { hits: [{ _id: "test_index" }] } } }); const spy = jest.spyOn(browserServicesMock.indexService, "searchPolicies"); render( @@ -49,7 +49,7 @@ describe(" spec", () => { }); it("adds danger toaster on safe error", async () => { - httpClientMock.post = jest.fn().mockResolvedValue({ ok: false, error: "some error" }); + httpClientMock.get = jest.fn().mockResolvedValue({ ok: false, error: "some error" }); const spy = jest.spyOn(browserServicesMock.indexService, "searchPolicies"); render( @@ -67,7 +67,7 @@ describe(" spec", () => { }); it("adds danger toaster on unsafe error", async () => { - httpClientMock.post = jest.fn().mockRejectedValue(new Error("testing error")); + httpClientMock.get = jest.fn().mockRejectedValue(new Error("testing error")); const spy = jest.spyOn(browserServicesMock.indexService, "searchPolicies"); render( diff --git a/public/pages/Indices/components/ApplyPolicyModal/ApplyPolicyModal.tsx b/public/pages/Indices/components/ApplyPolicyModal/ApplyPolicyModal.tsx index f59f0be9..2ce15a53 100644 --- a/public/pages/Indices/components/ApplyPolicyModal/ApplyPolicyModal.tsx +++ b/public/pages/Indices/components/ApplyPolicyModal/ApplyPolicyModal.tsx @@ -36,7 +36,7 @@ import { } from "@elastic/eui"; import { BrowserServices } from "../../../../models/interfaces"; import { PolicyOption } from "../../models/interfaces"; -import { Policy, State } from "../../../../../models/interfaces"; +import { DocumentPolicy, Policy, State } from "../../../../../models/interfaces"; import { getErrorMessage } from "../../../../utils/helpers"; import { DOCUMENTATION_URL } from "../../../../utils/constants"; import { CoreServicesContext } from "../../../../components/core_services"; @@ -136,9 +136,9 @@ export default class ApplyPolicyModal extends Component ({ - label: hit._id, - policy: hit._source.policy, + const policies = searchPoliciesResponse.response.policies.map((p: DocumentPolicy) => ({ + label: p.id, + policy: p.policy, })); this.setState({ policyOptions: policies }); } else { diff --git a/public/pages/ManagedIndices/containers/ManagedIndices/ManagedIndices.tsx b/public/pages/ManagedIndices/containers/ManagedIndices/ManagedIndices.tsx index 7ee72802..e23b0a9f 100644 --- a/public/pages/ManagedIndices/containers/ManagedIndices/ManagedIndices.tsx +++ b/public/pages/ManagedIndices/containers/ManagedIndices/ManagedIndices.tsx @@ -181,10 +181,8 @@ export default class ManagedIndices extends Component { let errorMessage: string | undefined = undefined; - if (!item.policy) { - if (!item.managedIndexMetaData) errorMessage = `Still initializing, please wait a moment`; - else errorMessage = `Failed to load the policy: ${item.policyId}`; - } + if (item.managedIndexMetaData?.policySeqNo == null) errorMessage = `Still initializing, please wait a moment`; + if (!item.policy) errorMessage = `Failed to load the policy: ${item.policyId}`; return ( diff --git a/public/pages/Policies/containers/Policies/Policies.tsx b/public/pages/Policies/containers/Policies/Policies.tsx index eb37624b..6f351aba 100644 --- a/public/pages/Policies/containers/Policies/Policies.tsx +++ b/public/pages/Policies/containers/Policies/Policies.tsx @@ -106,7 +106,7 @@ export default class Policies extends Component { ), }, { - field: "policy.policy.description", + field: "policy.description", name: "Description", sortable: true, truncateText: true, @@ -114,7 +114,7 @@ export default class Policies extends Component { width: "150px", }, { - field: "policy.policy.last_updated_time", + field: "policy.last_updated_time", name: "Last updated time", sortable: true, truncateText: false, diff --git a/public/services/IndexService.test.ts b/public/services/IndexService.test.ts index cd81e8a8..46555e2b 100644 --- a/public/services/IndexService.test.ts +++ b/public/services/IndexService.test.ts @@ -44,6 +44,6 @@ describe("IndexService spec", () => { const searchValue = "test"; await indexService.searchPolicies(searchValue); - expect(httpClientMock.post).toHaveBeenCalledTimes(1); + expect(httpClientMock.get).toHaveBeenCalledTimes(1); }); }); diff --git a/public/services/IndexService.ts b/public/services/IndexService.ts index 1417d39e..f8708594 100644 --- a/public/services/IndexService.ts +++ b/public/services/IndexService.ts @@ -15,7 +15,13 @@ import { HttpSetup } from "kibana/public"; import { INDEX } from "../../server/utils/constants"; -import { AcknowledgedResponse, ApplyPolicyResponse, GetIndicesResponse, SearchResponse } from "../../server/models/interfaces"; +import { + AcknowledgedResponse, + ApplyPolicyResponse, + GetIndicesResponse, + GetPoliciesResponse, + SearchResponse, +} from "../../server/models/interfaces"; import { ServerResponse } from "../../server/models/types"; import { NODE_API } from "../../utils/constants"; @@ -46,22 +52,11 @@ export default class IndexService { return response; }; - searchPolicies = async (searchValue: string, source: boolean = false): Promise>> => { + searchPolicies = async (searchValue: string, source: boolean = false): Promise> => { const str = searchValue.trim(); - const mustQuery = { - query_string: { - default_field: "policy.policy_id", - default_operator: "AND", - query: str ? `*${str.split(" ").join("* *")}*` : "*", - }, - }; - const body = { - index: INDEX.OPENDISTRO_ISM_CONFIG, - size: 10, - query: { _source: source, query: { bool: { must: [mustQuery, { exists: { field: "policy" } }] } } }, - }; - const url = `..${NODE_API._SEARCH}`; - const response = (await this.httpClient.post(url, { body: JSON.stringify(body) })) as ServerResponse; + const queryObject = { from: 0, size: 10, search: str, sortDirection: "desc", sortField: "id" }; + const url = `..${NODE_API.POLICIES}`; + const response = (await this.httpClient.get(url, { query: queryObject })) as ServerResponse; return response; }; } diff --git a/server/clusters/ism/ismPlugin.ts b/server/clusters/ism/ismPlugin.ts index 27ae08af..e9e1acc9 100644 --- a/server/clusters/ism/ismPlugin.ts +++ b/server/clusters/ism/ismPlugin.ts @@ -38,6 +38,13 @@ export default function ismPlugin(Client: any, config: any, components: any) { method: "GET", }); + ism.getPolicies = ca({ + url: { + fmt: `${API.POLICY_BASE}`, + }, + method: "GET", + }); + ism.createPolicy = ca({ url: { fmt: `${API.POLICY_BASE}/<%=policyId%>?refresh=wait_for`, @@ -100,6 +107,13 @@ export default function ismPlugin(Client: any, config: any, components: any) { method: "GET", }); + ism.explainAll = ca({ + url: { + fmt: `${API.EXPLAIN_BASE}`, + }, + method: "GET", + }); + ism.retry = ca({ url: { fmt: `${API.RETRY_BASE}/<%=index%>`, diff --git a/server/models/interfaces.ts b/server/models/interfaces.ts index 00cb4f9c..1075be34 100644 --- a/server/models/interfaces.ts +++ b/server/models/interfaces.ts @@ -34,6 +34,11 @@ export interface ExplainResponse { [index: string]: ExplainAPIManagedIndexMetaData | undefined; } +export interface ExplainAllResponse { + [index: string]: ExplainAPIManagedIndexMetaData | number; + total_managed_indices: number; +} + export interface GetManagedIndicesResponse { totalManagedIndices: number; managedIndices: ManagedIndexItem[]; @@ -170,10 +175,10 @@ export interface FailedIndex { } export interface ExplainAPIManagedIndexMetaData { - "opendistro.index_state_management.policy_id": string | null; - index?: string; - index_uuid?: string; - policy_id?: string; + "index.opendistro.index_state_management.policy_id": string | null; + index: string; + index_uuid: string; + policy_id: string; policy_seq_no?: number; policy_primary_term?: number; policy_completed?: boolean; @@ -183,6 +188,7 @@ export interface ExplainAPIManagedIndexMetaData { action?: { name: string; start_time: number; index: number; failed: boolean; consumed_retries: number }; retry_info?: { failed: boolean; consumed_retries: number }; info?: object; + enabled: boolean; } export interface IndexManagementApi { diff --git a/server/services/IndexService.ts b/server/services/IndexService.ts index 6b9d95f9..c11d7e29 100644 --- a/server/services/IndexService.ts +++ b/server/services/IndexService.ts @@ -15,7 +15,16 @@ import { RequestParams } from "@elastic/elasticsearch"; import { INDEX, Setting } from "../utils/constants"; -import { AcknowledgedResponse, ApplyPolicyResponse, AddResponse, CatIndex, GetIndicesResponse, SearchResponse } from "../models/interfaces"; +import { + AcknowledgedResponse, + ApplyPolicyResponse, + AddResponse, + CatIndex, + GetIndicesResponse, + SearchResponse, + ExplainResponse, + ExplainAPIManagedIndexMetaData, +} from "../models/interfaces"; import { ServerResponse } from "../models/types"; import { KibanaRequest, KibanaResponseFactory, IClusterClient, IKibanaResponse, RequestHandlerContext } from "../../../../src/core/server"; @@ -26,35 +35,6 @@ export default class IndexService { this.esDriver = esDriver; } - search = async ( - context: RequestHandlerContext, - request: KibanaRequest, - response: KibanaResponseFactory - ): Promise>> => { - try { - const { query, index, size = 0 } = request.body as { query: object; index: string; size?: number }; - const params: RequestParams.Search = { index, size, body: query }; - const { callAsCurrentUser: callWithRequest } = this.esDriver.asScoped(request); - const results: SearchResponse = await callWithRequest("search", params); - return response.custom({ - statusCode: 200, - body: { - ok: true, - response: results, - }, - }); - } catch (err) { - console.error("Index Management - IndexService - search", err); - return response.custom({ - statusCode: 200, - body: { - ok: false, - error: err.message, - }, - }); - } - }; - getIndices = async ( context: RequestHandlerContext, request: KibanaRequest, @@ -81,9 +61,9 @@ export default class IndexService { const fromNumber = parseInt(from, 10); const sizeNumber = parseInt(size, 10); const paginatedIndices = indicesResponse.slice(fromNumber, fromNumber + sizeNumber); - const indexUuids = paginatedIndices.map((value: CatIndex) => value.uuid); + const indexNames = paginatedIndices.map((value: CatIndex) => value.index); - const managedStatus = await this._getManagedStatus(request, indexUuids); + const managedStatus = await this._getManagedStatus(request, indexNames); // NOTE: Cannot use response.ok due to typescript type checking return response.custom({ @@ -91,7 +71,7 @@ export default class IndexService { body: { ok: true, response: { - indices: paginatedIndices.map((catIndex: CatIndex) => ({ ...catIndex, managed: managedStatus[catIndex.uuid] || "N/A" })), + indices: paginatedIndices.map((catIndex: CatIndex) => ({ ...catIndex, managed: managedStatus[catIndex.index] || "N/A" })), totalIndices: indicesResponse.length, }, }, @@ -121,30 +101,25 @@ export default class IndexService { } }; - // given a list of indexUuids return the managed status of each (true, false, N/A) - _getManagedStatus = async (request: KibanaRequest, indexUuids: string[]): Promise<{ [indexUuid: string]: string }> => { + _getManagedStatus = async (request: KibanaRequest, indexNames: string[]): Promise<{ [indexName: string]: string }> => { try { - const searchParams: RequestParams.Search = { - index: INDEX.OPENDISTRO_ISM_CONFIG, - size: indexUuids.length, - body: { _source: "_id", query: { ids: { values: indexUuids } } }, - }; - const { callAsCurrentUser: searchCallWithRequest } = this.esDriver.asScoped(request); - const results: SearchResponse = await searchCallWithRequest("search", searchParams); - const managed: { [indexUuid: string]: string } = results.hits.hits.reduce( - (accu: object, hit: { _id: string }) => ({ ...accu, [hit._id]: "Yes" }), - {} - ); - return indexUuids.reduce((accu: object, value: string) => ({ ...accu, [value]: managed[value] || "No" }), {}); - } catch (err) { - // If the config index does not exist then nothing is being managed - if (err.statusCode === 404 && err.body.error.type === "index_not_found_exception") { - return indexUuids.reduce((accu, value) => ({ ...accu, [value]: "No" }), {}); + const explainParamas = { index: indexNames.toString() }; + const { callAsCurrentUser: callWithRequest } = this.esDriver.asScoped(request); + const explainResponse: ExplainResponse = await callWithRequest("ism.explain", explainParamas); + + const managed: { [indexName: string]: string } = {}; + for (const indexName in explainResponse) { + if (indexName === "total_managed_indices") continue; + const explain = explainResponse[indexName] as ExplainAPIManagedIndexMetaData; + managed[indexName] = explain["index.opendistro.index_state_management.policy_id"] === null ? "No" : "Yes"; } + + return managed; + } catch (err) { // otherwise it could be an unauthorized access error to config index or some other error // in which case we will return managed status N/A console.error("Index Management - IndexService - _getManagedStatus:", err); - return indexUuids.reduce((accu, value) => ({ ...accu, [value]: "N/A" }), {}); + return indexNames.reduce((accu, value) => ({ ...accu, [value]: "N/A" }), {}); } }; diff --git a/server/services/ManagedIndexService.ts b/server/services/ManagedIndexService.ts index 123237ca..30ba3c15 100644 --- a/server/services/ManagedIndexService.ts +++ b/server/services/ManagedIndexService.ts @@ -20,6 +20,8 @@ import { INDEX } from "../utils/constants"; import { getMustQuery, transformManagedIndexMetaData } from "../utils/helpers"; import { ChangePolicyResponse, + ExplainAllResponse, + ExplainAPIManagedIndexMetaData, ExplainResponse, GetManagedIndicesResponse, RemovePolicyResponse, @@ -30,6 +32,7 @@ import { SearchResponse, } from "../models/interfaces"; import { ManagedIndicesSort, ServerResponse } from "../models/types"; +import { ManagedIndexItem } from "../../models/interfaces"; export default class ManagedIndexService { esDriver: IClusterClient; @@ -83,59 +86,53 @@ export default class ManagedIndexService { }; const managedIndexSorts: ManagedIndicesSort = { index: "managed_index.index", policyId: "managed_index.policy_id" }; - const searchParams: RequestParams.Search = { - index: INDEX.OPENDISTRO_ISM_CONFIG, - seq_no_primary_term: true, - body: { - size, - from, - sort: managedIndexSorts[sortField] ? [{ [managedIndexSorts[sortField]]: sortDirection }] : [], - query: { - bool: { - filter: [{ exists: { field: "managed_index" } }], - must: getMustQuery("managed_index.name", search), - }, - }, - }, + const explainParams = { + size, + from, + sortField: sortField ? managedIndexSorts[sortField] : null, + sortOrder: sortDirection, + queryString: search ? `*${search.split(" ").join("* *")}*` : null, }; const { callAsCurrentUser: callWithRequest } = this.esDriver.asScoped(request); - const searchResponse: SearchResponse = await callWithRequest("search", searchParams); + const explainAllResponse: ExplainAllResponse = await callWithRequest("ism.explainAll", explainParams); - const indices = searchResponse.hits.hits.map((hit) => hit._source.managed_index.index); - const totalManagedIndices = _.get(searchResponse, "hits.total.value", 0); - - if (!indices.length) { - return response.custom({ - statusCode: 200, - body: { - ok: true, - response: { managedIndices: [], totalManagedIndices: 0 }, - }, + const managedIndices: ManagedIndexItem[] = []; + for (const indexName in explainAllResponse) { + if (indexName == "total_managed_indices") continue; + const metadata = explainAllResponse[indexName] as ExplainAPIManagedIndexMetaData; + let policy, seqNo, primaryTerm, getResponse; + try { + getResponse = await callWithRequest("ism.getPolicy", { policyId: metadata.policy_id }); + } catch (err) { + if (err.statusCode === 404 && err.body.error.reason === "Policy not found") { + console.log("managed index with not existing policy"); + } else { + throw err; + } + } + policy = _.get(getResponse, "policy", null); + seqNo = _.get(getResponse, "_seq_no", null); + primaryTerm = _.get(getResponse, "_primary_term", null); + managedIndices.push({ + index: metadata.index, + indexUuid: metadata.index_uuid, + policyId: metadata.policy_id, + policySeqNo: seqNo, + policyPrimaryTerm: primaryTerm, + policy: policy, + enabled: metadata.enabled, + managedIndexMetaData: transformManagedIndexMetaData(metadata), }); } - const explainParams = { index: indices.join(",") }; - const explainResponse: ExplainResponse = await callWithRequest("ism.explain", explainParams); - const managedIndices = searchResponse.hits.hits.map((hit) => { - const index = hit._source.managed_index.index; - return { - index, - indexUuid: hit._source.managed_index.index_uuid, - policyId: hit._source.managed_index.policy_id, - policySeqNo: hit._source.managed_index.policy_seq_no, - policyPrimaryTerm: hit._source.managed_index.policy_primary_term, - policy: hit._source.managed_index.policy, - enabled: hit._source.managed_index.enabled, - managedIndexMetaData: transformManagedIndexMetaData(explainResponse[index]), // this will be undefined if we are initializing - }; - }); + const totalManagedIndices: number = explainAllResponse.total_managed_indices; return response.custom({ statusCode: 200, body: { ok: true, - response: { managedIndices, totalManagedIndices }, + response: { managedIndices: managedIndices, totalManagedIndices: totalManagedIndices }, }, }); } catch (err) { diff --git a/server/services/PolicyService.ts b/server/services/PolicyService.ts index 5e8f6a30..c1f059fb 100644 --- a/server/services/PolicyService.ts +++ b/server/services/PolicyService.ts @@ -160,7 +160,7 @@ export default class PolicyService { }; /** - * Performs a fuzzy search request on policy id + * Calls backend Get Policy API */ getPolicies = async ( context: RequestHandlerContext, @@ -168,9 +168,9 @@ export default class PolicyService { response: KibanaResponseFactory ): Promise>> => { try { - const { from, size, search, sortDirection, sortField } = request.query as { - from: string; - size: string; + const { from = 0, size = 20, search, sortDirection = "desc", sortField = "id" } = request.query as { + from: number; + size: number; search: string; sortDirection: string; sortField: string; @@ -181,33 +181,27 @@ export default class PolicyService { "policy.policy.description": "policy.description.keyword", "policy.policy.last_updated_time": "policy.last_updated_time", }; + const params = { - index: INDEX.OPENDISTRO_ISM_CONFIG, - seq_no_primary_term: true, - body: { - size, - from, - sort: policySorts[sortField] ? [{ [policySorts[sortField]]: sortDirection }] : [], - query: { - bool: { - filter: [{ exists: { field: "policy" } }], - must: getMustQuery("policy.policy_id", search), - }, - }, - }, + from, + size, + sortOrder: sortDirection, + sortField: policySorts[sortField] || policySorts.id, + queryString: search.trim() ? `*${search.trim().split(" ").join("* *")}*` : "*", }; const { callAsCurrentUser: callWithRequest } = this.esDriver.asScoped(request); - const searchResponse: SearchResponse = await callWithRequest("search", params); + const getResponse = await callWithRequest("ism.getPolicies", params); - const totalPolicies = searchResponse.hits.total.value; - const policies = searchResponse.hits.hits.map((hit) => ({ - seqNo: hit._seq_no as number, - primaryTerm: hit._primary_term as number, - id: hit._id, - policy: hit._source, + const policies: DocumentPolicy[] = getResponse.policies.map((p) => ({ + seqNo: p._seq_no, + primaryTerm: p._primary_term, + id: p._id, + policy: p.policy, })); + const totalPolicies: number = getResponse.total_policies; + return response.custom({ statusCode: 200, body: {