Skip to content

Commit

Permalink
Fix additional rebase issues
Browse files Browse the repository at this point in the history
Signed-off-by: Robert Downs <[email protected]>
  • Loading branch information
downsrob committed Mar 3, 2022
1 parent 7c254e1 commit 3ad2464
Show file tree
Hide file tree
Showing 16 changed files with 30 additions and 110 deletions.
Empty file.
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,7 @@ class IndexManagementPlugin : JobSchedulerExtension, NetworkPlugin, ActionPlugin
.registerSkipFlag(skipFlag)
.registerThreadPool(threadPool)
.registerExtensionChecker(extensionChecker)
.registerIndexMetadataProvider(indexMetadataProvider)

val metadataService = MetadataService(client, clusterService, skipFlag, indexManagementIndices)
val templateService = ISMTemplateService(client, clusterService, xContentRegistry, indexManagementIndices)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@ import org.opensearch.indexmanagement.indexstatemanagement.util.hasTimedOut
import org.opensearch.indexmanagement.indexstatemanagement.util.hasVersionConflict
import org.opensearch.indexmanagement.indexstatemanagement.util.isAllowed
import org.opensearch.indexmanagement.indexstatemanagement.util.isFailed
import org.opensearch.indexmanagement.indexstatemanagement.util.isMetadataMoved
import org.opensearch.indexmanagement.indexstatemanagement.util.isSafeToChange
import org.opensearch.indexmanagement.indexstatemanagement.util.isSuccessfulDelete
import org.opensearch.indexmanagement.indexstatemanagement.util.managedIndexConfigIndexRequest
Expand Down Expand Up @@ -252,7 +251,8 @@ object ManagedIndexRunner :
indexMetadata = null
// If the cluster state/default index type didn't have an index with a matching name and uuid combination, try all other index types
val nonDefaultIndexTypes = indexMetadataProvider.services.keys.filter { it != DEFAULT_INDEX_TYPE }
val multiTypeIndexNameToMetaData = indexMetadataProvider.getMultiTypeIndexMetadata(nonDefaultIndexTypes, listOf(managedIndexConfig.index))
val multiTypeIndexNameToMetaData =
indexMetadataProvider.getMultiTypeISMIndexMetadata(nonDefaultIndexTypes, listOf(managedIndexConfig.index))
val someTypeMatchedUuid = multiTypeIndexNameToMetaData.values.any {
it[managedIndexConfig.index]?.indexUuid == managedIndexConfig.indexUuid
}
Expand Down Expand Up @@ -841,7 +841,7 @@ object ManagedIndexRunner :
@Suppress("ReturnCount")
private suspend fun getIndexCreationDate(managedIndexConfig: ManagedIndexConfig): Long? {
try {
val multiTypeIndexNameToMetaData = indexMetadataProvider.getMultiTypeIndexMetadata(indexNames = listOf(managedIndexConfig.index))
val multiTypeIndexNameToMetaData = indexMetadataProvider.getMultiTypeISMIndexMetadata(indexNames = listOf(managedIndexConfig.index))
// the managedIndexConfig.indexUuid should be unique across all index types
val indexCreationDate = multiTypeIndexNameToMetaData.values.firstOrNull {
it[managedIndexConfig.index]?.indexUuid == managedIndexConfig.indexUuid
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -118,16 +118,15 @@ data class Policy(
}

/**
* Disallowed actions are specified in the [ManagedIndexSettings.BLOCKED_ACTIONS_LIST], or are not specified in the
* deprecated [ManagedIndexSettings.ALLOW_LIST] setting.
* Disallowed actions are ones that are not specified in the [ManagedIndexSettings.ALLOW_LIST] setting.
*/
fun getDisallowedActions(allowList: List<String>, blockedActionsList: List<String>): List<String> {
val allowedActionsSet = allowList.toSet() - blockedActionsList.toSet()
fun getDisallowedActions(allowList: List<String>): List<String> {
val allowListSet = allowList.toSet()
val disallowedActions = mutableListOf<String>()
this.states.forEach { state ->
state.actions.forEach { action ->
if (!allowedActionsSet.contains(action.type)) {
disallowedActions.add(action.type)
state.actions.forEach { actionConfig ->
if (!allowListSet.contains(actionConfig.type)) {
disallowedActions.add(actionConfig.type)
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.LEGACY_POL
import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.POLICY_BASE_URI
import org.opensearch.indexmanagement.indexstatemanagement.model.Policy
import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.ALLOW_LIST
import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings.Companion.BLOCKED_ACTIONS_LIST
import org.opensearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.IndexPolicyAction
import org.opensearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.IndexPolicyRequest
import org.opensearch.indexmanagement.indexstatemanagement.transport.action.indexpolicy.IndexPolicyResponse
Expand All @@ -42,11 +41,9 @@ class RestIndexPolicyAction(
) : BaseRestHandler() {

@Volatile private var allowList = ALLOW_LIST.get(settings)
@Volatile private var blockedActionsList = BLOCKED_ACTIONS_LIST.get(settings)

init {
clusterService.clusterSettings.addSettingsUpdateConsumer(ALLOW_LIST) { allowList = it }
clusterService.clusterSettings.addSettingsUpdateConsumer(BLOCKED_ACTIONS_LIST) { blockedActionsList = it }
}

override fun routes(): List<Route> {
Expand Down Expand Up @@ -88,7 +85,7 @@ class RestIndexPolicyAction(
WriteRequest.RefreshPolicy.IMMEDIATE
}

val disallowedActions = policy.getDisallowedActions(allowList, blockedActionsList)
val disallowedActions = policy.getDisallowedActions(allowList)
if (disallowedActions.isNotEmpty()) {
return RestChannelConsumer { channel ->
channel.sendResponse(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -179,8 +179,7 @@ class LegacyOpenDistroManagedIndexSettings {
ALLOW_LIST_ALL,
Function.identity(),
Setting.Property.NodeScope,
Setting.Property.Dynamic,
Setting.Property.Deprecated
Setting.Property.Dynamic
)

val SNAPSHOT_DENY_LIST: Setting<List<String>> = Setting.listSetting(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ class ManagedIndexSettings {
const val DEFAULT_JOB_INTERVAL = 5
const val DEFAULT_JITTER = 0.6
const val DEFAULT_RESTRICTED_PATTERN = "\\.opendistro_security|\\.kibana.*|\\$INDEX_MANAGEMENT_INDEX"
val DEFAULT_BLOCKED_ACTIONS = emptyList<String>()
val ALLOW_LIST_NONE = emptyList<String>()
val SNAPSHOT_DENY_LIST_NONE = emptyList<String>()
const val HOST_DENY_LIST = "opendistro.destination.host.deny_list"
Expand Down Expand Up @@ -173,15 +172,6 @@ class ManagedIndexSettings {
LegacyOpenDistroManagedIndexSettings.ALLOW_LIST,
Function.identity(),
Setting.Property.NodeScope,
Setting.Property.Dynamic,
Setting.Property.Deprecated
)

val BLOCKED_ACTIONS_LIST: Setting<List<String>> = Setting.listSetting(
"plugins.index_state_management.blocked_actions_list",
DEFAULT_BLOCKED_ACTIONS,
Function.identity(),
Setting.Property.NodeScope,
Setting.Property.Dynamic
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ class AttemptTransitionStep(private val action: TransitionsAction) : Step(name)
} else {
// And then finally check all other index types which may not be in the cluster
val nonDefaultIndexTypes = indexMetadataProvider.services.keys.filter { it != DEFAULT_INDEX_TYPE }
val multiTypeIndexNameToMetaData = indexMetadataProvider.getMultiTypeIndexMetadata(nonDefaultIndexTypes, listOf(indexName))
val multiTypeIndexNameToMetaData = indexMetadataProvider.getMultiTypeISMIndexMetadata(nonDefaultIndexTypes, listOf(indexName))
// the managedIndexConfig.indexUuid should be unique across all index types
val indexCreationDate = multiTypeIndexNameToMetaData.values.firstOrNull {
it[indexName]?.indexUuid == metadata.indexUuid
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -468,12 +468,9 @@ fun Policy.isSafeToChange(stateName: String?, newPolicy: Policy, changePolicy: C
}

/**
* Allowed actions are specified in the [ManagedIndexSettings.ALLOW_LIST] setting, and are not specified in the
* [ManagedIndexSettings.BLOCKED_ACTIONS_LIST] setting.
* Allowed actions are ones that are specified in the [ManagedIndexSettings.ALLOW_LIST] setting.
*/
fun Action.isAllowed(blockedActionsList: List<String>, allowList: List<String>): Boolean {
return allowList.contains(this.type) && !blockedActionsList.contains(this.type)
}
fun Action.isAllowed(allowList: List<String>): Boolean = allowList.contains(this.type)

/**
* Check if cluster state metadata has been moved to config index
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,6 @@ class IndexManagementSettingsTests : OpenSearchTestCase() {
ManagedIndexSettings.COORDINATOR_BACKOFF_COUNT,
ManagedIndexSettings.COORDINATOR_BACKOFF_MILLIS,
ManagedIndexSettings.ALLOW_LIST,
ManagedIndexSettings.BLOCKED_ACTIONS_LIST,
ManagedIndexSettings.SNAPSHOT_DENY_LIST,
ManagedIndexSettings.JITTER,
RollupSettings.ROLLUP_INGEST_BACKOFF_COUNT,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ class ActionRetryIT : IndexStateManagementRestTestCase() {
ManagedIndexMetaData.POLICY_SEQ_NO to policySeq::equals,
ManagedIndexMetaData.POLICY_PRIMARY_TERM to policyPrimaryTerm::equals,
ManagedIndexMetaData.ROLLED_OVER to false::equals,
ManagedIndexMetaData.INDEX_CREATION_DATE to fun(indexCreationDate: Any?): Boolean = (indexCreationDate as Long) > 1L,
StateMetaData.STATE to fun(stateMetaDataMap: Any?): Boolean =
assertStateEquals(StateMetaData("Ingest", Instant.now().toEpochMilli()), stateMetaDataMap),
ActionMetaData.ACTION to fun(actionMetaDataMap: Any?): Boolean =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

package org.opensearch.indexmanagement.indexstatemanagement.runner

import org.opensearch.indexmanagement.indexstatemanagement.ISMActionsParser
import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase
import org.opensearch.indexmanagement.indexstatemanagement.action.OpenAction
import org.opensearch.indexmanagement.indexstatemanagement.action.ReadOnlyAction
Expand Down Expand Up @@ -129,8 +130,8 @@ class ManagedIndexRunnerIT : IndexStateManagementRestTestCase() {
}
}

fun `test blocked action fails execution`() {
val indexName = "blocked_action_index"
fun `test allow list fails execution`() {
val indexName = "allow_list_index"

val firstState = randomState(
name = "first_state", actions = listOf(randomReadOnlyActionConfig()),
Expand Down Expand Up @@ -167,8 +168,11 @@ class ManagedIndexRunnerIT : IndexStateManagementRestTestCase() {
updateManagedIndexConfigStartTime(managedIndexConfig)
waitFor { assertEquals(AttemptTransitionStep.getSuccessMessage(indexName, firstState.name), getExplainManagedIndexMetaData(indexName).info?.get("message")) }

// block the read_only action
updateClusterSetting(ManagedIndexSettings.BLOCKED_ACTIONS_LIST.key, "[\"${ReadOnlyAction.name}\"]", escapeValue = false)
// remove read_only from the allowlist
val allowedActions = ISMActionsParser.instance.parsers.map { it.getActionType() }.toList()
.filter { actionType -> actionType != ReadOnlyAction.name }
.joinToString(prefix = "[", postfix = "]") { string -> "\"$string\"" }
updateClusterSetting(ManagedIndexSettings.ALLOW_LIST.key, allowedActions, escapeValue = false)

// speed up to fifth execution that should try to set index to read only and fail because the action is not allowed
updateManagedIndexConfigStartTime(managedIndexConfig)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ class ManagedIndexRunnerTests : OpenSearchTestCase() {
settingSet.add(ManagedIndexSettings.JOB_INTERVAL)
settingSet.add(ManagedIndexSettings.INDEX_STATE_MANAGEMENT_ENABLED)
settingSet.add(ManagedIndexSettings.ALLOW_LIST)
settingSet.add(ManagedIndexSettings.BLOCKED_ACTIONS_LIST)
val clusterSettings = ClusterSettings(settings, settingSet)
val originClusterService: ClusterService = ClusterServiceUtils.createClusterService(threadPool, discoveryNode, clusterSettings)
clusterService = Mockito.spy(originClusterService)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ class AttemptTransitionStepTests : OpenSearchTestCase() {
whenever(docsStats.count).doReturn(6L)
whenever(docsStats.totalSizeInBytes).doReturn(2)
val client = getClient(getAdminClient(getIndicesAdminClient(statsResponse, null)))
val indexMetadataProvider = IndexMetadataProvider(client, clusterService, mutableMapOf())
val indexMetadataProvider = IndexMetadataProvider(settings, client, clusterService, mutableMapOf())

runBlocking {
val managedIndexMetadata = ManagedIndexMetaData(indexName, indexUUID, "policy_id", null, null, null, null, null, null, null, null, null, null, null)
Expand All @@ -87,7 +87,7 @@ class AttemptTransitionStepTests : OpenSearchTestCase() {
whenever(indexMetadata.creationDate).doReturn(5L)
val exception = IllegalArgumentException("example")
val client = getClient(getAdminClient(getIndicesAdminClient(null, exception)))
val indexMetadataProvider = IndexMetadataProvider(client, clusterService, mutableMapOf())
val indexMetadataProvider = IndexMetadataProvider(settings, client, clusterService, mutableMapOf())

runBlocking {
val managedIndexMetadata = ManagedIndexMetaData(indexName, indexUUID, "policy_id", null, null, null, null, null, null, null, null, null, null, null)
Expand All @@ -105,7 +105,7 @@ class AttemptTransitionStepTests : OpenSearchTestCase() {
whenever(indexMetadata.creationDate).doReturn(5L)
val exception = RemoteTransportException("rte", IllegalArgumentException("nested"))
val client = getClient(getAdminClient(getIndicesAdminClient(null, exception)))
val indexMetadataProvider = IndexMetadataProvider(client, clusterService, mutableMapOf())
val indexMetadataProvider = IndexMetadataProvider(settings, client, clusterService, mutableMapOf())

runBlocking {
val managedIndexMetadata = ManagedIndexMetaData(indexName, indexUUID, "policy_id", null, null, null, null, null, null, null, null, null, null, null)
Expand All @@ -120,7 +120,7 @@ class AttemptTransitionStepTests : OpenSearchTestCase() {
}

fun `test step start time resetting between two transitions`() {
val indexMetadataProvider = IndexMetadataProvider(mock(), clusterService, mutableMapOf())
val indexMetadataProvider = IndexMetadataProvider(settings, mock(), clusterService, mutableMapOf())
runBlocking {
val completedStartTime = Instant.now()
val managedIndexMetadata = ManagedIndexMetaData(indexName, indexUUID, "policy_id", null, null, null, null, null, null, null, null, StepMetaData("attempt_transition", completedStartTime.toEpochMilli(), Step.StepStatus.COMPLETED), null, null)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,37 +13,6 @@ import org.opensearch.test.OpenSearchTestCase
class ExplainResponseTests : OpenSearchTestCase() {

fun `test explain response`() {
val indexNames = listOf("index1")
val indexPolicyIDs = listOf("policyID1")
val metadata = ManagedIndexMetaData(
index = "index1",
indexUuid = randomAlphaOfLength(10),
policyID = "policyID1",
policySeqNo = randomNonNegativeLong(),
policyPrimaryTerm = randomNonNegativeLong(),
policyCompleted = null,
rolledOver = null,
indexCreationDate = null,
transitionTo = randomAlphaOfLength(10),
stateMetaData = null,
actionMetaData = null,
stepMetaData = null,
policyRetryInfo = null,
info = null
)
val indexMetadatas = listOf(metadata)
val res = ExplainResponse(indexNames, indexPolicyIDs, indexMetadatas)

val out = BytesStreamOutput()
res.writeTo(out)
val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes)
val newRes = ExplainResponse(sin)
assertEquals(indexNames, newRes.indexNames)
assertEquals(indexPolicyIDs, newRes.indexPolicyIDs)
assertEquals(indexMetadatas, newRes.indexMetadatas)
}

fun `test explain all response`() {
val indexNames = listOf("index1")
val indexPolicyIDs = listOf("policyID1")
val metadata = ManagedIndexMetaData(
Expand All @@ -65,12 +34,12 @@ class ExplainResponseTests : OpenSearchTestCase() {
val indexMetadatas = listOf(metadata)
val totalManagedIndices = 1
val enabledState = mapOf("index1" to true)
val res = ExplainAllResponse(indexNames, indexPolicyIDs, indexMetadatas, totalManagedIndices, enabledState)
val res = ExplainResponse(indexNames, indexPolicyIDs, indexMetadatas, totalManagedIndices, enabledState)

val out = BytesStreamOutput()
res.writeTo(out)
val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes)
val newRes = ExplainAllResponse(sin)
val newRes = ExplainResponse(sin)
assertEquals(indexNames, newRes.indexNames)
assertEquals(indexPolicyIDs, newRes.indexPolicyIDs)
assertEquals(indexMetadatas, newRes.indexMetadatas)
Expand Down

0 comments on commit 3ad2464

Please sign in to comment.