From c06f95d7d83abc1873eb40da5c2fa303bab3f506 Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Tue, 10 Oct 2023 13:05:54 -0400 Subject: [PATCH 01/16] Implemented filtering on the ISM eplain API Signed-off-by: Joshua Au --- .../model/ExplainFilter.kt | 100 ++++++++++++++++++ .../resthandler/RestExplainAction.kt | 21 ++++ .../action/explain/ExplainRequest.kt | 6 ++ .../action/explain/TransportExplainAction.kt | 28 ++--- 4 files changed, 142 insertions(+), 13 deletions(-) create mode 100644 src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt new file mode 100644 index 000000000..8fb09d7e7 --- /dev/null +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt @@ -0,0 +1,100 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.model + +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.common.io.stream.Writeable +import org.opensearch.core.xcontent.XContentParser +import org.opensearch.core.xcontent.XContentParser.Token +import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken +import java.io.IOException + +data class ExplainFilter( + val filterPolicyID: String?, + val filterState: String?, + val filterAction: String? +) : Writeable { + + constructor() : this( + filterPolicyID = null, + filterState = null, + filterAction = null + ) + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + filterPolicyID = sin.readString(), + filterState = sin.readString(), + filterAction = sin.readString() + ) + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeString(filterPolicyID) + out.writeString(filterState) + out.writeString(filterAction) + } + + fun isValid(managedIndexConfig: ManagedIndexConfig): Boolean { + var valid = true + + if (filterPolicyID != null && filterPolicyID != managedIndexConfig.policyID) { + valid = false + } + + val policy = managedIndexConfig.policy + + if (filterState != null && policy != null && !policy.states.any { it.name == filterState }) { + valid = false + } + + if (filterAction != null && policy != null && !policy.states.any { state -> state.actions.any { it.type == filterAction } }) { + valid = false + } + + return valid + } + + companion object { + const val FILTER_FIELD = "filter" + const val POLICY_ID_FIELD = "policy_id" + const val STATE_FIELD = "state" + const val ACTION_FIELD = "action_type" + + @JvmStatic + @Throws(IOException::class) + fun parse(xcp: XContentParser): ExplainFilter { + var policyID: String? = null + var state: String? = null + var action: String? = null + + ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != Token.END_OBJECT) { + val fieldName = xcp.currentName() + xcp.nextToken() + + when (fieldName) { + FILTER_FIELD -> { + ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) + while (xcp.nextToken() != Token.END_OBJECT) { + val filter = xcp.currentName() + xcp.nextToken() + + when (filter) { + POLICY_ID_FIELD -> policyID = xcp.text() + STATE_FIELD -> state = xcp.text() + ACTION_FIELD -> action = xcp.text() + } + } + } + } + } + + return ExplainFilter(policyID, state, action) + } + } +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt index c9bde8bbf..68f8c353c 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt @@ -9,8 +9,11 @@ import org.apache.logging.log4j.LogManager import org.opensearch.client.node.NodeClient import org.opensearch.core.common.Strings import org.opensearch.common.logging.DeprecationLogger +import org.opensearch.core.xcontent.XContentParser.Token +import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.ISM_BASE_URI import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.LEGACY_ISM_BASE_URI +import org.opensearch.indexmanagement.indexstatemanagement.model.ExplainFilter import org.opensearch.indexmanagement.indexstatemanagement.transport.action.explain.ExplainAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.explain.ExplainRequest import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_EXPLAIN_VALIDATE_ACTION @@ -28,6 +31,7 @@ import org.opensearch.rest.RestHandler.ReplacedRoute import org.opensearch.rest.RestHandler.Route import org.opensearch.rest.RestRequest import org.opensearch.rest.RestRequest.Method.GET +import org.opensearch.rest.RestRequest.Method.POST import org.opensearch.rest.action.RestToXContentListener private val log = LogManager.getLogger(RestExplainAction::class.java) @@ -52,6 +56,14 @@ class RestExplainAction : BaseRestHandler() { ReplacedRoute( GET, "$EXPLAIN_BASE_URI/{index}", GET, "$LEGACY_EXPLAIN_BASE_URI/{index}" + ), + ReplacedRoute( + POST, EXPLAIN_BASE_URI, + POST, LEGACY_EXPLAIN_BASE_URI + ), + ReplacedRoute( + POST, "$EXPLAIN_BASE_URI/{index}", + POST, "$LEGACY_EXPLAIN_BASE_URI/{index}" ) ) } @@ -69,6 +81,14 @@ class RestExplainAction : BaseRestHandler() { val indexType = request.param(TYPE_PARAM_KEY, DEFAULT_INDEX_TYPE) + val explainFilter = if (request.method() == RestRequest.Method.POST) { + val xcp = request.contentParser() + ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp) + ExplainFilter.parse(xcp) + } else { + ExplainFilter() + } + val clusterManagerTimeout = parseClusterManagerTimeout( request, DeprecationLogger.getLogger(RestExplainAction::class.java), name ) @@ -78,6 +98,7 @@ class RestExplainAction : BaseRestHandler() { request.paramAsBoolean("local", false), clusterManagerTimeout, searchParams, + explainFilter, request.paramAsBoolean(SHOW_POLICY_QUERY_PARAM, DEFAULT_EXPLAIN_SHOW_POLICY), request.paramAsBoolean(SHOW_VALIDATE_ACTION, DEFAULT_EXPLAIN_VALIDATE_ACTION), indexType diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt index a34ce2020..ba0090442 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt @@ -12,6 +12,7 @@ import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput import org.opensearch.common.unit.TimeValue import org.opensearch.indexmanagement.common.model.rest.SearchParams +import org.opensearch.indexmanagement.indexstatemanagement.model.ExplainFilter import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import java.io.IOException @@ -21,6 +22,7 @@ class ExplainRequest : ActionRequest { val local: Boolean val clusterManagerTimeout: TimeValue val searchParams: SearchParams + val explainFilter: ExplainFilter val showPolicy: Boolean val validateAction: Boolean val indexType: String @@ -31,6 +33,7 @@ class ExplainRequest : ActionRequest { local: Boolean, clusterManagerTimeout: TimeValue, searchParams: SearchParams, + explainFilter: ExplainFilter, showPolicy: Boolean, validateAction: Boolean, indexType: String @@ -39,6 +42,7 @@ class ExplainRequest : ActionRequest { this.local = local this.clusterManagerTimeout = clusterManagerTimeout this.searchParams = searchParams + this.explainFilter = explainFilter this.showPolicy = showPolicy this.validateAction = validateAction this.indexType = indexType @@ -50,6 +54,7 @@ class ExplainRequest : ActionRequest { local = sin.readBoolean(), clusterManagerTimeout = sin.readTimeValue(), searchParams = SearchParams(sin), + explainFilter = ExplainFilter(sin), showPolicy = sin.readBoolean(), validateAction = sin.readBoolean(), indexType = sin.readString() @@ -72,6 +77,7 @@ class ExplainRequest : ActionRequest { out.writeBoolean(local) out.writeTimeValue(clusterManagerTimeout) searchParams.writeTo(out) + explainFilter.writeTo(out) out.writeBoolean(showPolicy) out.writeBoolean(validateAction) out.writeString(indexType) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt index ec32bcbd9..d8f140b7b 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt @@ -190,19 +190,21 @@ class TransportExplainAction @Inject constructor( } parseSearchHits(response.hits.hits).forEach { managedIndex -> - managedIndices.add(managedIndex.index) - enabledState[managedIndex.index] = managedIndex.enabled - managedIndicesMetaDataMap[managedIndex.index] = mapOf( - "index" to managedIndex.index, - "index_uuid" to managedIndex.indexUuid, - "policy_id" to managedIndex.policyID, - "enabled" to managedIndex.enabled.toString() - ) - if (showPolicy) { - managedIndex.policy?.let { appliedPolicies[managedIndex.index] = it } - } - if (validateAction) { - managedIndex.policy?.let { policiesforValidation[managedIndex.index] = it } + if (request.explainFilter.isValid(managedIndex)) { + managedIndices.add(managedIndex.index) + enabledState[managedIndex.index] = managedIndex.enabled + managedIndicesMetaDataMap[managedIndex.index] = mapOf( + "index" to managedIndex.index, + "index_uuid" to managedIndex.indexUuid, + "policy_id" to managedIndex.policyID, + "enabled" to managedIndex.enabled.toString() + ) + if (showPolicy) { + managedIndex.policy?.let { appliedPolicies[managedIndex.index] = it } + } + if (validateAction) { + managedIndex.policy?.let { policiesforValidation[managedIndex.index] = it } + } } } From c58d8d1d17f98e5af6c772fec92b247bba6fb241 Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Wed, 11 Oct 2023 14:15:52 -0400 Subject: [PATCH 02/16] Fixed tests for ExplainRequest Signed-off-by: Joshua Au --- .../indexstatemanagement/model/ExplainFilter.kt | 12 ++++++------ .../transport/action/explain/ExplainRequestTests.kt | 7 +++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt index 8fb09d7e7..ba562948f 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt @@ -27,16 +27,16 @@ data class ExplainFilter( @Throws(IOException::class) constructor(sin: StreamInput) : this( - filterPolicyID = sin.readString(), - filterState = sin.readString(), - filterAction = sin.readString() + filterPolicyID = sin.readOptionalString(), + filterState = sin.readOptionalString(), + filterAction = sin.readOptionalString() ) @Throws(IOException::class) override fun writeTo(out: StreamOutput) { - out.writeString(filterPolicyID) - out.writeString(filterState) - out.writeString(filterAction) + out.writeOptionalString(filterPolicyID) + out.writeOptionalString(filterState) + out.writeOptionalString(filterAction) } fun isValid(managedIndexConfig: ManagedIndexConfig): Boolean { diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequestTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequestTests.kt index 0c8d0e01e..832322a1a 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequestTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequestTests.kt @@ -9,6 +9,7 @@ import org.opensearch.common.io.stream.BytesStreamOutput import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.common.unit.TimeValue import org.opensearch.indexmanagement.common.model.rest.SearchParams +import org.opensearch.indexmanagement.indexstatemanagement.model.ExplainFilter import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import org.opensearch.test.OpenSearchTestCase @@ -19,9 +20,10 @@ class ExplainRequestTests : OpenSearchTestCase() { val local = true val clusterManagerTimeout = TimeValue.timeValueSeconds(30) val params = SearchParams(0, 20, "sort-field", "asc", "*") + val filter = ExplainFilter() val showPolicy = false val showValidationResult = false - val req = ExplainRequest(indices, local, clusterManagerTimeout, params, showPolicy, showValidationResult, DEFAULT_INDEX_TYPE) + val req = ExplainRequest(indices, local, clusterManagerTimeout, params, filter, showPolicy, showValidationResult, DEFAULT_INDEX_TYPE) val out = BytesStreamOutput() req.writeTo(out) @@ -36,9 +38,10 @@ class ExplainRequestTests : OpenSearchTestCase() { val local = true val clusterManagerTimeout = TimeValue.timeValueSeconds(30) val params = SearchParams(0, 20, "sort-field", "asc", "*") + val filter = ExplainFilter() val showPolicy = false val showValidationResult = false - val req = ExplainRequest(indices, local, clusterManagerTimeout, params, showPolicy, showValidationResult, "non-existent-index-type") + val req = ExplainRequest(indices, local, clusterManagerTimeout, params, filter, showPolicy, showValidationResult, "non-existent-index-type") val actualException: String? = req.validate()?.validationErrors()?.firstOrNull() val expectedException: String = ExplainRequest.MULTIPLE_INDICES_CUSTOM_INDEX_TYPE_ERROR From 2be92534800cd3cc578a3e8344963602864f946b Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Thu, 26 Oct 2023 12:33:29 -0400 Subject: [PATCH 03/16] Added filtering on query and metadata map Signed-off-by: Joshua Au --- .../model/StateMetaData.kt | 3 +- .../model/ExplainFilter.kt | 93 ++++++++++++++----- .../action/explain/TransportExplainAction.kt | 67 +++++++++---- .../util/RestHandlerUtils.kt | 1 + 4 files changed, 120 insertions(+), 44 deletions(-) diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt index 7c6174106..21a94d418 100644 --- a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt @@ -22,7 +22,8 @@ import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedInde import java.io.ByteArrayInputStream import java.nio.charset.StandardCharsets -data class StateMetaData( +data class +StateMetaData( val name: String, val startTime: Long ) : Writeable, ToXContentFragment { diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt index ba562948f..83ab63a58 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt @@ -5,58 +5,96 @@ package org.opensearch.indexmanagement.indexstatemanagement.model +import org.apache.logging.log4j.LogManager import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput import org.opensearch.core.common.io.stream.Writeable import org.opensearch.core.xcontent.XContentParser import org.opensearch.core.xcontent.XContentParser.Token import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.index.query.BoolQueryBuilder +import org.opensearch.index.query.Operator +import org.opensearch.index.query.QueryBuilders +import org.opensearch.indexmanagement.indexstatemanagement.util.MANAGED_INDEX_POLICY_ID_FIELD +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import java.io.IOException +private val log = LogManager.getLogger(ExplainFilter::class.java) + data class ExplainFilter( - val filterPolicyID: String?, - val filterState: String?, - val filterAction: String? + val policyID: String?, + val state: String?, + val actionType: String?, + val failed: Boolean? ) : Writeable { constructor() : this( - filterPolicyID = null, - filterState = null, - filterAction = null + policyID = null, + state = null, + actionType = null, + failed = null ) @Throws(IOException::class) constructor(sin: StreamInput) : this( - filterPolicyID = sin.readOptionalString(), - filterState = sin.readOptionalString(), - filterAction = sin.readOptionalString() + policyID = sin.readString(), + state = sin.readString(), + actionType = sin.readString(), + failed = sin.readBoolean() ) @Throws(IOException::class) override fun writeTo(out: StreamOutput) { - out.writeOptionalString(filterPolicyID) - out.writeOptionalString(filterState) - out.writeOptionalString(filterAction) + out.writeOptionalString(policyID) + out.writeOptionalString(state) + out.writeOptionalString(actionType) + out.writeOptionalBoolean(failed) + } + + fun convertToBoolQueryBuilder(): BoolQueryBuilder { + val boolQuery = QueryBuilders.boolQuery() + + if (policyID != null) { + boolQuery.must( + QueryBuilders + .queryStringQuery(policyID) + .field(MANAGED_INDEX_POLICY_ID_FIELD) + .defaultOperator(Operator.AND) + ) + } + + return boolQuery } - fun isValid(managedIndexConfig: ManagedIndexConfig): Boolean { - var valid = true + fun validMetaData(metaData: ManagedIndexMetaData): Boolean { + log.info("----FILTER---- $metaData") // : ${metaData.stateMetaData!!.name} ${metaData.actionMetaData!!.name} ${metaData.actionMetaData!!.failed}") - if (filterPolicyID != null && filterPolicyID != managedIndexConfig.policyID) { - valid = false + val stateMetaData = metaData.stateMetaData + if (stateMetaData != null) { + log.info("STATE META DATA: ${stateMetaData.name} : $state") } - val policy = managedIndexConfig.policy + if (state != null && (stateMetaData == null || stateMetaData.name != state)) { + log.info("filter error on state") + return false + } - if (filterState != null && policy != null && !policy.states.any { it.name == filterState }) { - valid = false + val actionMetaData = metaData.actionMetaData + if (actionMetaData != null) { + log.info("ACTION META DATA: ${actionMetaData.name} : $actionType, ${actionMetaData.failed} : $failed") } - if (filterAction != null && policy != null && !policy.states.any { state -> state.actions.any { it.type == filterAction } }) { - valid = false + if (actionType != null && (actionMetaData == null || actionMetaData.name != actionType)) { + log.info("filter error on action") + return false } - return valid + if (failed != null && (actionMetaData == null || actionMetaData.failed != failed)) { + log.info("filter error on failed") + return false + } + + return true } companion object { @@ -64,13 +102,15 @@ data class ExplainFilter( const val POLICY_ID_FIELD = "policy_id" const val STATE_FIELD = "state" const val ACTION_FIELD = "action_type" + const val FAILURE_FIELD = "failure" @JvmStatic @Throws(IOException::class) fun parse(xcp: XContentParser): ExplainFilter { var policyID: String? = null var state: String? = null - var action: String? = null + var actionType: String? = null + var failed: Boolean? = null ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp) while (xcp.nextToken() != Token.END_OBJECT) { @@ -87,14 +127,17 @@ data class ExplainFilter( when (filter) { POLICY_ID_FIELD -> policyID = xcp.text() STATE_FIELD -> state = xcp.text() - ACTION_FIELD -> action = xcp.text() + ACTION_FIELD -> actionType = xcp.text() + FAILURE_FIELD -> failed = xcp.booleanValue() } } } } } - return ExplainFilter(policyID, state, action) + log.info("[PARSE] pid: $policyID, s: $state, a: $actionType, f: $failed") + + return ExplainFilter(policyID, state, actionType, failed) } } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt index d8f140b7b..f12b8bc91 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt @@ -155,11 +155,24 @@ class TransportExplainAction @Inject constructor( private fun getSearchMetadataRequest(params: SearchParams, indexUUIDs: List, searchSize: Int): SearchRequest { val sortBuilder = params.getSortBuilder() - val queryBuilder = QueryBuilders.boolQuery() + // change query builder based off of the given json body +// val queryBuilder = QueryBuilders.boolQuery() +// .must( +// QueryBuilders +// .queryStringQuery(params.queryString) +// .defaultField(MANAGED_INDEX_NAME_KEYWORD_FIELD) +// .defaultOperator(Operator.AND) +// ).filter(QueryBuilders.termsQuery(MANAGED_INDEX_INDEX_UUID_FIELD, indexUUIDs)) + + log.info("QUERY STRING: ${params.queryString} :: $params") + + // can I string multiple bool queries together + + val queryBuilder = request.explainFilter.convertToBoolQueryBuilder() .must( QueryBuilders .queryStringQuery(params.queryString) - .defaultField(MANAGED_INDEX_NAME_KEYWORD_FIELD) + .field(MANAGED_INDEX_NAME_KEYWORD_FIELD) .defaultOperator(Operator.AND) ).filter(QueryBuilders.termsQuery(MANAGED_INDEX_INDEX_UUID_FIELD, indexUUIDs)) @@ -190,21 +203,19 @@ class TransportExplainAction @Inject constructor( } parseSearchHits(response.hits.hits).forEach { managedIndex -> - if (request.explainFilter.isValid(managedIndex)) { - managedIndices.add(managedIndex.index) - enabledState[managedIndex.index] = managedIndex.enabled - managedIndicesMetaDataMap[managedIndex.index] = mapOf( - "index" to managedIndex.index, - "index_uuid" to managedIndex.indexUuid, - "policy_id" to managedIndex.policyID, - "enabled" to managedIndex.enabled.toString() - ) - if (showPolicy) { - managedIndex.policy?.let { appliedPolicies[managedIndex.index] = it } - } - if (validateAction) { - managedIndex.policy?.let { policiesforValidation[managedIndex.index] = it } - } + managedIndices.add(managedIndex.index) + enabledState[managedIndex.index] = managedIndex.enabled + managedIndicesMetaDataMap[managedIndex.index] = mapOf( + "index" to managedIndex.index, + "index_uuid" to managedIndex.indexUuid, + "policy_id" to managedIndex.policyID, + "enabled" to managedIndex.enabled.toString() + ) + if (showPolicy) { + managedIndex.policy?.let { appliedPolicies[managedIndex.index] = it } + } + if (validateAction) { + managedIndex.policy?.let { policiesforValidation[managedIndex.index] = it } } } @@ -293,8 +304,28 @@ class TransportExplainAction @Inject constructor( mgetMetadataReq, object : ActionListener { override fun onResponse(response: MultiGetResponse) { - val metadataMap: Map = + var metadataMap: Map = response.responses.associate { it.id to getMetadata(it.response)?.toMap() } + + log.info("BEFORE METADATA MAP: $metadataMap") + + metadataMap = metadataMap.filter { (_, value) -> + var ok = false + + if (value != null) { + val metaData = ManagedIndexMetaData.fromMap(value) + ok = request.explainFilter.validMetaData(metaData) + if (!ok) { + indexNamesToUUIDs.remove(metaData.index) + } + } + + ok + } + + log.info("FILTERED METADATA MAP: $metadataMap") + log.info("FILTED INDEX NAMES TO UUID: $indexNamesToUUIDs") + buildResponse(indexNamesToUUIDs, metadataMap, clusterStateIndexMetadatas, threadContext) } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt index 4442ccc82..6c5a45694 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/util/RestHandlerUtils.kt @@ -39,6 +39,7 @@ const val MANAGED_INDEX_FIELD = "managed_index" const val MANAGED_INDEX_NAME_KEYWORD_FIELD = "$MANAGED_INDEX_FIELD.name.keyword" const val MANAGED_INDEX_INDEX_FIELD = "$MANAGED_INDEX_FIELD.index" const val MANAGED_INDEX_INDEX_UUID_FIELD = "$MANAGED_INDEX_FIELD.index_uuid" +const val MANAGED_INDEX_POLICY_ID_FIELD = "$MANAGED_INDEX_FIELD.policy_id" const val DEFAULT_JOB_SORT_FIELD = MANAGED_INDEX_INDEX_FIELD const val DEFAULT_POLICY_SORT_FIELD = "policy.policy_id.keyword" From 73af3b7d1040ea073e1a3984d1dfb69b60ed7fff Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Thu, 2 Nov 2023 12:54:22 -0400 Subject: [PATCH 04/16] Filtered on indexNames in metadata Signed-off-by: Joshua Au --- .../model/ExplainFilter.kt | 22 ++----------------- .../action/explain/TransportExplainAction.kt | 7 ++---- 2 files changed, 4 insertions(+), 25 deletions(-) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt index 83ab63a58..a7301eac8 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt @@ -5,7 +5,6 @@ package org.opensearch.indexmanagement.indexstatemanagement.model -import org.apache.logging.log4j.LogManager import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput import org.opensearch.core.common.io.stream.Writeable @@ -19,8 +18,6 @@ import org.opensearch.indexmanagement.indexstatemanagement.util.MANAGED_INDEX_PO import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import java.io.IOException -private val log = LogManager.getLogger(ExplainFilter::class.java) - data class ExplainFilter( val policyID: String?, val state: String?, @@ -67,30 +64,17 @@ data class ExplainFilter( } fun validMetaData(metaData: ManagedIndexMetaData): Boolean { - log.info("----FILTER---- $metaData") // : ${metaData.stateMetaData!!.name} ${metaData.actionMetaData!!.name} ${metaData.actionMetaData!!.failed}") - val stateMetaData = metaData.stateMetaData - if (stateMetaData != null) { - log.info("STATE META DATA: ${stateMetaData.name} : $state") - } - if (state != null && (stateMetaData == null || stateMetaData.name != state)) { - log.info("filter error on state") return false } val actionMetaData = metaData.actionMetaData - if (actionMetaData != null) { - log.info("ACTION META DATA: ${actionMetaData.name} : $actionType, ${actionMetaData.failed} : $failed") - } - if (actionType != null && (actionMetaData == null || actionMetaData.name != actionType)) { - log.info("filter error on action") return false } if (failed != null && (actionMetaData == null || actionMetaData.failed != failed)) { - log.info("filter error on failed") return false } @@ -102,7 +86,7 @@ data class ExplainFilter( const val POLICY_ID_FIELD = "policy_id" const val STATE_FIELD = "state" const val ACTION_FIELD = "action_type" - const val FAILURE_FIELD = "failure" + const val FAILED_FIELD = "failed" @JvmStatic @Throws(IOException::class) @@ -128,15 +112,13 @@ data class ExplainFilter( POLICY_ID_FIELD -> policyID = xcp.text() STATE_FIELD -> state = xcp.text() ACTION_FIELD -> actionType = xcp.text() - FAILURE_FIELD -> failed = xcp.booleanValue() + FAILED_FIELD -> failed = xcp.booleanValue() } } } } } - log.info("[PARSE] pid: $policyID, s: $state, a: $actionType, f: $failed") - return ExplainFilter(policyID, state, actionType, failed) } } diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt index f12b8bc91..6bfdc2927 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt @@ -307,8 +307,6 @@ class TransportExplainAction @Inject constructor( var metadataMap: Map = response.responses.associate { it.id to getMetadata(it.response)?.toMap() } - log.info("BEFORE METADATA MAP: $metadataMap") - metadataMap = metadataMap.filter { (_, value) -> var ok = false @@ -316,16 +314,15 @@ class TransportExplainAction @Inject constructor( val metaData = ManagedIndexMetaData.fromMap(value) ok = request.explainFilter.validMetaData(metaData) if (!ok) { + indexNames.remove(metaData.index) indexNamesToUUIDs.remove(metaData.index) + totalManagedIndices-- } } ok } - log.info("FILTERED METADATA MAP: $metadataMap") - log.info("FILTED INDEX NAMES TO UUID: $indexNamesToUUIDs") - buildResponse(indexNamesToUUIDs, metadataMap, clusterStateIndexMetadatas, threadContext) } From 871952ca08d1111685b20d0fb33694171e85dedc Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Wed, 8 Nov 2023 00:04:20 -0500 Subject: [PATCH 05/16] Fixed github workflow check errors Signed-off-by: Joshua Au --- .../model/ExplainFilter.kt | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt index a7301eac8..75856b566 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt @@ -34,10 +34,10 @@ data class ExplainFilter( @Throws(IOException::class) constructor(sin: StreamInput) : this( - policyID = sin.readString(), - state = sin.readString(), - actionType = sin.readString(), - failed = sin.readBoolean() + policyID = sin.readOptionalString(), + state = sin.readOptionalString(), + actionType = sin.readOptionalString(), + failed = sin.readOptionalBoolean() ) @Throws(IOException::class) @@ -64,21 +64,23 @@ data class ExplainFilter( } fun validMetaData(metaData: ManagedIndexMetaData): Boolean { + var ok = true + val stateMetaData = metaData.stateMetaData if (state != null && (stateMetaData == null || stateMetaData.name != state)) { - return false + ok = false } val actionMetaData = metaData.actionMetaData if (actionType != null && (actionMetaData == null || actionMetaData.name != actionType)) { - return false + ok = false } if (failed != null && (actionMetaData == null || actionMetaData.failed != failed)) { - return false + ok = false } - return true + return ok } companion object { From f75579c351cc4c51632e9c69b6b481f859fd6413 Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Wed, 8 Nov 2023 00:23:39 -0500 Subject: [PATCH 06/16] Removed debugging comments Signed-off-by: Joshua Au --- .../action/explain/TransportExplainAction.kt | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt index 6bfdc2927..8064e3a80 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt @@ -155,19 +155,6 @@ class TransportExplainAction @Inject constructor( private fun getSearchMetadataRequest(params: SearchParams, indexUUIDs: List, searchSize: Int): SearchRequest { val sortBuilder = params.getSortBuilder() - // change query builder based off of the given json body -// val queryBuilder = QueryBuilders.boolQuery() -// .must( -// QueryBuilders -// .queryStringQuery(params.queryString) -// .defaultField(MANAGED_INDEX_NAME_KEYWORD_FIELD) -// .defaultOperator(Operator.AND) -// ).filter(QueryBuilders.termsQuery(MANAGED_INDEX_INDEX_UUID_FIELD, indexUUIDs)) - - log.info("QUERY STRING: ${params.queryString} :: $params") - - // can I string multiple bool queries together - val queryBuilder = request.explainFilter.convertToBoolQueryBuilder() .must( QueryBuilders From d4ca2c8349a65b2e2b55ed4d0fa71e71a9b8b05e Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Fri, 10 Nov 2023 17:44:36 -0500 Subject: [PATCH 07/16] Updated code styling to make more clear Signed-off-by: Joshua Au --- .../model/StateMetaData.kt | 3 +- .../model/ExplainFilter.kt | 41 ++++++++++----- .../resthandler/RestExplainAction.kt | 2 +- .../action/explain/ExplainRequest.kt | 6 +-- .../action/explain/TransportExplainAction.kt | 51 ++++++++++++------- .../action/explain/ExplainRequestTests.kt | 5 +- 6 files changed, 67 insertions(+), 41 deletions(-) diff --git a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt index 21a94d418..7c6174106 100644 --- a/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt +++ b/spi/src/main/kotlin/org.opensearch.indexmanagement.spi/indexstatemanagement/model/StateMetaData.kt @@ -22,8 +22,7 @@ import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedInde import java.io.ByteArrayInputStream import java.nio.charset.StandardCharsets -data class -StateMetaData( +data class StateMetaData( val name: String, val startTime: Long ) : Writeable, ToXContentFragment { diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt index 75856b566..b819fd688 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt @@ -5,6 +5,7 @@ package org.opensearch.indexmanagement.indexstatemanagement.model +import org.apache.logging.log4j.LogManager import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput import org.opensearch.core.common.io.stream.Writeable @@ -18,6 +19,8 @@ import org.opensearch.indexmanagement.indexstatemanagement.util.MANAGED_INDEX_PO import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import java.io.IOException +private val log = LogManager.getLogger(ExplainFilter::class.java) + data class ExplainFilter( val policyID: String?, val state: String?, @@ -25,13 +28,6 @@ data class ExplainFilter( val failed: Boolean? ) : Writeable { - constructor() : this( - policyID = null, - state = null, - actionType = null, - failed = null - ) - @Throws(IOException::class) constructor(sin: StreamInput) : this( policyID = sin.readOptionalString(), @@ -63,24 +59,31 @@ data class ExplainFilter( return boolQuery } - fun validMetaData(metaData: ManagedIndexMetaData): Boolean { - var ok = true + fun byMetaData(metaData: ManagedIndexMetaData): Boolean { + var isValid = true + + log.info(metaData) val stateMetaData = metaData.stateMetaData if (state != null && (stateMetaData == null || stateMetaData.name != state)) { - ok = false + isValid = false } val actionMetaData = metaData.actionMetaData if (actionType != null && (actionMetaData == null || actionMetaData.name != actionType)) { - ok = false + isValid = false } - if (failed != null && (actionMetaData == null || actionMetaData.failed != failed)) { - ok = false + val retryInfoMetaData = metaData.policyRetryInfo + if (failed != null && (actionMetaData == null || actionMetaData.failed != failed) && + (retryInfoMetaData == null || retryInfoMetaData.failed != failed) + ) { + isValid = false } - return ok + log.info("IS VALID: $isValid") + + return isValid } companion object { @@ -125,3 +128,13 @@ data class ExplainFilter( } } } + +fun BoolQueryBuilder.filterByPolicyID(explainFilter: ExplainFilter?): BoolQueryBuilder { + if (explainFilter?.policyID != null) { + log.info("FILTER ON POLICY ID") + + return this.filter(QueryBuilders.termsQuery(MANAGED_INDEX_POLICY_ID_FIELD, explainFilter.policyID)) + } + + return this +} diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt index 68f8c353c..5877a3da0 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainAction.kt @@ -86,7 +86,7 @@ class RestExplainAction : BaseRestHandler() { ensureExpectedToken(Token.START_OBJECT, xcp.nextToken(), xcp) ExplainFilter.parse(xcp) } else { - ExplainFilter() + null } val clusterManagerTimeout = parseClusterManagerTimeout( diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt index ba0090442..47a486ab0 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt @@ -22,7 +22,7 @@ class ExplainRequest : ActionRequest { val local: Boolean val clusterManagerTimeout: TimeValue val searchParams: SearchParams - val explainFilter: ExplainFilter + val explainFilter: ExplainFilter? val showPolicy: Boolean val validateAction: Boolean val indexType: String @@ -33,7 +33,7 @@ class ExplainRequest : ActionRequest { local: Boolean, clusterManagerTimeout: TimeValue, searchParams: SearchParams, - explainFilter: ExplainFilter, + explainFilter: ExplainFilter?, showPolicy: Boolean, validateAction: Boolean, indexType: String @@ -77,7 +77,7 @@ class ExplainRequest : ActionRequest { out.writeBoolean(local) out.writeTimeValue(clusterManagerTimeout) searchParams.writeTo(out) - explainFilter.writeTo(out) + explainFilter?.writeTo(out) out.writeBoolean(showPolicy) out.writeBoolean(validateAction) out.writeString(indexType) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt index 8064e3a80..1e21181b8 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt @@ -155,13 +155,23 @@ class TransportExplainAction @Inject constructor( private fun getSearchMetadataRequest(params: SearchParams, indexUUIDs: List, searchSize: Int): SearchRequest { val sortBuilder = params.getSortBuilder() - val queryBuilder = request.explainFilter.convertToBoolQueryBuilder() - .must( - QueryBuilders - .queryStringQuery(params.queryString) - .field(MANAGED_INDEX_NAME_KEYWORD_FIELD) - .defaultOperator(Operator.AND) - ).filter(QueryBuilders.termsQuery(MANAGED_INDEX_INDEX_UUID_FIELD, indexUUIDs)) + var queryBuilder = QueryBuilders.boolQuery() + + if (request.explainFilter != null) { + queryBuilder = request.explainFilter.convertToBoolQueryBuilder() + } + + queryBuilder.must( + QueryBuilders + .queryStringQuery(params.queryString) + .field(MANAGED_INDEX_NAME_KEYWORD_FIELD) + .defaultOperator(Operator.AND) + ).filter(QueryBuilders.termsQuery(MANAGED_INDEX_INDEX_UUID_FIELD, indexUUIDs)) + +// if (request.explainFilter != null) { +// queryBuilder.filter(QueryBuilders.termsQuery(MANAGED_INDEX_POLICY_ID_FIELD, request.explainFilter.policyID)) +// } +// .filterByPolicyID(request.explainFilter) val searchSourceBuilder = SearchSourceBuilder() .from(params.from) @@ -294,20 +304,25 @@ class TransportExplainAction @Inject constructor( var metadataMap: Map = response.responses.associate { it.id to getMetadata(it.response)?.toMap() } - metadataMap = metadataMap.filter { (_, value) -> - var ok = false + log.info("METADATA MAP: $metadataMap") - if (value != null) { - val metaData = ManagedIndexMetaData.fromMap(value) - ok = request.explainFilter.validMetaData(metaData) - if (!ok) { - indexNames.remove(metaData.index) - indexNamesToUUIDs.remove(metaData.index) - totalManagedIndices-- + if (request.explainFilter != null) { + metadataMap = metadataMap.filter { (_, value) -> + var isValid = false + + if (value != null) { + val metaData = ManagedIndexMetaData.fromMap(value) + + isValid = request.explainFilter.byMetaData(metaData) + if (!isValid) { + indexNames.remove(metaData.index) + indexNamesToUUIDs.remove(metaData.index) + totalManagedIndices-- + } } - } - ok + isValid + } } buildResponse(indexNamesToUUIDs, metadataMap, clusterStateIndexMetadatas, threadContext) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequestTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequestTests.kt index 832322a1a..61d9d81e2 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequestTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequestTests.kt @@ -9,7 +9,6 @@ import org.opensearch.common.io.stream.BytesStreamOutput import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.common.unit.TimeValue import org.opensearch.indexmanagement.common.model.rest.SearchParams -import org.opensearch.indexmanagement.indexstatemanagement.model.ExplainFilter import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE import org.opensearch.test.OpenSearchTestCase @@ -20,7 +19,7 @@ class ExplainRequestTests : OpenSearchTestCase() { val local = true val clusterManagerTimeout = TimeValue.timeValueSeconds(30) val params = SearchParams(0, 20, "sort-field", "asc", "*") - val filter = ExplainFilter() + val filter = null val showPolicy = false val showValidationResult = false val req = ExplainRequest(indices, local, clusterManagerTimeout, params, filter, showPolicy, showValidationResult, DEFAULT_INDEX_TYPE) @@ -38,7 +37,7 @@ class ExplainRequestTests : OpenSearchTestCase() { val local = true val clusterManagerTimeout = TimeValue.timeValueSeconds(30) val params = SearchParams(0, 20, "sort-field", "asc", "*") - val filter = ExplainFilter() + val filter = null val showPolicy = false val showValidationResult = false val req = ExplainRequest(indices, local, clusterManagerTimeout, params, filter, showPolicy, showValidationResult, "non-existent-index-type") From f6ba9dde9f0cf9d336fafed50be2855375e16f7b Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Wed, 15 Nov 2023 02:13:08 -0500 Subject: [PATCH 08/16] Refactored code to match suggestions Signed-off-by: Joshua Au --- .../model/ExplainFilter.kt | 41 ++++--------------- .../action/explain/ExplainRequest.kt | 4 +- .../action/explain/TransportExplainAction.kt | 40 ++++++++---------- 3 files changed, 27 insertions(+), 58 deletions(-) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt index b819fd688..5749becb6 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt @@ -5,7 +5,6 @@ package org.opensearch.indexmanagement.indexstatemanagement.model -import org.apache.logging.log4j.LogManager import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput import org.opensearch.core.common.io.stream.Writeable @@ -13,19 +12,16 @@ import org.opensearch.core.xcontent.XContentParser import org.opensearch.core.xcontent.XContentParser.Token import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken import org.opensearch.index.query.BoolQueryBuilder -import org.opensearch.index.query.Operator import org.opensearch.index.query.QueryBuilders import org.opensearch.indexmanagement.indexstatemanagement.util.MANAGED_INDEX_POLICY_ID_FIELD import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData import java.io.IOException -private val log = LogManager.getLogger(ExplainFilter::class.java) - data class ExplainFilter( - val policyID: String?, - val state: String?, - val actionType: String?, - val failed: Boolean? + val policyID: String? = null, + val state: String? = null, + val actionType: String? = null, + val failed: Boolean? = null ) : Writeable { @Throws(IOException::class) @@ -44,26 +40,9 @@ data class ExplainFilter( out.writeOptionalBoolean(failed) } - fun convertToBoolQueryBuilder(): BoolQueryBuilder { - val boolQuery = QueryBuilders.boolQuery() - - if (policyID != null) { - boolQuery.must( - QueryBuilders - .queryStringQuery(policyID) - .field(MANAGED_INDEX_POLICY_ID_FIELD) - .defaultOperator(Operator.AND) - ) - } - - return boolQuery - } - fun byMetaData(metaData: ManagedIndexMetaData): Boolean { var isValid = true - log.info(metaData) - val stateMetaData = metaData.stateMetaData if (state != null && (stateMetaData == null || stateMetaData.name != state)) { isValid = false @@ -75,14 +54,12 @@ data class ExplainFilter( } val retryInfoMetaData = metaData.policyRetryInfo - if (failed != null && (actionMetaData == null || actionMetaData.failed != failed) && - (retryInfoMetaData == null || retryInfoMetaData.failed != failed) - ) { + val actionFailedNotValid = actionMetaData == null || actionMetaData.failed != failed + val retryFailedNotValid = retryInfoMetaData == null || retryInfoMetaData.failed != failed + if (failed != null && actionFailedNotValid && retryFailedNotValid) { isValid = false } - log.info("IS VALID: $isValid") - return isValid } @@ -131,9 +108,7 @@ data class ExplainFilter( fun BoolQueryBuilder.filterByPolicyID(explainFilter: ExplainFilter?): BoolQueryBuilder { if (explainFilter?.policyID != null) { - log.info("FILTER ON POLICY ID") - - return this.filter(QueryBuilders.termsQuery(MANAGED_INDEX_POLICY_ID_FIELD, explainFilter.policyID)) + this.filter(QueryBuilders.termsQuery(MANAGED_INDEX_POLICY_ID_FIELD, explainFilter.policyID)) } return this diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt index 47a486ab0..cbbddf03b 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/ExplainRequest.kt @@ -54,7 +54,7 @@ class ExplainRequest : ActionRequest { local = sin.readBoolean(), clusterManagerTimeout = sin.readTimeValue(), searchParams = SearchParams(sin), - explainFilter = ExplainFilter(sin), + explainFilter = sin.readOptionalWriteable(::ExplainFilter), showPolicy = sin.readBoolean(), validateAction = sin.readBoolean(), indexType = sin.readString() @@ -77,7 +77,7 @@ class ExplainRequest : ActionRequest { out.writeBoolean(local) out.writeTimeValue(clusterManagerTimeout) searchParams.writeTo(out) - explainFilter?.writeTo(out) + out.writeOptionalWriteable(explainFilter) out.writeBoolean(showPolicy) out.writeBoolean(validateAction) out.writeString(indexType) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt index 1e21181b8..fc4d47a72 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/transport/action/explain/TransportExplainAction.kt @@ -44,6 +44,7 @@ import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexCon import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.common.model.rest.SearchParams import org.opensearch.indexmanagement.indexstatemanagement.ManagedIndexRunner.actionValidation +import org.opensearch.indexmanagement.indexstatemanagement.model.filterByPolicyID import org.opensearch.indexmanagement.indexstatemanagement.transport.action.managedIndex.ManagedIndexAction import org.opensearch.indexmanagement.indexstatemanagement.transport.action.managedIndex.ManagedIndexRequest import org.opensearch.indexmanagement.indexstatemanagement.util.DEFAULT_INDEX_TYPE @@ -155,23 +156,14 @@ class TransportExplainAction @Inject constructor( private fun getSearchMetadataRequest(params: SearchParams, indexUUIDs: List, searchSize: Int): SearchRequest { val sortBuilder = params.getSortBuilder() - var queryBuilder = QueryBuilders.boolQuery() - - if (request.explainFilter != null) { - queryBuilder = request.explainFilter.convertToBoolQueryBuilder() - } - - queryBuilder.must( - QueryBuilders - .queryStringQuery(params.queryString) - .field(MANAGED_INDEX_NAME_KEYWORD_FIELD) - .defaultOperator(Operator.AND) - ).filter(QueryBuilders.termsQuery(MANAGED_INDEX_INDEX_UUID_FIELD, indexUUIDs)) - -// if (request.explainFilter != null) { -// queryBuilder.filter(QueryBuilders.termsQuery(MANAGED_INDEX_POLICY_ID_FIELD, request.explainFilter.policyID)) -// } -// .filterByPolicyID(request.explainFilter) + val queryBuilder = QueryBuilders.boolQuery() + .must( + QueryBuilders + .queryStringQuery(params.queryString) + .field(MANAGED_INDEX_NAME_KEYWORD_FIELD) + .defaultOperator(Operator.AND) + ).filter(QueryBuilders.termsQuery(MANAGED_INDEX_INDEX_UUID_FIELD, indexUUIDs)) + .filterByPolicyID(request.explainFilter) val searchSourceBuilder = SearchSourceBuilder() .from(params.from) @@ -304,20 +296,22 @@ class TransportExplainAction @Inject constructor( var metadataMap: Map = response.responses.associate { it.id to getMetadata(it.response)?.toMap() } - log.info("METADATA MAP: $metadataMap") - if (request.explainFilter != null) { metadataMap = metadataMap.filter { (_, value) -> - var isValid = false + var isValid = true if (value != null) { val metaData = ManagedIndexMetaData.fromMap(value) - isValid = request.explainFilter.byMetaData(metaData) - if (!isValid) { + if (!request.explainFilter.byMetaData(metaData)) { indexNames.remove(metaData.index) indexNamesToUUIDs.remove(metaData.index) - totalManagedIndices-- + + if (managedIndices.contains(metaData.index)) { + totalManagedIndices-- + } + + isValid = false } } From 1e6995f30fec8bdfb8126e0f6cda4d6168619375 Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Wed, 15 Nov 2023 02:41:09 -0500 Subject: [PATCH 09/16] Added test case for the ExplainFilter.byMetaData and parse methods Signed-off-by: Joshua Au --- .../model/ExplainFilter.kt | 18 +++++++++- .../indexstatemanagement/TestHelpers.kt | 15 ++++++++ .../model/ExplainFilterTests.kt | 35 +++++++++++++++++++ .../model/XContentTests.kt | 9 +++++ 4 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilterTests.kt diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt index 5749becb6..7a2f7c1ca 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt @@ -8,6 +8,9 @@ package org.opensearch.indexmanagement.indexstatemanagement.model import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput import org.opensearch.core.common.io.stream.Writeable +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.ToXContentObject +import org.opensearch.core.xcontent.XContentBuilder import org.opensearch.core.xcontent.XContentParser import org.opensearch.core.xcontent.XContentParser.Token import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken @@ -22,7 +25,7 @@ data class ExplainFilter( val state: String? = null, val actionType: String? = null, val failed: Boolean? = null -) : Writeable { +) : ToXContentObject, Writeable { @Throws(IOException::class) constructor(sin: StreamInput) : this( @@ -32,6 +35,19 @@ data class ExplainFilter( failed = sin.readOptionalBoolean() ) + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + builder.startObject() + builder.startObject(FILTER_FIELD) + + if (policyID != null) builder.field(POLICY_ID_FIELD, policyID) + if (state != null) builder.field(STATE_FIELD, state) + if (actionType != null) builder.field(ACTION_FIELD, actionType) + if (failed != null) builder.field(FAILED_FIELD, failed) + + builder.endObject() + return builder.endObject() + } + @Throws(IOException::class) override fun writeTo(out: StreamOutput) { out.writeOptionalString(policyID) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/TestHelpers.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/TestHelpers.kt index 7bb838f0a..ed4431af0 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/TestHelpers.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/TestHelpers.kt @@ -31,6 +31,7 @@ import org.opensearch.indexmanagement.indexstatemanagement.action.SnapshotAction import org.opensearch.indexmanagement.indexstatemanagement.model.ChangePolicy import org.opensearch.indexmanagement.indexstatemanagement.model.Conditions import org.opensearch.indexmanagement.indexstatemanagement.model.ErrorNotification +import org.opensearch.indexmanagement.indexstatemanagement.model.ExplainFilter import org.opensearch.indexmanagement.indexstatemanagement.model.ISMTemplate import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig import org.opensearch.indexmanagement.indexstatemanagement.model.Policy @@ -298,6 +299,15 @@ fun randomByteSizeValue(): ByteSizeValue = * End - Conditions helper functions */ +fun randomExplainFilter( + policyID: String? = if (OpenSearchRestTestCase.randomBoolean()) OpenSearchRestTestCase.randomAlphaOfLength(10) else null, + state: String? = if (OpenSearchRestTestCase.randomBoolean()) OpenSearchRestTestCase.randomAlphaOfLength(10) else null, + actionType: String? = if (OpenSearchRestTestCase.randomBoolean()) OpenSearchRestTestCase.randomAlphaOfLength(10) else null, + failed: Boolean? = if (OpenSearchRestTestCase.randomBoolean()) OpenSearchRestTestCase.randomBoolean() else null +): ExplainFilter { + return ExplainFilter(policyID, state, actionType, failed) +} + fun randomChangePolicy( policyID: String = OpenSearchRestTestCase.randomAlphaOfLength(10), state: String? = if (OpenSearchRestTestCase.randomBoolean()) OpenSearchRestTestCase.randomAlphaOfLength(10) else null, @@ -463,6 +473,11 @@ fun AllocationAction.toJsonString(): String { return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() } +fun ExplainFilter.toJsonString(): String { + val builder = XContentFactory.jsonBuilder() + return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() +} + fun ChangePolicy.toJsonString(): String { val builder = XContentFactory.jsonBuilder() return this.toXContent(builder, ToXContent.EMPTY_PARAMS).string() diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilterTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilterTests.kt new file mode 100644 index 000000000..36cac7ad8 --- /dev/null +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilterTests.kt @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.indexmanagement.indexstatemanagement.model + +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.PolicyRetryInfoMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaData +import org.opensearch.test.OpenSearchTestCase + +class ExplainFilterTests : OpenSearchTestCase() { + fun `test byMetadata`() { + val stateMetaData = StateMetaData("state", 100) + val actionMetaData = ActionMetaData("action", null, 0, false, 0, null, null) + val policyRetryInfoMetaData = PolicyRetryInfoMetaData(false, 1) + + val managedIndexMetaDataAllNull = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, null, null, null, null) + val managedIndexMetaDataNonNull = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, stateMetaData, actionMetaData, null, policyRetryInfoMetaData, null) + val managedIndexMetaDataSomeNull = ManagedIndexMetaData("test", "indexUuid", "policy_id", null, null, null, null, null, null, null, actionMetaData, null, null, null) + + val explainFilter = ExplainFilter(state = "state", actionType = "action", failed = false) + + var res = explainFilter.byMetaData(managedIndexMetaDataAllNull) + assertFalse(res) + + res = explainFilter.byMetaData(managedIndexMetaDataNonNull) + assertTrue(res) + + res = explainFilter.byMetaData(managedIndexMetaDataSomeNull) + assertFalse(res) + } +} diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/XContentTests.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/XContentTests.kt index defc5976e..fda797fac 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/XContentTests.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/XContentTests.kt @@ -20,6 +20,7 @@ import org.opensearch.indexmanagement.indexstatemanagement.randomChannel import org.opensearch.indexmanagement.indexstatemanagement.randomCloseActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomDeleteActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomDestination +import org.opensearch.indexmanagement.indexstatemanagement.randomExplainFilter import org.opensearch.indexmanagement.indexstatemanagement.randomForceMergeActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomIndexPriorityActionConfig import org.opensearch.indexmanagement.indexstatemanagement.randomManagedIndexConfig @@ -258,6 +259,14 @@ class XContentTests : OpenSearchTestCase() { assertEquals("Round tripping ManagedIndexMetaData doesn't work", metadata, parsedMetaData) } + fun `test explain filter parsing`() { + val explainFilter = randomExplainFilter() + + val explainFilterString = explainFilter.toJsonString() + val parsedExplainFilter = ExplainFilter.parse(parser(explainFilterString)) + assertEquals("Round tripping ExplainFilter doesn't work", explainFilter, parsedExplainFilter) + } + fun `test change policy parsing`() { val changePolicy = randomChangePolicy() From f559069c5829b2643efebbe94791fac07dbe85cc Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Wed, 22 Nov 2023 13:52:43 -0800 Subject: [PATCH 10/16] Started implementation of explain filter IT Signed-off-by: Joshua Au --- .../IndexStateManagementRestTestCase.kt | 13 ++- .../resthandler/RestExplainActionIT.kt | 110 ++++++++++++++++++ 2 files changed, 118 insertions(+), 5 deletions(-) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt index e6be5ce83..fa8caf6c8 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt @@ -37,12 +37,7 @@ import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_STAT import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.ISM_BASE_URI import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.POLICY_BASE_URI import org.opensearch.indexmanagement.IndexManagementRestTestCase -import org.opensearch.indexmanagement.indexstatemanagement.model.ChangePolicy -import org.opensearch.indexmanagement.indexstatemanagement.model.ISMTemplate -import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig -import org.opensearch.indexmanagement.indexstatemanagement.model.Policy import org.opensearch.indexmanagement.indexstatemanagement.model.Policy.Companion.POLICY_TYPE -import org.opensearch.indexmanagement.indexstatemanagement.model.StateFilter import org.opensearch.indexmanagement.indexstatemanagement.resthandler.RestExplainAction import org.opensearch.indexmanagement.indexstatemanagement.settings.ManagedIndexSettings import org.opensearch.indexmanagement.indexstatemanagement.util.FAILED_INDICES @@ -67,6 +62,12 @@ import org.opensearch.indexmanagement.waitFor import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule import org.opensearch.rest.RestRequest import org.opensearch.core.rest.RestStatus +import org.opensearch.indexmanagement.indexstatemanagement.model.ChangePolicy +import org.opensearch.indexmanagement.indexstatemanagement.model.ExplainFilter +import org.opensearch.indexmanagement.indexstatemanagement.model.ISMTemplate +import org.opensearch.indexmanagement.indexstatemanagement.model.ManagedIndexConfig +import org.opensearch.indexmanagement.indexstatemanagement.model.Policy +import org.opensearch.indexmanagement.indexstatemanagement.model.StateFilter import org.opensearch.indexmanagement.rollup.randomTermQuery import org.opensearch.test.OpenSearchTestCase import java.io.IOException @@ -469,6 +470,8 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase() protected fun ManagedIndexConfig.toHttpEntity(): HttpEntity = StringEntity(toJsonString(), ContentType.APPLICATION_JSON) + protected fun ExplainFilter.toHttpEntity(): HttpEntity = StringEntity(toJsonString(), ContentType.APPLICATION_JSON) + protected fun ChangePolicy.toHttpEntity(): HttpEntity { var string = "{\"${ChangePolicy.POLICY_ID_FIELD}\":\"$policyID\"," if (state != null) { diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt index 76dc83eb5..e5529d3f7 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt @@ -20,6 +20,8 @@ import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaDa import org.opensearch.indexmanagement.waitFor import org.opensearch.rest.RestRequest import org.opensearch.core.rest.RestStatus +import org.opensearch.indexmanagement.indexstatemanagement.model.ExplainFilter +import org.opensearch.indexmanagement.indexstatemanagement.toJsonString import java.time.Instant import java.util.Locale @@ -353,6 +355,114 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { } } + fun `test explain filter metadata filter`() { + // TODO: TEST ON INDEX WITH NULL POLICY: look at RestChangePolicyActionIT.kt +// val newPolicy = createPolicy(randomPolicy(states = listOf(updatedStateWithReadOnlyAction, stateWithDeleteAction)), "new_policy", true) + + val indexName1 = "${testIndexName}_filter1" + val indexName2 = "${testIndexName}_filter2" + val indexName3 = "${testIndexName}_filter3" + val policy1 = createRandomPolicy() + val policy2 = createRandomPolicy() // create random policy with actions? + + createIndex(indexName1, policy1.id) + createIndex(indexName2, policy1.id) + createIndex(indexName3, policy2.id) + + val indexName1Map = indexName1 to mapOf( + explainResponseOpendistroPolicyIdSetting to policy1.id, + explainResponseOpenSearchPolicyIdSetting to policy1.id, + "index" to indexName1, + "index_uuid" to getUuid(indexName1), + "policy_id" to policy1.id, + "enabled" to true + ) + + val indexName2Map = indexName2 to mapOf( + explainResponseOpendistroPolicyIdSetting to policy1.id, + explainResponseOpenSearchPolicyIdSetting to policy1.id, + "index" to indexName2, + "index_uuid" to getUuid(indexName2), + "policy_id" to policy1.id, + "enabled" to true + ) + + val indexName3Map = indexName3 to mapOf( + explainResponseOpendistroPolicyIdSetting to policy2.id, + explainResponseOpenSearchPolicyIdSetting to policy2.id, + "index" to indexName3, + "index_uuid" to getUuid(indexName3), + "policy_id" to policy2.id, + "enabled" to true + ) + + val filterPolicy1 = ExplainFilter( + policyID = policy1.id, + state = policy1.states[0].name + ) + + val filterPolicy2 = ExplainFilter( + state = policy2.states[0].name, + failed = false + ) + + val resp1Expected = mapOf( + indexName1Map, + indexName2Map, + TOTAL_MANAGED_INDICES to 2 + ) + + val resp2Expected = mapOf( + indexName3Map, + TOTAL_MANAGED_INDICES to 1 + ) + + logger.info("Policy1: $policy1") + logger.info("Policy2: $policy2") + + logger.info("filter policy 2: ${filterPolicy2.toJsonString()}") + + val resp1 = client().makeRequest( + RestRequest.Method.POST.toString(), + RestExplainAction.EXPLAIN_BASE_URI, emptyMap(), filterPolicy1.toHttpEntity() + ).asMap() + + val resp2 = client().makeRequest( + RestRequest.Method.POST.toString(), + RestExplainAction.EXPLAIN_BASE_URI, emptyMap(), filterPolicy2.toHttpEntity() + ).asMap() + + logger.info("Result1: $resp1") + logger.info("Result2: $resp2") + + // change the start time so the job will trigger in 2 seconds. + updateManagedIndexConfigStartTime(getExistingManagedIndexConfig(indexName1)) + updateManagedIndexConfigStartTime(getExistingManagedIndexConfig(indexName2)) + updateManagedIndexConfigStartTime(getExistingManagedIndexConfig(indexName3)) + + waitFor { assertResponseMap(resp1Expected, resp1) } + waitFor { assertResponseMap(resp2Expected, resp2) } + } + + fun `test explain filter on failed index`() { + // FORCE A FAIL + val indexName = "${testIndexName}_filter_fail" +// val policy = createRandomPoliclicy() + + updateManagedIndexConfigStartTime(getExistingManagedIndexConfig(indexName)) +// +// val resp = client().makeRequest( +// RestRequest.Method.POST.toString(), +// "$RestExplainAction.EXPLAIN_BASE_URI", emptyMap(), filterPolicy.toHttpEntity() +// ).asMap() +// +// val expected = mapOf( +// TOTAL_MANAGED_INDICES to 0 +// ) +// +// waitFor { assertResponseMap(expected, resp) } + } + @Suppress("UNCHECKED_CAST") // Do assertion of the response map here so we don't have many places to do suppression. private fun assertResponseMap(expected: Map, actual: Map) { assertEquals("Explain Map does not match", expected.size, actual.size) From 26eee41fe47046d43da5f8033389caeb21ec887c Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Fri, 1 Dec 2023 18:18:54 -0500 Subject: [PATCH 11/16] Implemented test explain filter method Signed-off-by: Joshua Au --- .../resthandler/RestExplainActionIT.kt | 199 +++++++++++------- .../RestRetryFailedManagedIndexActionIT.kt | 20 +- 2 files changed, 138 insertions(+), 81 deletions(-) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt index e5529d3f7..f9d017e0b 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt @@ -9,9 +9,6 @@ import org.opensearch.common.xcontent.XContentFactory import org.opensearch.indexmanagement.IndexManagementPlugin import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase import org.opensearch.indexmanagement.indexstatemanagement.model.ChangePolicy -import org.opensearch.indexmanagement.indexstatemanagement.util.SHOW_POLICY_QUERY_PARAM -import org.opensearch.indexmanagement.indexstatemanagement.util.TOTAL_MANAGED_INDICES -import org.opensearch.indexmanagement.indexstatemanagement.util.XCONTENT_WITHOUT_TYPE_AND_USER import org.opensearch.indexmanagement.makeRequest import org.opensearch.indexmanagement.opensearchapi.toMap import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData @@ -20,8 +17,17 @@ import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaDa import org.opensearch.indexmanagement.waitFor import org.opensearch.rest.RestRequest import org.opensearch.core.rest.RestStatus +import org.opensearch.indexmanagement.indexstatemanagement.action.DeleteAction +import org.opensearch.indexmanagement.indexstatemanagement.action.OpenAction +import org.opensearch.indexmanagement.indexstatemanagement.action.ReadOnlyAction import org.opensearch.indexmanagement.indexstatemanagement.model.ExplainFilter -import org.opensearch.indexmanagement.indexstatemanagement.toJsonString +import org.opensearch.indexmanagement.indexstatemanagement.model.Transition +import org.opensearch.indexmanagement.indexstatemanagement.randomPolicy +import org.opensearch.indexmanagement.indexstatemanagement.randomState +import org.opensearch.indexmanagement.indexstatemanagement.util.SHOW_POLICY_QUERY_PARAM +import org.opensearch.indexmanagement.indexstatemanagement.util.TOTAL_MANAGED_INDICES +import org.opensearch.indexmanagement.indexstatemanagement.util.XCONTENT_WITHOUT_TYPE_AND_USER +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData import java.time.Instant import java.util.Locale @@ -311,6 +317,10 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { waitFor { val expectedInfoString = mapOf("message" to "Fail to load policy: ${changePolicy.policyID}").toString() + logger.info("updated config: $managedIndexConfig") + + val explainMap = getExplainMap(indexName) + logger.info("map: $explainMap") assertPredicatesOnMetaData( listOf( indexName to listOf( @@ -326,7 +336,7 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { ManagedIndexMetaData.ENABLED to true::equals ) ), - getExplainMap(indexName) + explainMap ) } } @@ -355,93 +365,138 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { } } - fun `test explain filter metadata filter`() { - // TODO: TEST ON INDEX WITH NULL POLICY: look at RestChangePolicyActionIT.kt -// val newPolicy = createPolicy(randomPolicy(states = listOf(updatedStateWithReadOnlyAction, stateWithDeleteAction)), "new_policy", true) - + fun `test explain filter`() { val indexName1 = "${testIndexName}_filter1" val indexName2 = "${testIndexName}_filter2" - val indexName3 = "${testIndexName}_filter3" - val policy1 = createRandomPolicy() - val policy2 = createRandomPolicy() // create random policy with actions? - createIndex(indexName1, policy1.id) - createIndex(indexName2, policy1.id) - createIndex(indexName3, policy2.id) + val stateWithReadOnlyAction = randomState(actions = listOf(ReadOnlyAction(index = 0))) + val policy1 = createPolicy(randomPolicy(states = listOf(stateWithReadOnlyAction))) - val indexName1Map = indexName1 to mapOf( - explainResponseOpendistroPolicyIdSetting to policy1.id, - explainResponseOpenSearchPolicyIdSetting to policy1.id, - "index" to indexName1, - "index_uuid" to getUuid(indexName1), - "policy_id" to policy1.id, - "enabled" to true + val stateWithDeleteAction = randomState(actions = listOf(DeleteAction(index = 0))) + val updatedStateWithReadOnlyAction = stateWithReadOnlyAction.copy( + actions = listOf(stateWithReadOnlyAction.actions.first(), OpenAction(index = 1)), + transitions = listOf(Transition(stateWithDeleteAction.name, null)) ) + val policy2 = createPolicy(randomPolicy(states = listOf(stateWithDeleteAction, updatedStateWithReadOnlyAction))) - val indexName2Map = indexName2 to mapOf( - explainResponseOpendistroPolicyIdSetting to policy1.id, - explainResponseOpenSearchPolicyIdSetting to policy1.id, - "index" to indexName2, - "index_uuid" to getUuid(indexName2), - "policy_id" to policy1.id, - "enabled" to true + createIndex(indexName1, policy1.id) + createIndex(indexName2, policy2.id) + + val managedIndexConfig1 = getExistingManagedIndexConfig(indexName1) + val managedIndexConfig2 = getExistingManagedIndexConfig(indexName2) + + // init policy on job + updateManagedIndexConfigStartTime(managedIndexConfig1) + + // verify we have policy + waitFor { assertEquals(policy1.id, getExplainManagedIndexMetaData(indexName1).policyID) } + + // do the same for index2 + updateManagedIndexConfigStartTime(managedIndexConfig2) + waitFor { assertEquals(policy2.id, getExplainManagedIndexMetaData(indexName2).policyID) } + + // speed up to execute set read only step + updateManagedIndexConfigStartTime(managedIndexConfig1) + + val index1Predicates = indexName1 to listOf( + explainResponseOpendistroPolicyIdSetting to policy1.id::equals, + explainResponseOpenSearchPolicyIdSetting to policy1.id::equals, + ManagedIndexMetaData.INDEX to managedIndexConfig1.index::equals, + ManagedIndexMetaData.INDEX_UUID to managedIndexConfig1.indexUuid::equals, + ManagedIndexMetaData.POLICY_ID to managedIndexConfig1.policyID::equals, + ManagedIndexMetaData.POLICY_SEQ_NO to policy1.seqNo.toInt()::equals, + ManagedIndexMetaData.POLICY_PRIMARY_TERM to policy1.primaryTerm.toInt()::equals, + ManagedIndexMetaData.INDEX_CREATION_DATE to fun(indexCreationDate: Any?): Boolean = (indexCreationDate as Long) > 1L, + StateMetaData.STATE to fun(stateMetaDataMap: Any?): Boolean = assertStateEquals( + StateMetaData(policy1.defaultState, Instant.now().toEpochMilli()), + stateMetaDataMap + ), + ActionMetaData.ACTION to fun(actionMetaDataMap: Any?): Boolean = assertActionEquals( + ActionMetaData( + name = "read_only", startTime = Instant.now().toEpochMilli(), failed = false, + index = 0, consumedRetries = 0, lastRetryTime = null, actionProperties = null + ), + actionMetaDataMap + ), + PolicyRetryInfoMetaData.RETRY_INFO to fun(retryInfoMetaDataMap: Any?): Boolean = + assertRetryInfoEquals(PolicyRetryInfoMetaData(false, 0), retryInfoMetaDataMap), + ManagedIndexMetaData.ENABLED to true::equals ) - val indexName3Map = indexName3 to mapOf( - explainResponseOpendistroPolicyIdSetting to policy2.id, - explainResponseOpenSearchPolicyIdSetting to policy2.id, - "index" to indexName3, - "index_uuid" to getUuid(indexName3), - "policy_id" to policy2.id, - "enabled" to true + val index2Predicates = indexName2 to listOf( + explainResponseOpendistroPolicyIdSetting to policy2.id::equals, + explainResponseOpenSearchPolicyIdSetting to policy2.id::equals, + ManagedIndexMetaData.INDEX to managedIndexConfig2.index::equals, + ManagedIndexMetaData.INDEX_UUID to managedIndexConfig2.indexUuid::equals, + ManagedIndexMetaData.POLICY_ID to managedIndexConfig2.policyID::equals, + ManagedIndexMetaData.POLICY_SEQ_NO to policy2.seqNo.toInt()::equals, + ManagedIndexMetaData.POLICY_PRIMARY_TERM to policy2.primaryTerm.toInt()::equals, + ManagedIndexMetaData.INDEX_CREATION_DATE to fun(indexCreationDate: Any?): Boolean = (indexCreationDate as Long) > 1L, + StateMetaData.STATE to fun(stateMetaDataMap: Any?): Boolean = assertStateEquals( + StateMetaData(policy2.defaultState, Instant.now().toEpochMilli()), + stateMetaDataMap + ), + ActionMetaData.ACTION to fun(actionMetaDataMap: Any?): Boolean = assertActionEquals( + ActionMetaData( + name = "delete", startTime = Instant.now().toEpochMilli(), failed = false, + index = 0, consumedRetries = 0, lastRetryTime = null, actionProperties = null + ), + actionMetaDataMap + ), + PolicyRetryInfoMetaData.RETRY_INFO to fun(retryInfoMetaDataMap: Any?): Boolean = + assertRetryInfoEquals(PolicyRetryInfoMetaData(false, 0), retryInfoMetaDataMap), + ManagedIndexMetaData.ENABLED to true::equals ) - val filterPolicy1 = ExplainFilter( - policyID = policy1.id, - state = policy1.states[0].name - ) + // check metadata for result from filtering on the first policy and its state + waitFor { + val filterPolicy = ExplainFilter( + policyID = policy1.id, + state = policy1.states[0].name, + failed = false + ) - val filterPolicy2 = ExplainFilter( - state = policy2.states[0].name, - failed = false - ) + val resp = client().makeRequest( + RestRequest.Method.POST.toString(), + RestExplainAction.EXPLAIN_BASE_URI, emptyMap(), filterPolicy.toHttpEntity() + ) - val resp1Expected = mapOf( - indexName1Map, - indexName2Map, - TOTAL_MANAGED_INDICES to 2 - ) + assertEquals("Unexpected RestStatus", RestStatus.OK, resp.restStatus()) - val resp2Expected = mapOf( - indexName3Map, - TOTAL_MANAGED_INDICES to 1 - ) + assertPredicatesOnMetaData( + listOf(index1Predicates), + resp.asMap(), false + ) + } - logger.info("Policy1: $policy1") - logger.info("Policy2: $policy2") + // check metadata on filtering for the delete action - logger.info("filter policy 2: ${filterPolicy2.toJsonString()}") + // speed up to execute set read only step + updateManagedIndexConfigStartTime(managedIndexConfig1) + updateManagedIndexConfigStartTime(managedIndexConfig2) - val resp1 = client().makeRequest( - RestRequest.Method.POST.toString(), - RestExplainAction.EXPLAIN_BASE_URI, emptyMap(), filterPolicy1.toHttpEntity() - ).asMap() + waitFor { + val filterPolicy = ExplainFilter( + actionType = "delete", + failed = false + ) - val resp2 = client().makeRequest( - RestRequest.Method.POST.toString(), - RestExplainAction.EXPLAIN_BASE_URI, emptyMap(), filterPolicy2.toHttpEntity() - ).asMap() + val resp = client().makeRequest( + RestRequest.Method.POST.toString(), + RestExplainAction.EXPLAIN_BASE_URI, emptyMap(), filterPolicy.toHttpEntity() + ) - logger.info("Result1: $resp1") - logger.info("Result2: $resp2") + assertEquals("Unexpected RestStatus", RestStatus.OK, resp.restStatus()) - // change the start time so the job will trigger in 2 seconds. - updateManagedIndexConfigStartTime(getExistingManagedIndexConfig(indexName1)) - updateManagedIndexConfigStartTime(getExistingManagedIndexConfig(indexName2)) - updateManagedIndexConfigStartTime(getExistingManagedIndexConfig(indexName3)) + logger.info(resp.asMap()) - waitFor { assertResponseMap(resp1Expected, resp1) } - waitFor { assertResponseMap(resp2Expected, resp2) } + assertPredicatesOnMetaData( + listOf( + index2Predicates + ), + resp.asMap(), false + ) + } } fun `test explain filter on failed index`() { diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRetryFailedManagedIndexActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRetryFailedManagedIndexActionIT.kt index 0cf0e97cf..fe3663db2 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRetryFailedManagedIndexActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRetryFailedManagedIndexActionIT.kt @@ -272,20 +272,22 @@ class RestRetryFailedManagedIndexActionIT : IndexStateManagementRestTestCase() { updateManagedIndexConfigStartTime(managedIndexConfig) waitFor { + val explainMap = getExplainMap(indexName) + logger.info("explain map: $explainMap") + assertPredicatesOnMetaData( listOf( indexName to listOf( - ActionMetaData.ACTION to fun(actionMetaDataMap: Any?): Boolean = - assertActionEquals( - ActionMetaData( - name = "force_merge", startTime = Instant.now().toEpochMilli(), failed = false, - index = 0, consumedRetries = 0, lastRetryTime = null, actionProperties = null - ), - actionMetaDataMap - ) + ActionMetaData.ACTION to fun(actionMetaDataMap: Any?): Boolean = assertActionEquals( + ActionMetaData( + name = "force_merge", startTime = Instant.now().toEpochMilli(), failed = false, + index = 0, consumedRetries = 0, lastRetryTime = null, actionProperties = null + ), + actionMetaDataMap + ) ) ), - getExplainMap(indexName), false + explainMap, false ) } From 5d1909330590332ddaf33ab25cd6f4845564a218 Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Fri, 1 Dec 2023 19:25:24 -0500 Subject: [PATCH 12/16] Implemented explain filter test on failure Signed-off-by: Joshua Au --- .../resthandler/RestExplainActionIT.kt | 80 ++++++++++++++----- 1 file changed, 61 insertions(+), 19 deletions(-) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt index f9d017e0b..b6241c5e4 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt @@ -11,12 +11,10 @@ import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementR import org.opensearch.indexmanagement.indexstatemanagement.model.ChangePolicy import org.opensearch.indexmanagement.makeRequest import org.opensearch.indexmanagement.opensearchapi.toMap -import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData -import org.opensearch.indexmanagement.spi.indexstatemanagement.model.PolicyRetryInfoMetaData -import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaData import org.opensearch.indexmanagement.waitFor import org.opensearch.rest.RestRequest import org.opensearch.core.rest.RestStatus +import org.opensearch.indexmanagement.indexstatemanagement.action.AllocationAction import org.opensearch.indexmanagement.indexstatemanagement.action.DeleteAction import org.opensearch.indexmanagement.indexstatemanagement.action.OpenAction import org.opensearch.indexmanagement.indexstatemanagement.action.ReadOnlyAction @@ -27,7 +25,13 @@ import org.opensearch.indexmanagement.indexstatemanagement.randomState import org.opensearch.indexmanagement.indexstatemanagement.util.SHOW_POLICY_QUERY_PARAM import org.opensearch.indexmanagement.indexstatemanagement.util.TOTAL_MANAGED_INDICES import org.opensearch.indexmanagement.indexstatemanagement.util.XCONTENT_WITHOUT_TYPE_AND_USER +import org.opensearch.indexmanagement.spi.indexstatemanagement.Step import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionRetry +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.PolicyRetryInfoMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaData +import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData import java.time.Instant import java.util.Locale @@ -500,22 +504,60 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { } fun `test explain filter on failed index`() { - // FORCE A FAIL - val indexName = "${testIndexName}_filter_fail" -// val policy = createRandomPoliclicy() - - updateManagedIndexConfigStartTime(getExistingManagedIndexConfig(indexName)) -// -// val resp = client().makeRequest( -// RestRequest.Method.POST.toString(), -// "$RestExplainAction.EXPLAIN_BASE_URI", emptyMap(), filterPolicy.toHttpEntity() -// ).asMap() -// -// val expected = mapOf( -// TOTAL_MANAGED_INDICES to 0 -// ) -// -// waitFor { assertResponseMap(expected, resp) } + val indexName = "${testIndexName}_failed" + val config = AllocationAction(require = mapOf("..//" to "value"), exclude = emptyMap(), include = emptyMap(), index = 0) + config.configRetry = ActionRetry(0) + val states = listOf( + randomState().copy( + transitions = listOf(), + actions = listOf(config) + ) + ) + val invalidPolicy = randomPolicy().copy( + states = states, + defaultState = states[0].name + ) + createPolicy(invalidPolicy, invalidPolicy.id) + createIndex(indexName, invalidPolicy.id) + + val managedIndexConfig = getExistingManagedIndexConfig(indexName) + // change the start time so the job will trigger in 2 seconds. + updateManagedIndexConfigStartTime(managedIndexConfig) + waitFor { assertEquals(invalidPolicy.id, getExplainManagedIndexMetaData(indexName).policyID) } + + // Change the start time so that we attempt allocation that is intended to fail + updateManagedIndexConfigStartTime(managedIndexConfig) + waitFor { + val explainFilter = ExplainFilter( + failed = true + ) + + val resp = client().makeRequest( + RestRequest.Method.POST.toString(), + RestExplainAction.EXPLAIN_BASE_URI, emptyMap(), explainFilter.toHttpEntity() + ) + + assertEquals("Unexpected RestStatus", RestStatus.OK, resp.restStatus()) + + assertPredicatesOnMetaData( + listOf( + indexName to listOf( + explainResponseOpendistroPolicyIdSetting to invalidPolicy.id::equals, + explainResponseOpenSearchPolicyIdSetting to invalidPolicy.id::equals, + ManagedIndexMetaData.INDEX to managedIndexConfig.index::equals, + ManagedIndexMetaData.INDEX_UUID to managedIndexConfig.indexUuid::equals, + ManagedIndexMetaData.POLICY_ID to managedIndexConfig.policyID::equals, + ManagedIndexMetaData.INDEX_CREATION_DATE to fun(indexCreationDate: Any?): Boolean = (indexCreationDate as Long) > 1L, + StepMetaData.STEP to fun(stepMetaDataMap: Any?): Boolean = assertStepEquals( + StepMetaData("attempt_allocation", Instant.now().toEpochMilli(), Step.StepStatus.FAILED), + stepMetaDataMap + ), + ManagedIndexMetaData.ENABLED to true::equals + ) + ), + resp.asMap(), false + ) + } } @Suppress("UNCHECKED_CAST") // Do assertion of the response map here so we don't have many places to do suppression. From f3e3bee31a4db7de4efd968c7d58d94c51553a2f Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Fri, 1 Dec 2023 19:27:37 -0500 Subject: [PATCH 13/16] Cleaned up log statements Signed-off-by: Joshua Au --- .../indexstatemanagement/resthandler/RestExplainActionIT.kt | 2 -- .../resthandler/RestRetryFailedManagedIndexActionIT.kt | 1 - 2 files changed, 3 deletions(-) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt index b6241c5e4..6e5f7f9e5 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt @@ -321,10 +321,8 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { waitFor { val expectedInfoString = mapOf("message" to "Fail to load policy: ${changePolicy.policyID}").toString() - logger.info("updated config: $managedIndexConfig") val explainMap = getExplainMap(indexName) - logger.info("map: $explainMap") assertPredicatesOnMetaData( listOf( indexName to listOf( diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRetryFailedManagedIndexActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRetryFailedManagedIndexActionIT.kt index fe3663db2..2640c14c6 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRetryFailedManagedIndexActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestRetryFailedManagedIndexActionIT.kt @@ -273,7 +273,6 @@ class RestRetryFailedManagedIndexActionIT : IndexStateManagementRestTestCase() { waitFor { val explainMap = getExplainMap(indexName) - logger.info("explain map: $explainMap") assertPredicatesOnMetaData( listOf( From 44e553c5983f5b1ff5b127aec59384c1005d9bdd Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Sat, 2 Dec 2023 14:52:21 -0500 Subject: [PATCH 14/16] Added explain filter test for success Signed-off-by: Joshua Au --- .../resthandler/RestExplainActionIT.kt | 71 +++++++++++++++---- 1 file changed, 58 insertions(+), 13 deletions(-) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt index 6e5f7f9e5..c63c1b628 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt @@ -490,8 +490,6 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { assertEquals("Unexpected RestStatus", RestStatus.OK, resp.restStatus()) - logger.info(resp.asMap()) - assertPredicatesOnMetaData( listOf( index2Predicates @@ -501,8 +499,11 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { } } - fun `test explain filter on failed index`() { - val indexName = "${testIndexName}_failed" + fun `test explain filter failed index`() { + val indexName1 = "${testIndexName}_failed" + val indexName2 = "${testIndexName}_success" + + // for failed index val config = AllocationAction(require = mapOf("..//" to "value"), exclude = emptyMap(), include = emptyMap(), index = 0) config.configRetry = ActionRetry(0) val states = listOf( @@ -515,16 +516,27 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { states = states, defaultState = states[0].name ) + + // for successful index + val stateWithReadOnlyAction = randomState(actions = listOf(ReadOnlyAction(index = 0))) + val validPolicy = createPolicy(randomPolicy(states = listOf(stateWithReadOnlyAction))) + createPolicy(invalidPolicy, invalidPolicy.id) - createIndex(indexName, invalidPolicy.id) + createIndex(indexName1, invalidPolicy.id) + createIndex(indexName2, validPolicy.id) + + val managedIndexConfig1 = getExistingManagedIndexConfig(indexName1) + val managedIndexConfig2 = getExistingManagedIndexConfig(indexName2) - val managedIndexConfig = getExistingManagedIndexConfig(indexName) // change the start time so the job will trigger in 2 seconds. - updateManagedIndexConfigStartTime(managedIndexConfig) - waitFor { assertEquals(invalidPolicy.id, getExplainManagedIndexMetaData(indexName).policyID) } + updateManagedIndexConfigStartTime(managedIndexConfig1) + waitFor { assertEquals(invalidPolicy.id, getExplainManagedIndexMetaData(indexName1).policyID) } + + updateManagedIndexConfigStartTime(managedIndexConfig2) + waitFor { assertEquals(validPolicy.id, getExplainManagedIndexMetaData(indexName2).policyID) } // Change the start time so that we attempt allocation that is intended to fail - updateManagedIndexConfigStartTime(managedIndexConfig) + updateManagedIndexConfigStartTime(managedIndexConfig1) waitFor { val explainFilter = ExplainFilter( failed = true @@ -539,12 +551,12 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { assertPredicatesOnMetaData( listOf( - indexName to listOf( + indexName1 to listOf( explainResponseOpendistroPolicyIdSetting to invalidPolicy.id::equals, explainResponseOpenSearchPolicyIdSetting to invalidPolicy.id::equals, - ManagedIndexMetaData.INDEX to managedIndexConfig.index::equals, - ManagedIndexMetaData.INDEX_UUID to managedIndexConfig.indexUuid::equals, - ManagedIndexMetaData.POLICY_ID to managedIndexConfig.policyID::equals, + ManagedIndexMetaData.INDEX to managedIndexConfig1.index::equals, + ManagedIndexMetaData.INDEX_UUID to managedIndexConfig1.indexUuid::equals, + ManagedIndexMetaData.POLICY_ID to managedIndexConfig1.policyID::equals, ManagedIndexMetaData.INDEX_CREATION_DATE to fun(indexCreationDate: Any?): Boolean = (indexCreationDate as Long) > 1L, StepMetaData.STEP to fun(stepMetaDataMap: Any?): Boolean = assertStepEquals( StepMetaData("attempt_allocation", Instant.now().toEpochMilli(), Step.StepStatus.FAILED), @@ -556,6 +568,39 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { resp.asMap(), false ) } + + updateManagedIndexConfigStartTime(managedIndexConfig2) + waitFor { + val explainFilter = ExplainFilter( + failed = false + ) + + val resp = client().makeRequest( + RestRequest.Method.POST.toString(), + RestExplainAction.EXPLAIN_BASE_URI, emptyMap(), explainFilter.toHttpEntity() + ) + + assertEquals("Unexpected RestStatus", RestStatus.OK, resp.restStatus()) + + assertPredicatesOnMetaData( + listOf( + indexName2 to listOf( + explainResponseOpendistroPolicyIdSetting to validPolicy.id::equals, + explainResponseOpenSearchPolicyIdSetting to validPolicy.id::equals, + ManagedIndexMetaData.INDEX to managedIndexConfig2.index::equals, + ManagedIndexMetaData.INDEX_UUID to managedIndexConfig2.indexUuid::equals, + ManagedIndexMetaData.POLICY_ID to managedIndexConfig2.policyID::equals, + ManagedIndexMetaData.INDEX_CREATION_DATE to fun(indexCreationDate: Any?): Boolean = (indexCreationDate as Long) > 1L, + StepMetaData.STEP to fun(stepMetaDataMap: Any?): Boolean = assertStepEquals( + StepMetaData("set_read_only", Instant.now().toEpochMilli(), Step.StepStatus.STARTING), + stepMetaDataMap + ), + ManagedIndexMetaData.ENABLED to true::equals + ) + ), + resp.asMap(), false + ) + } } @Suppress("UNCHECKED_CAST") // Do assertion of the response map here so we don't have many places to do suppression. From 19796f0bd86f2dc8105d5f2c4f2392e175085228 Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Thu, 14 Dec 2023 17:57:46 -0500 Subject: [PATCH 15/16] Fixed lint errors Signed-off-by: Joshua Au --- .../indexstatemanagement/model/ExplainFilter.kt | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt index ce3ff3982..7a2f7c1ca 100644 --- a/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt +++ b/src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/model/ExplainFilter.kt @@ -25,8 +25,7 @@ data class ExplainFilter( val state: String? = null, val actionType: String? = null, val failed: Boolean? = null -) : Writeable { - +) : ToXContentObject, Writeable { @Throws(IOException::class) constructor(sin: StreamInput) : this( @@ -36,6 +35,19 @@ data class ExplainFilter( failed = sin.readOptionalBoolean() ) + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + builder.startObject() + builder.startObject(FILTER_FIELD) + + if (policyID != null) builder.field(POLICY_ID_FIELD, policyID) + if (state != null) builder.field(STATE_FIELD, state) + if (actionType != null) builder.field(ACTION_FIELD, actionType) + if (failed != null) builder.field(FAILED_FIELD, failed) + + builder.endObject() + return builder.endObject() + } + @Throws(IOException::class) override fun writeTo(out: StreamOutput) { out.writeOptionalString(policyID) From 00b6bd3c943acc624c8b03cd4a025b671b98e51f Mon Sep 17 00:00:00 2001 From: Joshua Au Date: Sun, 17 Dec 2023 23:27:47 -0500 Subject: [PATCH 16/16] Removed policy from index to fix flaky tests Signed-off-by: Joshua Au --- .../indexstatemanagement/resthandler/RestExplainActionIT.kt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt index c63c1b628..5c91f9364 100644 --- a/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt +++ b/src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestExplainActionIT.kt @@ -497,6 +497,9 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { resp.asMap(), false ) } + + removePolicyFromIndex(indexName1) + removePolicyFromIndex(indexName2) } fun `test explain filter failed index`() { @@ -601,6 +604,9 @@ class RestExplainActionIT : IndexStateManagementRestTestCase() { resp.asMap(), false ) } + + removePolicyFromIndex(indexName1) + removePolicyFromIndex(indexName2) } @Suppress("UNCHECKED_CAST") // Do assertion of the response map here so we don't have many places to do suppression.