[Backport 2.x] IndicesHandler and conversationIndexMemory (#1762)

client/build.gradle

@@ -15,7 +15,7 @@ plugins {
 
 dependencies {
     implementation project(path: ":${rootProject.name}-spi", configuration: 'shadow')
-    implementation project(':opensearch-ml-common')
+    implementation project(path: ":${rootProject.name}-common", configuration: 'shadow')
     compileOnly group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}"
     testImplementation group: 'junit', name: 'junit', version: '4.13.2'
     testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.7.0'
@@ -122,4 +122,5 @@ publishing {
     }
 
 }
-
+compileJava.dependsOn(':opensearch-ml-common:shadowJar')
+delombok.dependsOn(':opensearch-ml-common:shadowJar')

common/build.gradle

@@ -6,8 +6,11 @@
 //TODO: cleanup gradle config file, some overlap
 plugins {
     id 'java'
+    id 'com.github.johnrengelman.shadow'
     id 'jacoco'
     id "io.freefair.lombok"
+    id 'maven-publish'
+    id 'signing'
 }
 
 dependencies {
@@ -21,6 +24,15 @@ dependencies {
     compileOnly group: 'org.apache.commons', name: 'commons-text', version: '1.10.0'
     compileOnly group: 'com.google.code.gson', name: 'gson', version: '2.10.1'
     compileOnly group: 'org.json', name: 'json', version: '20231013'
+
+    implementation('com.google.guava:guava:32.1.2-jre') {
+        exclude group: 'com.google.guava', module: 'failureaccess'
+        exclude group: 'com.google.code.findbugs', module: 'jsr305'
+        exclude group: 'org.checkerframework', module: 'checker-qual'
+        exclude group: 'com.google.errorprone', module: 'error_prone_annotations'
+        exclude group: 'com.google.j2objc', module: 'j2objc-annotations'
+        exclude group: 'com.google.guava', module: 'listenablefuture'
+    }
 }
 
 lombok {
@@ -53,3 +65,75 @@ jacocoTestCoverageVerification {
     dependsOn jacocoTestReport
 }
 check.dependsOn jacocoTestCoverageVerification
+
+shadowJar {
+    destinationDirectory = file("${project.buildDir}/distributions")
+    archiveClassifier.set(null)
+    exclude 'META-INF/maven/com.google.guava/**'
+    exclude 'com/google/thirdparty/**'
+    relocate 'com.google.common', 'org.opensearch.ml.repackage.com.google.common' // dependency of cron-utils
+}
+
+jar {
+    enabled false
+}
+
+task sourcesJar(type: Jar) {
+    archiveClassifier.set 'sources'
+    from sourceSets.main.allJava
+}
+
+task javadocJar(type: Jar) {
+    archiveClassifier.set 'javadoc'
+    from javadoc.destinationDir
+    dependsOn javadoc
+}
+
+publishing {
+    repositories {
+        maven {
+            name = 'staging'
+            url = "${rootProject.buildDir}/local-staging-repo"
+        }
+        maven {
+            name = "Snapshots" //  optional target repository name
+            url = "https://aws.oss.sonatype.org/content/repositories/snapshots"
+            credentials {
+                username "$System.env.SONATYPE_USERNAME"
+                password "$System.env.SONATYPE_PASSWORD"
+            }
+        }
+    }
+    publications {
+        shadow(MavenPublication) { publication ->
+            project.shadow.component(publication)
+            artifact sourcesJar
+            artifact javadocJar
+
+            pom {
+                name = "OpenSearch ML Commons Comm"
+                packaging = "jar"
+                url = "https://github.com/opensearch-project/ml-commons"
+                description = "OpenSearch ML Common"
+                scm {
+                    connection = "scm:[email protected]:opensearch-project/ml-commons.git"
+                    developerConnection = "scm:[email protected]:opensearch-project/ml-commons.git"
+                    url = "[email protected]:opensearch-project/ml-commons.git"
+                }
+                licenses {
+                    license {
+                        name = "The Apache License, Version 2.0"
+                        url = "http://www.apache.org/licenses/LICENSE-2.0.txt"
+                    }
+                }
+                developers {
+                    developer {
+                        name = "OpenSearch"
+                        url = "https://github.com/opensearch-project/ml-commons"
+                    }
+                }
+            }
+        }
+    }
+}
+publishShadowPublicationToMavenLocal.mustRunAfter shadowJar

memory/build.gradle

@@ -24,7 +24,7 @@ plugins {
 }
 
 dependencies {
-    implementation project(":opensearch-ml-common")
+    implementation project(path: ":${rootProject.name}-common", configuration: 'shadow')
     implementation group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}"
     implementation group: 'org.apache.httpcomponents.core5', name: 'httpcore5', version: '5.2.1'
     implementation "org.opensearch:common-utils:${common_utils_version}"

ml-algorithms/build.gradle

@@ -17,10 +17,10 @@ repositories {
 }
 
 dependencies {
-    compileOnly group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}"
     implementation project(path: ":${rootProject.name}-spi", configuration: 'shadow')
-    implementation project(':opensearch-ml-common')
+    implementation project(path: ":${rootProject.name}-common", configuration: 'shadow')
     implementation project(':opensearch-ml-memory')
+    compileOnly group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}"
     implementation "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
     testImplementation "org.opensearch.test:framework:${opensearch_version}"
     implementation "org.opensearch:common-utils:${common_utils_version}"
@@ -103,6 +103,7 @@ jacocoTestCoverageVerification {
     dependsOn jacocoTestReport
 }
 check.dependsOn jacocoTestCoverageVerification
+compileJava.dependsOn(':opensearch-ml-common:shadowJar')
 
 spotless {
     java {

plugin/src/main/java/org/opensearch/ml/indices/MLIndex.java → ml-algorithms/src/main/java/org/opensearch/ml/engine/indices/MLIndex.java

@@ -3,14 +3,23 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.ml.indices;
+package org.opensearch.ml.engine.indices;
 
+import static org.opensearch.ml.common.CommonValue.ML_AGENT_INDEX;
+import static org.opensearch.ml.common.CommonValue.ML_AGENT_INDEX_MAPPING;
+import static org.opensearch.ml.common.CommonValue.ML_AGENT_INDEX_SCHEMA_VERSION;
 import static org.opensearch.ml.common.CommonValue.ML_CONFIG_INDEX;
 import static org.opensearch.ml.common.CommonValue.ML_CONFIG_INDEX_MAPPING;
 import static org.opensearch.ml.common.CommonValue.ML_CONFIG_INDEX_SCHEMA_VERSION;
 import static org.opensearch.ml.common.CommonValue.ML_CONNECTOR_INDEX;
 import static org.opensearch.ml.common.CommonValue.ML_CONNECTOR_INDEX_MAPPING;
 import static org.opensearch.ml.common.CommonValue.ML_CONNECTOR_SCHEMA_VERSION;
+import static org.opensearch.ml.common.CommonValue.ML_MEMORY_MESSAGE_INDEX;
+import static org.opensearch.ml.common.CommonValue.ML_MEMORY_MESSAGE_INDEX_MAPPING;
+import static org.opensearch.ml.common.CommonValue.ML_MEMORY_MESSAGE_INDEX_SCHEMA_VERSION;
+import static org.opensearch.ml.common.CommonValue.ML_MEMORY_META_INDEX;
+import static org.opensearch.ml.common.CommonValue.ML_MEMORY_META_INDEX_MAPPING;
+import static org.opensearch.ml.common.CommonValue.ML_MEMORY_META_INDEX_SCHEMA_VERSION;
 import static org.opensearch.ml.common.CommonValue.ML_MODEL_GROUP_INDEX;
 import static org.opensearch.ml.common.CommonValue.ML_MODEL_GROUP_INDEX_MAPPING;
 import static org.opensearch.ml.common.CommonValue.ML_MODEL_GROUP_INDEX_SCHEMA_VERSION;
@@ -26,7 +35,10 @@ public enum MLIndex {
     MODEL(ML_MODEL_INDEX, false, ML_MODEL_INDEX_MAPPING, ML_MODEL_INDEX_SCHEMA_VERSION),
     TASK(ML_TASK_INDEX, false, ML_TASK_INDEX_MAPPING, ML_TASK_INDEX_SCHEMA_VERSION),
     CONNECTOR(ML_CONNECTOR_INDEX, false, ML_CONNECTOR_INDEX_MAPPING, ML_CONNECTOR_SCHEMA_VERSION),
-    CONFIG(ML_CONFIG_INDEX, false, ML_CONFIG_INDEX_MAPPING, ML_CONFIG_INDEX_SCHEMA_VERSION);
+    CONFIG(ML_CONFIG_INDEX, false, ML_CONFIG_INDEX_MAPPING, ML_CONFIG_INDEX_SCHEMA_VERSION),
+    AGENT(ML_AGENT_INDEX, false, ML_AGENT_INDEX_MAPPING, ML_AGENT_INDEX_SCHEMA_VERSION),
+    MEMORY_META(ML_MEMORY_META_INDEX, false, ML_MEMORY_META_INDEX_MAPPING, ML_MEMORY_META_INDEX_SCHEMA_VERSION),
+    MEMORY_MESSAGE(ML_MEMORY_MESSAGE_INDEX, false, ML_MEMORY_MESSAGE_INDEX_MAPPING, ML_MEMORY_MESSAGE_INDEX_SCHEMA_VERSION);
 
     private final String indexName;
     // whether we use an alias for the index

plugin/src/main/java/org/opensearch/ml/indices/MLIndicesHandler.java → ml-algorithms/src/main/java/org/opensearch/ml/engine/indices/MLIndicesHandler.java

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.ml.indices;
+package org.opensearch.ml.engine.indices;
 
 import static org.opensearch.ml.common.CommonValue.META;
 import static org.opensearch.ml.common.CommonValue.SCHEMA_VERSION_FIELD;
@@ -62,10 +62,22 @@ public void initMLConnectorIndex(ActionListener<Boolean> listener) {
         initMLIndexIfAbsent(MLIndex.CONNECTOR, listener);
     }
 
+    public void initMemoryMetaIndex(ActionListener<Boolean> listener) {
+        initMLIndexIfAbsent(MLIndex.MEMORY_META, listener);
+    }
+
+    public void initMemoryMessageIndex(ActionListener<Boolean> listener) {
+        initMLIndexIfAbsent(MLIndex.MEMORY_MESSAGE, listener);
+    }
+
     public void initMLConfigIndex(ActionListener<Boolean> listener) {
         initMLIndexIfAbsent(MLIndex.CONFIG, listener);
     }
 
+    public void initMLAgentIndex(ActionListener<Boolean> listener) {
+        initMLIndexIfAbsent(MLIndex.AGENT, listener);
+    }
+
     public void initMLIndexIfAbsent(MLIndex index, ActionListener<Boolean> listener) {
         String indexName = index.getIndexName();
         String mapping = index.getMapping();

plugin/src/main/java/org/opensearch/ml/indices/MLInputDatasetHandler.java → ml-algorithms/src/main/java/org/opensearch/ml/engine/indices/MLInputDatasetHandler.java

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-package org.opensearch.ml.indices;
+package org.opensearch.ml.engine.indices;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -35,19 +35,6 @@
 public class MLInputDatasetHandler {
     Client client;
 
-    // /**
-    // * Retrieve DataFrame from DataFrameInputDataset
-    // * @param mlInputDataset MLInputDataset
-    // * @return DataFrame
-    // */
-    // public DataFrame parseDataFrameInput(MLInputDataset mlInputDataset) {
-    // if (!mlInputDataset.getInputDataType().equals(MLInputDataType.DATA_FRAME)) {
-    // throw new IllegalArgumentException("Input dataset is not DATA_FRAME type.");
-    // }
-    // DataFrameInputDataset inputDataset = (DataFrameInputDataset) mlInputDataset;
-    // return inputDataset.getDataFrame();
-    // }
-
     /**
      * Create DataFrame based on given search query
      * @param mlInputDataset MLInputDataset

ml-algorithms/src/main/java/org/opensearch/ml/engine/memory/BaseMessage.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.ml.engine.memory;
+
+import java.io.IOException;
+
+import org.opensearch.core.xcontent.ToXContentObject;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.ml.common.spi.memory.Message;
+
+import lombok.Builder;
+import lombok.Getter;
+import lombok.Setter;
+
+public class BaseMessage implements Message, ToXContentObject {
+
+    @Getter
+    @Setter
+    protected String type;
+    @Getter
+    @Setter
+    protected String content;
+
+    @Builder
+    public BaseMessage(String type, String content) {
+        this.type = type;
+        this.content = content;
+    }
+
+    @Override
+    public String toString() {
+        return type + ": " + content;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        builder.field("type", type);
+        builder.field("content", content);
+        builder.endObject();
+        return builder;
+    }
+}