forked from opensearch-project/ml-commons
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'opensearch-project:main' into main
- Loading branch information
Showing
150 changed files
with
9,255 additions
and
722 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,9 @@ | ||
| .gradle/ | ||
| build/ | ||
| .idea/ | ||
| .project | ||
| .classpath | ||
| .settings | ||
| client/build/ | ||
| common/build/ | ||
| ml-algorithms/build/ | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -6,8 +6,11 @@ | |
| //TODO: cleanup gradle config file, some overlap | ||
| plugins { | ||
| id 'java' | ||
| id 'com.github.johnrengelman.shadow' | ||
| id 'jacoco' | ||
| id "io.freefair.lombok" | ||
| id 'maven-publish' | ||
| id 'signing' | ||
| } | ||
|
|
||
| dependencies { | ||
|
|
@@ -21,6 +24,15 @@ dependencies { | |
| compileOnly group: 'org.apache.commons', name: 'commons-text', version: '1.10.0' | ||
| compileOnly group: 'com.google.code.gson', name: 'gson', version: '2.10.1' | ||
| compileOnly group: 'org.json', name: 'json', version: '20231013' | ||
|
|
||
| implementation('com.google.guava:guava:32.1.2-jre') { | ||
| exclude group: 'com.google.guava', module: 'failureaccess' | ||
| exclude group: 'com.google.code.findbugs', module: 'jsr305' | ||
| exclude group: 'org.checkerframework', module: 'checker-qual' | ||
| exclude group: 'com.google.errorprone', module: 'error_prone_annotations' | ||
| exclude group: 'com.google.j2objc', module: 'j2objc-annotations' | ||
| exclude group: 'com.google.guava', module: 'listenablefuture' | ||
| } | ||
| } | ||
|
|
||
| lombok { | ||
|
|
@@ -54,5 +66,74 @@ jacocoTestCoverageVerification { | |
| } | ||
| check.dependsOn jacocoTestCoverageVerification | ||
|
|
||
| shadowJar { | ||
| destinationDirectory = file("${project.buildDir}/distributions") | ||
| archiveClassifier.set(null) | ||
| exclude 'META-INF/maven/com.google.guava/**' | ||
| exclude 'com/google/thirdparty/**' | ||
| relocate 'com.google.common', 'org.opensearch.ml.repackage.com.google.common' // dependency of cron-utils | ||
| } | ||
|
|
||
| jar { | ||
| enabled false | ||
| } | ||
|
|
||
| task sourcesJar(type: Jar) { | ||
| archiveClassifier.set 'sources' | ||
| from sourceSets.main.allJava | ||
| } | ||
|
|
||
| task javadocJar(type: Jar) { | ||
| archiveClassifier.set 'javadoc' | ||
| from javadoc.destinationDir | ||
| dependsOn javadoc | ||
| } | ||
|
|
||
| publishing { | ||
| repositories { | ||
| maven { | ||
| name = 'staging' | ||
| url = "${rootProject.buildDir}/local-staging-repo" | ||
| } | ||
| maven { | ||
| name = "Snapshots" // optional target repository name | ||
| url = "https://aws.oss.sonatype.org/content/repositories/snapshots" | ||
| credentials { | ||
| username "$System.env.SONATYPE_USERNAME" | ||
| password "$System.env.SONATYPE_PASSWORD" | ||
| } | ||
| } | ||
| } | ||
| publications { | ||
| shadow(MavenPublication) { publication -> | ||
| project.shadow.component(publication) | ||
| artifact sourcesJar | ||
| artifact javadocJar | ||
|
|
||
| pom { | ||
| name = "OpenSearch ML Commons Comm" | ||
| packaging = "jar" | ||
| url = "https://github.com/opensearch-project/ml-commons" | ||
| description = "OpenSearch ML Common" | ||
| scm { | ||
| connection = "scm:[email protected]:opensearch-project/ml-commons.git" | ||
| developerConnection = "scm:[email protected]:opensearch-project/ml-commons.git" | ||
| url = "[email protected]:opensearch-project/ml-commons.git" | ||
| } | ||
| licenses { | ||
| license { | ||
| name = "The Apache License, Version 2.0" | ||
| url = "http://www.apache.org/licenses/LICENSE-2.0.txt" | ||
| } | ||
| } | ||
| developers { | ||
| developer { | ||
| name = "OpenSearch" | ||
| url = "https://github.com/opensearch-project/ml-commons" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| publishShadowPublicationToMavenLocal.mustRunAfter shadowJar | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
102 changes: 102 additions & 0 deletions
102
common/src/main/java/org/opensearch/ml/common/agent/LLMSpec.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.ml.common.agent; | ||
|
|
||
| import lombok.Builder; | ||
| import lombok.Getter; | ||
| import org.opensearch.core.common.io.stream.StreamInput; | ||
| import org.opensearch.core.common.io.stream.StreamOutput; | ||
| import org.opensearch.core.xcontent.ToXContentObject; | ||
| import org.opensearch.core.xcontent.XContentBuilder; | ||
| import org.opensearch.core.xcontent.XContentParser; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Map; | ||
|
|
||
| import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; | ||
| import static org.opensearch.ml.common.utils.StringUtils.getParameterMap; | ||
|
|
||
|
|
||
| @Getter | ||
| public class LLMSpec implements ToXContentObject { | ||
| public static final String MODEL_ID_FIELD = "model_id"; | ||
| public static final String PARAMETERS_FIELD = "parameters"; | ||
|
|
||
| private String modelId; | ||
| private Map<String, String> parameters; | ||
|
|
||
|
|
||
| @Builder(toBuilder = true) | ||
| public LLMSpec(String modelId, Map<String, String> parameters) { | ||
| if (modelId == null) { | ||
| throw new IllegalArgumentException("model id is null"); | ||
| } | ||
| this.modelId = modelId; | ||
| this.parameters = parameters; | ||
| } | ||
|
|
||
| public LLMSpec(StreamInput input) throws IOException{ | ||
| modelId = input.readString(); | ||
| if (input.readBoolean()) { | ||
| parameters = input.readMap(StreamInput::readString, StreamInput::readOptionalString); | ||
| } | ||
| } | ||
|
|
||
| public void writeTo(StreamOutput out) throws IOException { | ||
| out.writeString(modelId); | ||
| if (parameters != null && parameters.size() > 0) { | ||
| out.writeBoolean(true); | ||
| out.writeMap(parameters, StreamOutput::writeString, StreamOutput::writeOptionalString); | ||
| } else { | ||
| out.writeBoolean(false); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { | ||
| builder.startObject(); | ||
| if (modelId != null) { | ||
| builder.field(MODEL_ID_FIELD, modelId); | ||
| } | ||
| if (parameters != null && parameters.size() > 0) { | ||
| builder.field(PARAMETERS_FIELD, parameters); | ||
| } | ||
| builder.endObject(); | ||
| return builder; | ||
| } | ||
|
|
||
| public static LLMSpec parse(XContentParser parser) throws IOException { | ||
| String modelId = null; | ||
| Map<String, String> parameters = null; | ||
|
|
||
| ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser); | ||
| while (parser.nextToken() != XContentParser.Token.END_OBJECT) { | ||
| String fieldName = parser.currentName(); | ||
| parser.nextToken(); | ||
|
|
||
| switch (fieldName) { | ||
| case MODEL_ID_FIELD: | ||
| modelId = parser.text(); | ||
| break; | ||
| case PARAMETERS_FIELD: | ||
| parameters = getParameterMap(parser.map()); | ||
| break; | ||
| default: | ||
| parser.skipChildren(); | ||
| break; | ||
| } | ||
| } | ||
| return LLMSpec.builder() | ||
| .modelId(modelId) | ||
| .parameters(parameters) | ||
| .build(); | ||
| } | ||
|
|
||
| public static LLMSpec fromStream(StreamInput in) throws IOException { | ||
| LLMSpec toolSpec = new LLMSpec(in); | ||
| return toolSpec; | ||
| } | ||
| } |
Oops, something went wrong.