Skip to content

Commit

Permalink
Merge branch 'opensearch-project:main' into gradle-check-retry
Browse files Browse the repository at this point in the history
  • Loading branch information
kotwanikunal authored Mar 29, 2022
2 parents 8353a75 + 8f4aec1 commit 811ee66
Show file tree
Hide file tree
Showing 40 changed files with 1,165 additions and 54 deletions.
2 changes: 1 addition & 1 deletion buildSrc/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ dependencies {
api 'de.thetaphi:forbiddenapis:3.2'
api 'com.avast.gradle:gradle-docker-compose-plugin:0.14.12'
api 'org.apache.maven:maven-model:3.6.2'
api 'com.networknt:json-schema-validator:1.0.67'
api 'com.networknt:json-schema-validator:1.0.68'
api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson')}"

testFixturesApi "junit:junit:${props.getProperty('junit')}"
Expand Down
3 changes: 3 additions & 0 deletions buildSrc/reaper/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@

apply plugin: 'java'

targetCompatibility = JavaVersion.VERSION_11
sourceCompatibility = JavaVersion.VERSION_11

jar {
archiveFileName = "${project.name}.jar"
manifest {
Expand Down
2 changes: 1 addition & 1 deletion buildSrc/src/testKit/thirdPartyAudit/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ repositories {
}

dependencies {
forbiddenApisCliJar 'de.thetaphi:forbiddenapis:3.2'
forbiddenApisCliJar 'de.thetaphi:forbiddenapis:3.3'
jdkJarHell 'org.opensearch:opensearch-core:current'
compileOnly "org.${project.properties.compileOnlyGroup}:${project.properties.compileOnlyVersion}"
implementation "org.${project.properties.compileGroup}:${project.properties.compileVersion}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ public class TaskInfo {
private TaskId parentTaskId;
private final Map<String, Object> status = new HashMap<>();
private final Map<String, String> headers = new HashMap<>();
private final Map<String, Object> resourceStats = new HashMap<>();

public TaskInfo(TaskId taskId) {
this.taskId = taskId;
Expand Down Expand Up @@ -150,6 +151,14 @@ public Map<String, Object> getStatus() {
return status;
}

void setResourceStats(Map<String, Object> resourceStats) {
this.resourceStats.putAll(resourceStats);
}

public Map<String, Object> getResourceStats() {
return resourceStats;
}

private void noOpParse(Object s) {}

public static final ObjectParser.NamedObjectParser<TaskInfo, Void> PARSER;
Expand All @@ -170,6 +179,7 @@ private void noOpParse(Object s) {}
parser.declareBoolean(TaskInfo::setCancelled, new ParseField("cancelled"));
parser.declareString(TaskInfo::setParentTaskId, new ParseField("parent_task_id"));
parser.declareObject(TaskInfo::setHeaders, (p, c) -> p.mapStrings(), new ParseField("headers"));
parser.declareObject(TaskInfo::setResourceStats, (p, c) -> p.map(), new ParseField("resource_stats"));
PARSER = (XContentParser p, Void v, String name) -> parser.parse(p, new TaskInfo(new TaskId(name)), null);
}

Expand All @@ -188,7 +198,8 @@ && isCancelled() == taskInfo.isCancelled()
&& Objects.equals(getDescription(), taskInfo.getDescription())
&& Objects.equals(getParentTaskId(), taskInfo.getParentTaskId())
&& Objects.equals(status, taskInfo.status)
&& Objects.equals(getHeaders(), taskInfo.getHeaders());
&& Objects.equals(getHeaders(), taskInfo.getHeaders())
&& Objects.equals(getResourceStats(), taskInfo.getResourceStats());
}

@Override
Expand All @@ -204,7 +215,8 @@ public int hashCode() {
isCancelled(),
getParentTaskId(),
status,
getHeaders()
getHeaders(),
getResourceStats()
);
}

Expand Down Expand Up @@ -236,6 +248,8 @@ public String toString() {
+ status
+ ", headers="
+ headers
+ ", resource_stats="
+ resourceStats
+ '}';
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,16 @@
import org.opensearch.common.xcontent.ToXContent;
import org.opensearch.common.xcontent.XContentBuilder;
import org.opensearch.tasks.RawTaskStatus;
import org.opensearch.tasks.TaskResourceStats;
import org.opensearch.tasks.TaskResourceUsage;
import org.opensearch.tasks.Task;
import org.opensearch.tasks.TaskId;
import org.opensearch.tasks.TaskInfo;
import org.opensearch.test.OpenSearchTestCase;

import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;

import static org.opensearch.test.AbstractXContentTestCase.xContentTester;
Expand All @@ -57,7 +60,7 @@ public void testFromXContent() throws IOException {
)
.assertEqualsConsumer(this::assertEqualInstances)
.assertToXContentEquivalence(true)
.randomFieldsExcludeFilter(field -> field.endsWith("headers") || field.endsWith("status"))
.randomFieldsExcludeFilter(field -> field.endsWith("headers") || field.endsWith("status") || field.contains("resource_stats"))
.test();
}

Expand Down Expand Up @@ -106,7 +109,8 @@ static TaskInfo randomTaskInfo() {
cancellable,
cancelled,
parentTaskId,
headers
headers,
randomResourceStats()
);
}

Expand All @@ -127,4 +131,14 @@ private static RawTaskStatus randomRawTaskStatus() {
throw new IllegalStateException(e);
}
}

private static TaskResourceStats randomResourceStats() {
return randomBoolean() ? null : new TaskResourceStats(new HashMap<String, TaskResourceUsage>() {
{
for (int i = 0; i < randomInt(5); i++) {
put(randomAlphaOfLength(5), new TaskResourceUsage(randomNonNegativeLong(), randomNonNegativeLong()));
}
}
});
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,8 @@ protected CancelTasksResponseTests.ByNodeCancelTasksResponse createServerTestIns
cancellable,
cancelled,
new TaskId("node1", randomLong()),
Collections.singletonMap("x-header-of", "some-value")
Collections.singletonMap("x-header-of", "some-value"),
null
)
);
}
Expand Down
32 changes: 32 additions & 0 deletions jenkins/jenkinsfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
pipeline {
agent {
docker {
label 'AL2-X64'
/* See
https://github.com/opensearch-project/opensearch-build/blob/main/docker/ci/dockerfiles/build.ubuntu18.opensearch.x64.dockerfile
for docker image
*/
image 'opensearchstaging/ci-runner:ci-runner-ubuntu1804-build-v1'
alwaysPull true
}
}

environment {
JAVA11_HOME="/opt/java/openjdk-11"
JAVA14_HOME="/opt/java/openjdk-14"
JAVA17_HOME="/opt/java/openjdk-17"
JAVA8_HOME="/opt/java/openjdk-8"
JAVA_HOME="/opt/java/openjdk-14"
}

stages {
stage('gradle-check') {
steps {
script {
sh 'echo gradle check'
sh './gradlew check --no-daemon --no-scan'
}
}
}
}
}
2 changes: 1 addition & 1 deletion modules/lang-painless/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ testClusters.all {
dependencies {
api 'org.antlr:antlr4-runtime:4.9.3'
api 'org.ow2.asm:asm-util:9.2'
api 'org.ow2.asm:asm-tree:7.2'
api 'org.ow2.asm:asm-tree:9.2'
api 'org.ow2.asm:asm-commons:9.2'
api 'org.ow2.asm:asm-analysis:7.2'
api 'org.ow2.asm:asm:9.2'
Expand Down
1 change: 0 additions & 1 deletion modules/lang-painless/licenses/asm-tree-7.2.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions modules/lang-painless/licenses/asm-tree-9.2.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
d96c99a30f5e1a19b0e609dbb19a44d8518ac01e
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,8 @@ public void testRethrottleSuccessfulResponse() {
true,
false,
new TaskId("test", task.getId()),
Collections.emptyMap()
Collections.emptyMap(),
null
)
);
sliceStatuses.add(new BulkByScrollTask.StatusOrException(status));
Expand Down Expand Up @@ -167,7 +168,8 @@ public void testRethrottleWithSomeSucceeded() {
true,
false,
new TaskId("test", task.getId()),
Collections.emptyMap()
Collections.emptyMap(),
null
)
);
sliceStatuses.add(new BulkByScrollTask.StatusOrException(status));
Expand Down
2 changes: 1 addition & 1 deletion plugins/discovery-azure-classic/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ dependencies {
api "com.sun.jersey:jersey-client:${versions.jersey}"
api "com.sun.jersey:jersey-core:${versions.jersey}"
api "com.sun.jersey:jersey-json:${versions.jersey}"
api 'org.codehaus.jettison:jettison:1.1'
api 'org.codehaus.jettison:jettison:1.4.1'
api 'com.sun.xml.bind:jaxb-impl:2.2.3-1'

// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
8d16bbcbac93446942c9e5da04530159afbe3e65
2 changes: 1 addition & 1 deletion plugins/repository-gcs/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ dependencies {
api 'com.google.cloud:google-cloud-core-http:1.93.3'
api 'com.google.auth:google-auth-library-credentials:0.20.0'
api 'com.google.auth:google-auth-library-oauth2-http:0.20.0'
api 'com.google.oauth-client:google-oauth-client:1.31.0'
api 'com.google.oauth-client:google-oauth-client:1.33.1'
api 'com.google.api-client:google-api-client:1.30.10'
api 'com.google.http-client:google-http-client-appengine:1.35.0'
api 'com.google.http-client:google-http-client-jackson2:1.35.0'
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
0a431f1a677c5f89507591ab47a7ccdb0b18b6f7
2 changes: 1 addition & 1 deletion plugins/repository-hdfs/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ dependencies {
api "org.apache.hadoop:hadoop-client-api:${versions.hadoop3}"
runtimeOnly "org.apache.hadoop:hadoop-client-runtime:${versions.hadoop3}"
api "org.apache.hadoop:hadoop-hdfs:${versions.hadoop3}"
api 'org.apache.htrace:htrace-core4:4.1.0-incubating'
api 'org.apache.htrace:htrace-core4:4.2.0-incubating'
api "org.apache.logging.log4j:log4j-core:${versions.log4j}"
api 'org.apache.avro:avro:1.10.2'
api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
94b3f1966922bc45d0f8a86a2aa867a4b0df288b
Original file line number Diff line number Diff line change
Expand Up @@ -907,7 +907,8 @@ public void testNodeNotFoundButTaskFound() throws Exception {
false,
false,
TaskId.EMPTY_TASK_ID,
Collections.emptyMap()
Collections.emptyMap(),
null
),
new RuntimeException("test")
),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,8 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
IndexSettings.FINAL_PIPELINE,
MetadataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING,
ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_SETTING,
IndexSettings.INDEX_MERGE_ON_FLUSH_ENABLED,
IndexSettings.INDEX_MERGE_ON_FLUSH_MAX_FULL_FLUSH_MERGE_WAIT_TIME,

// validate that built-in similarities don't get redefined
Setting.groupSetting("index.similarity.", (s) -> {
Expand Down
50 changes: 50 additions & 0 deletions server/src/main/java/org/opensearch/index/IndexSettings.java
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,27 @@ public final class IndexSettings {
Setting.Property.IndexScope
);

/**
* Expert: sets the amount of time to wait for merges (during {@link org.apache.lucene.index.IndexWriter#commit}
* or {@link org.apache.lucene.index.IndexWriter#getReader(boolean, boolean)}) returned by MergePolicy.findFullFlushMerges(...).
* If this time is reached, we proceed with the commit based on segments merged up to that point. The merges are not
* aborted, and will still run to completion independent of the commit or getReader call, like natural segment merges.
*/
public static final Setting<TimeValue> INDEX_MERGE_ON_FLUSH_MAX_FULL_FLUSH_MERGE_WAIT_TIME = Setting.timeSetting(
"index.merge_on_flush.max_full_flush_merge_wait_time",
new TimeValue(10, TimeUnit.SECONDS),
new TimeValue(0, TimeUnit.MILLISECONDS),
Property.Dynamic,
Property.IndexScope
);

public static final Setting<Boolean> INDEX_MERGE_ON_FLUSH_ENABLED = Setting.boolSetting(
"index.merge_on_flush.enabled",
false,
Property.IndexScope,
Property.Dynamic
);

private final Index index;
private final Version version;
private final Logger logger;
Expand Down Expand Up @@ -584,6 +605,15 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
*/
private volatile int maxRegexLength;

/**
* The max amount of time to wait for merges
*/
private volatile TimeValue maxFullFlushMergeWaitTime;
/**
* Is merge of flush enabled or not
*/
private volatile boolean mergeOnFlushEnabled;

/**
* Returns the default search fields for this index.
*/
Expand Down Expand Up @@ -696,6 +726,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
mappingTotalFieldsLimit = scopedSettings.get(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING);
mappingDepthLimit = scopedSettings.get(INDEX_MAPPING_DEPTH_LIMIT_SETTING);
mappingFieldNameLengthLimit = scopedSettings.get(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING);
maxFullFlushMergeWaitTime = scopedSettings.get(INDEX_MERGE_ON_FLUSH_MAX_FULL_FLUSH_MERGE_WAIT_TIME);
mergeOnFlushEnabled = scopedSettings.get(INDEX_MERGE_ON_FLUSH_ENABLED);

scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING, mergePolicyConfig::setNoCFSRatio);
scopedSettings.addSettingsUpdateConsumer(
Expand Down Expand Up @@ -765,6 +797,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, this::setMappingTotalFieldsLimit);
scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit);
scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit);
scopedSettings.addSettingsUpdateConsumer(INDEX_MERGE_ON_FLUSH_MAX_FULL_FLUSH_MERGE_WAIT_TIME, this::setMaxFullFlushMergeWaitTime);
scopedSettings.addSettingsUpdateConsumer(INDEX_MERGE_ON_FLUSH_ENABLED, this::setMergeOnFlushEnabled);
}

private void setSearchIdleAfter(TimeValue searchIdleAfter) {
Expand Down Expand Up @@ -1328,4 +1362,20 @@ public long getMappingFieldNameLengthLimit() {
private void setMappingFieldNameLengthLimit(long value) {
this.mappingFieldNameLengthLimit = value;
}

private void setMaxFullFlushMergeWaitTime(TimeValue timeValue) {
this.maxFullFlushMergeWaitTime = timeValue;
}

private void setMergeOnFlushEnabled(boolean enabled) {
this.mergeOnFlushEnabled = enabled;
}

public TimeValue getMaxFullFlushMergeWaitTime() {
return this.maxFullFlushMergeWaitTime;
}

public boolean isMergeOnFlushEnabled() {
return mergeOnFlushEnabled;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import org.apache.lucene.index.ShuffleForcedMergePolicy;
import org.apache.lucene.index.SoftDeletesRetentionMergePolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.sandbox.index.MergeOnFlushMergePolicy;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DocIdSetIterator;
Expand Down Expand Up @@ -2425,6 +2426,21 @@ private IndexWriterConfig getIndexWriterConfig() {
// to enable it.
mergePolicy = new ShuffleForcedMergePolicy(mergePolicy);
}

if (config().getIndexSettings().isMergeOnFlushEnabled()) {
final long maxFullFlushMergeWaitMillis = config().getIndexSettings().getMaxFullFlushMergeWaitTime().millis();
if (maxFullFlushMergeWaitMillis > 0) {
iwc.setMaxFullFlushMergeWaitMillis(maxFullFlushMergeWaitMillis);
mergePolicy = new MergeOnFlushMergePolicy(mergePolicy);
} else {
logger.warn(
"The {} is enabled but {} is set to 0, merge on flush will not be activated",
IndexSettings.INDEX_MERGE_ON_FLUSH_ENABLED.getKey(),
IndexSettings.INDEX_MERGE_ON_FLUSH_MAX_FULL_FLUSH_MERGE_WAIT_TIME.getKey()
);
}
}

iwc.setMergePolicy(new OpenSearchMergePolicy(mergePolicy));
iwc.setSimilarity(engineConfig.getSimilarity());
iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac());
Expand Down
Loading

0 comments on commit 811ee66

Please sign in to comment.