Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

semantic_text: Add index metadata information for inference field mappers #107147

Merged
merged 8 commits into from
Apr 10, 2024
Merged

semantic_text: Add index metadata information for inference field mappers #107147

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
17f1fde
semantic_text: Add cluster metadata information for inference field m…
carlosdelest Apr 5, 2024
4025d2c
Add javadoc
carlosdelest Apr 5, 2024
d78acc3
Fix test helper
carlosdelest Apr 5, 2024
7a2b70b
PR Review comments
carlosdelest Apr 5, 2024
dc46e88
Add test coverage for null constructor args
carlosdelest Apr 8, 2024
531d1b1
Merge branch 'main' into carlosdelest/semantic-text-index-metadata-ch…
carlosdelest Apr 9, 2024
b5871c8
PR review comments
carlosdelest Apr 10, 2024
7c9ab2c
Merge branch 'refs/heads/main' into carlosdelest/semantic-text-index-…
carlosdelest Apr 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion server/src/internalClusterTest/java/org/elasticsearch/cluster/ClusterStateDiffIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
import static java.util.Collections.emptyList;
import static java.util.Collections.emptySet;
import static org.elasticsearch.cluster.metadata.AliasMetadata.newAliasMetadataBuilder;
import static org.elasticsearch.cluster.metadata.IndexMetadataTests.randomInferenceFields;
import static org.elasticsearch.cluster.routing.RandomShardRoutingMutator.randomChange;
import static org.elasticsearch.cluster.routing.TestShardRouting.shardRoutingBuilder;
import static org.elasticsearch.cluster.routing.UnassignedInfoTests.randomUnassignedInfo;
Expand Down Expand Up @@ -571,7 +572,7 @@ public IndexMetadata randomCreate(String name) {
@Override
public IndexMetadata randomChange(IndexMetadata part) {
IndexMetadata.Builder builder = IndexMetadata.builder(part);
switch (randomIntBetween(0, 2)) {
switch (randomIntBetween(0, 3)) {
case 0:
builder.settings(Settings.builder().put(part.getSettings()).put(randomSettings(Settings.EMPTY)));
break;
Expand All @@ -585,6 +586,9 @@ public IndexMetadata randomChange(IndexMetadata part) {
case 2:
builder.settings(Settings.builder().put(part.getSettings()).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0));
break;
case 3:
builder.putInferenceFields(randomInferenceFields());
break;
default:
throw new IllegalArgumentException("Shouldn't be here");
}
Expand Down
1 change: 1 addition & 0 deletions server/src/main/java/org/elasticsearch/TransportVersions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@ static TransportVersion def(int id) {
public static final TransportVersion INDEXING_PRESSURE_DOCUMENT_REJECTIONS_COUNT = def(8_625_00_0);
public static final TransportVersion ALIAS_ACTION_RESULTS = def(8_626_00_0);
public static final TransportVersion HISTOGRAM_AGGS_KEY_SORTED = def(8_627_00_0);
public static final TransportVersion INFERENCE_FIELDS_METADATA = def(8_628_00_0);

/*
* STOP! READ THIS FIRST! No, really,
Expand Down
70 changes: 70 additions & 0 deletions server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -540,6 +540,8 @@ public Iterator<Setting<?>> settings() {

public static final String KEY_SHARD_SIZE_FORECAST = "shard_size_forecast";

public static final String KEY_INFERENCE_FIELDS = "field_inference";

public static final String INDEX_STATE_FILE_PREFIX = "state-";

static final TransportVersion SYSTEM_INDEX_FLAG_ADDED = TransportVersions.V_7_10_0;
Expand Down Expand Up @@ -574,6 +576,8 @@ public Iterator<Setting<?>> settings() {
@Nullable
private final MappingMetadata mapping;

private final ImmutableOpenMap<String, InferenceFieldMetadata> inferenceFields;

private final ImmutableOpenMap<String, DiffableStringMap> customData;

private final Map<Integer, Set<String>> inSyncAllocationIds;
Expand Down Expand Up @@ -642,6 +646,7 @@ private IndexMetadata(
final int numberOfReplicas,
final Settings settings,
final MappingMetadata mapping,
final ImmutableOpenMap<String, InferenceFieldMetadata> inferenceFields,
final ImmutableOpenMap<String, AliasMetadata> aliases,
final ImmutableOpenMap<String, DiffableStringMap> customData,
final Map<Integer, Set<String>> inSyncAllocationIds,
Expand Down Expand Up @@ -692,6 +697,7 @@ private IndexMetadata(
this.totalNumberOfShards = numberOfShards * (numberOfReplicas + 1);
this.settings = settings;
this.mapping = mapping;
this.inferenceFields = inferenceFields;
this.customData = customData;
this.aliases = aliases;
this.inSyncAllocationIds = inSyncAllocationIds;
Expand Down Expand Up @@ -748,6 +754,7 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) {
this.numberOfReplicas,
this.settings,
mapping,
this.inferenceFields,
this.aliases,
this.customData,
this.inSyncAllocationIds,
Expand Down Expand Up @@ -806,6 +813,7 @@ public IndexMetadata withInSyncAllocationIds(int shardId, Set<String> inSyncSet)
this.numberOfReplicas,
this.settings,
this.mapping,
this.inferenceFields,
this.aliases,
this.customData,
Maps.copyMapWithAddedOrReplacedEntry(this.inSyncAllocationIds, shardId, Set.copyOf(inSyncSet)),
Expand Down Expand Up @@ -862,6 +870,7 @@ public IndexMetadata withIncrementedPrimaryTerm(int shardId) {
this.numberOfReplicas,
this.settings,
this.mapping,
this.inferenceFields,
this.aliases,
this.customData,
this.inSyncAllocationIds,
Expand Down Expand Up @@ -918,6 +927,7 @@ public IndexMetadata withTimestampRange(IndexLongFieldRange timestampRange) {
this.numberOfReplicas,
this.settings,
this.mapping,
this.inferenceFields,
this.aliases,
this.customData,
this.inSyncAllocationIds,
Expand Down Expand Up @@ -970,6 +980,7 @@ public IndexMetadata withIncrementedVersion() {
this.numberOfReplicas,
this.settings,
this.mapping,
this.inferenceFields,
this.aliases,
this.customData,
this.inSyncAllocationIds,
Expand Down Expand Up @@ -1193,6 +1204,10 @@ public MappingMetadata mapping() {
return mapping;
}

public Map<String, InferenceFieldMetadata> getInferenceFields() {
return inferenceFields;
}

@Nullable
public IndexMetadataStats getStats() {
return stats;
Expand Down Expand Up @@ -1403,6 +1418,9 @@ public boolean equals(Object o) {
if (rolloverInfos.equals(that.rolloverInfos) == false) {
return false;
}
if (inferenceFields.equals(that.inferenceFields) == false) {
return false;
}
if (isSystem != that.isSystem) {
return false;
}
Expand All @@ -1423,6 +1441,7 @@ public int hashCode() {
result = 31 * result + Arrays.hashCode(primaryTerms);
result = 31 * result + inSyncAllocationIds.hashCode();
result = 31 * result + rolloverInfos.hashCode();
result = 31 * result + inferenceFields.hashCode();
result = 31 * result + Boolean.hashCode(isSystem);
return result;
}
Expand Down Expand Up @@ -1469,6 +1488,7 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
@Nullable
private final Diff<Settings> settingsDiff;
private final Diff<ImmutableOpenMap<String, MappingMetadata>> mappings;
private final Diff<ImmutableOpenMap<String, InferenceFieldMetadata>> inferenceFields;
private final Diff<ImmutableOpenMap<String, AliasMetadata>> aliases;
private final Diff<ImmutableOpenMap<String, DiffableStringMap>> customData;
private final Diff<Map<Integer, Set<String>>> inSyncAllocationIds;
Expand Down Expand Up @@ -1500,6 +1520,7 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
: ImmutableOpenMap.<String, MappingMetadata>builder(1).fPut(MapperService.SINGLE_MAPPING_NAME, after.mapping).build(),
DiffableUtils.getStringKeySerializer()
);
inferenceFields = DiffableUtils.diff(before.inferenceFields, after.inferenceFields, DiffableUtils.getStringKeySerializer());
aliases = DiffableUtils.diff(before.aliases, after.aliases, DiffableUtils.getStringKeySerializer());
customData = DiffableUtils.diff(before.customData, after.customData, DiffableUtils.getStringKeySerializer());
inSyncAllocationIds = DiffableUtils.diff(
Expand All @@ -1524,6 +1545,8 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
new DiffableUtils.DiffableValueReader<>(DiffableStringMap::readFrom, DiffableStringMap::readDiffFrom);
private static final DiffableUtils.DiffableValueReader<String, RolloverInfo> ROLLOVER_INFO_DIFF_VALUE_READER =
new DiffableUtils.DiffableValueReader<>(RolloverInfo::new, RolloverInfo::readDiffFrom);
private static final DiffableUtils.DiffableValueReader<String, InferenceFieldMetadata> INFERENCE_FIELDS_METADATA_DIFF_VALUE_READER =
new DiffableUtils.DiffableValueReader<>(InferenceFieldMetadata::new, InferenceFieldMetadata::readDiffFrom);

IndexMetadataDiff(StreamInput in) throws IOException {
index = in.readString();
Expand All @@ -1546,6 +1569,15 @@ private static class IndexMetadataDiff implements Diff<IndexMetadata> {
}
primaryTerms = in.readVLongArray();
mappings = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), MAPPING_DIFF_VALUE_READER);
if (in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_FIELDS_METADATA)) {
inferenceFields = DiffableUtils.readImmutableOpenMapDiff(
in,
DiffableUtils.getStringKeySerializer(),
INFERENCE_FIELDS_METADATA_DIFF_VALUE_READER
);
} else {
inferenceFields = DiffableUtils.emptyDiff();
}
aliases = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), ALIAS_METADATA_DIFF_VALUE_READER);
customData = DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), CUSTOM_DIFF_VALUE_READER);
inSyncAllocationIds = DiffableUtils.readJdkMapDiff(
Expand Down Expand Up @@ -1595,6 +1627,9 @@ public void writeTo(StreamOutput out) throws IOException {
}
out.writeVLongArray(primaryTerms);
mappings.writeTo(out);
if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_FIELDS_METADATA)) {
inferenceFields.writeTo(out);
}
aliases.writeTo(out);
customData.writeTo(out);
inSyncAllocationIds.writeTo(out);
Expand Down Expand Up @@ -1628,6 +1663,7 @@ public IndexMetadata apply(IndexMetadata part) {
builder.mapping = mappings.apply(
ImmutableOpenMap.<String, MappingMetadata>builder(1).fPut(MapperService.SINGLE_MAPPING_NAME, part.mapping).build()
).get(MapperService.SINGLE_MAPPING_NAME);
builder.inferenceFields.putAllFromMap(inferenceFields.apply(part.inferenceFields));
builder.aliases.putAllFromMap(aliases.apply(part.aliases));
builder.customMetadata.putAllFromMap(customData.apply(part.customData));
builder.inSyncAllocationIds.putAll(inSyncAllocationIds.apply(part.inSyncAllocationIds));
Expand Down Expand Up @@ -1673,6 +1709,10 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function<String,
builder.putMapping(new MappingMetadata(in));
}
}
if (in.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_FIELDS_METADATA)) {
var fields = in.readCollectionAsImmutableList(InferenceFieldMetadata::new);
fields.stream().forEach(f -> builder.putInferenceField(f));
}
int aliasesSize = in.readVInt();
for (int i = 0; i < aliasesSize; i++) {
AliasMetadata aliasMd = new AliasMetadata(in);
Expand Down Expand Up @@ -1733,6 +1773,9 @@ public void writeTo(StreamOutput out, boolean mappingsAsHash) throws IOException
mapping.writeTo(out);
}
}
if (out.getTransportVersion().onOrAfter(TransportVersions.INFERENCE_FIELDS_METADATA)) {
out.writeCollection(inferenceFields.values());
}
out.writeCollection(aliases.values());
out.writeMap(customData, StreamOutput::writeWriteable);
out.writeMap(
Expand Down Expand Up @@ -1788,6 +1831,7 @@ public static class Builder {
private long[] primaryTerms = null;
private Settings settings = Settings.EMPTY;
private MappingMetadata mapping;
private final ImmutableOpenMap.Builder<String, InferenceFieldMetadata> inferenceFields;
private final ImmutableOpenMap.Builder<String, AliasMetadata> aliases;
private final ImmutableOpenMap.Builder<String, DiffableStringMap> customMetadata;
private final Map<Integer, Set<String>> inSyncAllocationIds;
Expand All @@ -1802,6 +1846,7 @@ public static class Builder {

public Builder(String index) {
this.index = index;
this.inferenceFields = ImmutableOpenMap.builder();
this.aliases = ImmutableOpenMap.builder();
this.customMetadata = ImmutableOpenMap.builder();
this.inSyncAllocationIds = new HashMap<>();
Expand All @@ -1819,6 +1864,7 @@ public Builder(IndexMetadata indexMetadata) {
this.settings = indexMetadata.getSettings();
this.primaryTerms = indexMetadata.primaryTerms.clone();
this.mapping = indexMetadata.mapping;
this.inferenceFields = ImmutableOpenMap.builder(indexMetadata.inferenceFields);
this.aliases = ImmutableOpenMap.builder(indexMetadata.aliases);
this.customMetadata = ImmutableOpenMap.builder(indexMetadata.customData);
this.routingNumShards = indexMetadata.routingNumShards;
Expand Down Expand Up @@ -2059,6 +2105,16 @@ public Builder shardSizeInBytesForecast(Long shardSizeInBytesForecast) {
return this;
}

public Builder putInferenceField(InferenceFieldMetadata value) {
this.inferenceFields.put(value.getName(), value);
return this;
}

public Builder putInferenceFields(Map<String, InferenceFieldMetadata> values) {
this.inferenceFields.putAllFromMap(values);
return this;
}

public IndexMetadata build() {
return build(false);
}
Expand Down Expand Up @@ -2221,6 +2277,7 @@ IndexMetadata build(boolean repair) {
numberOfReplicas,
settings,
mapping,
inferenceFields.build(),
aliasesMap,
newCustomMetadata,
Map.ofEntries(denseInSyncAllocationIds),
Expand Down Expand Up @@ -2379,6 +2436,14 @@ public static void toXContent(IndexMetadata indexMetadata, XContentBuilder build
builder.field(KEY_SHARD_SIZE_FORECAST, indexMetadata.shardSizeInBytesForecast);
}

if (indexMetadata.getInferenceFields().isEmpty() == false) {
builder.startObject(KEY_INFERENCE_FIELDS);
for (InferenceFieldMetadata field : indexMetadata.getInferenceFields().values()) {
field.toXContent(builder, params);
}
builder.endObject();
}

builder.endObject();
}

Expand Down Expand Up @@ -2456,6 +2521,11 @@ public static IndexMetadata fromXContent(XContentParser parser, Map<String, Mapp
case KEY_STATS:
builder.stats(IndexMetadataStats.fromXContent(parser));
break;
case KEY_INFERENCE_FIELDS:
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
builder.putInferenceField(InferenceFieldMetadata.fromXContent(parser));
}
break;
default:
// assume it's custom index metadata
builder.putCustom(currentFieldName, parser.mapStrings());
Expand Down
Loading