Skip to content

Commit

Permalink
Optimize SLM Policy Queries (#79341)
Browse files Browse the repository at this point in the history 
Same as #79321 but for SLM policies. Enhances RepositoryData accordingly
to enable the optimization.
  • Loading branch information
@original-brownbear
original-brownbear authored Oct 19, 2021
1 parent 3d35144 commit 12e2f55
Show file tree
Hide file tree
Showing 8 changed files with 180 additions and 74 deletions.
3 changes: 2 additions & 1 deletion ...nternalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,8 @@ public void testEnforcedCooldownPeriod() throws IOException {
SnapshotState.SUCCESS,
SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION.minimumCompatibilityVersion(),
0L, // -1 would refresh RepositoryData and find the real version
0L // -1 would refresh RepositoryData and find the real version
0L, // -1 would refresh RepositoryData and find the real version,
"" // null would refresh RepositoryData and find the real version
)));
final BytesReference serialized = BytesReference.bytes(modifiedRepositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(),
SnapshotsService.OLD_SNAPSHOT_FORMAT));
Expand Down
2 changes: 1 addition & 1 deletion .../internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ public void testHandlingMissingRootLevelSnapshotMetadata() throws Exception {
.collect(
Collectors.toMap(
SnapshotId::getUUID,
s -> new RepositoryData.SnapshotDetails(repositoryData.getSnapshotState(s), null, -1, -1)
s -> new RepositoryData.SnapshotDetails(repositoryData.getSnapshotState(s), null, -1, -1, null)
)
),
Collections.emptyMap(),
Expand Down
96 changes: 72 additions & 24 deletions ...ava/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -666,7 +666,7 @@ private static Predicate<SnapshotInfo> buildAfterPredicate(
}
}

private static Predicate<SnapshotInfo> filterBySLMPolicies(String[] slmPolicies) {
private static SnapshotPredicate filterBySLMPolicies(String[] slmPolicies) {
final List<String> includePatterns = new ArrayList<>();
final List<String> excludePatterns = new ArrayList<>();
boolean seenWildcard = false;
Expand All @@ -686,25 +686,47 @@ private static Predicate<SnapshotInfo> filterBySLMPolicies(String[] slmPolicies)
final String[] includes = includePatterns.toArray(Strings.EMPTY_ARRAY);
final String[] excludes = excludePatterns.toArray(Strings.EMPTY_ARRAY);
final boolean matchWithoutPolicy = matchNoPolicy;
return snapshotInfo -> {
final Map<String, Object> metadata = snapshotInfo.userMetadata();
final String policy;
if (metadata == null) {
policy = null;
} else {
final Object policyFound = metadata.get(SnapshotsService.POLICY_ID_METADATA_FIELD);
policy = policyFound instanceof String ? (String) policyFound : null;
}
if (policy == null) {
return matchWithoutPolicy;
return new SnapshotPredicate() {
@Override
public boolean matchesPreflight(SnapshotId snapshotId, RepositoryData repositoryData) {
final RepositoryData.SnapshotDetails details = repositoryData.getSnapshotDetails(snapshotId);
final String policy;
if (details == null || (details.getSlmPolicy() == null)) {
// no SLM policy recorded
return true;
} else {
final String policyFound = details.getSlmPolicy();
// empty string means that snapshot was not created by an SLM policy
policy = policyFound.isEmpty() ? null : policyFound;
}
return matchPolicy(includes, excludes, matchWithoutPolicy, policy);
}
if (Regex.simpleMatch(includes, policy) == false) {
return false;

@Override
public boolean matches(SnapshotInfo snapshotInfo) {
final Map<String, Object> metadata = snapshotInfo.userMetadata();
final String policy;
if (metadata == null) {
policy = null;
} else {
final Object policyFound = metadata.get(SnapshotsService.POLICY_ID_METADATA_FIELD);
policy = policyFound instanceof String ? (String) policyFound : null;
}
return matchPolicy(includes, excludes, matchWithoutPolicy, policy);
}
return excludes.length == 0 || Regex.simpleMatch(excludes, policy) == false;
};
}

private static boolean matchPolicy(String[] includes, String[] excludes, boolean matchWithoutPolicy, @Nullable String policy) {
if (policy == null) {
return matchWithoutPolicy;
}
if (Regex.simpleMatch(includes, policy) == false) {
return false;
}
return excludes.length == 0 || Regex.simpleMatch(excludes, policy) == false;
}

private static Predicate<SnapshotInfo> filterByLongOffset(ToLongFunction<SnapshotInfo> extractor, long after, SortOrder order) {
return order == SortOrder.ASC ? info -> after <= extractor.applyAsLong(info) : info -> after >= extractor.applyAsLong(info);
}
Expand Down Expand Up @@ -759,19 +781,23 @@ private static final class SnapshotPredicates {
Predicate<SnapshotInfo> snapshotPredicate = null;
final String[] slmPolicies = request.policies();
final String fromSortValue = request.fromSortValue();
BiPredicate<SnapshotId, RepositoryData> preflightPredicate = null;
if (slmPolicies.length > 0) {
snapshotPredicate = filterBySLMPolicies(slmPolicies);
final SnapshotPredicate predicate = filterBySLMPolicies(slmPolicies);
snapshotPredicate = predicate::matches;
preflightPredicate = predicate::matchesPreflight;
}
final GetSnapshotsRequest.SortBy sortBy = request.sort();
final SortOrder order = request.order();
if (fromSortValue == null) {
preflightPredicate = null;
this.preflightPredicate = preflightPredicate;
} else {
final Predicate<SnapshotInfo> fromSortValuePredicate;
final BiPredicate<SnapshotId, RepositoryData> preflightPred;
switch (sortBy) {
case START_TIME:
final long after = Long.parseLong(fromSortValue);
preflightPredicate = order == SortOrder.ASC ? (snapshotId, repositoryData) -> {
preflightPred = order == SortOrder.ASC ? (snapshotId, repositoryData) -> {
final long startTime = getStartTime(snapshotId, repositoryData);
return startTime == -1 || after <= startTime;
} : (snapshotId, repositoryData) -> {
Expand All @@ -781,14 +807,14 @@ private static final class SnapshotPredicates {
fromSortValuePredicate = filterByLongOffset(SnapshotInfo::startTime, after, order);
break;
case NAME:
preflightPredicate = order == SortOrder.ASC
preflightPred = order == SortOrder.ASC
? (snapshotId, repositoryData) -> fromSortValue.compareTo(snapshotId.getName()) <= 0
: (snapshotId, repositoryData) -> fromSortValue.compareTo(snapshotId.getName()) >= 0;
fromSortValuePredicate = null;
break;
case DURATION:
final long afterDuration = Long.parseLong(fromSortValue);
preflightPredicate = order == SortOrder.ASC ? (snapshotId, repositoryData) -> {
preflightPred = order == SortOrder.ASC ? (snapshotId, repositoryData) -> {
final long duration = getDuration(snapshotId, repositoryData);
return duration == -1 || afterDuration <= duration;
} : (snapshotId, repositoryData) -> {
Expand All @@ -799,22 +825,22 @@ private static final class SnapshotPredicates {
break;
case INDICES:
final int afterIndexCount = Integer.parseInt(fromSortValue);
preflightPredicate = order == SortOrder.ASC
preflightPred = order == SortOrder.ASC
? (snapshotId, repositoryData) -> afterIndexCount <= indexCount(snapshotId, repositoryData)
: (snapshotId, repositoryData) -> afterIndexCount >= indexCount(snapshotId, repositoryData);
fromSortValuePredicate = null;
break;
case REPOSITORY:
// already handled in #maybeFilterRepositories
preflightPredicate = null;
preflightPred = null;
fromSortValuePredicate = null;
break;
case SHARDS:
preflightPredicate = null;
preflightPred = null;
fromSortValuePredicate = filterByLongOffset(SnapshotInfo::totalShards, Integer.parseInt(fromSortValue), order);
break;
case FAILED_SHARDS:
preflightPredicate = null;
preflightPred = null;
fromSortValuePredicate = filterByLongOffset(SnapshotInfo::failedShards, Integer.parseInt(fromSortValue), order);
break;
default:
Expand All @@ -826,6 +852,15 @@ private static final class SnapshotPredicates {
} else if (fromSortValuePredicate != null) {
snapshotPredicate = fromSortValuePredicate.and(snapshotPredicate);
}
if (preflightPredicate == null) {
this.preflightPredicate = preflightPred;
} else {
if (preflightPred != null) {
this.preflightPredicate = preflightPredicate.and(preflightPred);
} else {
this.preflightPredicate = preflightPredicate;
}
}
}
this.snapshotPredicate = snapshotPredicate;
}
Expand All @@ -842,6 +877,19 @@ public BiPredicate<SnapshotId, RepositoryData> preflightPredicate() {

}

private interface SnapshotPredicate {

/**
* Checks if a snapshot matches the predicate by testing its {@link SnapshotId} for a given {@link RepositoryData}.
*/
boolean matchesPreflight(SnapshotId snapshotId, RepositoryData repositoryData);

/**
* Checks if a snapshot matches the predicate by testing its {@link SnapshotInfo}.
*/
boolean matches(SnapshotInfo snapshotInfo);
}

private static final class SnapshotsInRepo {

private final List<SnapshotInfo> snapshotInfos;
Expand Down
42 changes: 36 additions & 6 deletions server/src/main/java/org/elasticsearch/repositories/RepositoryData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,8 @@ public boolean hasMissingDetails(SnapshotId snapshotId) {
return snapshotDetails == null
|| snapshotDetails.getVersion() == null
|| snapshotDetails.getStartTimeMillis() == -1
|| snapshotDetails.getEndTimeMillis() == -1;
|| snapshotDetails.getEndTimeMillis() == -1
|| snapshotDetails.getSlmPolicy() == null;
}

/**
Expand Down Expand Up @@ -639,6 +640,7 @@ public Map<String, IndexId> resolveNewIndices(List<String> indicesToResolve, Map
private static final String MIN_VERSION = "min_version";
private static final String START_TIME_MILLIS = "start_time_millis";
private static final String END_TIME_MILLIS = "end_time_millis";
private static final String SLM_POLICY = "slm_policy";

/**
* Writes the snapshots metadata and the related indices metadata to x-content.
Expand Down Expand Up @@ -725,6 +727,9 @@ public XContentBuilder snapshotsToXContent(final XContentBuilder builder, final
if (snapshotDetails.getEndTimeMillis() != -1) {
builder.field(END_TIME_MILLIS, snapshotDetails.getEndTimeMillis());
}
if (snapshotDetails.getSlmPolicy() != null) {
builder.field(SLM_POLICY, snapshotDetails.getSlmPolicy());
}

builder.endObject();
}
Expand Down Expand Up @@ -891,6 +896,7 @@ private static void parseSnapshots(
Version version = null;
long startTimeMillis = -1;
long endTimeMillis = -1;
String slmPolicy = null;
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
String currentFieldName = parser.currentName();
parser.nextToken();
Expand Down Expand Up @@ -918,12 +924,14 @@ private static void parseSnapshots(
assert endTimeMillis == -1;
endTimeMillis = parser.longValue();
break;
case SLM_POLICY:
slmPolicy = stringDeduplicator.computeIfAbsent(parser.text(), Function.identity());
}
}
assert (startTimeMillis == -1) == (endTimeMillis == -1) : "unexpected: " + startTimeMillis + ", " + endTimeMillis + ", ";
final SnapshotId snapshotId = new SnapshotId(name, uuid);
if (state != null || version != null) {
snapshotsDetails.put(uuid, new SnapshotDetails(state, version, startTimeMillis, endTimeMillis));
snapshotsDetails.put(uuid, new SnapshotDetails(state, version, startTimeMillis, endTimeMillis, slmPolicy));
}
snapshots.put(uuid, snapshotId);
if (metaGenerations != null && metaGenerations.isEmpty() == false) {
Expand Down Expand Up @@ -1037,7 +1045,7 @@ private static String parseLegacySnapshotUUID(XContentParser parser) throws IOEx
*/
public static class SnapshotDetails {

public static SnapshotDetails EMPTY = new SnapshotDetails(null, null, -1, -1);
public static SnapshotDetails EMPTY = new SnapshotDetails(null, null, -1, -1, null);

@Nullable // TODO forbid nulls here, this only applies to very old repositories
private final SnapshotState snapshotState;
Expand All @@ -1051,11 +1059,23 @@ public static class SnapshotDetails {
// May be -1 if unknown, which happens if the snapshot was taken before 7.14 and hasn't been updated yet
private final long endTimeMillis;

public SnapshotDetails(@Nullable SnapshotState snapshotState, @Nullable Version version, long startTimeMillis, long endTimeMillis) {
// May be null if unknown, which happens if the snapshot was taken before 7.16 and hasn't been updated yet. Empty string indicates
// that this snapshot was not created by an SLM policy.
@Nullable
private final String slmPolicy;

public SnapshotDetails(
@Nullable SnapshotState snapshotState,
@Nullable Version version,
long startTimeMillis,
long endTimeMillis,
@Nullable String slmPolicy
) {
this.snapshotState = snapshotState;
this.version = version;
this.startTimeMillis = startTimeMillis;
this.endTimeMillis = endTimeMillis;
this.slmPolicy = slmPolicy;
}

@Nullable
Expand All @@ -1082,6 +1102,15 @@ public long getEndTimeMillis() {
return endTimeMillis;
}

/**
* @return the SLM policy that the snapshot was created by or an empty string if it was not created by an SLM policy or
* {@code null} if unknown.
*/
@Nullable
public String getSlmPolicy() {
return slmPolicy;
}

@Override
public boolean equals(Object o) {
if (this == o) return true;
Expand All @@ -1090,12 +1119,13 @@ public boolean equals(Object o) {
return startTimeMillis == that.startTimeMillis
&& endTimeMillis == that.endTimeMillis
&& snapshotState == that.snapshotState
&& Objects.equals(version, that.version);
&& Objects.equals(version, that.version)
&& Objects.equals(slmPolicy, that.slmPolicy);
}

@Override
public int hashCode() {
return Objects.hash(snapshotState, version, startTimeMillis, endTimeMillis);
return Objects.hash(snapshotState, version, startTimeMillis, endTimeMillis, slmPolicy);
}

}
Expand Down
Loading

0 comments on commit 12e2f55

Please sign in to comment.