Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: query by within + Text.contains #1803

Merged
merged 7 commits into from
Apr 19, 2022
Merged

fix: query by within + Text.contains #1803

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2e0deac
fix query by within + Text.contains
javeme Mar 29, 2022
7926f6b
add test
javeme Mar 29, 2022
76a7322
improve resultfilter
javeme Mar 31, 2022
e6029e2
improve test
javeme Mar 31, 2022
6a6d99c
fix warnings
javeme Mar 31, 2022
f293145
improve test: add mayCommitTx()
javeme Mar 31, 2022
3e1e0ac
check limit query with uncommitted insert
javeme Apr 1, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions hugegraph-core/src/main/java/com/baidu/hugegraph/backend/page/QueryList.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,7 @@ public QueryResults<R> iterator() {
private QueryResults<R> each(IdHolder holder) {
assert !holder.paging();
Query bindQuery = holder.query();
this.updateResultsFilter(bindQuery);
this.updateOffsetIfNeeded(bindQuery);

// Iterate by all
Expand Down Expand Up @@ -267,6 +268,8 @@ public PageResults<R> iterator(int index, String page, long pageSize) {
E.checkArgument(0 <= index && index <= this.holders.size(),
"Invalid page index %s", index);
IdHolder holder = this.holders.get(index);
Query bindQuery = holder.query();
this.updateResultsFilter(bindQuery);
PageIds pageIds = holder.fetchNext(page, pageSize);
if (pageIds.empty()) {
return PageResults.emptyIterator();
Expand Down Expand Up @@ -298,6 +301,16 @@ private void updateOffsetIfNeeded(Query query) {
query.copyOffset(parent);
}

private void updateResultsFilter(Query query) {
while (query != null) {
if (query instanceof ConditionQuery) {
((ConditionQuery) query).updateResultsFilter();
return;
}
query = query.originQuery();
}
}

private QueryResults<R> queryByIndexIds(Set<Id> ids) {
return this.queryByIndexIds(ids, false);
}
Expand Down
1 change: 1 addition & 0 deletions hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/Condition.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ public enum RelationType implements BiPredicate<Object, Object> {
((Id) v1).asBytes());
}),
TEXT_CONTAINS("textcontains", String.class, String.class, (v1, v2) -> {
// TODO: support collection-property textcontains
return v1 != null && ((String) v1).contains((String) v2);
}),
TEXT_CONTAINS_ANY("textcontainsany", String.class, Collection.class,
Expand Down
128 changes: 80 additions & 48 deletions hugegraph-core/src/main/java/com/baidu/hugegraph/backend/query/ConditionQuery.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@
import com.baidu.hugegraph.util.InsertionOrderUtil;
import com.baidu.hugegraph.util.LongEncoding;
import com.baidu.hugegraph.util.NumericUtil;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
Expand Down Expand Up @@ -79,7 +78,7 @@ public class ConditionQuery extends IdQuery {
private List<Condition> conditions = EMPTY_CONDITIONS;

private OptimizedType optimizedType = OptimizedType.NONE;
private Function<HugeElement, Boolean> resultsFilter = null;
private ResultsFilter resultsFilter = null;
private Element2IndexValueMap element2IndexValueMap = null;

public ConditionQuery(HugeType resultType) {
Expand Down Expand Up @@ -173,20 +172,11 @@ public void resetConditions() {
}

public void recordIndexValue(Id propertyId, Id id, Object indexValue) {
this.ensureElement2IndexValueMap();
this.element2IndexValueMap.addIndexValue(propertyId, id, indexValue);
this.element2IndexValueMap().addIndexValue(propertyId, id, indexValue);
}

public void selectedIndexField(Id indexField) {
this.ensureElement2IndexValueMap();
this.element2IndexValueMap.selectedIndexField(indexField);
}

public Set<LeftIndex> getElementLeftIndex(Id elementId) {
if (this.element2IndexValueMap == null) {
return null;
}
return this.element2IndexValueMap.getLeftIndex(elementId);
this.element2IndexValueMap().selectedIndexField(indexField);
}

public void removeElementLeftIndex(Id elementId) {
Expand All @@ -197,7 +187,21 @@ public void removeElementLeftIndex(Id elementId) {
}

public boolean existLeftIndex(Id elementId) {
return this.getElementLeftIndex(elementId) != null;
return this.getLeftIndexOfElement(elementId) != null;
}

public Set<LeftIndex> getLeftIndexOfElement(Id elementId) {
if (this.element2IndexValueMap == null) {
return null;
}
return this.element2IndexValueMap.getLeftIndex(elementId);
}

private Element2IndexValueMap element2IndexValueMap() {
if (this.element2IndexValueMap == null) {
this.element2IndexValueMap = new Element2IndexValueMap();
}
return this.element2IndexValueMap;
}

public List<Condition.Relation> relations() {
Expand Down Expand Up @@ -548,14 +552,26 @@ public boolean test(HugeElement element) {
return false;
}

if (this.resultsFilter != null) {
return this.resultsFilter.apply(element);
/*
* Currently results-filter is used to filter unmatched results returned
* by search index, and there may be multiple results-filter for every
* sub-query like within() + Text.contains().
* We can't use sub-query results-filter here for fresh element which is
* not committed to backend store, because it's not from a sub-query.
*/
if (this.resultsFilter != null && !element.fresh()) {
return this.resultsFilter.test(element);
}

/*
* NOTE: seems need to keep call checkRangeIndex() for each condition,
* so don't break early even if test() return false.
*/
boolean valid = true;
for (Condition cond : this.conditions) {
valid &= cond.test(element);
valid &= (this.element2IndexValueMap == null ||
this.element2IndexValueMap.validRangeIndex(element, cond));
valid &= this.element2IndexValueMap == null ||
this.element2IndexValueMap.checkRangeIndex(element, cond);
}
return valid;
}
Expand Down Expand Up @@ -615,14 +631,29 @@ public OptimizedType optimized() {
return this.optimizedType;
}

public void registerResultsFilter(Function<HugeElement, Boolean> filter) {
public void registerResultsFilter(ResultsFilter filter) {
assert this.resultsFilter == null;
this.resultsFilter = filter;
}

public void updateResultsFilter() {
Query originQuery = this.originQuery();
if (originQuery instanceof ConditionQuery) {
ConditionQuery cq = ((ConditionQuery) originQuery);
cq.registerResultsFilter(filter);
ConditionQuery originCQ = ((ConditionQuery) originQuery);
if (this.resultsFilter != null) {
originCQ.updateResultsFilter(this.resultsFilter);
} else {
originCQ.updateResultsFilter();
}
}
}

protected void updateResultsFilter(ResultsFilter filter) {
this.resultsFilter = filter;
Query originQuery = this.originQuery();
if (originQuery instanceof ConditionQuery) {
ConditionQuery originCQ = ((ConditionQuery) originQuery);
originCQ.updateResultsFilter(filter);
}
}

Expand All @@ -638,12 +669,6 @@ public ConditionQuery originConditionQuery() {
return (ConditionQuery) originQuery;
}

private void ensureElement2IndexValueMap() {
if (this.element2IndexValueMap == null) {
this.element2IndexValueMap = new Element2IndexValueMap();
}
}

public static String concatValues(List<?> values) {
assert !values.isEmpty();
List<Object> newValues = new ArrayList<>(values.size());
Expand Down Expand Up @@ -717,7 +742,7 @@ public Element2IndexValueMap() {
public void addIndexValue(Id indexField, Id elementId,
Object indexValue) {
if (!this.filed2IndexValues.containsKey(indexField)) {
this.filed2IndexValues.put(indexField, new HashMap<>());
this.filed2IndexValues.putIfAbsent(indexField, new HashMap<>());
}
Map<Id, Set<Object>> element2IndexValueMap =
this.filed2IndexValues.get(indexField);
Expand All @@ -733,15 +758,15 @@ public void selectedIndexField(Id indexField) {
this.selectedIndexField = indexField;
}

public Set<Object> removeIndexValues(Id indexField, Id elementId) {
public Set<Object> toRemoveIndexValues(Id indexField, Id elementId) {
if (!this.filed2IndexValues.containsKey(indexField)) {
return null;
}
return this.filed2IndexValues.get(indexField).get(elementId);
}

public void addLeftIndex(Id indexField, Set<Object> indexValues,
Id elementId) {
public void addLeftIndex(Id elementId, Id indexField,
Set<Object> indexValues) {
LeftIndex leftIndex = new LeftIndex(indexValues, indexField);
if (this.leftIndexMap.containsKey(elementId)) {
this.leftIndexMap.get(elementId).add(leftIndex);
Expand All @@ -758,7 +783,7 @@ public void removeElementLeftIndex(Id elementId) {
this.leftIndexMap.remove(elementId);
}

public boolean validRangeIndex(HugeElement element, Condition cond) {
public boolean checkRangeIndex(HugeElement element, Condition cond) {
// Not UserpropRelation
if (!(cond instanceof Condition.UserpropRelation)) {
return true;
Expand All @@ -767,35 +792,36 @@ public boolean validRangeIndex(HugeElement element, Condition cond) {
Condition.UserpropRelation propRelation =
(Condition.UserpropRelation) cond;
Id propId = propRelation.key();
Set<Object> fieldValues = this.removeIndexValues(propId,
element.id());
Set<Object> fieldValues = this.toRemoveIndexValues(propId,
element.id());
if (fieldValues == null) {
// Not range index
return true;
}

HugeProperty<Object> hugeProperty = element.getProperty(propId);
if (hugeProperty == null) {
// Property value has been deleted
this.addLeftIndex(propId, fieldValues, element.id());
HugeProperty<Object> property = element.getProperty(propId);
if (property == null) {
// Property value has been deleted, so it's not matched
this.addLeftIndex(element.id(), propId, fieldValues);
return false;
}

/*
* NOTE: If success remove means has correct index,
* we should add left index values to left index map
* waiting to be removed
* NOTE: If removing successfully means there is correct index,
* else we should add left-index values to left index map to
* wait the left-index to be removed.
*/
boolean hasRightValue = removeValue(fieldValues, hugeProperty.value());
boolean hasRightValue = removeFieldValue(fieldValues,
property.value());
if (fieldValues.size() > 0) {
this.addLeftIndex(propId, fieldValues, element.id());
this.addLeftIndex(element.id(), propId, fieldValues);
}

/*
* NOTE: When query by more than one range index field,
* if current field is not the selected one, it can only be used to
* determine whether the index values matched, can't determine
* the element is valid or not
* the element is valid or not.
*/
if (this.selectedIndexField != null) {
return !propId.equals(this.selectedIndexField) || hasRightValue;
Expand All @@ -804,10 +830,11 @@ public boolean validRangeIndex(HugeElement element, Condition cond) {
return hasRightValue;
}

private static boolean removeValue(Set<Object> values, Object value){
for (Object compareValue : values) {
if (numberEquals(compareValue, value)) {
values.remove(compareValue);
private static boolean removeFieldValue(Set<Object> values,
Object value){
for (Object elem : values) {
if (numberEquals(elem, value)) {
values.remove(elem);
return true;
}
}
Expand Down Expand Up @@ -847,4 +874,9 @@ public Id indexField() {
return this.indexField;
}
}

public static interface ResultsFilter {

public boolean test(HugeElement element);
}
}
Loading