Skip to content

Commit

Permalink
SOLR-13731: javabin must support a 1:1 mapping of the JSON update format
Browse files Browse the repository at this point in the history
  • Loading branch information
noblepaul committed Oct 14, 2019
1 parent 845c3e9 commit 41779e5
Show file tree
Hide file tree
Showing 8 changed files with 224 additions and 62 deletions.
4 changes: 3 additions & 1 deletion solr/CHANGES.txt
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,9 @@ New Features

Improvements
---------------------
(No changes)

*SOLR-13731: javabin must support a 1:1 mapping of the JSON update format (noble)


Optimizations
---------------------
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import static org.apache.solr.common.params.CommonParams.CHILDDOC;
import static org.apache.solr.common.util.ByteArrayUtf8CharSequence.convertCharSeq;

/**
Expand Down Expand Up @@ -81,7 +82,7 @@ public void marshal(UpdateRequest updateRequest, OutputStream os) throws IOExcep
if(updateRequest.getDocIterator() != null){
docIter = updateRequest.getDocIterator();
}

Map<SolrInputDocument,Map<String,Object>> docMap = updateRequest.getDocumentsMap();

nl.add("params", params);// 0: params
Expand Down Expand Up @@ -125,7 +126,7 @@ public UpdateRequest unmarshal(InputStream is, final StreamingUpdateHandler hand
try (JavaBinCodec codec = new StreamingCodec(namedList, updateRequest, handler)) {
codec.unmarshal(is);
}

// NOTE: if the update request contains only delete commands the params
// must be loaded now
if(updateRequest.getParams()==null) {
Expand All @@ -145,11 +146,11 @@ public UpdateRequest unmarshal(InputStream is, final StreamingUpdateHandler hand
} else {
docMap = (List<Entry<SolrInputDocument, Map<Object, Object>>>) docsMapObj;
}


// we don't add any docs, because they were already processed
// deletes are handled later, and must be passed back on the UpdateRequest

if (delById != null) {
for (String s : delById) {
updateRequest.deleteById(s);
Expand All @@ -167,15 +168,15 @@ public UpdateRequest unmarshal(InputStream is, final StreamingUpdateHandler hand
} else {
updateRequest.deleteById(entry.getKey());
}

}
}
if (delByQ != null) {
for (String s : delByQ) {
updateRequest.deleteByQuery(s);
}
}

return updateRequest;
}

Expand Down Expand Up @@ -282,7 +283,9 @@ public List readIterator(DataInputInputStream fis) throws IOException {


private List readOuterMostDocIterator(DataInputInputStream fis) throws IOException {
if(namedList[0] == null) namedList[0] = new NamedList();
NamedList params = (NamedList) namedList[0].get("params");
if (params == null) params = new NamedList();
updateRequest.setParams(new ModifiableSolrParams(params.toSolrParams()));
if (handler == null) return super.readIterator(fis);
Integer commitWithin = null;
Expand Down Expand Up @@ -313,8 +316,10 @@ private List readOuterMostDocIterator(DataInputInputStream fis) throws IOExcepti
commitWithin = (Integer) p.get(UpdateRequest.COMMIT_WITHIN);
overwrite = (Boolean) p.get(UpdateRequest.OVERWRITE);
}
} else {
} else if (o instanceof SolrInputDocument) {
sdoc = (SolrInputDocument) o;
} else if (o instanceof Map) {
sdoc = convertMapToSolrInputDoc((Map) o);
}

// peek at the next object to see if we're at the end
Expand All @@ -333,5 +338,26 @@ private List readOuterMostDocIterator(DataInputInputStream fis) throws IOExcepti
}
}

private SolrInputDocument convertMapToSolrInputDoc(Map m) {
SolrInputDocument result = createSolrInputDocument(m.size());
m.forEach((k, v) -> {
if (CHILDDOC.equals(k.toString())) {
if (v instanceof List) {
List list = (List) v;
for (Object o : list) {
if (o instanceof Map) {
result.addChildDocument(convertMapToSolrInputDoc((Map) o));
}
}
} else if (v instanceof Map) {
result.addChildDocument(convertMapToSolrInputDoc((Map) v));
}
} else {
result.addField(k.toString(), v);
}
});
return result;
}

}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.solr.common;

import java.io.IOException;
import java.util.Set;
import java.util.function.Predicate;

public class ConditionalKeyMapWriter implements MapWriter {
private final MapWriter delegate;
private final Predicate<CharSequence> predicate;

public ConditionalKeyMapWriter(MapWriter delegate, Predicate<CharSequence> predicate) {
this.delegate = delegate;
this.predicate = predicate;
}

public static class EntryWriterWrapper implements EntryWriter {
private final EntryWriter delegate;
private final Predicate<CharSequence> predicate;

public EntryWriterWrapper(EntryWriter delegate, Predicate<CharSequence> predicate) {
this.delegate = delegate;
this.predicate = predicate;
}

@Override
public EntryWriter put(CharSequence k, Object v) throws IOException {
if (predicate.test(k)) delegate.put(k, v);
return this;
}

@Override
public EntryWriter put(CharSequence k, int v) throws IOException {
if (predicate.test(k)) delegate.put(k, v);
return this;
}

@Override
public EntryWriter put(CharSequence k, long v) throws IOException {
if (predicate.test(k)) delegate.put(k, v);
return this;
}

@Override
public EntryWriter put(CharSequence k, float v) throws IOException {
if (predicate.test(k)) delegate.put(k, v);
return this;
}

@Override
public EntryWriter put(CharSequence k, double v) throws IOException {
if (predicate.test(k)) delegate.put(k, v);
return this;
}

@Override
public EntryWriter put(CharSequence k, boolean v) throws IOException {
if (predicate.test(k)) delegate.put(k, v);
return this;
}
}

@Override
public void writeMap(EntryWriter ew) throws IOException {
if (delegate != null) delegate.writeMap(new EntryWriterWrapper(ew, predicate));
}

public static Predicate<CharSequence> dedupeKeyPredicate(Set<CharSequence> keys) {
return (k) -> keys.add(k);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import java.util.Set;
import java.util.function.BiConsumer;

import org.apache.solr.common.params.CommonParams;

/**
* Represent the field-value information needed to construct and index
* a Lucene Document. Like the SolrDocument, the field values should
Expand Down Expand Up @@ -57,6 +59,9 @@ public void writeMap(EntryWriter ew) throws IOException {
bc.accept(k, o);
};
_fields.forEach(wrapper);
if (_childDocuments != null) {
ew.put(CommonParams.CHILDDOC, _childDocuments);
}
}

public SolrInputDocument(Map<String,SolrInputField> fields) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ public static EchoParamStyle get( String v ) {
}
return null;
}
};
}

/** which parameters to log (if not supplied all parameters will be logged) **/
String LOG_PARAMS_LIST = "logParamsList";
Expand Down Expand Up @@ -299,5 +299,7 @@ public static EchoParamStyle get( String v ) {
String FILE = "file";
String FILES = "files";

String CHILDDOC = "_childDocuments_";

}

Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.function.Predicate;

import org.apache.solr.common.ConditionalMapWriter;
import org.apache.solr.common.ConditionalKeyMapWriter;
import org.apache.solr.common.EnumFieldValue;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.IteratorWriter.ItemWriter;
Expand All @@ -49,6 +50,7 @@
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.params.CommonParams;
import org.noggit.CharArr;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -531,7 +533,7 @@ public byte[] readByteArray(DataInputInputStream dis) throws IOException {
//use this to ignore the writable interface because , child docs will ignore the fl flag
// is it a good design?
private boolean ignoreWritable =false;
private ConditionalMapWriter.EntryWriterWrapper cew;
private MapWriter.EntryWriter cew;

public void writeSolrDocument(SolrDocument doc) throws IOException {
List<SolrDocument> children = doc.getChildDocuments();
Expand All @@ -546,7 +548,7 @@ public void writeSolrDocument(SolrDocument doc) throws IOException {
int sz = fieldsCount + (children==null ? 0 : children.size());
writeTag(SOLRDOC);
writeTag(ORDERED_MAP, sz);
if (cew == null) cew = new ConditionalMapWriter.EntryWriterWrapper(ew, (k, o) -> toWrite(k.toString()));
if (cew == null) cew = new ConditionalKeyMapWriter.EntryWriterWrapper(ew, (k) -> toWrite(k.toString()));
doc.writeMap(cew);
if (children != null) {
try {
Expand Down Expand Up @@ -649,13 +651,14 @@ public SolrInputDocument readSolrInputDocument(DataInputInputStream dis) throws
protected SolrInputDocument createSolrInputDocument(int sz) {
return new SolrInputDocument(new LinkedHashMap<>(sz));
}
static final Predicate<CharSequence> IGNORECHILDDOCS = it -> !CommonParams.CHILDDOC.equals(it.toString());

public void writeSolrInputDocument(SolrInputDocument sdoc) throws IOException {
List<SolrInputDocument> children = sdoc.getChildDocuments();
int sz = sdoc.size() + (children==null ? 0 : children.size());
writeTag(SOLRINPUTDOC, sz);
writeFloat(1f); // document boost
sdoc.writeMap(ew);
sdoc.writeMap(new ConditionalKeyMapWriter.EntryWriterWrapper(ew,IGNORECHILDDOCS));
if (children != null) {
for (SolrInputDocument child : children) {
writeSolrInputDocument(child);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -417,6 +417,9 @@ public NamedList<T> getImmutableCopy() {
}

public Map<String,T> asShallowMap() {
return asShallowMap(false);
}
public Map<String,T> asShallowMap(boolean allowDps) {
return new Map<String, T>() {
@Override
public int size() {
Expand Down Expand Up @@ -444,13 +447,17 @@ public T get(Object key) {

@Override
public T put(String key, T value) {
if (allowDps) {
NamedList.this.add(key, value);
return null;
}
int idx = NamedList.this.indexOf(key, 0);
if (idx == -1) {
NamedList.this.add(key, value);
} else {
NamedList.this.setVal(idx, value);
}
return null;
return null;
}

@Override
Expand Down
Loading

0 comments on commit 41779e5

Please sign in to comment.