Skip to content

Commit

Permalink
Merge branch 'release/29.x'
Browse files Browse the repository at this point in the history 
* release/29.x:
  #4209 - Improve query speed for large KBs when using RDF4J Lucene FTS
  #4212 - Unable to mark local KB as read-only
  • Loading branch information
@reckart
reckart committed Sep 28, 2023
2 parents 8152e97 + f9cc0e9 commit 8e2fdc4
Show file tree
Hide file tree
Showing 3 changed files with 74 additions and 26 deletions.
93 changes: 70 additions & 23 deletions .../inception-kb/src/main/java/de/tudarmstadt/ukp/inception/kb/KnowledgeBaseServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import static org.apache.commons.lang3.StringUtils.isNotBlank;
import static org.apache.commons.lang3.StringUtils.substringAfter;
import static org.apache.commons.lang3.StringUtils.substringBefore;
import static org.apache.commons.lang3.reflect.FieldUtils.readField;
import static org.apache.commons.lang3.reflect.FieldUtils.writeField;
import static org.eclipse.rdf4j.sparqlbuilder.rdf.Rdf.iri;

import java.io.BufferedInputStream;
Expand Down Expand Up @@ -78,6 +80,7 @@
import org.eclipse.rdf4j.repository.manager.RepositoryManager;
import org.eclipse.rdf4j.repository.manager.RepositoryProvider;
import org.eclipse.rdf4j.repository.sail.SailRepository;
import org.eclipse.rdf4j.repository.sail.SailRepositoryConnection;
import org.eclipse.rdf4j.repository.sail.config.SailRepositoryConfig;
import org.eclipse.rdf4j.repository.sparql.SPARQLRepository;
import org.eclipse.rdf4j.repository.sparql.config.SPARQLRepositoryConfig;
Expand All @@ -87,6 +90,8 @@
import org.eclipse.rdf4j.rio.Rio;
import org.eclipse.rdf4j.sail.SailException;
import org.eclipse.rdf4j.sail.lucene.LuceneSail;
import org.eclipse.rdf4j.sail.lucene.LuceneSailConnection;
import org.eclipse.rdf4j.sail.lucene.impl.LuceneIndex;
import org.eclipse.rdf4j.sail.lucene.impl.config.LuceneSailConfig;
import org.eclipse.rdf4j.sail.nativerdf.config.NativeStoreConfig;
import org.eclipse.rdf4j.sparqlbuilder.constraint.propertypath.builder.PropertyPathBuilder;
Expand Down Expand Up @@ -150,6 +155,8 @@
public class KnowledgeBaseServiceImpl
implements KnowledgeBaseService, DisposableBean
{
private static final int LOCAL_FUZZY_PREFIX_LENGTH = 3;

private final Logger log = LoggerFactory.getLogger(getClass());

private @PersistenceContext EntityManager entityManager;
Expand Down Expand Up @@ -299,33 +306,40 @@ public void registerKnowledgeBase(KnowledgeBase aKB, RepositoryImplConfig aCfg)

// We want to have a separate Lucene index for every local repo, so we need to hack the
// index dir in here because this is the place where we finally know the repo ID.
setIndexDir(aKB, aCfg);
syncIndexParameters(aKB, aCfg);

repoManager.addRepositoryConfig(new RepositoryConfig(repositoryId, aCfg));
entityManager.persist(aKB);
}

@Override
public void defineBaseProperties(KnowledgeBase akb)
public void defineBaseProperties(KnowledgeBase aKB)
{
// KB will initialize base properties with base IRI schema properties defined by user
if (akb.getType() == RepositoryType.LOCAL) {
ValueFactory vf = SimpleValueFactory.getInstance();

createBaseProperty(akb, new KBProperty(akb.getSubclassIri(),
vf.createIRI(akb.getSubclassIri()).getLocalName()));
createBaseProperty(akb,
new KBProperty(akb.getLabelIri(),
vf.createIRI(akb.getLabelIri()).getLocalName(), null,
XSD.STRING.stringValue()));
createBaseProperty(akb,
new KBProperty(akb.getDescriptionIri(),
vf.createIRI(akb.getDescriptionIri()).getLocalName(), null,
XSD.STRING.stringValue()));
createBaseProperty(akb, new KBProperty(akb.getTypeIri(),
vf.createIRI(akb.getTypeIri()).getLocalName()));
createBaseProperty(akb, new KBProperty(akb.getSubPropertyIri(),
vf.createIRI(akb.getSubPropertyIri()).getLocalName()));
if (aKB.getType() == RepositoryType.LOCAL) {
var readOnly = aKB.isReadOnly();
aKB.setReadOnly(false);
try {
ValueFactory vf = SimpleValueFactory.getInstance();

createBaseProperty(aKB, new KBProperty(aKB.getSubclassIri(),
vf.createIRI(aKB.getSubclassIri()).getLocalName()));
createBaseProperty(aKB,
new KBProperty(aKB.getLabelIri(),
vf.createIRI(aKB.getLabelIri()).getLocalName(), null,
XSD.STRING.stringValue()));
createBaseProperty(aKB,
new KBProperty(aKB.getDescriptionIri(),
vf.createIRI(aKB.getDescriptionIri()).getLocalName(), null,
XSD.STRING.stringValue()));
createBaseProperty(aKB, new KBProperty(aKB.getTypeIri(),
vf.createIRI(aKB.getTypeIri()).getLocalName()));
createBaseProperty(aKB, new KBProperty(aKB.getSubPropertyIri(),
vf.createIRI(aKB.getSubPropertyIri()).getLocalName()));
}
finally {
aKB.setReadOnly(readOnly);
}
}
}

Expand Down Expand Up @@ -492,6 +506,8 @@ public RepositoryConnection getConnection(KnowledgeBase kb)
{
{
skipCertificateChecks(kb.isSkipSslValidation());

syncIndexParameters(kb, getDelegate());
}

@Override
Expand Down Expand Up @@ -1366,25 +1382,56 @@ void reconfigureLocalKnowledgeBase(KnowledgeBase aKB)
*/

RepositoryImplConfig config = getNativeConfig();
setIndexDir(aKB, config);
syncIndexParameters(aKB, config);
repoManager.addRepositoryConfig(new RepositoryConfig(aKB.getRepositoryId(), config));
}

private void setIndexDir(KnowledgeBase aKB, RepositoryImplConfig aCfg)
private void syncIndexParameters(KnowledgeBase aKB, RepositoryImplConfig aCfg)
{
assertRegistration(aKB);

// We want to have a separate Lucene index for every local repo, so we need to hack the
// index dir in here because this is the place where we finally know the repo ID.
if (aCfg instanceof SailRepositoryConfig) {
SailRepositoryConfig cfg = (SailRepositoryConfig) aCfg;
if (cfg.getSailImplConfig() instanceof LuceneSailConfig) {
LuceneSailConfig luceneSailCfg = (LuceneSailConfig) cfg.getSailImplConfig();

// We want to have a separate Lucene index for every local repo, so we need to hack
// the index dir in here because this is the place where we finally know the repo
// ID.
luceneSailCfg.setIndexDir(
new File(kbRepositoriesRoot, "indexes/" + aKB.getRepositoryId())
.getAbsolutePath());

// Apply the FTS results limit to the KB
luceneSailCfg.setParameter(LuceneSail.MAX_DOCUMENTS_KEY,
Integer.toString(aKB.getMaxResults()));

// Improve fuzzy search speed
luceneSailCfg.setParameter(LuceneSail.FUZZY_PREFIX_LENGTH_KEY,
Integer.toString(LOCAL_FUZZY_PREFIX_LENGTH));
}
}
}

private void syncIndexParameters(KnowledgeBase kb, RepositoryConnection aConn)
{
try {
if (aConn instanceof SailRepositoryConnection) {
var sailRepo = (SailRepositoryConnection) aConn;
var sailConnection = sailRepo.getSailConnection();
if (sailConnection instanceof LuceneSailConnection) {
var luceneSailConnection = (LuceneSailConnection) sailConnection;
var luceneIndex = (LuceneIndex) readField(luceneSailConnection, "luceneIndex",
true);
writeField(luceneIndex, "maxDocs", kb.getMaxResults(), true);
writeField(luceneIndex, "fuzzyPrefixLength", LOCAL_FUZZY_PREFIX_LENGTH, true);
}
}
}
catch (Exception e) {
throw new RuntimeException("Unable to sync query parameters into live index - "
+ "maybe the RDF4J Lucene index implementation has changed.", e);
}
}

@Override
Expand Down
3 changes: 0 additions & 3 deletions ...src/main/java/de/tudarmstadt/ukp/inception/ui/kb/project/AccessSpecificSettingsPanel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,6 @@ public void applyState()
KnowledgeBase kb = getModel().getObject().getKb();
switch (kb.getType()) {
case LOCAL:
// local knowledge bases are editable by default
kb.setReadOnly(false);

// We need to handle this manually here because the onSubmit method of the upload
// form is only called *after* the upload component has already been detached and
// as a consequence all uploaded files have been cleared
Expand Down
4 changes: 4 additions & 0 deletions ...n/java/de/tudarmstadt/ukp/inception/ui/kb/project/wizard/KnowledgeBaseCreationWizard.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,10 +158,14 @@ public void applyState()

switch (kbModel.getObject().getKb().getType()) {
case LOCAL:
// Local KBs are writeable by default
kbModel.getObject().getKb().setReadOnly(false);
// local KBs are always RDF4J + Lucene, so we can set the FTS mode accordingly
kbModel.getObject().getKb().setFullTextSearchIri(FTS_LUCENE.stringValue());
break;
case REMOTE:
// Local KBs are read-only
kbModel.getObject().getKb().setReadOnly(true);
// remote KBs are by default not using FTS but if we apply a remote DB profile,
// then it will set the FTS according to the setting in the profile
kbModel.getObject().getKb().setFullTextSearchIri(FTS_NONE.stringValue());
Expand Down

0 comments on commit 8e2fdc4

Please sign in to comment.