From e16b3c160664057d4e00f2b4030cb6cb0d32fabd Mon Sep 17 00:00:00 2001 From: Yifan Yang Date: Mon, 7 Sep 2020 21:55:38 +0800 Subject: [PATCH] Use standalone EmbeddedSolrServer for Solr integration tests (#1368) --- .../solr/AclAnthologyEndToEndTest.java | 5 + .../integration/solr/CoreEndToEndTest.java | 5 + .../integration/solr/SolrEndToEndTest.java | 128 ++++++++++++------ .../integration/solr/TrecEndToEndTest.java | 5 + 4 files changed, 103 insertions(+), 40 deletions(-) diff --git a/src/test/java/io/anserini/integration/solr/AclAnthologyEndToEndTest.java b/src/test/java/io/anserini/integration/solr/AclAnthologyEndToEndTest.java index fb31a0ed33..84c131470b 100644 --- a/src/test/java/io/anserini/integration/solr/AclAnthologyEndToEndTest.java +++ b/src/test/java/io/anserini/integration/solr/AclAnthologyEndToEndTest.java @@ -22,6 +22,11 @@ import io.anserini.search.SearchSolr; public class AclAnthologyEndToEndTest extends SolrEndToEndTest { + @Override + protected String getCollectionName() { + return "AclAnthology"; + } + @Override protected String getSchemaAdjustmentFile() { return "solr/schemas/acl-anthology.json"; diff --git a/src/test/java/io/anserini/integration/solr/CoreEndToEndTest.java b/src/test/java/io/anserini/integration/solr/CoreEndToEndTest.java index 1817d07564..430390b2f9 100644 --- a/src/test/java/io/anserini/integration/solr/CoreEndToEndTest.java +++ b/src/test/java/io/anserini/integration/solr/CoreEndToEndTest.java @@ -22,6 +22,11 @@ import io.anserini.search.SearchSolr; public class CoreEndToEndTest extends SolrEndToEndTest { + @Override + protected String getCollectionName() { + return "Core"; + } + @Override protected String getSchemaAdjustmentFile() { return "solr/schemas/core.json"; diff --git a/src/test/java/io/anserini/integration/solr/SolrEndToEndTest.java b/src/test/java/io/anserini/integration/solr/SolrEndToEndTest.java index 12e74034c0..7b9d124ffc 100644 --- a/src/test/java/io/anserini/integration/solr/SolrEndToEndTest.java +++ b/src/test/java/io/anserini/integration/solr/SolrEndToEndTest.java @@ -26,29 +26,98 @@ import org.apache.commons.pool2.impl.DefaultPooledObject; import org.apache.commons.pool2.impl.GenericObjectPool; import org.apache.commons.pool2.impl.GenericObjectPoolConfig; -import org.apache.solr.EmbeddedSolrServerTestBase; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; +import org.apache.solr.client.solrj.request.CoreAdminRequest; import org.apache.solr.client.solrj.request.json.DirectJsonQueryRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.params.CommonParams; +import org.apache.solr.core.NodeConfig; +import org.apache.solr.core.SolrResourceLoader; +import org.junit.After; import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.lang.reflect.Field; +import java.net.URL; import java.nio.file.Files; -public abstract class SolrEndToEndTest extends EmbeddedSolrServerTestBase { +import static org.apache.solr.SolrTestCaseJ4.params; + +@LuceneTestCase.SuppressSysoutChecks(bugUrl = "None") +public abstract class SolrEndToEndTest extends LuceneTestCase { + private static final Logger LOG = LogManager.getLogger(SolrEndToEndTest.class); + protected ObjectPool stubSolrPool; protected final String searchOutputPrefix = "e2eTestSearch"; + protected EmbeddedSolrServer client; + + protected static File getFile(String path) { + final URL url = SolrEndToEndTest.class.getClassLoader().getResource(path); + if (url != null) { + try { + return new File(url.toURI()); + } catch (Exception e) { + throw new RuntimeException("Resource was found on classpath, but cannot be resolved to a normal file: " + path); + } + } + final File file = new File(path); + if (file.exists()) { + return file; + } + throw new RuntimeException("Cannot find resource in classpath or in file-system (relative to CWD): " + path); + } + + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + + final File solrHome = createTempDir().toFile(); + final File configSetBaseDir = new File(solrHome.toPath() + File.separator + "configsets"); + FileUtils.copyDirectory(getFile("solr/anserini"), new File(configSetBaseDir + File.separator + "anserini")); + + SolrResourceLoader loader = new SolrResourceLoader(solrHome.toPath()); + NodeConfig config = new NodeConfig.NodeConfigBuilder("embeddedSolrServerNode", loader) + .setConfigSetBaseDirectory(configSetBaseDir.getAbsolutePath()).build(); + client = new EmbeddedSolrServer(config, getCollectionName()); + LOG.info("Created Embedded Solr Server"); + + CoreAdminRequest.Create createRequest = new CoreAdminRequest.Create(); + createRequest.setCoreName(getCollectionName()); + createRequest.setConfigSet("anserini"); + createRequest.process(client); + client.commit(); + LOG.info("Created Solr Core: " + getCollectionName()); + + GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig<>(); + poolConfig.setMaxTotal(1); // only 1 EmbeddedSolrServer instance will be created by getSolrClient + poolConfig.setMinIdle(1); + stubSolrPool = new GenericObjectPool<>(new StubSolrClientFactory(client), poolConfig); + } + + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + + client.deleteByQuery("*:*"); + client.commit(); + client.close(); + stubSolrPool.close(); + } + protected IndexArgs createDefaultIndexArgs() { IndexArgs args = new IndexArgs(); - args.solrIndex = DEFAULT_CORE_NAME; + args.solrIndex = getCollectionName(); args.threads = 1; args.storePositions = true; args.storeDocvectors = true; @@ -64,8 +133,8 @@ protected IndexArgs createDefaultIndexArgs() { protected SearchSolr.Args createSearchArgs(String topicReader, String topicFile) { SearchSolr.Args args = new SearchSolr.Args(); - args.solrIndex = DEFAULT_CORE_NAME; - args.output = this.searchOutputPrefix + topicReader; + args.solrIndex = getCollectionName(); + args.output = searchOutputPrefix + topicReader; args.topicReader = topicReader; args.topics = new String[]{topicFile}; args.zkUrl = "localhost"; // SearchSolr initialization workaround @@ -73,51 +142,29 @@ protected SearchSolr.Args createSearchArgs(String topicReader, String topicFile) return args; } - protected class StubSolrClientFactory extends BasePooledObjectFactory { + protected static class StubSolrClientFactory extends BasePooledObjectFactory { + final SolrClient client; + + public StubSolrClientFactory(SolrClient client) { + this.client = client; + } + @Override public SolrClient create() { - return getSolrClient(); + return this.client; } @Override public PooledObject wrap(SolrClient solrClient) { return new DefaultPooledObject<>(solrClient); } - - @Override - public void destroyObject(PooledObject pooled) throws Exception { - pooled.getObject().close(); - } - } - - @BeforeClass - public static void beforeClass() throws Exception { - File testHome = createTempDir().toFile(); - String collectionPath = testHome.getAbsolutePath() + File.separator + DEFAULT_CORE_NAME; - FileUtils.copyDirectory(getFile("solr/anserini"), new File(collectionPath)); - initCore("solrconfig.xml", "managed-schema", testHome.getAbsolutePath(), DEFAULT_CORE_NAME); - } - - @Before - @Override - public void setUp() throws Exception { - super.setUp(); - - clearIndex(); - assertU(commit()); - assertU(optimize()); - - GenericObjectPoolConfig config = new GenericObjectPoolConfig<>(); - config.setMaxTotal(1); // only 1 EmbeddedSolrServer instance will be created by getSolrClient - config.setMinIdle(1); - this.stubSolrPool = new GenericObjectPool<>(new StubSolrClientFactory(), config); } protected IndexCollection getIndexRunner(IndexArgs args) throws Exception { IndexCollection runner = new IndexCollection(args); Field f = runner.getClass().getDeclaredField("solrPool"); f.setAccessible(true); - f.set(runner, this.stubSolrPool); + f.set(runner, stubSolrPool); return runner; } @@ -126,10 +173,12 @@ protected SearchSolr getSearchRunner(SearchSolr.Args args) throws Exception { Field f = runner.getClass().getDeclaredField("client"); f.setAccessible(true); ((SolrClient) f.get(runner)).close(); // close the old client - f.set(runner, getSolrClient()); + f.set(runner, client); return runner; } + protected abstract String getCollectionName(); + protected abstract String getSchemaAdjustmentFile(); protected abstract IndexArgs getIndexArgs(); @@ -145,14 +194,13 @@ public void testIndexAndSearch() throws Exception { // update schema, much like curl -X POST -H 'Content-type:application/json' --data-binary SCHEMA_NAME.json http://localhost:8983/solr/COLLECTION_NAME/schema String schemaJson = Files.readString(getFile(schemaAdjustmentFile).toPath()); DirectJsonQueryRequest schemaRequest = new DirectJsonQueryRequest(schemaJson, params(CommonParams.QT, "/schema")); - QueryResponse response = schemaRequest.process(getSolrClient(), DEFAULT_CORE_NAME); + QueryResponse response = schemaRequest.process(client, getCollectionName()); assertEquals(0, response.getStatus()); } IndexArgs indexArgs = getIndexArgs(); IndexCollection indexRunner = getIndexRunner(indexArgs); indexRunner.run(); - this.stubSolrPool.close(); SearchSolr.Args searchArgs = getSearchArgs(); SearchSolr searchRunner = getSearchRunner(searchArgs); diff --git a/src/test/java/io/anserini/integration/solr/TrecEndToEndTest.java b/src/test/java/io/anserini/integration/solr/TrecEndToEndTest.java index eafda1c70a..fb28daa380 100644 --- a/src/test/java/io/anserini/integration/solr/TrecEndToEndTest.java +++ b/src/test/java/io/anserini/integration/solr/TrecEndToEndTest.java @@ -21,6 +21,11 @@ import io.anserini.search.SearchSolr; public class TrecEndToEndTest extends SolrEndToEndTest { + @Override + protected String getCollectionName() { + return "Trec"; + } + @Override protected String getSchemaAdjustmentFile() { return null; // no need to adjust schema