Merge branch 'v2-alpha' into gh-981-gafferpy-fishbowl-maintain

gchq · Jul 13, 2022 · bde92b1 · bde92b1
2 parents 77acde0 + 5e1a304
commit bde92b1
Show file tree

Hide file tree

Showing 16 changed files with 135 additions and 94 deletions.
diff --git a/NOTICES b/NOTICES
@@ -20,7 +20,7 @@ Gaffer tools is built using maven. This process will automatically pull in depen
  projects below.
 
 
-Gaffer (uk.gov.gchq.gaffer:gaffer2:2.0.0-alpha-0.2):
+Gaffer (uk.gov.gchq.gaffer:gaffer2:2.0.0-alpha-0.3):
 
 - Apache License, Version 2.0
 

diff --git a/mini-accumulo-cluster/pom.xml b/mini-accumulo-cluster/pom.xml
@@ -84,6 +84,22 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.apache.accumulo</groupId>
+            <artifactId>accumulo-shell</artifactId>
+            <version>${accumulo.version}</version>
+            <exclusions>
+                <!-- Conflicts with Reload4j -->
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/...c/test/java/uk/gov/gchq/gaffer/miniaccumulocluster/MiniAccumuloClusterControllerTest.java b/...c/test/java/uk/gov/gchq/gaffer/miniaccumulocluster/MiniAccumuloClusterControllerTest.java
@@ -16,13 +16,11 @@
 
 package uk.gov.gchq.gaffer.miniaccumulocluster;
 
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
 
 import java.io.File;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
+import static org.assertj.core.api.Assertions.assertThat;
 
 public class MiniAccumuloClusterControllerTest {
     @Test
@@ -58,9 +56,9 @@ public void shouldStartAndStopCluster() throws Exception {
             Thread.sleep(5000);
 
             // Then
-            assertNull(null != accumuloException[0] ? accumuloException[0].getMessage() : "", accumuloException[0]);
-            assertTrue("store.properties was not generated", new File(miniAccumuloClusterName + "/store.properties").exists());
-            assertEquals(1024, runner[0].getHeapSize());
+            assertThat(accumuloException[0]).withFailMessage(null != accumuloException[0] ? accumuloException[0].getMessage() : "").isNull();
+            assertThat(new File(miniAccumuloClusterName + "/store.properties").exists()).withFailMessage("store.properties was not generated").isTrue();
+            assertThat(1024).isEqualTo(runner[0].getHeapSize());
         } finally {
             if (null != runner[0]) {
                 runner[0].stop();

diff --git a/...fer/accumulostore/performancetesting/ingest/TestAccumuloStoreRandomElementIngestTest.java b/...fer/accumulostore/performancetesting/ingest/TestAccumuloStoreRandomElementIngestTest.java
@@ -16,14 +16,12 @@
 package uk.gov.gchq.gaffer.accumulostore.performancetesting.ingest;
 
 import org.apache.hadoop.conf.Configuration;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
 
 import uk.gov.gchq.gaffer.accumulostore.AccumuloProperties;
 import uk.gov.gchq.gaffer.accumulostore.AccumuloStore;
 import uk.gov.gchq.gaffer.accumulostore.MiniAccumuloStore;
-import uk.gov.gchq.gaffer.commonutil.CommonTestConstants;
 import uk.gov.gchq.gaffer.commonutil.StreamUtil;
 import uk.gov.gchq.gaffer.graph.Graph;
 import uk.gov.gchq.gaffer.graph.GraphConfig;
@@ -33,12 +31,13 @@
 import uk.gov.gchq.gaffer.store.StoreException;
 import uk.gov.gchq.gaffer.store.schema.Schema;
 
+import java.io.File;
 import java.io.IOException;
 
 public class TestAccumuloStoreRandomElementIngestTest {
 
-    @Rule
-    public TemporaryFolder tempFolder = new TemporaryFolder(CommonTestConstants.TMP_DIRECTORY);
+    @TempDir
+    public File tempFolder;
 
     @Test
     public void testAccumuloStoreRandomElementIngestTestRuns() throws StoreException, OperationException, IOException {
@@ -49,7 +48,7 @@ public void testAccumuloStoreRandomElementIngestTestRuns() throws StoreException
         testProperties.setElementSupplierClass(RmatElementSupplier.class.getName());
         testProperties.setRmatProbabilities(Constants.RMAT_PROBABILITIES);
         testProperties.setRmatMaxNodeId(100L);
-        testProperties.setTempDirectory(tempFolder.newFolder().getCanonicalPath());
+        testProperties.setTempDirectory(tempFolder.getCanonicalPath());
 
         final Schema schema = Schema.fromJson(StreamUtil.schemas(Constants.class));
         final AccumuloProperties storeProperties = AccumuloProperties.loadStoreProperties(

diff --git a/performance-testing/performance-testing-aws/pom.xml b/performance-testing/performance-testing-aws/pom.xml
@@ -60,6 +60,22 @@
             <artifactId>httpcore</artifactId>
             <version>4.4.4</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.accumulo</groupId>
+            <artifactId>accumulo-core</artifactId>
+            <version>${accumulo.version}</version>
+            <exclusions>
+                <!-- Conflicts with Reload4j -->
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>log4j</groupId>
+                    <artifactId>log4j</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
     </dependencies>
 
     <build>

diff --git a/...in/java/uk/gov/gchq/gaffer/performancetesting/aws/PublishAccumuloMetricsToCloudWatch.java b/...in/java/uk/gov/gchq/gaffer/performancetesting/aws/PublishAccumuloMetricsToCloudWatch.java
@@ -24,17 +24,19 @@
 import com.amazonaws.services.cloudwatch.model.PutMetricDataRequest;
 import com.amazonaws.services.cloudwatch.model.PutMetricDataResult;
 import com.amazonaws.services.cloudwatch.model.StandardUnit;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.client.impl.MasterClient;
-import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.clientImpl.MasterClient;
+import org.apache.accumulo.core.clientImpl.Tables;
+import org.apache.accumulo.core.conf.SiteConfiguration;
+import org.apache.accumulo.core.data.TableId;
 import org.apache.accumulo.core.master.thrift.MasterClientService;
 import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
 import org.apache.accumulo.core.master.thrift.TableInfo;
 import org.apache.accumulo.core.master.thrift.TabletServerStatus;
-import org.apache.accumulo.core.trace.Tracer;
+import org.apache.accumulo.core.trace.TraceUtil;
 import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.server.AccumuloServerContext;
-import org.apache.accumulo.server.conf.ServerConfigurationFactory;
+import org.apache.accumulo.server.ServerContext;
 import org.apache.thrift.TException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -56,12 +58,12 @@ public class PublishAccumuloMetricsToCloudWatch implements Runnable {
     private final String zookeepers;
     private int interval = 60;
 
-    private ZooKeeperInstance instance;
-    private AccumuloServerContext context;
+    private AccumuloClient instance;
+    private ServerContext context;
     private MasterClientService.Client client;
     private AmazonCloudWatch cloudwatch;
 
-    private Map<String, String> tableIdToNameMap;
+    private Map<TableId, String> tableIdToNameMap;
     private Date previousCaptureDate = null;
     private Map<Pair<String, List<Dimension>>, Double> previousMetrics = null;
 
@@ -77,9 +79,8 @@ public PublishAccumuloMetricsToCloudWatch(final String instanceName, final Strin
 
     private void connect() {
         // Connect to Accumulo Master
-        this.instance = new ZooKeeperInstance(this.instanceName, this.zookeepers);
-        ServerConfigurationFactory config = new ServerConfigurationFactory(this.instance);
-        this.context = new AccumuloServerContext(config);
+        this.instance = Accumulo.newClient().to(instanceName, zookeepers).as("admin", "admin").build();
+        this.context = new ServerContext(new SiteConfiguration());
         this.client = MasterClient.getConnection(this.context);
 
         // Set up connection to AWS CloudWatch
@@ -96,7 +97,7 @@ private void disconnect() {
     private String getTableNameForId(final String id) {
         if (this.tableIdToNameMap == null || !this.tableIdToNameMap.containsKey(id)) {
             // Refresh the cache as a table may have just been created
-            this.tableIdToNameMap = Tables.getIdToNameMap(this.instance);
+            this.tableIdToNameMap = Tables.getIdToNameMap(this.context);
         }
 
         return this.tableIdToNameMap.get(id);
@@ -106,7 +107,7 @@ private String getTableNameForId(final String id) {
     private List<MetricDatum> gatherMetrics() throws TException {
         final String awsEmrJobFlowId = AwsEmrUtils.getJobFlowId();
 
-        final MasterMonitorInfo stats = this.client.getMasterStats(Tracer.traceInfo(), this.context.rpcCreds());
+        final MasterMonitorInfo stats = this.client.getMasterStats(TraceUtil.traceInfo(), this.context.rpcCreds());
         LOGGER.trace(stats.toString());
 
         final Date now = new Date();

diff --git a/...ore/src/test/java/uk/gov/gchq/gaffer/performancetesting/ingest/TestElementIngestTest.java b/...ore/src/test/java/uk/gov/gchq/gaffer/performancetesting/ingest/TestElementIngestTest.java
@@ -16,13 +16,11 @@
 package uk.gov.gchq.gaffer.performancetesting.ingest;
 
 import org.apache.commons.io.FileUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
 
 import uk.gov.gchq.gaffer.accumulostore.AccumuloProperties;
 import uk.gov.gchq.gaffer.accumulostore.MiniAccumuloStore;
-import uk.gov.gchq.gaffer.commonutil.CommonTestConstants;
 import uk.gov.gchq.gaffer.commonutil.StreamUtil;
 import uk.gov.gchq.gaffer.graph.Graph;
 import uk.gov.gchq.gaffer.graph.GraphConfig;
@@ -34,12 +32,12 @@
 import java.io.IOException;
 import java.util.List;
 
-import static org.junit.Assert.assertTrue;
+import static org.assertj.core.api.Assertions.assertThat;
 
 public class TestElementIngestTest {
 
-    @Rule
-    public TemporaryFolder folder = new TemporaryFolder(CommonTestConstants.TMP_DIRECTORY);
+    @TempDir
+    public File folder;
 
     @Test
     public void testElementIngestTestRuns() {
@@ -65,7 +63,7 @@ public void testElementIngestTestRuns() {
         final double result = test.run();
 
         // Then
-        assertTrue(result > 0.0D);
+        assertThat(result > 0.0D).isTrue();
     }
 
     @Test
@@ -78,7 +76,7 @@ public void testElementIngestTestOutputsToListener() throws IOException {
         testProperties.setRmatProbabilities(Constants.RMAT_PROBABILITIES);
         testProperties.setRmatMaxNodeId(100L);
         testProperties.setMetricsListenerClass(FileWriterMetricsListener.class.getName());
-        final File metricsResults = folder.newFile();
+        final File metricsResults = new File(folder, "metricsResults.txt");
         final String metricsResultsFilename = metricsResults.getPath();
         testProperties.setProperty(FileWriterMetricsListener.FILENAME, metricsResultsFilename);
         final AccumuloProperties storeProperties = AccumuloProperties.loadStoreProperties(
@@ -97,15 +95,15 @@ public void testElementIngestTestOutputsToListener() throws IOException {
         final List<String> lines = FileUtils.readLines(new File(metricsResultsFilename));
 
         // Then
-        assertTrue(lines.size() > 0);
+        assertThat(lines.size() > 0).isTrue();
         final int offsetBatch = IngestMetrics.ELEMENTS_PER_SECOND_BATCH.length() + 1;
         final int offsetOverall = IngestMetrics.ELEMENTS_PER_SECOND_OVERALL.length() + 1;
         for (int i = 0; i < lines.size() - 2; i++) {
             final String[] tokens = lines.get(i).split(",");
-            assertTrue(tokens[0].replaceAll(" ", "").startsWith(IngestMetrics.ELEMENTS_PER_SECOND_BATCH + ":"));
-            assertTrue(nullOrPositive(tokens[0].substring(offsetBatch)));
-            assertTrue(tokens[1].replaceAll(" ", "").startsWith(IngestMetrics.ELEMENTS_PER_SECOND_OVERALL + ":"));
-            assertTrue(nullOrPositive(tokens[1].substring(offsetOverall)));
+            assertThat(tokens[0].replaceAll(" ", "").startsWith(IngestMetrics.ELEMENTS_PER_SECOND_BATCH + ":")).isTrue();
+            assertThat(nullOrPositive(tokens[0].substring(offsetBatch))).isTrue();
+            assertThat(tokens[1].replaceAll(" ", "").startsWith(IngestMetrics.ELEMENTS_PER_SECOND_OVERALL + ":")).isTrue();
+            assertThat(nullOrPositive(tokens[1].substring(offsetOverall))).isTrue();
         }
     }
 

diff --git a/...esting-core/src/test/java/uk/gov/gchq/gaffer/performancetesting/ingest/TestQueryTest.java b/...esting-core/src/test/java/uk/gov/gchq/gaffer/performancetesting/ingest/TestQueryTest.java
@@ -16,13 +16,11 @@
 package uk.gov.gchq.gaffer.performancetesting.ingest;
 
 import org.apache.commons.io.FileUtils;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
 
 import uk.gov.gchq.gaffer.accumulostore.AccumuloProperties;
 import uk.gov.gchq.gaffer.accumulostore.MiniAccumuloStore;
-import uk.gov.gchq.gaffer.commonutil.CommonTestConstants;
 import uk.gov.gchq.gaffer.commonutil.StreamUtil;
 import uk.gov.gchq.gaffer.graph.Graph;
 import uk.gov.gchq.gaffer.graph.GraphConfig;
@@ -43,12 +41,12 @@
 import java.util.List;
 import java.util.stream.Stream;
 
-import static org.junit.Assert.assertTrue;
+import static org.assertj.core.api.Assertions.assertThat;
 
 public class TestQueryTest {
 
-    @Rule
-    public TemporaryFolder folder = new TemporaryFolder(CommonTestConstants.TMP_DIRECTORY);
+    @TempDir
+    public File folder;
 
     @Test
     public void testQueryTestRuns() throws OperationException {
@@ -79,7 +77,7 @@ public void testQueryTestRuns() throws OperationException {
         final double result = test.run();
 
         // Then
-        assertTrue(result > 0.0D);
+        assertThat(result > 0.0D).isTrue();
     }
 
     @Test
@@ -92,7 +90,7 @@ public void testQueryTestOutputsToListener() throws IOException, OperationExcept
         testProperties.setRmatProbabilities(Constants.RMAT_PROBABILITIES);
         testProperties.setRmatMaxNodeId(100L);
         testProperties.setMetricsListenerClass(FileWriterMetricsListener.class.getName());
-        final File metricsResults = folder.newFile();
+        final File metricsResults = new File(folder, "metricsResults.txt");
         final String metricsResultsFilename = metricsResults.getPath();
         testProperties.setProperty(FileWriterMetricsListener.FILENAME, metricsResultsFilename);
         final AccumuloProperties storeProperties = AccumuloProperties.loadStoreProperties(
@@ -116,13 +114,13 @@ public void testQueryTestOutputsToListener() throws IOException, OperationExcept
         final List<String> lines = FileUtils.readLines(new File(metricsResultsFilename));
 
         // Then
-        assertTrue(lines.size() > 0);
+        assertThat(lines.size() > 0).isTrue();
         lines.forEach(line -> {
             final String[] fields = line.split(", ");
-            assertTrue(fields[0].startsWith(QueryMetrics.RESULTS_PER_SECOND));
-            assertTrue(Double.parseDouble(fields[0].split(":")[1]) > 0.0D);
-            assertTrue(fields[1].startsWith(QueryMetrics.SEEDS_PER_SECOND));
-            assertTrue(Double.parseDouble(fields[1].split(":")[1]) > 0.0D);
+            assertThat(fields[0].startsWith(QueryMetrics.RESULTS_PER_SECOND)).isTrue();
+            assertThat(Double.parseDouble(fields[0].split(":")[1]) > 0.0D).isTrue();
+            assertThat(fields[1].startsWith(QueryMetrics.SEEDS_PER_SECOND)).isTrue();
+            assertThat(Double.parseDouble(fields[1].split(":")[1]) > 0.0D).isTrue();
         });
     }
 }
diff --git a/pom.xml b/pom.xml
@@ -21,7 +21,7 @@
     <parent>
         <groupId>uk.gov.gchq.gaffer</groupId>
         <artifactId>gaffer2</artifactId>
-        <version>2.0.0-alpha-0.2</version>
+        <version>2.0.0-alpha-0.3</version>
     </parent>
 
     <artifactId>gaffer-tools</artifactId>
@@ -36,7 +36,7 @@
     </modules>
 
     <properties>
-        <gaffer.version>2.0.0-alpha-0.2</gaffer.version>
+        <gaffer.version>2.0.0-alpha-0.3</gaffer.version>
         <scm.url>
             https://github.com/gchq/gaffer-tools
         </scm.url>

diff --git a/python-shell/src/__init__.py b/python-shell/src/__init__.py
@@ -0,0 +1,11 @@
+__version__ = "2.0.0-alpha-0.3"
+
+__title__ = "gafferpy"
+__description__ = "Gaffer Python Shell"
+__uri__ = "https://github.com/gchq/gafferpy-tools"
+__doc__ = __description__ + " <" + __uri__ + ">"
+
+__author__ = "GCHQ"
+
+__license__ = "Apache 2.0"
+__copyright__ = "Crown Copyright (c) 2016-2017"
diff --git a/python-shell/src/examples/__init__.py b/python-shell/src/examples/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "2.0.0-alpha-0.2"
+__version__ = "2.0.0-alpha-0.3"
 
 __title__ = "gafferpy"
 __description__ = "Gaffer Python Shell"

diff --git a/python-shell/src/gafferpy/__init__.py b/python-shell/src/gafferpy/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "2.0.0-alpha-0.2"
+__version__ = "2.0.0-alpha-0.3"
 
 __title__ = "gafferpy"
 __description__ = "Gaffer Python Shell"