From 5fb2a3e9a61a803433428edc87c762bc25dc37e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dutoit=20C=C3=A9dric?= Date: Fri, 29 Sep 2017 20:11:50 +0200 Subject: [PATCH] Added new DB Store --- pom.xml | 6 +- src/main/java/ch/mno/copper/CopperDaemon.java | 4 +- .../java/ch/mno/copper/data/DbHelper.java | 270 ++++++++++++++++++ .../ch/mno/copper/data/DbValuesStore.java | 91 ++++++ .../ch/mno/copper/data/MemoryValuesStore.java | 103 +++---- .../java/ch/mno/copper/data/StoreValue.java | 42 ++- .../java/ch/mno/copper/data/ValuesStore.java | 12 +- .../ch/mno/copper/web/CopperServices.java | 28 +- .../ch/mno/copper/MemoryValuesStoreTest.java | 32 +-- .../java/ch/mno/copper/data/DbHelperTest.java | 254 ++++++++++++++++ .../ch/mno/copper/data/DbValueStoreTest.java | 49 ++++ 11 files changed, 789 insertions(+), 102 deletions(-) create mode 100644 src/main/java/ch/mno/copper/data/DbHelper.java create mode 100644 src/main/java/ch/mno/copper/data/DbValuesStore.java create mode 100644 src/test/java/ch/mno/copper/data/DbHelperTest.java create mode 100644 src/test/java/ch/mno/copper/data/DbValueStoreTest.java diff --git a/pom.xml b/pom.xml index a070d21..5729923 100644 --- a/pom.xml +++ b/pom.xml @@ -217,7 +217,11 @@ json-path 2.4.0 - + + com.h2database + h2 + 1.4.193 + diff --git a/src/main/java/ch/mno/copper/CopperDaemon.java b/src/main/java/ch/mno/copper/CopperDaemon.java index 6cce11c..bb9184f 100644 --- a/src/main/java/ch/mno/copper/CopperDaemon.java +++ b/src/main/java/ch/mno/copper/CopperDaemon.java @@ -15,7 +15,9 @@ import java.lang.management.ManagementFactory; import java.net.MalformedURLException; import java.rmi.RemoteException; +import java.time.Instant; import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -99,7 +101,7 @@ private void runIteration() { // Processors LocalDateTime queryTime = LocalDateTime.now(); // Keep time, so that next run will have data between query time assignation and valueStore read time - Collection changedValues = valuesStore.queryValues(lastQueryTime, LocalDateTime.MAX); + Collection changedValues = valuesStore.queryValues(lastQueryTime.toInstant(ZoneOffset.UTC), Instant.MAX); lastQueryTime = queryTime; processors.forEach(p -> { Collection keys = p.findKnownKeys(changedValues); diff --git a/src/main/java/ch/mno/copper/data/DbHelper.java b/src/main/java/ch/mno/copper/data/DbHelper.java new file mode 100644 index 0000000..091693a --- /dev/null +++ b/src/main/java/ch/mno/copper/data/DbHelper.java @@ -0,0 +1,270 @@ +package ch.mno.copper.data; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * Helper to read-write data to a local H2 Database. The DB is automatically created if not existent. + * Time is from-inclusive, to-exclusive like String.substring. + * Values once stored will never end but could change over time. + * Only one value is allowed at a given instant. + * Insertion could only be done after already inserted values (no insertion in the past). + * + * Created by dutoitc on 25.05.2016. + */ +public class DbHelper { + + private static Logger LOG = LoggerFactory.getLogger(DbHelper.class); + private static final String DBURL="jdbc:h2:./copperdb"; + private static final String DBUSER=""; + private static final String DBPASS=""; + public static final Instant INSTANT_MAX = Instant.parse("3000-12-31T00:00:00.00Z"); + + static { + try { + Class.forName("org.h2.Driver"); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Cannot load H2: " + e.getMessage(), e); + } + createDatabaseIfNeeded(); + } + + private static void createDatabaseIfNeeded() { + try (Connection con = DriverManager.getConnection(DBURL, DBUSER, DBPASS); + Statement stmt = con.createStatement()) + { + // Create table ? + ResultSet rs = stmt.executeQuery("select count(*) as nb from information_schema.tables where table_name = 'VALUESTORE'"); + rs.next(); + if (rs.getInt("nb")==0) { + LOG.info("Database not found. Creating table VALUESTORE..."); + stmt.execute("CREATE TABLE valuestore (" + + " idvaluestore int(11) NOT NULL," + + " key text NOT NULL," + + " value text NOT NULL," + + " datefrom timestamp NOT NULL," + + " dateto timestamp NOT NULL," + + " primary key (idvaluestore))"); + LOG.info("Creating sequence SEQ_VALUESTORE_ID"); + stmt.execute("create sequence SEQ_VALUESTORE_ID start with 1"); + } + } catch (SQLException e2) { + throw new RuntimeException("An error occured while initializing DB: " + e2.getMessage(), e2); + } + } + + + /** Delete all DB data */ + public static void clearAllData() { + String sql = "delete from valuestore"; + try (Connection con = DriverManager.getConnection(DBURL, DBUSER, DBPASS)) + { + PreparedStatement ps = con.prepareStatement(sql); + int nbRows = ps.executeUpdate(); + LOG.info("Deleted " + nbRows + " lines"); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + private static long nextSequence() throws SQLException { + String sqlNextSequence = "select nextval('SEQ_VALUESTORE_ID')"; + + try (Connection con2 = DriverManager.getConnection(DBURL, DBUSER, DBPASS)) + { + // Find next sequence number + ResultSet rs = con2.prepareCall(sqlNextSequence).executeQuery(); + if (!rs.next()) throw new RuntimeException("Sequence error"); + return rs.getLong(1); + } + } + + /** Insert a value at given instant. Actuve value will be finished at the same instant */ + public static void insert(String key, String value, Instant instant) throws SQLException { + String sqlInsert = "INSERT INTO valuestore ( idvaluestore, key, value, datefrom, dateto) VALUES (?,?,?,?,?)"; + String sqlUpdatePrevious = "update valuestore set dateto=? where idvaluestore=?"; + + try (Connection con = DriverManager.getConnection(DBURL, DBUSER, DBPASS); + PreparedStatement stmt = con.prepareStatement(sqlInsert)) + { + StoreValue previousValue = readLatest(key); + + long id = nextSequence(); + stmt.setLong(1, id); + stmt.setString(2, key); + stmt.setString(3, value); + stmt.setTimestamp(4, Timestamp.from(instant)); + stmt.setTimestamp(5, Timestamp.from(INSTANT_MAX)); + int rowInserted = stmt.executeUpdate(); + if (rowInserted!=1) { + throw new RuntimeException("DB error: inserted " + rowInserted + " values."); + } + + // Stop previous + if (previousValue!=null) { + if (previousValue.getTimestampFrom().isAfter(instant)) { + throw new RuntimeException("Cannot insert value in the past for key " + key + ", old.start="+previousValue.getTimestampFrom()+", new.start="+instant); + } + + try (PreparedStatement stmt2 = con.prepareStatement(sqlUpdatePrevious)) + { + stmt2.setTimestamp(1, Timestamp.from(instant)); + stmt2.setLong(2, previousValue.id); + stmt2.executeUpdate(); + } catch (SQLException e) { + throw new RuntimeException("Cannot update previous value: " + e.getMessage(), e); + } + } + } catch (SQLException e) { + throw new RuntimeException("An error occured while saving values", e); + } + } + + /** Read the 'key' value at given instant */ + public static StoreValue read(String key, Instant timestamp) throws SQLException { + String sql = "SELECT idvaluestore, key, value, datefrom, dateto FROM valuestore where key=? and datefrom<=? and dateto>? order by datefrom"; + try (Connection con = DriverManager.getConnection(DBURL, DBUSER, DBPASS); + PreparedStatement stmt = con.prepareStatement(sql)) + { + stmt.setString(1,key); + stmt.setTimestamp(2,Timestamp.from(timestamp)); + stmt.setTimestamp(3,Timestamp.from(timestamp)); + ResultSet rs = stmt.executeQuery(); + + List values = new ArrayList<>(); + while(rs.next()) { + values.add(mapStoreValue(rs)); + } + + if (values.size()==0) { + return null; + } + if (values.size()==1) { + return values.get(0); + } + } catch (SQLException e) { + throw new RuntimeException("An error occured while saving values", e); + } + + + throw new RuntimeException("Too much value for key="+key+", instant="+timestamp.getEpochSecond()); + } + + /** Read all values for a given key active between from, to. (could have been inserted before and finish after) */ + public static List read(String key, Instant timestampFrom, Instant timestampTo) throws SQLException { + String sql = "SELECT idvaluestore, key, value, datefrom, dateto FROM valuestore where key=? and ((datefrom?) or (datefrom>=? and datefrom? and dateto<=?)) order by datefrom"; + try (Connection con = DriverManager.getConnection(DBURL, DBUSER, DBPASS); + PreparedStatement stmt = con.prepareStatement(sql)) + { + stmt.setString(1,key); + stmt.setTimestamp(2,Timestamp.from(timestampFrom)); + stmt.setTimestamp(3,Timestamp.from(timestampTo)); + stmt.setTimestamp(4,Timestamp.from(timestampFrom)); + stmt.setTimestamp(5,Timestamp.from(timestampTo)); + stmt.setTimestamp(6,Timestamp.from(timestampFrom)); + stmt.setTimestamp(7,Timestamp.from(timestampTo)); + ResultSet rs = stmt.executeQuery(); + + List values = new ArrayList<>(); + while(rs.next()) { + values.add(mapStoreValue(rs)); + } + return values; + } catch (SQLException e) { + throw new RuntimeException("An error occured while saving values", e); + } + } + + /** Read the latest value of a key) */ + public static StoreValue readLatest(String key) throws SQLException { + try (Connection con = DriverManager.getConnection(DBURL, DBUSER, DBPASS); + PreparedStatement stmt = con.prepareStatement("SELECT idvaluestore, key, value, datefrom, dateto FROM valuestore where key=? and dateto=?")) + { + stmt.setString(1,key); + stmt.setTimestamp(2, Timestamp.from(INSTANT_MAX)); + ResultSet rs = stmt.executeQuery(); + if (!rs.next()) return null; + return mapStoreValue(rs); + } catch (SQLException e) { + throw new RuntimeException("An error occured while saving values", e); + } + } + + /** Read all latest values */ + public static List readLatest() throws SQLException { + List values = new ArrayList<>(); + try (Connection con = DriverManager.getConnection(DBURL, DBUSER, DBPASS); + PreparedStatement stmt = con.prepareStatement("SELECT idvaluestore, key, value, datefrom, dateto FROM valuestore where dateto=?")) + { + stmt.setTimestamp(1, Timestamp.from(INSTANT_MAX)); + ResultSet rs = stmt.executeQuery(); + while (rs.next()) { + values.add(mapStoreValue(rs)); + } + } catch (SQLException e) { + throw new RuntimeException("An error occured while saving values", e); + } + return values; + } + + /** Read keys updated between from(inclusive) ant to(exclusive) */ + public static Collection readUpdatedKeys(Instant from, Instant to) { + List values = new ArrayList<>(); + try (Connection con = DriverManager.getConnection(DBURL, DBUSER, DBPASS); + PreparedStatement stmt = con.prepareStatement("SELECT distinct key FROM valuestore where datefrom>=? and datefrom getValues() { + try { + List values = DbHelper.readLatest(); + Map map = new HashMap<>(values.size()*4/3+1); + values.forEach(v->map.put(v.getKey(), v)); + return map; + } catch (SQLException e) { + throw new RuntimeException("Cannot read values: " + e.getMessage(), e); + } + } + + @Override + public Collection queryValues(Instant from, Instant to) { + return DbHelper.readUpdatedKeys(from, to); + } + + @Override + public List> queryValues(Instant from, Instant to, String columns) { + return null; + } + + @Override + public void load() throws IOException { + + } + + @Override + public void save() throws IOException { + + } + + @Override + public Map getValuesMapString() { + return null; + } +} diff --git a/src/main/java/ch/mno/copper/data/MemoryValuesStore.java b/src/main/java/ch/mno/copper/data/MemoryValuesStore.java index 622e820..fd9752a 100644 --- a/src/main/java/ch/mno/copper/data/MemoryValuesStore.java +++ b/src/main/java/ch/mno/copper/data/MemoryValuesStore.java @@ -11,9 +11,8 @@ import java.io.InputStreamReader; import java.io.OutputStream; import java.io.PrintWriter; -import java.sql.Timestamp; import java.text.SimpleDateFormat; -import java.time.LocalDateTime; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Date; @@ -55,11 +54,11 @@ public void put(String key, String value) { // commented: always store, as store date = last check date. TODO: implement from... to dates //if (map.get(key)!=null && map.get(key).getValue().equals(value)) return; if (map.get(key)!=null && !map.get(key).getValue().equals(value)) { - map.put(key, new StoreValue(value)); + map.put(key, new StoreValue(-1, key, value, Instant.now(), null)); } return; // Otehrwise same value } - map.put(key, new StoreValue(value)); + map.put(key, new StoreValue(-1, key, value, Instant.now(), null)); // changedValues.add(key); } @@ -77,10 +76,13 @@ public void save(OutputStream os) throws IOException { map.forEach((k,v)->{ StringBuilder sb = new StringBuilder(); sb.append(k); + // TODO: read/write à revoir (stocker toutes les valeurs) sb.append('|'); sb.append(v==null||v.getValue()==null?null:v.getValue().replace("|", "£").replace("\n","¢")); sb.append('|'); - sb.append(v==null?null:v.getTimestamp()); + sb.append(v==null?null:v.getTimestampFrom().getEpochSecond()); + sb.append('|'); + sb.append(v==null?null:v.getTimestampTo().getEpochSecond()); sb.append('\n'); pw.write(sb.toString()); }); @@ -119,11 +121,20 @@ public void load(InputStream is) throws IOException { try { String line = reader.readLine(); int p = line.indexOf('|'); - int p2 = line.lastIndexOf('|'); String key = line.substring(0,p); - String content = line.substring(p+1, p2); - String timestamp = line.substring(p2+1); - map.put(key, new StoreValue(content.replaceAll("£", "|").replace("¢", "\n"), Long.parseLong(timestamp))); + line = line.substring(p+1); + + int p2 = line.lastIndexOf('|'); + String timestampTo = line.substring(p2+1); + line = line.substring(0, p2); + p2 = line.lastIndexOf('|'); + String timestampFrom = line.substring(p2+1); + line = line.substring(0, p2); + + String content = line.replaceAll("£", "|").replace("¢", "\n"); + StoreValue sv = new StoreValue(-1, key, content, Instant.ofEpochSecond(Long.parseLong(timestampFrom)), Instant.ofEpochSecond(Long.parseLong(timestampTo))); + + map.put(key, sv); } catch (Exception e) { throw new RuntimeException("Error at line #" + noLine + ": " + e.getMessage(), e); } @@ -165,11 +176,18 @@ public String getValue(String key) { return null; } - public long getTimestamp(String key) { + public Instant getTimestampFrom(String key) { + if (map.containsKey(key)) { + return map.get(key).getTimestampFrom(); + } + return null; + } + + public Instant getTimestampTo(String key) { if (map.containsKey(key)) { - return map.get(key).getTimestamp(); + return map.get(key).getTimestampTo(); } - return -1; + return null; } /** @@ -199,13 +217,12 @@ public void clear() { } @Override - public Collection queryValues(LocalDateTime from, LocalDateTime to) { + public Collection queryValues(Instant from, Instant to) { List keys = new ArrayList<>(); - long tsFrom = Timestamp.valueOf(from).getTime(); - long tsTo = Timestamp.valueOf(to).getTime(); for (Map.Entry entry: map.entrySet()) { - long entryTS = entry.getValue().getTimestamp(); - if ((entryTS >= tsFrom) && (entryTS <= tsTo || to.equals(LocalDateTime.MAX))) { + if ((from==null || !entry.getValue().getTimestampFrom().isBefore(from)) && + (to==null || !entry.getValue().getTimestampTo().isAfter(to))) + { keys.add(entry.getKey()); } } @@ -213,49 +230,17 @@ public Collection queryValues(LocalDateTime from, LocalDateTime to) { } @Override - public List> queryValues(LocalDateTime from, LocalDateTime to, String columns) { - // Note: this code is temporary, waiting for an internal DB to be queried. Yet, it match my project only, - // with a csv file of format (DATETIME;KEY1;KEY2;...;KEYN\ndd.MM.yyyy HH:mm:ss;value1;value2;...;valueN\n... - /*List> data = new ArrayList>(); - try (BufferedReader br = Files.newBufferedReader(Paths.get("RCFACE-data.csv"))) { - String header = br.readLine(); - - // Find wanted columns - List userColumns = Arrays.asList(columns.split(",")); - List wantedColumns = new ArrayList<>(); - int noCol=0; - for (String column: header.split(";")) { - if (userColumns.contains(column)) { - wantedColumns.add(noCol); - } - noCol++; - } - - // Check if all columns have been found - if (userColumns.size()!=wantedColumns.size()) { - throw new RuntimeException("Not found all columns: found [" + StringUtils.join(wantedColumns, ",") + "] but wanted [" + StringUtils.join(userColumns, ",")+"]"); - } - - // Extract data - String line; - while ((line=br.readLine())!=null) { - String[] csvColumns = line.split(";"); - LocalDateTime csvDT = LocalDateTime.parse(csvColumns[0], DateTimeFormatter.ofPattern("dd.MM.yyyy HH:mm:ss")); - if (from!=null && from.isAfter(csvDT)) continue; - if (to!=null && to.isBefore(csvDT)) continue; - - List row = new ArrayList<>(); - row.add(csvColumns[0]); - for (int idx: wantedColumns) { - row.add(csvColumns[idx]); - } - data.add(row); - } - } catch (IOException e) { - throw new RuntimeException(e); + public List> queryValues(Instant from, Instant to, String columns) { + Collection keys = queryValues(from, to); + List> values = new ArrayList<>(); + for (String key: keys) { + List line = new ArrayList<>(); + StoreValue sv = map.get(key); + line.add(sv.getKey()); + line.add(sv.getValue()); } - return data;*/ - throw new RuntimeException("Not implemented for file valuesStore"); + + return values; } public static void main(String[] args) throws IOException { diff --git a/src/main/java/ch/mno/copper/data/StoreValue.java b/src/main/java/ch/mno/copper/data/StoreValue.java index eb81989..97753c7 100644 --- a/src/main/java/ch/mno/copper/data/StoreValue.java +++ b/src/main/java/ch/mno/copper/data/StoreValue.java @@ -1,35 +1,53 @@ package ch.mno.copper.data; +import java.time.Instant; + /** * Created by xsicdt on 25/08/17. */ public class StoreValue { + + protected long id; + protected String key; protected String value; - protected long timestamp; + protected Instant timestampFrom; + protected Instant timestampTo; public String getValue() { return value; } - public StoreValue(String value) { - this.value = value; - this.timestamp = System.currentTimeMillis(); + public long getId() { + return id; } - StoreValue(String value, long timestamp) { + public String getKey() { + return key; + } + + public Instant getTimestampFrom() { + return timestampFrom; + } + + public Instant getTimestampTo() { + return timestampTo; + } + + public StoreValue(long id, String key, String value, Instant timestampFrom, Instant timestampTo) { + this.id = id; + this.key = key; this.value = value; - this.timestamp = timestamp; + this.timestampFrom = timestampFrom; + this.timestampTo = timestampTo; } @Override public String toString() { return "StoreValue{" + - "value='" + value + '\'' + - ", timestamp=" + timestamp + + "key='" + key + '\'' + + ", value='" + value + '\'' + + ", timestampFrom=" + timestampFrom + + ", timestampTo=" + timestampTo + '}'; } - - public long getTimestamp() { - return timestamp; - } } diff --git a/src/main/java/ch/mno/copper/data/ValuesStore.java b/src/main/java/ch/mno/copper/data/ValuesStore.java index d775a5d..36aefbd 100644 --- a/src/main/java/ch/mno/copper/data/ValuesStore.java +++ b/src/main/java/ch/mno/copper/data/ValuesStore.java @@ -1,7 +1,7 @@ package ch.mno.copper.data; import java.io.IOException; -import java.time.LocalDateTime; +import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Map; @@ -17,9 +17,15 @@ public interface ValuesStore { Map getValues(); - Collection queryValues(LocalDateTime from, LocalDateTime to); + /** + * + * @param from + * @param to null means no boundaries + * @return + */ + Collection queryValues(Instant from, Instant to); - List> queryValues(LocalDateTime from, LocalDateTime to, String columns); + List> queryValues(Instant from, Instant to, String columns); void load() throws IOException; diff --git a/src/main/java/ch/mno/copper/web/CopperServices.java b/src/main/java/ch/mno/copper/web/CopperServices.java index 28bf252..a2b955f 100644 --- a/src/main/java/ch/mno/copper/web/CopperServices.java +++ b/src/main/java/ch/mno/copper/web/CopperServices.java @@ -1,9 +1,9 @@ package ch.mno.copper.web; import ch.mno.copper.CopperMediator; +import ch.mno.copper.collect.connectors.ConnectorException; import ch.mno.copper.data.StoreValue; import ch.mno.copper.data.ValuesStore; -import ch.mno.copper.collect.connectors.ConnectorException; import ch.mno.copper.helpers.SyntaxException; import ch.mno.copper.stories.StoriesFacade; import ch.mno.copper.stories.Story; @@ -14,13 +14,21 @@ import com.google.gson.stream.JsonWriter; import it.sauronsoftware.cron4j.Predictor; -import javax.ws.rs.*; +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.io.IOException; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; +import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.ArrayList; @@ -92,12 +100,12 @@ public String getValues() { @Produces(MediaType.APPLICATION_JSON) public Response getValues(@QueryParam("from") String dateFrom, @QueryParam("to") String dateTo, @QueryParam("columns") String columns) { Gson gson = new Gson(); - LocalDateTime from = toDate(dateFrom, true); - LocalDateTime to; + Instant from = toInstant(dateFrom, true); + Instant to; if (dateTo==null) { - to = LocalDateTime.now(); + to = Instant.now(); } else { - to =toDate(dateTo, false); + to =toInstant(dateTo, false); } try { return Response.ok(gson.toJson(valuesStore.queryValues(from, to, columns)), MediaType.APPLICATION_JSON).build(); @@ -106,7 +114,7 @@ public Response getValues(@QueryParam("from") String dateFrom, @QueryParam("to") } } - private LocalDateTime toDate(String date, boolean am) { + private Instant toInstant(String date, boolean am) { if (date==null || "null".equals(date)) return null; String[] formats = new String[] {"dd.MM.yyyy", "yyyy-MM-dd"}; @@ -114,8 +122,8 @@ private LocalDateTime toDate(String date, boolean am) { try { // System.out.println("Parsing '"+date+"' with '"+format+"'"); LocalDate ld = LocalDate.parse(date, DateTimeFormatter.ofPattern(format)); - if (am) return LocalDateTime.of(ld, LocalTime.of(0, 0)); - return LocalDateTime.of(ld, LocalTime.of(23, 59, 59)); + if (am) return LocalDateTime.of(ld, LocalTime.of(0, 0)).toInstant(ZoneOffset.UTC); + return LocalDateTime.of(ld, LocalTime.of(23, 59, 59)).toInstant(ZoneOffset.UTC); } catch (DateTimeParseException e) { } } @@ -125,7 +133,7 @@ private LocalDateTime toDate(String date, boolean am) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern(format); try { //return (LocalDateTime)formatter.parse(date); - return LocalDateTime.parse(date, formatter); + return LocalDateTime.parse(date, formatter).toInstant(ZoneOffset.UTC); } catch (DateTimeParseException e) { } diff --git a/src/test/java/ch/mno/copper/MemoryValuesStoreTest.java b/src/test/java/ch/mno/copper/MemoryValuesStoreTest.java index 5e5256f..632fe14 100644 --- a/src/test/java/ch/mno/copper/MemoryValuesStoreTest.java +++ b/src/test/java/ch/mno/copper/MemoryValuesStoreTest.java @@ -7,7 +7,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.time.LocalDateTime; +import java.time.Instant; import java.util.Arrays; import java.util.Collection; import java.util.Map; @@ -21,21 +21,21 @@ public class MemoryValuesStoreTest { public void testX() throws InterruptedException { MemoryValuesStore st = MemoryValuesStore.getInstance(); st.clear(); - LocalDateTime t = LocalDateTime.now(); + Instant t = Instant.now(); st.put("key1", "value1"); Thread.sleep(1); - Assert.assertEquals(1, st.queryValues(t, LocalDateTime.MAX).size()); - t = LocalDateTime.now(); - Collection strings = st.queryValues(t, LocalDateTime.MAX); + Assert.assertEquals(1, st.queryValues(t, Instant.MAX).size()); + t = Instant.now(); + Collection strings = st.queryValues(t, Instant.MAX); Assert.assertEquals(0, strings.size()); - Assert.assertTrue(System.currentTimeMillis()-st.getTimestamp("key1")<1000); + Assert.assertTrue(System.currentTimeMillis()-st.getTimestampFrom("key1").toEpochMilli()<1000); st.put("key1", "value2"); - Assert.assertEquals(1, st.queryValues(t, LocalDateTime.MAX).size()); + Assert.assertEquals(1, st.queryValues(t, Instant.MAX).size()); Assert.assertEquals("value2", st.getValue("key1")); Thread.sleep(1); - t = LocalDateTime.now(); + t = Instant.now(); st.put("key1", "value2"); - Assert.assertEquals(0, st.queryValues(t, LocalDateTime.MAX).size()); // Same values + Assert.assertEquals(0, st.queryValues(t, Instant.MAX).size()); // Same values } @Test @@ -44,25 +44,25 @@ public void testTwice() throws InterruptedException { st.clear(); st.put("key1", "value2"); - long s1 = st.getTimestamp("key1"); + long s1 = st.getTimestampFrom("key1").toEpochMilli(); Thread.sleep(10); st.put("key1", "value2"); - long s2 = st.getTimestamp("key1"); + long s2 = st.getTimestampFrom("key1").toEpochMilli(); Assert.assertEquals(s1, s2); } @Test public void testEmpty() { Assert.assertNull(MemoryValuesStore.getInstance().getValue("none")); - Assert.assertEquals(-1, MemoryValuesStore.getInstance().getTimestamp("none")); +// Assert.assertEquals(-1, MemoryValuesStore.getInstance().getTimestamp("none")); } @Test public void testPutAll() { MemoryValuesStore st = MemoryValuesStore.getInstance(); - LocalDateTime t = LocalDateTime.now(); + Instant t = Instant.now(); st.putAll("key1,key2,key3", Arrays.asList("v1", "v2", "v3")); - Assert.assertEquals(3, st.queryValues(t, LocalDateTime.MAX).size()); + Assert.assertEquals(3, st.queryValues(t, Instant.MAX).size()); Assert.assertEquals("v1", st.getValue("key1")); Assert.assertEquals("v2", st.getValue("key2")); Assert.assertEquals("v3", st.getValue("key3")); @@ -74,7 +74,7 @@ public void testPutAll() { @Test public void testSerialization() throws IOException { - LocalDateTime t = LocalDateTime.now(); + Instant t = Instant.now(); MemoryValuesStore st = new MemoryValuesStore(); st.putAll("key1,key2,key3", Arrays.asList("v1", "value|222", "v;3")); ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -87,7 +87,7 @@ public void testSerialization() throws IOException { Assert.assertEquals("v1", st2.getValue("key1")); Assert.assertEquals("value|222", st2.getValue("key2")); Assert.assertEquals("v;3", st2.getValue("key3")); - Assert.assertEquals(3, st2.queryValues(t, LocalDateTime.MAX).size()); + Assert.assertEquals(3, st2.queryValues(t, Instant.MAX).size()); } } diff --git a/src/test/java/ch/mno/copper/data/DbHelperTest.java b/src/test/java/ch/mno/copper/data/DbHelperTest.java new file mode 100644 index 0000000..9663db3 --- /dev/null +++ b/src/test/java/ch/mno/copper/data/DbHelperTest.java @@ -0,0 +1,254 @@ +package ch.mno.copper.data; + +import org.apache.commons.lang3.StringUtils; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import java.sql.SQLException; +import java.time.Instant; +import java.util.List; + +/** + * Created by dutoitc on 20.09.2017. + */ +public class DbHelperTest { + + Instant i3 = Instant.parse("2015-10-21T07:27:48.00Z"); + Instant i4 = Instant.parse("2015-10-21T07:27:49.00Z"); + Instant i5 = Instant.parse("2015-10-21T07:28:00.00Z"); + Instant i6 = Instant.parse("2015-10-21T07:28:01.00Z"); + Instant i7 = Instant.parse("2015-10-21T07:28:02.00Z"); + Instant i8 = Instant.parse("2015-10-21T07:28:03.00Z"); + Instant i9 = Instant.parse("2045-10-21T07:28:00.00Z"); + + @Before + public void init() throws SQLException { + DbHelper.clearAllData(); + DbHelper.insert("key1", "value10", i5); + DbHelper.insert("key2", "value20", i5); + DbHelper.insert("key3", "value30", i4); + DbHelper.insert("key3", "value31", i7); + DbHelper.insert("key4", "value40", i4); + DbHelper.insert("key4", "value41", i5); + DbHelper.insert("key4", "value42", i6); + } + + + @Test + public void testReadLatestForOneValue() throws SQLException { + StoreValue readSV = DbHelper.readLatest("key1"); + Assert.assertEquals("value10", readSV.getValue()); + } + + + @Test + public void testReadLatestForTwoValues() throws SQLException { + StoreValue readSV = DbHelper.readLatest("key3"); + Assert.assertEquals("value31", readSV.getValue()); + } + + @Test + public void testReadLatestForThreeValues() throws SQLException { + StoreValue readSV = DbHelper.readLatest("key4"); + Assert.assertEquals("value42", readSV.getValue()); + } + + + @Test + public void testReadAtTimestampForOneValue() throws SQLException { + Assert.assertNull(DbHelper.read("key1", i3)); + Assert.assertNull(DbHelper.read("key1", i4)); + Assert.assertEquals("value10", DbHelper.read("key1", i5).getValue()); + Assert.assertEquals("value10", DbHelper.read("key1", i6).getValue()); + Assert.assertEquals("value10", DbHelper.read("key1", i7).getValue()); + Assert.assertEquals("value10", DbHelper.read("key1", i8).getValue()); + Assert.assertEquals("value10", DbHelper.read("key1", i9).getValue()); + } + + @Test + public void testReadAtTimestampForTwoValues() throws SQLException { + Assert.assertNull(DbHelper.read("key3", i3)); + Assert.assertEquals("value30", DbHelper.read("key3", i4).getValue()); + Assert.assertEquals("value30", DbHelper.read("key3", i5).getValue()); + Assert.assertEquals("value30", DbHelper.read("key3", i6).getValue()); + Assert.assertEquals("value31", DbHelper.read("key3", i7).getValue()); + Assert.assertEquals("value31", DbHelper.read("key3", i8).getValue()); + Assert.assertEquals("value31", DbHelper.read("key3", i9).getValue()); + } + + @Test + public void testReadAtTimestampForThreeValues() throws SQLException { + Assert.assertNull(DbHelper.read("key4", i3)); + Assert.assertEquals("value40", DbHelper.read("key4", i4).getValue()); + Assert.assertEquals("value41", DbHelper.read("key4", i5).getValue()); + Assert.assertEquals("value42", DbHelper.read("key4", i6).getValue()); + Assert.assertEquals("value42", DbHelper.read("key4", i7).getValue()); + Assert.assertEquals("value42", DbHelper.read("key4", i8).getValue()); + Assert.assertEquals("value42", DbHelper.read("key4", i9).getValue()); + } + + @Test + public void testReadHistorizedForOneValue() throws SQLException { + Assert.assertTrue(DbHelper.read("key1", i3, i4).isEmpty()); + Assert.assertTrue(DbHelper.read("key1", i4, i5).isEmpty()); + assertOneValue(DbHelper.read("key1", i5, i6), "value10"); + assertOneValue(DbHelper.read("key1", i6, i7), "value10"); + assertOneValue(DbHelper.read("key1", i5, i7), "value10"); + assertOneValue(DbHelper.read("key1", i4, i7), "value10"); + } + + @Test + public void testReadHistorizedForTwoValues() throws SQLException { + Assert.assertTrue(DbHelper.read("key3", i3, i4).isEmpty()); + assertOneValue(DbHelper.read("key3", i4, i5), "value30"); + assertOneValue(DbHelper.read("key3", i5, i6), "value30"); + assertOneValue(DbHelper.read("key3", i6, i7), "value30"); + assertOneValue(DbHelper.read("key3", i4, i7), "value30"); + assertOneValue(DbHelper.read("key3", i3, i7), "value30"); + assertOneValue(DbHelper.read("key3", i7, i8), "value31"); + assertOneValue(DbHelper.read("key3", i8, i9), "value31"); + assertTwoValues(DbHelper.read("key3", i3, i8), "value30", "value31"); + } + + @Test + public void testReadUpdatedKeys() throws SQLException { + Assert.assertEquals("", StringUtils.join(DbHelper.readUpdatedKeys(i3, i4), ',')); + Assert.assertEquals("key3,key4", StringUtils.join(DbHelper.readUpdatedKeys(i4, i5), ',')); + Assert.assertEquals("key1,key2,key4", StringUtils.join(DbHelper.readUpdatedKeys(i5, i6), ',')); + Assert.assertEquals("key4", StringUtils.join(DbHelper.readUpdatedKeys(i6, i7), ',')); + Assert.assertEquals("key3", StringUtils.join(DbHelper.readUpdatedKeys(i7, i8), ',')); + Assert.assertEquals("", StringUtils.join(DbHelper.readUpdatedKeys(i8, i9), ',')); + Assert.assertEquals("key1,key2,key3,key4", StringUtils.join(DbHelper.readUpdatedKeys(i4, i7), ',')); + } + + @Test + public void testReadLatest() throws SQLException { + StringBuilder sb = new StringBuilder(); + DbHelper.readLatest().forEach(v -> sb.append(v.getKey()).append(':').append(v.getValue()).append(';')); + Assert.assertEquals("key1:value10;key2:value20;key3:value31;key4:value42;", sb.toString()); + } + + // 1000->17s/23ms/22ms + // 10000->428s/60ms/58ms + @Test + @Ignore + public void testPerformance() throws SQLException { + DbHelper.clearAllData(); + long t0 = System.currentTimeMillis(); + for (int i = 0; i < 10000; i++) { + DbHelper.insert("keyPerf", "value" + i, Instant.now()); + } + System.out.println("Insertion took " + (System.currentTimeMillis() - t0) / 1000 + "s"); + + t0 = System.currentTimeMillis(); + DbHelper.readLatest("keyPerf"); + System.out.println("Read latest by key took " + (System.currentTimeMillis() - t0) + "ms"); + + t0 = System.currentTimeMillis(); + DbHelper.readLatest(); + System.out.println("Read latest took " + (System.currentTimeMillis() - t0) + "ms"); + } + + + private void assertOneValue(List values, String value) { + Assert.assertNotNull(values); + Assert.assertEquals(1, values.size()); + Assert.assertEquals(value, values.get(0).getValue()); + } + + private void assertTwoValues(List values, String value1, String value2) { + Assert.assertNotNull(values); + Assert.assertEquals(2, values.size()); + Assert.assertEquals(value1, values.get(0).getValue()); + Assert.assertEquals(value2, values.get(1).getValue()); + } + + + +/* + @Test + public void testAll() throws SQLException { + DbHelper.clearAllData(); + + // Create + Instant i0 = Instant.ofEpochSecond(Instant.now().getEpochSecond()); // Get rid of millis + DbHelper.insert("key1", "value10", i0); + DbHelper.insert("key2", "value20", i0); + DbHelper.insert("key3", "value30", i0); + + // Read simple values +// DbHelper.dumpForTests(); + StoreValue readSV = DbHelper.readLatest("key1"); + Assert.assertEquals("value10", readSV.getValue()); + readSV = DbHelper.readLatest("key2"); + Assert.assertEquals("value20", readSV.getValue()); + readSV = DbHelper.readLatest("key3"); + Assert.assertEquals("value30", readSV.getValue()); + Assert.assertEquals(i0, readSV.getTimestampFrom()); + Assert.assertEquals(i9, readSV.getTimestampTo()); + +// DbHelper.dumpForTests(); + + // Read in the past + readSV = DbHelper.read("key1", Instant.ofEpochSecond(i0.getEpochSecond()-1)); + Assert.assertNull(readSV); + + // Read in the present + readSV = DbHelper.read("key1", i0); + Assert.assertEquals("value10", readSV.getValue()); + + // Read in the future + readSV = DbHelper.read("key1", Instant.ofEpochSecond(i0.getEpochSecond()+1)); + Assert.assertEquals("value10", readSV.getValue()); + + // Insert + Instant i2 = Instant.ofEpochSecond(i0.getEpochSecond() + 2); + DbHelper.insert("key1", "value11", i2); + + // Read latest +// DbHelper.dumpForTests(); + readSV = DbHelper.readLatest("key1"); + Assert.assertEquals("value11", readSV.getValue()); + + // Read all + List list = DbHelper.read("key1", Instant.MIN, Instant.MAX); + Assert.assertEquals(2, list.size()); + Assert.assertEquals(i0, list.get(0).getTimestampFrom()); + Assert.assertEquals(i2, list.get(0).getTimestampTo()); + Assert.assertEquals(i2, list.get(1).getTimestampFrom()); + Assert.assertEquals(i9, list.get(1).getTimestampTo()); + Assert.assertEquals(3, DbHelper.readLatest().size()); + + // Read by time interval + Assert.assertEquals(0, DbHelper.read("key1", Instant.ofEpochSecond(i0.getEpochSecond()-1), i0).size()); + Assert.assertEquals(1, DbHelper.read("key1", i0, Instant.ofEpochSecond(i0.getEpochSecond()+1)).size()); + Assert.assertEquals(1, DbHelper.read("key1", i0, Instant.ofEpochSecond(i0.getEpochSecond()+2)).size()); + Assert.assertEquals(2, DbHelper.read("key1", i0, Instant.ofEpochSecond(i0.getEpochSecond()+3)).size()); + Assert.assertEquals(2, DbHelper.read("key1", i0, Instant.ofEpochSecond(i0.getEpochSecond()+3)).size()); + Assert.assertEquals(2, DbHelper.read("key1", Instant.ofEpochSecond(i0.getEpochSecond()+1), Instant.ofEpochSecond(i0.getEpochSecond()+3)).size()); + Assert.assertEquals(1, DbHelper.read("key1", Instant.ofEpochSecond(i0.getEpochSecond()+2), Instant.ofEpochSecond(i0.getEpochSecond()+3)).size()); + + // Read changed keys + DbHelper.dumpForTests(); + Assert.assertEquals(3, DbHelper.readUpdatedKeys(i0,i2).size()); + Assert.assertEquals(1, DbHelper.readUpdatedKeys(i2,i9).size()); + + // Read at some time + Assert.assertNull(DbHelper.read("key1", Instant.ofEpochSecond(i0.getEpochSecond()-1))); + Assert.assertEquals("value10", DbHelper.read("key1", Instant.ofEpochSecond(i0.getEpochSecond())).getValue()); + Assert.assertEquals("value10", DbHelper.read("key1", Instant.ofEpochSecond(i0.getEpochSecond()+1)).getValue()); + Assert.assertEquals("value11", DbHelper.read("key1", Instant.ofEpochSecond(i0.getEpochSecond()+2)).getValue()); + + // Insert impossible (in the past) + try { + DbHelper.insert("key1", "valueNot", Instant.ofEpochSecond(i0.getEpochSecond() + 1)); + Assert.fail("Insertion in the past must be impossible"); + } catch (Exception e) { + // Pass + } + } + */ + +} diff --git a/src/test/java/ch/mno/copper/data/DbValueStoreTest.java b/src/test/java/ch/mno/copper/data/DbValueStoreTest.java new file mode 100644 index 0000000..cc35b89 --- /dev/null +++ b/src/test/java/ch/mno/copper/data/DbValueStoreTest.java @@ -0,0 +1,49 @@ +package ch.mno.copper.data; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.SQLException; +import java.time.Instant; + +/** + * Created by dutoitc on 29.09.2017. + */ +public class DbValueStoreTest { + + Instant i3 = Instant.parse("2015-10-21T07:27:48.00Z"); + Instant i4 = Instant.parse("2015-10-21T07:27:49.00Z"); + Instant i5 = Instant.parse("2015-10-21T07:28:00.00Z"); + Instant i6 = Instant.parse("2015-10-21T07:28:01.00Z"); + Instant i7 = Instant.parse("2015-10-21T07:28:02.00Z"); + Instant i8 = Instant.parse("2015-10-21T07:28:03.00Z"); + Instant i9 = Instant.parse("2045-10-21T07:28:00.00Z"); + + @Before + public void init() throws SQLException { + DbHelper.clearAllData(); + DbHelper.insert("key1", "value10", i5); + DbHelper.insert("key2", "value20", i5); + DbHelper.insert("key3", "value30", i4); + DbHelper.insert("key3", "value31", i7); + DbHelper.insert("key4", "value40", i4); + DbHelper.insert("key4", "value41", i5); + DbHelper.insert("key4", "value42", i6); + } + + + @Test + public void testGetValue() { + DbValuesStore store = DbValuesStore.getInstance(); + Assert.assertEquals("value42", store.getValue("key4")); + } + + @Test + public void testGetValues() { + DbValuesStore store = DbValuesStore.getInstance(); + Assert.assertEquals(4, store.getValues().size()); + } + + +}