Deprecated old MemoryValuesStore, replaced by DbValuesStore

Added story error on html
Added values filtering

CHANGELOG

@@ -2,6 +2,9 @@ Version v1.2.0
 ==============
 New:
 - Added database store in replacement of memory datastore, with data historization and querying
+- Added story error on web story page
+- Added values filtering on web values page
+- Added an history screen with table and graph for values, multicolumns
 
 Version v1.1.0
 ==============

src/main/java/ch/mno/copper/CopperDaemon.java

@@ -100,7 +100,7 @@ private void runIteration() {
         }
 
         // Processors
-        LocalDateTime queryTime = LocalDateTime.now(); // Keep time, so that next run will have data between query time assignation and valueStore read time
+        LocalDateTime queryTime = LocalDateTime.now(); // Keep time, so that next run will have data between query time assignation and valueStore readInstant time
         Collection<String> changedValues = valuesStore.queryValues(lastQueryTime.toInstant(ZoneOffset.UTC), Instant.MAX);
         lastQueryTime = queryTime;
         processors.forEach(p -> {

src/main/java/ch/mno/copper/collect/JdbcCollectorWrapper.java

@@ -98,7 +98,7 @@ public static JdbcCollectorWrapper buildCollector(StoryGrammar grammar, String s
             String query = matcher2.group(4);
             return new JdbcCollectorWrapper(url, username, password, query);
         } else {
-            throw new RuntimeException("Cannot read COLLECTOR_JDBC body in <" + collectorJdbcData + ">");
+            throw new RuntimeException("Cannot readInstant COLLECTOR_JDBC body in <" + collectorJdbcData + ">");
         }
     }
 

src/main/java/ch/mno/copper/collect/JmxCollectorWrapper.java

@@ -84,7 +84,7 @@ public static JmxCollectorWrapper buildCollector(StoryGrammar grammar, String st
             }
             return new JmxCollectorWrapper(url, username, password, jmxQueries, names);
         } else {
-            throw new RuntimeException("Cannot read COLLECTOR_JMX body in <" + collectorJmxData + ">");
+            throw new RuntimeException("Cannot readInstant COLLECTOR_JMX body in <" + collectorJmxData + ">");
         }
     }
 

src/main/java/ch/mno/copper/collect/WebCollectorWrapper.java

@@ -81,7 +81,7 @@ public static WebCollectorWrapper buildCollector(StoryGrammar grammar, String st
                 url = matcher2.group(1);
                 queries = matcher2.group(2);
             } else {
-                throw new RuntimeException("Cannot read COLLECTOR_WEB body in <" + collectorWebData + ">");
+                throw new RuntimeException("Cannot readInstant COLLECTOR_WEB body in <" + collectorWebData + ">");
             }
         }
 

src/main/java/ch/mno/copper/data/DbHelper.java

@@ -16,7 +16,7 @@
 import java.util.List;
 
 /**
- * Helper to read-write data to a local H2 Database. The DB is automatically created if not existent.
+ * Helper to readInstant-write data to a local H2 Database. The DB is automatically created if not existent.
  * Time is from-inclusive, to-exclusive like String.substring.
  * Values once stored will never end but could change over time.
  * Only one value is allowed at a given instant.
@@ -52,7 +52,7 @@ private static void createDatabaseIfNeeded() {
                 LOG.info("Database not found. Creating table VALUESTORE...");
                 stmt.execute("CREATE TABLE valuestore (" +
                         "  idvaluestore int(11) NOT NULL," +
-                        "  key text NOT NULL," +
+                        "  key varchar(50) NOT NULL," +
                         "  value text NOT NULL," +
                         "  datefrom timestamp NOT NULL," +
                         "  dateto timestamp NOT NULL," +
@@ -91,15 +91,27 @@ private static long nextSequence() throws SQLException {
         }
     }
 
-    /** Insert a value at given instant. Actuve value will be finished at the same instant */
+    /** Insert a value at given instant. Actuve value will be finished at the same instant. no data is inserted if current value is the same value */
     public static void insert(String key, String value, Instant instant) throws SQLException {
         String sqlInsert = "INSERT INTO valuestore ( idvaluestore, key, value, datefrom, dateto) VALUES (?,?,?,?,?)";
         String sqlUpdatePrevious = "update valuestore set dateto=? where idvaluestore=?";
 
         try (Connection con = DriverManager.getConnection(DBURL, DBUSER, DBPASS);
              PreparedStatement stmt = con.prepareStatement(sqlInsert))
         {
+            // no data is inserted if current value is the same value
             StoreValue previousValue = readLatest(key);
+            if (previousValue!=null) {
+                if (previousValue.getValue() == null) {
+                    if (value == null) {
+                        return; // No update
+                    }
+                } else {
+                    if (previousValue.getValue().equals(value)) {
+                        return; // No update
+                    }
+                }
+            }
 
             long id = nextSequence();
             stmt.setLong(1, id);
@@ -162,8 +174,57 @@ public static StoreValue read(String key, Instant timestamp) throws SQLException
         throw new RuntimeException("Too much value for key="+key+", instant="+timestamp.getEpochSecond());
     }
 
+    public static List<InstantValues> readInstant(List<String> keys, Instant timestampFrom, Instant timestampTo, long intervalSeconds) throws SQLException {
+        String sql = "select ts,c1, value, idValueStore from ( " +
+                "select ts, c1 from ( " +
+                "WITH RECURSIVE T(ts) AS ( " +
+                "    SELECT ? " +
+                "    UNION ALL " +
+                "    SELECT dateadd('second', ?, ts) FROM T WHERE ts<? " +
+                ") " +
+                "SELECT * FROM T) t,  (values (XXX)) " +
+                ") left outer join valuestore vs on ts>=vs.datefrom and ts<vs.dateto and key =c1 " +
+                "order by ts,key";
+        String s = "?";
+        for (int i=1; i<keys.size(); i++) {
+            s+="),(?";
+        }
+        sql = sql.replace("XXX", s);
+        try (Connection con = DriverManager.getConnection(DBURL, DBUSER, DBPASS);
+             PreparedStatement stmt = con.prepareStatement(sql))
+        {
+            stmt.setTimestamp(1, Timestamp.from(timestampFrom));
+            stmt.setLong(2, intervalSeconds);
+            stmt.setTimestamp(3, Timestamp.from(timestampTo));
+            for (int i=0; i<keys.size(); i++) {
+                stmt.setString(4+i, keys.get(i));
+            }
+            ResultSet rs = stmt.executeQuery();
+
+            List<InstantValues> values = new ArrayList<>();
+            InstantValues last = null;
+            while(rs.next()) {
+                InstantValue instantValue = mapInstantValue(rs);
+                if (last==null || !instantValue.getTimestamp().equals(last.timestamp)) {
+                    last = new InstantValues();
+                    last.setTimestamp(instantValue.getTimestamp());
+                    values.add(last);
+                }
+                last.put(instantValue.getKey(), instantValue);
+            }
+            return values;
+        } catch (SQLException e) {
+            throw new RuntimeException("An error occured while saving values", e);
+        }
+    }
+
+
+
     /** Read all values for a given key active between from, to. (could have been inserted before and finish after) */
     public static List<StoreValue> read(String key, Instant timestampFrom, Instant timestampTo) throws SQLException {
+        if (timestampTo.isAfter(Instant.now())) {
+            timestampTo = Instant.now();
+        }
         String sql = "SELECT idvaluestore, key, value, datefrom, dateto FROM valuestore where key=? and ((datefrom<? and dateto>?) or (datefrom>=? and datefrom<?) or (dateto>? and dateto<=?)) order by datefrom";
         try (Connection con = DriverManager.getConnection(DBURL, DBUSER, DBPASS);
              PreparedStatement stmt = con.prepareStatement(sql))
@@ -246,6 +307,16 @@ private static StoreValue mapStoreValue(ResultSet rs) throws SQLException {
         return new StoreValue(idValueStore, dbKey, value, from, to);
     }
 
+    private static InstantValue mapInstantValue(ResultSet rs) throws SQLException {
+        long idValueStore = rs.getLong("idValueStore");
+        if (idValueStore==0) idValueStore=-1; // TODO: check if id is null, not zero
+        String dbKey = rs.getString("c1");
+        String value = rs.getString("value");
+        if (value==null) value="";
+        Instant ts = rs.getTimestamp("ts").toInstant();
+        return new InstantValue(idValueStore, dbKey, value, ts);
+    }
+
 
 //    public static void dumpForTests() {
 //        try (Connection con = DriverManager.getConnection(DBURL, DBUSER, DBPASS);

src/main/java/ch/mno/copper/data/DbValuesStore.java

@@ -3,6 +3,7 @@
 import java.io.IOException;
 import java.sql.SQLException;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
@@ -16,11 +17,13 @@ public class DbValuesStore implements ValuesStore {
     private static DbValuesStore instance;
 
 
-    /** Singleton factory */
+    /**
+     * Singleton factory
+     */
     public static DbValuesStore getInstance() {
-        if (instance==null) {
+        if (instance == null) {
             synchronized (DbValuesStore.class) {
-                if (instance==null) {
+                if (instance == null) {
                     instance = new DbValuesStore();
                 }
             }
@@ -44,9 +47,9 @@ public String getValue(String key) {
         try {
             storeValue = DbHelper.readLatest(key);
         } catch (SQLException e) {
-            throw new RuntimeException("Cannot read value " + key + ": " + e.getMessage());
+            throw new RuntimeException("Cannot readInstant value " + key + ": " + e.getMessage());
         }
-        if (storeValue==null) {
+        if (storeValue == null) {
             return ""; // no data
         }
         return storeValue.getValue();
@@ -56,11 +59,11 @@ public String getValue(String key) {
     public Map<String, StoreValue> getValues() {
         try {
             List<StoreValue> values = DbHelper.readLatest();
-            Map<String, StoreValue> map = new HashMap<>(values.size()*4/3+1);
-            values.forEach(v->map.put(v.getKey(), v));
+            Map<String, StoreValue> map = new HashMap<>(values.size() * 4 / 3 + 1);
+            values.forEach(v -> map.put(v.getKey(), v));
             return map;
         } catch (SQLException e) {
-            throw new RuntimeException("Cannot read values: " + e.getMessage(), e);
+            throw new RuntimeException("Cannot readInstant values: " + e.getMessage(), e);
         }
     }
 
@@ -70,8 +73,25 @@ public Collection<String> queryValues(Instant from, Instant to) {
     }
 
     @Override
-    public List<List<String>> queryValues(Instant from, Instant to, String columns) {
-        return null;
+    public List<StoreValue> queryValues(Instant from, Instant to, List<String> columns) {
+        List<StoreValue> values = new ArrayList<>();
+        for (String key : columns) {
+            try {
+                values.addAll(DbHelper.read(key, from, to));
+            } catch (SQLException e) {
+                throw new RuntimeException(e.getMessage(), e);
+            }
+        }
+        return values;
+    }
+
+    @Override
+    public List<InstantValues> queryValues(Instant from, Instant to, long intervalSecond, List<String> columns) {
+        try {
+            return DbHelper.readInstant(columns, from, to, intervalSecond);
+        } catch (SQLException e) {
+            throw new RuntimeException(e.getMessage(), e);
+        }
     }
 
     @Override

src/main/java/ch/mno/copper/data/MemoryValuesStore.java

@@ -76,7 +76,7 @@ public void save(OutputStream os) throws IOException {
         map.forEach((k, v) -> {
             StringBuilder sb = new StringBuilder();
             sb.append(k);
-            // TODO: read/write à revoir (stocker toutes les valeurs)
+            // TODO: readInstant/write à revoir (stocker toutes les valeurs)
             sb.append('|');
             sb.append(v == null || v.getValue() == null ? null : v.getValue().replace("|", "£").replace("\n", "¢"));
             sb.append('|');
@@ -244,19 +244,22 @@ public Collection<String> queryValues(Instant from, Instant to) {
     }
 
     @Override
-    public List<List<String>> queryValues(Instant from, Instant to, String columns) {
+    public List<StoreValue> queryValues(Instant from, Instant to, List<String> columns) {
         Collection<String> keys = queryValues(from, to);
-        List<List<String>> values = new ArrayList<>();
+        List<StoreValue> values = new ArrayList<>();
         for (String key : keys) {
-            List<String> line = new ArrayList<>();
             StoreValue sv = map.get(key);
-            line.add(sv.getKey());
-            line.add(sv.getValue());
+            values.add(sv);
         }
 
         return values;
     }
 
+    @Override
+    public List<InstantValues> queryValues(Instant from, Instant to, long intervalSecond, List<String> columns) {
+        return null;
+    }
+
     public static void main(String[] args) throws IOException {
         new MemoryValuesStore().load(new FileInputStream("/tmp/valuesStore.tmp"));
     }

src/main/java/ch/mno/copper/data/ValuesStore.java

@@ -25,7 +25,9 @@ public interface ValuesStore {
      */
     Collection<String> queryValues(Instant from, Instant to);
 
-    List<List<String>> queryValues(Instant from, Instant to, String columns);
+    List<StoreValue> queryValues(Instant from, Instant to, List<String> columns);
+
+    List<InstantValues> queryValues(Instant from, Instant to, long intervalSecond, List<String> columns);
 
     void load() throws IOException;
 

src/main/java/ch/mno/copper/stories/StoriesFacade.java

@@ -70,7 +70,7 @@ public StoryTask buildStoryTask(Story story, ValuesStore valuesStore) {
 
                 // Collect
                 AbstractCollectorWrapper collectorWrapper = story.getCollectorWrapper();
-                if (collectorWrapper == null) { // Null means to read value store
+                if (collectorWrapper == null) { // Null means to readInstant value store
                     values = valuesStore.getValuesMapString();
                 } else {
                     values = collectorWrapper.execute();

src/main/java/ch/mno/copper/web/CopperServices.java

@@ -2,6 +2,7 @@
 
 import ch.mno.copper.CopperMediator;
 import ch.mno.copper.collect.connectors.ConnectorException;
+import ch.mno.copper.data.InstantValues;
 import ch.mno.copper.data.StoreValue;
 import ch.mno.copper.data.ValuesStore;
 import ch.mno.copper.helpers.SyntaxException;
@@ -32,6 +33,7 @@
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 @Path("/ws")
@@ -100,16 +102,49 @@ public String getValues() {
     @Produces(MediaType.APPLICATION_JSON)
     public Response getValues(@QueryParam("from") String dateFrom, @QueryParam("to") String dateTo, @QueryParam("columns") String columns) {
         Gson gson = new Gson();
-        Instant from = toInstant(dateFrom, true);
+        Instant from;
+        if (dateFrom==null) {
+            from = Instant.parse("2000-01-01T00:00:00.00Z");
+        } else {
+            from = toInstant(dateFrom, true);
+        }
+        Instant to;
+        if (dateTo==null) {
+            to = Instant.now();
+        }  else {
+            to =toInstant(dateTo, false);
+        }
+        try {
+            List<String> cols = Arrays.asList(columns.split(","));
+            return Response.ok(gson.toJson(valuesStore.queryValues(from, to, cols)), MediaType.APPLICATION_JSON).build();
+        } catch (RuntimeException e) {
+            return Response.serverError().entity("SERVER ERROR\n" + e.getMessage()).build();
+        }
+    }
+
+    @GET
+    @Path("instants/query")
+    @Produces(MediaType.APPLICATION_JSON)
+    public Response getValues(@QueryParam("from") String dateFrom, @QueryParam("to") String dateTo, @QueryParam("columns") String columns, @QueryParam("intervalSeconds") long intervalSeconds) {
+        Gson gson = new Gson();
+        Instant from;
+        if (dateFrom==null) {
+            from = Instant.parse("2000-01-01T00:00:00.00Z");
+        } else {
+            from = toInstant(dateFrom, true);
+        }
         Instant to;
         if (dateTo==null) {
             to = Instant.now();
         }  else {
             to =toInstant(dateTo, false);
         }
         try {
-            return Response.ok(gson.toJson(valuesStore.queryValues(from, to, columns)), MediaType.APPLICATION_JSON).build();
+            List<String> cols = Arrays.asList(columns.split(","));
+            List<InstantValues> values = valuesStore.queryValues(from, to, intervalSeconds, cols);
+            return Response.ok(gson.toJson(values), MediaType.APPLICATION_JSON).build();
         } catch (RuntimeException e) {
+            e.printStackTrace();
             return Response.serverError().entity("SERVER ERROR\n" + e.getMessage()).build();
         }
     }

src/main/resources/WEB-INF/app/app.js

@@ -7,18 +7,20 @@ var copperApp = angular.module('copperApp', [
   'copperApp.stories',
   'copperApp.story',
   'copperApp.values',
-  'ui.bootstrap'
+  'copperApp.history',
+  'ui.bootstrap',
+  'chart.js'
 ])
-.config(['$routeProvider', '$httpProvider', function($routeProvider, $httpProvider) {
+.config(['$routeProvider', '$httpProvider', '$locationProvider', function($routeProvider, $httpProvider, $locationProvider) {
     $routeProvider
     .when('/help', {
         templateUrl: 'app/views/help.html'
     })
     .otherwise({redirectTo: '/overview'});
+    $locationProvider.hashPrefix('');
 }])
     .filter('trusted', ['$sce', function($sce){
         return function(text) {
             return $sce.trustAsHtml(text);
         };
 }]);
-