server: improve loading logs for multiple pieces of material

server/lib/BlackMaple.MachineFramework/db/EventLog.cs

@@ -409,52 +409,66 @@ public string OriginalMessageByForeignID(string foreignID)
       return "";
     }
 
-    public List<LogEntry> GetLogForMaterial(long materialID, bool includeInvalidatedCycles = true)
+    public IEnumerable<LogEntry> GetLogForMaterial(long materialID, bool includeInvalidatedCycles = true)
     {
       if (materialID < 0)
-        return new List<LogEntry>();
-      using (var trans = _connection.BeginTransaction())
-      using (var cmd = _connection.CreateCommand())
       {
-        cmd.Transaction = trans;
-        cmd.CommandText =
-          "SELECT s.Counter, s.Pallet, s.StationLoc, s.StationNum, s.Program, s.Start, s.TimeUTC, s.Result, s.EndOfRoute, s.Elapsed, s.ActiveTime, s.StationName "
-          + " FROM stations s "
-          + " WHERE s.Counter IN (SELECT m.Counter FROM stations_mat m WHERE m.MaterialID = $mat)"
-          + (includeInvalidatedCycles ? "" : " AND " + ignoreInvalidEventCondition)
-          + " ORDER BY s.Counter ASC";
-        cmd.Parameters.Add("mat", SqliteType.Integer).Value = materialID;
+        yield break;
+      }
 
-        using (var reader = cmd.ExecuteReader())
-        {
-          return LoadLog(reader, trans).ToList();
-        }
+      using var trans = _connection.BeginTransaction();
+      using var cmd = _connection.CreateCommand();
+      cmd.Transaction = trans;
+      cmd.CommandText =
+        "SELECT s.Counter, s.Pallet, s.StationLoc, s.StationNum, s.Program, s.Start, s.TimeUTC, s.Result, s.EndOfRoute, s.Elapsed, s.ActiveTime, s.StationName "
+        + " FROM stations s "
+        + " WHERE s.Counter IN (SELECT m.Counter FROM stations_mat m WHERE m.MaterialID = $mat)"
+        + (includeInvalidatedCycles ? "" : " AND " + ignoreInvalidEventCondition)
+        + " ORDER BY s.Counter ASC";
+      cmd.Parameters.Add("mat", SqliteType.Integer).Value = materialID;
+
+      using var reader = cmd.ExecuteReader();
+
+      foreach (var e in LoadLog(reader, trans))
+      {
+        yield return e;
       }
     }
 
-    public List<LogEntry> GetLogForMaterial(IEnumerable<long> materialIDs)
+    public IEnumerable<LogEntry> GetLogForMaterial(
+      IEnumerable<long> materialIDs,
+      bool includeInvalidatedCycles = true
+    )
     {
-      var ret = new List<LogEntry>();
-      using (var cmd = _connection.CreateCommand())
-      using (var trans = _connection.BeginTransaction())
+      using var cmd = _connection.CreateCommand();
+      using var trans = _connection.BeginTransaction();
+
+      cmd.Transaction = trans;
+      cmd.CommandText = "CREATE TEMP TABLE temp_mat_ids (MaterialID INTEGER)";
+      cmd.ExecuteNonQuery();
+
+      cmd.CommandText = "INSERT INTO temp_mat_ids (MaterialID) VALUES ($mat)";
+      cmd.Parameters.Add("mat", SqliteType.Integer);
+      foreach (var mat in materialIDs)
       {
-        cmd.Transaction = trans;
-        cmd.CommandText =
-          "SELECT Counter, Pallet, StationLoc, StationNum, Program, Start, TimeUTC, Result, EndOfRoute, Elapsed, ActiveTime, StationName "
-          + " FROM stations WHERE Counter IN (SELECT Counter FROM stations_mat WHERE MaterialID = $mat) ORDER BY Counter ASC";
-        var param = cmd.Parameters.Add("mat", SqliteType.Integer);
+        cmd.Parameters[0].Value = mat;
+        cmd.ExecuteNonQuery();
+      }
 
-        foreach (var matId in materialIDs)
-        {
-          param.Value = matId;
-          using (var reader = cmd.ExecuteReader())
-          {
-            ret.AddRange(LoadLog(reader, trans));
-          }
-        }
-        trans.Commit();
+      cmd.CommandText =
+        "SELECT s.Counter, s.Pallet, s.StationLoc, s.StationNum, s.Program, s.Start, s.TimeUTC, s.Result, s.EndOfRoute, s.Elapsed, s.ActiveTime, s.StationName "
+        + " FROM stations s WHERE s.Counter IN "
+        + "     (SELECT m.Counter FROM stations_mat m WHERE m.MaterialID IN "
+        + "        (SELECT t.MaterialID FROM temp_mat_ids t))"
+        + (includeInvalidatedCycles ? "" : " AND " + ignoreInvalidEventCondition)
+        + " ORDER BY s.Counter ASC";
+      cmd.Parameters.Clear();
+
+      using var reader = cmd.ExecuteReader();
+      foreach (var e in LoadLog(reader, trans))
+      {
+        yield return e;
       }
-      return ret;
     }
 
     public IEnumerable<LogEntry> GetLogForSerial(string serial)

server/lib/BlackMaple.MachineFramework/db/IRepository.cs

@@ -50,8 +50,11 @@ public interface IRepository : IDisposable
     IEnumerable<LogEntry> GetLogOfAllCompletedParts(DateTime startUTC, DateTime endUTC);
     IEnumerable<LogEntry> GetLogForJobUnique(string jobUnique);
     IEnumerable<LogEntry> CompletedUnloadsSince(long counter);
-    List<LogEntry> GetLogForMaterial(long materialID, bool includeInvalidatedCycles = true);
-    List<LogEntry> GetLogForMaterial(IEnumerable<long> materialIDs);
+    IEnumerable<LogEntry> GetLogForMaterial(long materialID, bool includeInvalidatedCycles = true);
+    IEnumerable<LogEntry> GetLogForMaterial(
+      IEnumerable<long> materialIDs,
+      bool includeInvalidatedCycles = true
+    );
     IEnumerable<LogEntry> GetLogForSerial(string serial);
     IEnumerable<LogEntry> GetLogForWorkorder(string workorder);
     List<LogEntry> CurrentPalletLog(int pallet, bool includeLastPalletCycleEvt = false);

server/lib/BlackMaple.MachineFramework/http/Controllers/LogsController.cs

@@ -141,19 +141,28 @@ public IEnumerable<LogEntry> Recent(
     }
 
     [HttpGet("events/for-material/{materialID}")]
-    public List<LogEntry> LogForMaterial(long materialID)
+    public IEnumerable<LogEntry> LogForMaterial(long materialID)
     {
       using var db = repo.OpenConnection();
-      return db.GetLogForMaterial(materialID);
+      foreach (var e in db.GetLogForMaterial(materialID))
+      {
+        yield return e;
+      }
     }
 
     [HttpGet("events/for-material")]
-    public List<LogEntry> LogForMaterials([FromQuery] List<long> id)
+    public IEnumerable<LogEntry> LogForMaterials([FromQuery] List<long> id)
     {
       if (id == null || id.Count == 0)
-        return [];
+      {
+        yield break;
+      }
+
       using var db = repo.OpenConnection();
-      return db.GetLogForMaterial(id);
+      foreach (var e in db.GetLogForMaterial(id))
+      {
+        yield return e;
+      }
     }
 
     [HttpGet("events/for-serial/{serial}")]

server/test/InspectionTest.cs

@@ -585,7 +585,6 @@ private void CheckDecision(
       Assert.Equal(forcedCnt, forced ? 1 : 0);
 
       using var _insp = _repoCfg.OpenConnection();
-      var log = _insp.GetLogForMaterial(matID);
       int inspEntries = 0;
       int forceEntries = 0;
       foreach (var entry in _insp.GetLogForMaterial(matID))

server/test/JobLogTest.cs

@@ -809,13 +809,14 @@ public void AddLog()
 
       Assert.False(_jobLog.CycleExists(DateTime.Parse("4/6/2011"), 123, LogType.MachineCycle, "MC", 3));
 
-      CheckLog(logsForMat1, _jobLog.GetLogForMaterial(1), start);
+      CheckLog(logsForMat1, _jobLog.GetLogForMaterial(1).ToList(), start);
       CheckLog(
         logsForMat1.Concat(logsForMat2).ToList(),
-        _jobLog.GetLogForMaterial(new[] { 1, mat2.MaterialID }),
+        _jobLog.GetLogForMaterial(new[] { 1, mat2.MaterialID }).ToList(),
         start
       );
       _jobLog.GetLogForMaterial(18).Should().BeEmpty();
+      _jobLog.GetLogForMaterial([18, 19]).Should().BeEmpty();
 
       var markLog = _jobLog.RecordSerialForMaterialID(
         EventLogMaterial.FromLogMat(mat1),
@@ -1120,7 +1121,14 @@ public void LookupByPallet()
       CheckLog(pal1Initial, _jobLog.CurrentPalletLog(1), DateTime.UtcNow.AddHours(-10));
       CheckLog(
         pal1Initial,
-        _jobLog.GetLogForMaterial(mat1.MaterialID, includeInvalidatedCycles: false),
+        _jobLog.GetLogForMaterial(mat1.MaterialID, includeInvalidatedCycles: false).ToList(),
+        DateTime.UtcNow.AddHours(-10)
+      );
+      CheckLog(
+        pal1Initial,
+        _jobLog
+          .GetLogForMaterial([mat1.MaterialID, mat2.MaterialID], includeInvalidatedCycles: false)
+          .ToList(),
         DateTime.UtcNow.AddHours(-10)
       );
       _jobLog.CurrentPalletLog(2).Should().BeEmpty();
@@ -1252,7 +1260,14 @@ public void LookupByPallet()
       CheckLog(pal2Cycle, _jobLog.CurrentPalletLog(2), DateTime.UtcNow.AddHours(-10));
       CheckLog(
         pal1Initial.Concat(pal1Cycle).Concat(pal2Cycle).Where(e => !e.Material.IsEmpty),
-        _jobLog.GetLogForMaterial(mat1.MaterialID, includeInvalidatedCycles: false),
+        _jobLog.GetLogForMaterial(mat1.MaterialID, includeInvalidatedCycles: false).ToList(),
+        DateTime.UtcNow.AddHours(-10)
+      );
+      CheckLog(
+        pal1Initial.Concat(pal1Cycle).Concat(pal2Cycle).Where(e => !e.Material.IsEmpty),
+        _jobLog
+          .GetLogForMaterial([mat1.MaterialID, mat2.MaterialID], includeInvalidatedCycles: false)
+          .ToList(),
         DateTime.UtcNow.AddHours(-10)
       );
 
@@ -1271,7 +1286,14 @@ public void LookupByPallet()
       CheckLog(pal1Cycle, _jobLog.CurrentPalletLog(1), DateTime.UtcNow.AddHours(-10));
       CheckLog(
         pal1Initial.Concat(pal1Cycle).Concat(pal2Cycle).Where(e => !e.Material.IsEmpty),
-        _jobLog.GetLogForMaterial(mat1.MaterialID, includeInvalidatedCycles: false),
+        _jobLog.GetLogForMaterial(mat1.MaterialID, includeInvalidatedCycles: false).ToList(),
+        DateTime.UtcNow.AddHours(-10)
+      );
+      CheckLog(
+        pal1Initial.Concat(pal1Cycle).Concat(pal2Cycle).Where(e => !e.Material.IsEmpty),
+        _jobLog
+          .GetLogForMaterial([mat1.MaterialID, mat2.MaterialID], includeInvalidatedCycles: false)
+          .ToList(),
         DateTime.UtcNow.AddHours(-10)
       );
 
@@ -1309,7 +1331,7 @@ public void LookupByPallet()
 
       CheckLog(
         pal1Initial.Concat(pal1Cycle).Concat(pal2Cycle).Where(e => !e.Material.IsEmpty),
-        _jobLog.GetLogForMaterial(mat1.MaterialID, includeInvalidatedCycles: false),
+        _jobLog.GetLogForMaterial(mat1.MaterialID, includeInvalidatedCycles: false).ToList(),
         DateTime.UtcNow.AddHours(-10)
       );
     }