From ecdc3dcc2ca07765aabe0702a935f396d57c457e Mon Sep 17 00:00:00 2001 From: Mark Otway Date: Wed, 6 Mar 2024 12:30:29 +0000 Subject: [PATCH] Improvements --- Damselfly.Core.Utils/ML/ImageDetectResult.cs | 3 +- .../Services/ImageRecognitionService.cs | 36 +++++-------- Damselfly.ML.FaceONNX/FaceONNXService.cs | 53 ++++++++++--------- Damselfly.Web.Client/wwwroot/version.js | 2 +- 4 files changed, 45 insertions(+), 49 deletions(-) diff --git a/Damselfly.Core.Utils/ML/ImageDetectResult.cs b/Damselfly.Core.Utils/ML/ImageDetectResult.cs index 34baa359..03042d08 100644 --- a/Damselfly.Core.Utils/ML/ImageDetectResult.cs +++ b/Damselfly.Core.Utils/ML/ImageDetectResult.cs @@ -10,6 +10,7 @@ public class ImageDetectResult public string ServiceModel { get; set; } public float Score { get; set; } public float[] Embeddings { get; set; } - + public bool IsNewPerson { get; set; } + public string PersonGuid { get; set; } public bool IsFace => string.Compare(Tag, "Face", true) == 0; } \ No newline at end of file diff --git a/Damselfly.Core/Services/ImageRecognitionService.cs b/Damselfly.Core/Services/ImageRecognitionService.cs index a992c178..91df4bfa 100644 --- a/Damselfly.Core/Services/ImageRecognitionService.cs +++ b/Damselfly.Core/Services/ImageRecognitionService.cs @@ -124,10 +124,10 @@ public async Task UpdateName(ImageObject faceObject, string name) await db.SaveChangesAsync("SetName"); - if (faceObject.Person.Embeddings != null) + if (faceObject.Person.PersonGuid != null) { // Add/update the cache - _peopleCache[faceObject.Person.Embeddings] = faceObject.Person; + _peopleCache[faceObject.Person.PersonGuid] = faceObject.Person; } _imageCache.Evict(faceObject.ImageId); @@ -146,7 +146,7 @@ public async Task UpdatePerson(Person person, string name) await db.SaveChangesAsync("SetName"); // Add/update the cache - _peopleCache[person.Embeddings] = person; + _peopleCache[person.PersonGuid] = person; } public JobPriorities Priority => JobPriorities.ImageRecognition; @@ -287,34 +287,30 @@ private async Task LoadPersonCache(bool force = false) /// Create the DB entries for people who we don't know about, /// and then pre-populate the cache with their entries. /// - /// + /// /// - public async Task CreateMissingPeople(IEnumerable embeddingsToAdd) + public async Task CreateMissingPeople(IEnumerable detectedFaces) { using var scope = _scopeFactory.CreateScope(); using var db = scope.ServiceProvider.GetService(); try { - if ( embeddingsToAdd != null && embeddingsToAdd.Any() ) + if ( detectedFaces != null ) { - // Find the people that aren't already in the cache and add new ones - // Be careful - filter out empty ones (shouldn't ever happen, but belt - // and braces - var newEmbeddings = embeddingsToAdd.Select(x => x.Trim()) - .Where(x => !string.IsNullOrEmpty(x) && !_peopleCache.ContainsKey(x)) - .ToList(); + var newFaces = detectedFaces.Where( x => x.IsNewPerson ).ToList(); - if ( newEmbeddings.Any() ) + if ( newFaces.Any() ) { - Logging.Log($"Adding {newEmbeddings.Count()} person records."); + Logging.Log($"Adding {newFaces.Count()} person records."); - var newPeople = newEmbeddings.Select(x => new Person + var newPeople = newFaces.Select(x => new Person { Name = "Unknown", State = Person.PersonState.Unknown, LastUpdated = DateTime.UtcNow, - Embeddings = x + Embeddings = string.Join( ",", x.Embeddings), + PersonGuid = x.PersonGuid }).ToList(); if ( newPeople.Any() ) @@ -322,7 +318,7 @@ public async Task CreateMissingPeople(IEnumerable embeddingsToAdd) await db.BulkInsert(db.People, newPeople); // Add or replace the new people in the cache (this should always add) - newPeople.ForEach(x => _peopleCache[x.Embeddings] = x); + newPeople.ForEach(x => _peopleCache[x.PersonGuid] = x); } } } @@ -423,12 +419,8 @@ private async Task DetectObjects(ImageMetaData metadata) var newTags = await CreateNewTags(faces); - // Get a list of the new Embeddings - // TODO this logic may be wrong - var embeddingStrings = faces.Select(x => string.Join(",",x.Embeddings)); - // Create any new ones, or pull existing ones back from the cache - await CreateMissingPeople(embeddingStrings); + await CreateMissingPeople(faces); var newFaces = faces.Select(x => new ImageObject { diff --git a/Damselfly.ML.FaceONNX/FaceONNXService.cs b/Damselfly.ML.FaceONNX/FaceONNXService.cs index c78cdcdd..5432b834 100644 --- a/Damselfly.ML.FaceONNX/FaceONNXService.cs +++ b/Damselfly.ML.FaceONNX/FaceONNXService.cs @@ -67,7 +67,7 @@ public void LoadFaceEmbeddings(Dictionary embeddings) private class FaceONNXFace { - public string? PersonGuid { get; set; } + public string PersonGuid { get; set; } public Rectangle Rectangle { get; set; } public float[] Embeddings { get; set; } public float Score { get; set; } @@ -137,55 +137,58 @@ private IEnumerable GetFacesFromImage(Image image) /// public async Task> DetectFaces(Image image) { - List detectedFaces; - + List detected = new(); var watch = new Stopwatch("FaceOnnxDetection"); try { - detectedFaces = GetFacesFromImage(image).ToList(); + var detectedFaces = GetFacesFromImage(image).ToList(); foreach( var face in detectedFaces ) { // For each result, loop through and see if we have a match var (personGuid, similarity) = _embeddings.FromSimilarity(face.Embeddings); - if( personGuid != null ) - { - // Looks like we have a match. Yay! - face.PersonGuid = personGuid; - } - else + bool isNewPerson = true; + + if( personGuid == null || similarity < 0.75 ) { // No match, so create a new person GUID face.PersonGuid = Guid.NewGuid().ToString(); // Add it to the embeddings DB _embeddings.Add(face.PersonGuid, face.Embeddings); } + else + { + // Looks like we have a match. Yay! + face.PersonGuid = personGuid; + isNewPerson = false; + } + + detected.Add( new ImageDetectResult + { + // TODO - need to disambiguate rectangles... + Rect = new System.Drawing.Rectangle(face.Rectangle.X, face.Rectangle.Y, + face.Rectangle.Width, face.Rectangle.Height), + Score = face.Score, + Tag = "Face", + Service = "FaceONNX", + IsNewPerson = isNewPerson, + PersonGuid = face.PersonGuid, + Embeddings = face.Embeddings + } ); } } catch ( Exception ex ) { Logging.LogError($"Exception during FaceONNX face detection: {ex}"); - detectedFaces = new(); } watch.Stop(); - if ( detectedFaces.Any() ) - Logging.Log($" FaceONNAX Detected {detectedFaces.Count()} faces in {watch.ElapsedTime}ms"); + if ( detected.Any() ) + Logging.Log($" FaceONNAX Detected {detected.Count()} faces in {watch.ElapsedTime}ms"); - // Convert to the caller's type. - var result = detectedFaces.Select( x => new ImageDetectResult - { - // TODO - need to disambiguate rectangles... - Rect = new System.Drawing.Rectangle(x.Rectangle.X, x.Rectangle.Y, x.Rectangle.Width, x.Rectangle.Height), - Score = x.Score, - Tag = "Face", - Service = "FaceONNX", - Embeddings = x.Embeddings - }).ToList(); - - return result; + return detected; } } \ No newline at end of file diff --git a/Damselfly.Web.Client/wwwroot/version.js b/Damselfly.Web.Client/wwwroot/version.js index 67819813..7fba3a7f 100644 --- a/Damselfly.Web.Client/wwwroot/version.js +++ b/Damselfly.Web.Client/wwwroot/version.js @@ -1 +1 @@ -const CACHE_VERSION='4.1.0-20240306085257' +const CACHE_VERSION='4.1.0-20240306121028'