From d4c950e8e38a0d62fc8cb4c776bbd959e2b11588 Mon Sep 17 00:00:00 2001 From: Mike Phares Date: Tue, 30 Aug 2022 20:13:05 -0700 Subject: [PATCH] Finished Testing Infineon Pictures --- Instance/DlibDotNet.cs | 5 +- Instance/Models/_E_Distance.cs | 10 +-- Map/Models/MapLogic.cs | 81 ++++++++++++------- .../Stateless/SetByDeterministicHashCode.cs | 18 +++-- Shared/Models/Stateless/ILocation.cs | 2 + 5 files changed, 74 insertions(+), 42 deletions(-) diff --git a/Instance/DlibDotNet.cs b/Instance/DlibDotNet.cs index 305420d..2e083a2 100644 --- a/Instance/DlibDotNet.cs +++ b/Instance/DlibDotNet.cs @@ -679,10 +679,11 @@ public class DlibDotNet private void MapLogic(string argZero, Container[] containers, long ticks, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, MapLogic mapLogic, string outputResolution) { + int totalNotMapped = 0; mapLogic.UseKeyValuePairsSaveFaceEncoding(containers); foreach (Container container in containers) { - mapLogic.AddToMapping(container.Items); + totalNotMapped += mapLogic.AddToMapping(container.Items); if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution)) mapLogic.SaveShortcuts(_Configuration.JuliePhares, container.Items); } @@ -693,7 +694,7 @@ public class DlibDotNet dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()"); mapLogic.SaveNotMappedTicks(); _ = LogDeltaInMinutes(ticks, nameof(mapLogic.SaveNotMappedTicks)); - Dictionary> keyValuePairs = _Distance.ParallelWork(_AppSettings.MaxDegreeOfParallelism, _Configuration.IgnoreRelativePaths, argZero, ticks, mapLogic, containers); + Dictionary> keyValuePairs = _Distance.ParallelWork(_AppSettings.MaxDegreeOfParallelism, _Configuration.IgnoreRelativePaths, argZero, ticks, mapLogic, containers, totalNotMapped); _ = LogDeltaInSeconds(ticks, nameof(E_Distance.ParallelWork)); Dictionary> strippedKeyValuePairs = Strip(keyValuePairs); List<(string, int, Mapping, DateTime, bool?, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>)> collection = Convert(keyValuePairs); diff --git a/Instance/Models/_E_Distance.cs b/Instance/Models/_E_Distance.cs index 140df1a..1106cb6 100644 --- a/Instance/Models/_E_Distance.cs +++ b/Instance/Models/_E_Distance.cs @@ -621,14 +621,13 @@ internal class E_Distance } } - private Dictionary> GetThreeSigmaFaceEncodings(int maxDegreeOfParallelism, long ticks, Dictionary> keyValuePairs) + private Dictionary> GetThreeSigmaFaceEncodings(int maxDegreeOfParallelism, long ticks, Random random, Dictionary> keyValuePairs) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); Dictionary> results = new(); int totalSeconds; int selectedIndex; - Random random = new(); List faceDistances; MappingContainer mappingContainer; int keyValuePairsCount = keyValuePairs.Count; @@ -662,12 +661,13 @@ internal class E_Distance return results; } - internal Dictionary> ParallelWork(int maxDegreeOfParallelism, string[] ignoreRelativePaths, string argZero, long ticks, Map.Models.MapLogic mapLogic, Container[] containers) + internal Dictionary> ParallelWork(int maxDegreeOfParallelism, string[] ignoreRelativePaths, string argZero, long ticks, Map.Models.MapLogic mapLogic, Container[] containers, int totalNotMapped) { Dictionary> results; - mapLogic.ForceSingleImage(ignoreRelativePaths, argZero, containers); + Random random = new((int)ticks); + mapLogic.ForceSingleImage(ignoreRelativePaths, argZero, containers, totalNotMapped, random); Dictionary> keyValuePairs = Map.Models.Stateless.IMapLogic.GetKeyValuePairs(ignoreRelativePaths, argZero, containers); - results = GetThreeSigmaFaceEncodings(maxDegreeOfParallelism, ticks, keyValuePairs); + results = GetThreeSigmaFaceEncodings(maxDegreeOfParallelism, ticks, random, keyValuePairs); return results; } diff --git a/Map/Models/MapLogic.cs b/Map/Models/MapLogic.cs index e8d01eb..1955736 100644 --- a/Map/Models/MapLogic.cs +++ b/Map/Models/MapLogic.cs @@ -319,8 +319,9 @@ public class MapLogic } } - public void AddToMapping(List items) + public int AddToMapping(List items) { + int result = 0; long ticks; Mapping mapping; bool forced = false; @@ -343,7 +344,10 @@ public class MapLogic continue; deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item, face); if (!_DeterministicHashCodeKeyValuePairs.ContainsKey(deterministicHashCodeKey)) + { + result += 1; continue; + } personBirthdays.AddRange(_DeterministicHashCodeKeyValuePairs[deterministicHashCodeKey]); for (int i = 0; i < personBirthdays.Count; i++) { @@ -377,6 +381,7 @@ public class MapLogic } } } + return result; } private void SaveContainers(List saveContainers) @@ -677,10 +682,10 @@ public class MapLogic key = string.Concat(personKey, dateKey); if (match.Filtered is null) directory = Path.Combine(_ZPropertyHolderContentTicksDirectory, $"{facePopulatedKey}Null", personKey, dateKey); - else if (!match.Filtered.Value) - directory = Path.Combine(_ZPropertyHolderContentTicksDirectory, $"{facePopulatedKey}Okay", personKey, dateKey); - else + else if (match.Filtered.Value) directory = Path.Combine(_ZPropertyHolderContentTicksDirectory, $"{facePopulatedKey}OutOfControl", personKey, dateKey); + else + continue; personDirectory = Path.Combine(directory, match.DisplayDirectoryName[..1], "lnk"); saveContainer = new(personDirectory); results.Add(saveContainer); @@ -744,9 +749,9 @@ public class MapLogic if (match.Filtered is null) directory = Path.Combine(_ZPropertyHolderContentTicksDirectory, $"{facePopulatedKey}WithButNull", personKey, dateKey); else if (!match.Filtered.Value) - directory = Path.Combine(_ZPropertyHolderContentTicksDirectory, $"{facePopulatedKey}WithAndOkay", personKey, dateKey); - else directory = Path.Combine(_ZPropertyHolderContentTicksDirectory, $"{facePopulatedKey}WithButOutOfControl", personKey, dateKey); + else + continue; personDirectory = Path.Combine(directory, match.DisplayDirectoryName[..1], "lnk"); saveContainer = new(personDirectory); results.Add(saveContainer); @@ -755,7 +760,7 @@ public class MapLogic deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item, face); checkFile = Path.Combine(directory, $"{deterministicHashCodeKey}{item.ImageFileHolder.ExtensionLowered}"); faceFileHolder = new(Path.Combine(facesDirectory, $"{deterministicHashCodeKey}{item.ImageFileHolder.ExtensionLowered}{_FacesFilenameExtension}")); - facePartsFileHolder = new(Path.Combine(facesDirectory, $"{deterministicHashCodeKey}{item.ImageFileHolder.ExtensionLowered}{_FacesHiddenFilenameExtension}")); + hiddenFaceFileHolder = new(Path.Combine(facesDirectory, $"{deterministicHashCodeKey}{item.ImageFileHolder.ExtensionLowered}{_FacesHiddenFilenameExtension}")); facePartsFileHolder = new(Path.Combine(facePartsDirectory, $"{deterministicHashCodeKey}{item.ImageFileHolder.ExtensionLowered}{_FacePartsFilenameExtension}")); if (string.IsNullOrEmpty(personDirectory)) shortcutFile = string.Empty; @@ -925,54 +930,70 @@ public class MapLogic SaveContainers(saveContainers); } - public void ForceSingleImage(string[] ignoreRelativePaths, string argZero, Container[] containers) + public void ForceSingleImage(string[] ignoreRelativePaths, string argZero, Container[] containers, int totalNotMapped, Random random) { + int count = 0; Closest closest; Mapping mapping; bool? isWrongYear; long? ticks = null; const int zero = 0; bool forced = true; + int randomSelection; DateTime minimumDateTime; bool aboveTolerance = false; int? approximateYears = null; PersonBirthday personBirthday; const string displayDirectoryName = Property.Models.Stateless.IResult.AllInOne; - foreach (Container container in containers) + for (int i = 0; i < int.MaxValue; i++) { - if (!_NotMappedTicks.Any()) - break; - if (!container.Items.Any()) - continue; - if (!container.SourceDirectory.StartsWith(argZero)) - continue; - if (ignoreRelativePaths.Contains(Path.GetFileName(container.SourceDirectory))) - continue; - foreach (Item item in container.Items) + count = 0; + randomSelection = random.Next(0, totalNotMapped); + foreach (Container container in containers) { - if (item.ImageFileHolder is null || item.Property?.Id is null || item.Mapping.Any()) + if (!_NotMappedTicks.Any()) + break; + if (!container.Items.Any()) continue; - foreach (Face face in item.Faces) + if (!container.SourceDirectory.StartsWith(argZero)) + continue; + if (ignoreRelativePaths.Contains(Path.GetFileName(container.SourceDirectory))) + continue; + foreach (Item item in container.Items) { - if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null) + if (item.ImageFileHolder is null || item.Property?.Id is null || item.Mapping.Any()) continue; - ticks = _NotMappedTicks[zero]; - minimumDateTime = Shared.Models.Stateless.Methods.IProperty.GetMinimumDateTime(item.Property); - (isWrongYear, _) = item.Property.IsWrongYear(item.ImageFileHolder.FullName, minimumDateTime); - personBirthday = Shared.Models.Stateless.Methods.IPersonBirthday.GetPersonBirthday(ticks.Value); - mapping = new(approximateYears, displayDirectoryName, forced, face.Location.NormalizedPixelPercentage, personBirthday); - closest = new(aboveTolerance, zero, face.Location.NormalizedPixelPercentage.Value, isWrongYear, mapping, zero, minimumDateTime, null); - item.Closest.Add(closest); - item.Mapping.Add(mapping); + foreach (Face face in item.Faces) + { + if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null) + continue; + if (face.Location.Confidence < ILocation.MinimumConfidence) + continue; + count += 1; + if (count < randomSelection) + continue; + ticks = _NotMappedTicks[zero]; + minimumDateTime = Shared.Models.Stateless.Methods.IProperty.GetMinimumDateTime(item.Property); + (isWrongYear, _) = item.Property.IsWrongYear(item.ImageFileHolder.FullName, minimumDateTime); + personBirthday = Shared.Models.Stateless.Methods.IPersonBirthday.GetPersonBirthday(ticks.Value); + mapping = new(approximateYears, displayDirectoryName, forced, face.Location.NormalizedPixelPercentage, personBirthday); + closest = new(aboveTolerance, zero, face.Location.NormalizedPixelPercentage.Value, isWrongYear, mapping, zero, minimumDateTime, null); + item.Closest.Add(closest); + item.Mapping.Add(mapping); + if (ticks is not null) + break; + } if (ticks is not null) break; } if (ticks is not null) break; } - if (ticks is not null) + if (ticks is not null || count == 0) break; } + if (ticks is null && count != 0) + throw new Exception(); } public void SavePropertyHolders(string argZero, Container[] containers) diff --git a/Map/Models/Stateless/SetByDeterministicHashCode.cs b/Map/Models/Stateless/SetByDeterministicHashCode.cs index ba4ad74..e0c2ea6 100644 --- a/Map/Models/Stateless/SetByDeterministicHashCode.cs +++ b/Map/Models/Stateless/SetByDeterministicHashCode.cs @@ -148,11 +148,14 @@ public class ByDeterministicHashCode foreach (string personKeyDirectory in personKeyDirectories) { personKey = Path.GetFileName(personKeyDirectory); - if (personKey == nameof(Closest)) - throw new Exception($"Move personKey directories up one from {nameof(Closest)} and delete {nameof(Closest)} directory!"); - personBirthday = Shared.Models.Stateless.Methods.IPersonBirthday.GetPersonBirthday(personKey); - if (personBirthday is null) - continue; + if (personKey.Length != Shared.Models.Stateless.IPersonBirthday.Format.Length) + personBirthday = null; + else + { + personBirthday = Shared.Models.Stateless.Methods.IPersonBirthday.GetPersonBirthday(personKey); + if (personBirthday is null) + continue; + } yearDirectories = Directory.GetDirectories(personKeyDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string yearDirectory in yearDirectories) { @@ -167,6 +170,10 @@ public class ByDeterministicHashCode continue; personFirstInitialDirectory = Path.Combine(yearDirectory, personFirstInitial); files = Directory.GetFiles(personNameDirectory, "*", SearchOption.TopDirectoryOnly); + if (personKey == nameof(Closest) && files.Any()) + throw new Exception($"Move personKey directories up one from {nameof(Closest)} and delete {nameof(Closest)} directory!"); + if (personBirthday is null) + continue; foreach (string file in files) { if (file.EndsWith(".lnk") || file.EndsWith(".json")) @@ -198,6 +205,7 @@ public class ByDeterministicHashCode _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(personKeyDirectory); } _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(ticksDirectory); + _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(ticksDirectory); } } diff --git a/Shared/Models/Stateless/ILocation.cs b/Shared/Models/Stateless/ILocation.cs index 8051d28..c6965d0 100644 --- a/Shared/Models/Stateless/ILocation.cs +++ b/Shared/Models/Stateless/ILocation.cs @@ -3,7 +3,9 @@ public interface ILocation { + const double MinimumConfidence = 0.8d; const int Digits = 6; + const double Tolerance = 0.6d; const int Factor = 1000000; } \ No newline at end of file