diff --git a/Distance/Models/_E_Distance.cs b/Distance/Models/_E_Distance.cs index d9809e8..d93e9ef 100644 --- a/Distance/Models/_E_Distance.cs +++ b/Distance/Models/_E_Distance.cs @@ -315,59 +315,6 @@ public partial class E_Distance : IDistance _DuplicateMappedFaceFiles.Clear(); } - public ReadOnlyCollection GetMissingFaceDistanceContainer(int maxDegreeOfParallelism, long ticks, string dFacesCollectionDirectory, ReadOnlyDictionary>> missingIdThenWholePercentagesToPersonContainers) - { - List results = new(); - string[] files; - List? faces; - int wholePercentages; - int confidencePercent; - bool? isWrongYear = null; - FaceDistance faceDistance; - List<(int id, string json)> collection = new(); - FaceDistanceContainer faceDistanceContainer; - foreach (KeyValuePair>> keyValuePair in missingIdThenWholePercentagesToPersonContainers) - { - files = Directory.GetFiles(dFacesCollectionDirectory, $"{keyValuePair.Key}*.json", SearchOption.TopDirectoryOnly); - if (files.Length != 1) - continue; - collection.Add(new(keyValuePair.Key, Shared.Models.Stateless.Methods.IFace.GetJson(files[0]))); - } - int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); - ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; - string message = $") {collection.Count:000} Setting missing distance containers - {totalSeconds} total second(s)"; - ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; - using ProgressBar progressBar = new(collection.Count, message, options); - _ = Parallel.For(0, collection.Count, parallelOptions, (i, state) => - { - progressBar.Tick(); - int id = collection[i].id; - string json = collection[i].json; - faces = JsonSerializer.Deserialize>(json); - if (faces is null) - throw new NullReferenceException(nameof(faces)); - foreach (Face face in faces) - { - if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null) - continue; - confidencePercent = Shared.Models.Stateless.Methods.ILocation.GetConfidencePercent(_FaceConfidencePercent, _RangeFaceConfidence, face.Location.Confidence); - wholePercentages = Shared.Models.Stateless.Methods.ILocation.GetWholePercentages(face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution); - if (face.FaceDistance?.Encoding is not null && face.FaceDistance.Encoding is FaceRecognitionDotNet.FaceEncoding faceEncoding) - faceDistance = new(confidencePercent, face.DateTime, faceEncoding, id, isWrongYear, wholePercentages); - else - { - faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding); - faceDistance = new(confidencePercent, face.DateTime, faceEncoding, id, isWrongYear, wholePercentages); - face.SetFaceDistance(faceDistance); - } - faceDistanceContainer = new(face, faceDistance); - lock (results) - results.Add(faceDistanceContainer); - } - }); - return new(results); - } - public static void SaveFaceDistances(Property.Models.Configuration configuration, SortingContainer[] sortingContainers) { string eDistanceContentCollectionDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(E_Distance), "([])"); @@ -387,7 +334,7 @@ public partial class E_Distance : IDistance { if (face.Mapping?.MappingFromFilter is null) throw new NotSupportedException(); - if (face.Mapping.MappingFromFilter.IsFocusPerson is not null && !face.Mapping.MappingFromFilter.IsFocusPerson.Value) + if (face.Mapping.MappingFromFilter.IsUsed is not null && face.Mapping.MappingFromFilter.IsUsed.Value && face.Mapping.MappingFromFilter.IsFocusPerson is not null && !face.Mapping.MappingFromFilter.IsFocusPerson.Value) continue; if (face.FaceEncoding is not null && face.FaceDistance?.Encoding is not null && face.FaceDistance.Encoding is FaceRecognitionDotNet.FaceEncoding _) continue; @@ -443,11 +390,11 @@ public partial class E_Distance : IDistance return results; } - private static List GetSortingCollection(Map.Models.MapLogic mapLogic, ReadOnlyCollection faceDistanceEncodings, int i, FaceDistance faceDistanceEncoding) + private static List GetSortingCollection(Map.Models.MapLogic mapLogic, ReadOnlyCollection faceDistanceEncodings, int i, FaceDistance faceDistanceEncoding, Face face) { List results; List faceDistanceLengths = FaceRecognition.FaceDistances(faceDistanceEncodings, faceDistanceEncoding); - results = mapLogic.GetSortingCollection(i, faceDistanceEncoding, faceDistanceLengths); + results = mapLogic.GetSortingCollection(i, faceDistanceEncoding, face, faceDistanceLengths); return results; } @@ -471,24 +418,6 @@ public partial class E_Distance : IDistance return results; } - public static ReadOnlyCollection GetFaceDistanceEncodings(ReadOnlyCollection faceDistanceContainers, ReadOnlyCollection missingFaceDistanceContainers) - { - List faceDistanceEncodings = new(); - foreach (FaceDistanceContainer faceDistanceContainer in faceDistanceContainers) - { - if (faceDistanceContainer.FaceDistance.Encoding is null) - continue; - faceDistanceEncodings.Add(faceDistanceContainer.FaceDistance); - } - foreach (FaceDistanceContainer faceDistanceContainer in missingFaceDistanceContainers) - { - if (faceDistanceContainer.FaceDistance.Encoding is null) - continue; - faceDistanceEncodings.Add(faceDistanceContainer.FaceDistance); - } - return new(faceDistanceEncodings); - } - public static FaceDistanceContainer[] FilteredFaceDistanceContainers(Map.Models.MapLogic mapLogic, ReadOnlyCollection faceDistanceContainers, long? skipOlderThan, DistanceLimits distanceLimits) { List results = new(); @@ -531,7 +460,7 @@ public partial class E_Distance : IDistance progressBar.Tick(); FaceDistance faceDistanceEncoding = filteredFaceDistanceContainers[i].FaceDistance; Face face = filteredFaceDistanceContainers[i].Face; - List sortingCollection = GetSortingCollection(mapLogic, faceDistanceEncodings, i, faceDistanceEncoding); + List sortingCollection = GetSortingCollection(mapLogic, faceDistanceEncodings, i, faceDistanceEncoding, face); if (sortingCollection.Count == 0) return; List sortingContainers = GetSortingContainers(mapConfiguration, distanceLimits, face, faceDistanceEncoding, sortingCollection); diff --git a/Instance/DlibDotNet.cs b/Instance/DlibDotNet.cs index b7959b1..dab7ca6 100644 --- a/Instance/DlibDotNet.cs +++ b/Instance/DlibDotNet.cs @@ -816,15 +816,21 @@ public partial class DlibDotNet } } - private void SaveFaceDistances(long ticks, MapLogic mapLogic, ReadOnlyCollection distinctFilteredFaces, ReadOnlyCollection mappingCollection, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, string dFacesCollectionDirectory, ReadOnlyDictionary> idToWholePercentagesToMapping) + private void SaveFaceDistances(long ticks, MapLogic mapLogic, ReadOnlyCollection distinctFilteredFaces, ReadOnlyCollection mappingCollection, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, ReadOnlyDictionary> idToWholePercentagesToMapping) { E_Distance.SetFaceDistances(_AppSettings.MaxDegreeOfParallelism, ticks, distinctFilteredFaces); ReadOnlyCollection faceDistanceContainers = E_Distance.GetFaceDistanceContainers(distinctFilteredFaces); - ReadOnlyDictionary>> missingIdThenWholePercentagesToPersonContainers = mapLogic.GetMissing(idToWholePercentagesToMapping); - ReadOnlyCollection missingFaceDistanceContainers = _Distance.GetMissingFaceDistanceContainer(_AppSettings.MaxDegreeOfParallelism, ticks, dFacesCollectionDirectory, missingIdThenWholePercentagesToPersonContainers); - ReadOnlyCollection faceDistanceEncodings = E_Distance.GetFaceDistanceEncodings(faceDistanceContainers, missingFaceDistanceContainers); if (faceDistanceContainers.Count > 0) - SaveFaceDistances(ticks, mapLogic, mappingCollection, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, faceDistanceEncodings, faceDistanceContainers); + { + List faceDistanceEncodings = new(); + foreach (FaceDistanceContainer faceDistanceContainer in faceDistanceContainers) + { + if (faceDistanceContainer.FaceDistance.Encoding is null) + continue; + faceDistanceEncodings.Add(faceDistanceContainer.FaceDistance); + } + SaveFaceDistances(ticks, mapLogic, mappingCollection, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, new(faceDistanceEncodings), faceDistanceContainers); + } } private void MapLogic(long ticks, ReadOnlyCollection containers, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string fPhotoPrismContentDirectory, MapLogic mapLogic, string outputResolution, ReadOnlyDictionary> personKeyToIds, ReadOnlyCollection distinctFilteredFaces, ReadOnlyCollection distinctFilteredMappingCollection) @@ -832,7 +838,6 @@ public partial class DlibDotNet string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent); string d2FacePartsContentDirectory = Path.Combine(d2ResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent); string d2FacePartsContentCollectionDirectory = Path.Combine(d2ResultsFullGroupDirectory, "[()]"); - string dFacesCollectionDirectory = Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultCollection, _Configuration.PropertyConfiguration.ResultAllInOne); if (distinctFilteredMappingCollection.Count > 0) { Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(d2FacePartsContentDirectory, ticks); @@ -848,7 +853,7 @@ public partial class DlibDotNet if (_Configuration.SaveMappedForOutputResolutions.Contains(outputResolution)) mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, personKeyToIds, distinctFilteredMappingCollection, idToWholePercentagesToMapping); if (_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution)) - SaveFaceDistances(ticks, mapLogic, distinctFilteredFaces, distinctFilteredMappingCollection, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, dFacesCollectionDirectory, idToWholePercentagesToMapping); + SaveFaceDistances(ticks, mapLogic, distinctFilteredFaces, distinctFilteredMappingCollection, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping); } private string SaveUrlAndGetNewRootDirectory(string[] files) diff --git a/Map/Models/MapLogic.cs b/Map/Models/MapLogic.cs index cd8f7b4..5f3cff1 100644 --- a/Map/Models/MapLogic.cs +++ b/Map/Models/MapLogic.cs @@ -582,7 +582,7 @@ public class MapLogic : Shared.Models.Methods.IMapLogic Stateless.MapLogic.SaveMappingShortcuts(mappingDirectory); } - public List GetSortingCollection(int i, FaceDistance faceDistanceEncoding, List faceDistanceLengths) + public List GetSortingCollection(int i, FaceDistance faceDistanceEncoding, Face face, List faceDistanceLengths) { if (_Configuration is null) throw new NullReferenceException(nameof(_Configuration)); @@ -591,22 +591,20 @@ public class MapLogic : Shared.Models.Methods.IMapLogic FaceDistance faceDistanceLength; List? wholePercentagesCollection; bool skipNotSkipCollectionAny = _SkipNotSkipCollection.Count > 0; - ReadOnlyDictionary>? wholePercentagesToPersonContainers; for (int j = 0; j < faceDistanceLengths.Count; j++) { if (faceDistanceEncoding.WholePercentages is null) throw new NotSupportedException(); + if (face.Mapping?.MappingFromFilter is null) + throw new NotSupportedException(); if (j == i) continue; if (_SkipCollection.TryGetValue(faceDistanceEncoding.Id, out wholePercentagesCollection) && wholePercentagesCollection.Contains(faceDistanceEncoding.WholePercentages.Value)) continue; if (skipNotSkipCollectionAny && (!_SkipNotSkipCollection.TryGetValue(faceDistanceEncoding.Id, out wholePercentagesCollection) || !wholePercentagesCollection.Contains(faceDistanceEncoding.WholePercentages.Value))) continue; - if (_IdThenWholePercentagesToPersonContainers.TryGetValue(faceDistanceEncoding.Id, out wholePercentagesToPersonContainers)) - { - if (wholePercentagesToPersonContainers.ContainsKey(faceDistanceEncoding.WholePercentages.Value)) - continue; - } + if (face.Mapping.MappingFromFilter.IsUsed is not null && face.Mapping.MappingFromFilter.IsUsed.Value) + continue; faceDistanceLength = faceDistanceLengths[j]; if (faceDistanceLength.WholePercentages is null || faceDistanceLength.Length is null) throw new NotSupportedException(); @@ -1223,18 +1221,6 @@ public class MapLogic : Shared.Models.Methods.IMapLogic } } - public ReadOnlyDictionary>> GetMissing(ReadOnlyDictionary> idToWholePercentagesToMapping) - { - Dictionary>> results = new(); - foreach (KeyValuePair>> idToCollection in _IdThenWholePercentagesToPersonContainers) - { - if (idToWholePercentagesToMapping.ContainsKey(idToCollection.Key)) - continue; - results.Add(idToCollection.Key, idToCollection.Value); - } - return new(results); - } - public ReadOnlyDictionary>? GetWholePercentagesToPersonContainers(int? id) { ReadOnlyDictionary>? result; @@ -1245,23 +1231,26 @@ public class MapLogic : Shared.Models.Methods.IMapLogic return result; } - public bool IsUsed(bool ignoreXMatches, int id, ReadOnlyDictionary>? wholePercentagesToPersonContainers, MappingFromLocation mappingFromLocation) + private bool IsUsed(bool ignoreXMatches, int id, ReadOnlyDictionary>? wholePercentagesToPersonContainers, int wholePercentages) { bool result; List? wholePercentagesCollection; ReadOnlyCollection? personContainers; - result = _SkipCollection.TryGetValue(id, out wholePercentagesCollection) && wholePercentagesCollection.Contains(mappingFromLocation.WholePercentages); + result = _SkipCollection.TryGetValue(id, out wholePercentagesCollection) && wholePercentagesCollection.Contains(wholePercentages); if (!result && wholePercentagesToPersonContainers is not null) { - if (wholePercentagesToPersonContainers.TryGetValue(mappingFromLocation.WholePercentages, out personContainers)) + if (wholePercentagesToPersonContainers.TryGetValue(wholePercentages, out personContainers)) { - if (!ignoreXMatches || personContainers.All(l => IPerson.IsDefaultName(l))) + if (!ignoreXMatches || !personContainers.Any(l => IPerson.IsDefaultName(l))) result = true; } } return result; } + public bool IsUsed(bool ignoreXMatches, int id, ReadOnlyDictionary>? wholePercentagesToPersonContainers, MappingFromLocation mappingFromLocation) => + IsUsed(ignoreXMatches, id, wholePercentagesToPersonContainers, mappingFromLocation.WholePercentages); + public bool InSkipCollection(int id, MappingFromLocation mappingFromLocation) => _SkipCollection.TryGetValue(id, out List? wholePercentagesCollection) && wholePercentagesCollection.Contains(mappingFromLocation.WholePercentages);