From 06a1207285b1128adbb4c9b70f6f1b947cce8ad3 Mon Sep 17 00:00:00 2001 From: Mike Phares Date: Thu, 29 Dec 2022 15:24:23 -0700 Subject: [PATCH] SaveRandomForOutputResolutions --- Distance/Models/MapLogicSupport.cs | 25 ++--- Instance/DlibDotNet.cs | 63 ++++--------- Instance/Models/Binder/Configuration.cs | 9 +- Instance/Models/Configuration.cs | 9 +- Instance/Models/_F_Random.cs | 66 +++++++++---- Instance/appsettings.Development.json | 12 ++- Instance/appsettings.json | 13 +-- Map/Models/MapLogic.cs | 119 ++++++++++++++++++++---- PhotoPrism/Models/_F_PhotoPrism.cs | 2 +- 9 files changed, 209 insertions(+), 109 deletions(-) diff --git a/Distance/Models/MapLogicSupport.cs b/Distance/Models/MapLogicSupport.cs index 86f0319..eaf09b7 100644 --- a/Distance/Models/MapLogicSupport.cs +++ b/Distance/Models/MapLogicSupport.cs @@ -173,7 +173,7 @@ public class MapLogicSupport : Shared.Models.Methods.IMapLogicSupport return results; } - private static FaceDistanceContainer[] GetFaceDistanceContainers(Face[] distinctFilteredFaces) + private static FaceDistanceContainer[] GetFaceDistanceContainers(List distinctFilteredFaces) { FaceDistanceContainer[] results; FaceDistance faceDistance; @@ -211,7 +211,7 @@ public class MapLogicSupport : Shared.Models.Methods.IMapLogicSupport return faceDistanceEncodings; } - public SortingContainer[] SetFaceMappingSortingCollectionThenGetSortingContainers(int maxDegreeOfParallelism, long ticks, MapLogic mapLogic, Face[] distinctFilteredFaces, List missingFaceDistanceContainers, int? useFiltersCounter) + public SortingContainer[] SetFaceMappingSortingCollectionThenGetSortingContainers(int maxDegreeOfParallelism, long ticks, MapLogic mapLogic, List distinctFilteredFaces, List missingFaceDistanceContainers, int? useFiltersCounter) { SortingContainer[] results; List collection = new(); @@ -248,22 +248,22 @@ public class MapLogicSupport : Shared.Models.Methods.IMapLogicSupport return results; } - public static Mapping[] GetSelectedMappingCollection(Face[] distinctFilteredFaces) + public static Mapping[] GetSelectedMappingCollection(List distinctFilteredFaces) { Mapping[] results; - IEnumerable collection = from l in distinctFilteredFaces orderby l.Mapping?.MappingFromItem.MinimumDateTime descending select l.Mapping; + IEnumerable collection = from l in distinctFilteredFaces orderby l.Mapping?.MappingFromItem.Id select l.Mapping; results = (from l in collection where l is not null select l).ToArray(); return results; } - public static void SetFaceDistances(int maxDegreeOfParallelism, long ticks, Face[] distinctFilteredFaces) + public static void SetFaceDistances(int maxDegreeOfParallelism, long ticks, List distinctFilteredFaces) { int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; - string message = $") {distinctFilteredFaces.Length:000} Load Face Encoding - {totalSeconds} total second(s)"; + string message = $") {distinctFilteredFaces.Count:000} Load Face Encoding - {totalSeconds} total second(s)"; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; - using ProgressBar progressBar = new(distinctFilteredFaces.Length, message, options); - _ = Parallel.For(0, distinctFilteredFaces.Length, parallelOptions, (i, state) => + using ProgressBar progressBar = new(distinctFilteredFaces.Count, message, options); + _ = Parallel.For(0, distinctFilteredFaces.Count, parallelOptions, (i, state) => { Face face = distinctFilteredFaces[i]; if (face.FaceEncoding is null || face.Mapping is null) @@ -331,12 +331,15 @@ public class MapLogicSupport : Shared.Models.Methods.IMapLogicSupport public static Dictionary> GetIdToNormalizedRectangleToFace(Mapping[] mappingCollection) { Dictionary> results = new(); - Dictionary keyValuePairs; + Dictionary? keyValuePairs; foreach (Mapping mapping in mappingCollection) { - if (!results.ContainsKey(mapping.MappingFromItem.Id)) + if (!results.TryGetValue(mapping.MappingFromItem.Id, out keyValuePairs)) + { results.Add(mapping.MappingFromItem.Id, new()); - keyValuePairs = results[mapping.MappingFromItem.Id]; + if (!results.TryGetValue(mapping.MappingFromItem.Id, out keyValuePairs)) + throw new Exception(); + } if (keyValuePairs.ContainsKey(mapping.MappingFromLocation.NormalizedRectangle)) continue; keyValuePairs.Add(mapping.MappingFromLocation.NormalizedRectangle, mapping); diff --git a/Instance/DlibDotNet.cs b/Instance/DlibDotNet.cs index cae65f5..a7ee044 100644 --- a/Instance/DlibDotNet.cs +++ b/Instance/DlibDotNet.cs @@ -796,13 +796,12 @@ public partial class DlibDotNet } } - private Shared.Models.Face[] GetFilteredDistinctFaces(string argZero, Container[] containers) + private List GetFilteredDistinctFaces(string argZero, Container[] containers) { - Shared.Models.Face[] results; + List results = new(); Item[] filteredItems; bool isIgnoreRelativePath; List distinct = new(); - List collection = new(); foreach (Container container in containers) { if (!container.Items.Any()) @@ -826,11 +825,10 @@ public partial class DlibDotNet { if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null) continue; - collection.Add(face); + results.Add(face); } } } - results = (from l in collection orderby l.Mapping?.MappingFromItem.Id select l).ToArray(); return results; } @@ -860,16 +858,15 @@ public partial class DlibDotNet return items; } - private void MapLogic(string argZero, long ticks, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string fPhotoPrismContentDirectory, MapLogicSupport mapLogicSupport, MapLogic mapLogic, string outputResolution) + private void MapLogic(string argZero, long ticks, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string fPhotoPrismContentDirectory, MapLogicSupport mapLogicSupport, MapLogic mapLogic, string outputResolution, List distinctFilteredFaces, Mapping[] mappingCollection, int totalNotMapped) { int? useFiltersCounter = null; SortingContainer[] sortingContainers; string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()"); string d2FacePartsContentDirectory = Path.Combine(d2ResultsFullGroupDirectory, "()"); - Shared.Models.Face[] distinctFilteredFaces = GetFilteredDistinctFaces(argZero, containers); - Mapping[] mappingCollection = MapLogicSupport.GetSelectedMappingCollection(distinctFilteredFaces); + Dictionary personKeyToCount = mapLogic.GetPersonKeyToCount(mappingCollection); string dFacesCollectionDirectory = Path.Combine(dResultsFullGroupDirectory, "[]", _Configuration.PropertyConfiguration.ResultAllInOne); - (Dictionary personKeyToCount, int totalNotMapped) = mapLogic.AddToMapping(mappingCollection); + Dictionary> idToNormalizedRectangleToMapping = MapLogicSupport.GetIdToNormalizedRectangleToFace(mappingCollection); if (Directory.Exists(fPhotoPrismContentDirectory)) F_PhotoPrism.WriteMatches(fPhotoPrismContentDirectory, _Configuration.PersonBirthdayFormat, ticks, distinctFilteredFaces, mapLogic); if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution)) @@ -879,7 +876,6 @@ public partial class DlibDotNet } if (_Configuration.PersonCharactersToCopyTo.Length == 1 && _Configuration.PersonCharacters.ToArray().Contains(_Configuration.PersonCharactersToCopyTo[0])) mapLogic.CopyAtLeastOneMappedFiles(_Configuration.PersonCharactersToCopyTo[0], dFacesContentDirectory, a2PeopleSingletonDirectory, mappingCollection); - Dictionary> idToNormalizedRectangleToMapping = MapLogicSupport.GetIdToNormalizedRectangleToFace(mappingCollection); mapLogic.CopyManualFiles(dFacesContentDirectory, idToNormalizedRectangleToMapping); if (_Configuration.SaveNotMappedForOutputResolutions.Contains(outputResolution)) mapLogic.CopyNotMappedFaces(_Configuration.RangeFaceAreaPermilleTolerance, dFacesContentDirectory, idToNormalizedRectangleToMapping); @@ -906,7 +902,7 @@ public partial class DlibDotNet MapLogicSupport.SaveFaceDistances(_Configuration.PropertyConfiguration, sortingContainers); if (totalNotMapped > 0) { - bool saveNullPerson = !personKeyToCount.Any() || _Configuration.RangeDistanceTolerance[1] < (_Configuration.RangeDistanceTolerance[2] * .5); + bool saveNullPerson = !personKeyToCount.Any() || _Configuration.RangeDistanceTolerance[1] < (_Configuration.RangeDistanceTolerance[2] * .66); int updated = mapLogic.UpdateFromSortingContainers(sortingContainers, saveNullPerson); List saveContainers = mapLogic.GetSaveContainers(dFacesContentDirectory, d2FacePartsContentDirectory, mappingCollection, idToNormalizedRectangleToMapping, useFiltersCounter, saveNullPerson); mapLogic.SaveContainers(totalNotMapped, updated, saveContainers); @@ -984,33 +980,6 @@ public partial class DlibDotNet return result; } - private List GetMappingFromItemCollection(Container[] containers) - { - List results = new(); - Item[] filteredItems; - DateTime[] containerDateTimes; - MappingFromItem mappingFromItem; - foreach (Container container in containers) - { - if (!container.Items.Any()) - continue; - if (_Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory)) - continue; - filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container); - if (!filteredItems.Any()) - continue; - containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems); - foreach (Item item in filteredItems) - { - if (item.Property?.Id is null) - continue; - mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item); - results.Add(mappingFromItem); - } - } - return results; - } - private void ParallelFor(string eDistanceContentDirectory, List<(bool, string, int, int, IReadOnlyList)> collection, string file) { const string lnk = ".lnk"; @@ -1082,12 +1051,14 @@ public partial class DlibDotNet private void Search(long ticks, string argZero, string propertyRoot) { - int j; int f; + int j; int t; + int totalNotMapped; Container[] containers; - string? a2PeopleContentDirectory; + Mapping[] mappingCollection; string eDistanceContentDirectory; + string? a2PeopleContentDirectory; string aResultsFullGroupDirectory; string bResultsFullGroupDirectory; string cResultsFullGroupDirectory; @@ -1095,6 +1066,7 @@ public partial class DlibDotNet string d2ResultsFullGroupDirectory; string fPhotoPrismContentDirectory; string fPhotoPrismSingletonDirectory; + List distinctFilteredFaces; string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "{}"); int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); string message = $") Building Container(s) - {totalSeconds} total second(s)"; @@ -1150,6 +1122,10 @@ public partial class DlibDotNet (aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); if (_Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution)) MapLogicSupport.BeforeSaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, a2PeopleSingletonDirectory); + SetMapping(_FileNameToCollection, argZero, containers); + distinctFilteredFaces = GetFilteredDistinctFaces(argZero, containers); + mappingCollection = MapLogicSupport.GetSelectedMappingCollection(distinctFilteredFaces); + totalNotMapped = mapLogic.UpdateMappingFromPerson(mappingCollection); if (_ArgZeroIsConfigurationRootDirectory && _Configuration.SaveResizedSubfiles && outputResolution == _Configuration.OutputResolutions[0] @@ -1158,18 +1134,17 @@ public partial class DlibDotNet { if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any()) break; - SetMapping(_FileNameToCollection, argZero, containers); - MapLogic(argZero, ticks, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, fPhotoPrismContentDirectory, mapLogicSupport, mapLogic, outputResolution); + MapLogic(argZero, ticks, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, fPhotoPrismContentDirectory, mapLogicSupport, mapLogic, outputResolution, distinctFilteredFaces, mappingCollection, totalNotMapped); if (_IsEnvironment.Development) continue; - List mappingFromItemCollection = GetMappingFromItemCollection(containers); - _Random.Random(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0], mappingFromItemCollection); G2_Identify identify = new(_Configuration); List identifiedCollection = identify.GetIdentifiedCollection(_IsEnvironment, _Configuration.PropertyConfiguration, _Faces.FileNameExtension); identify.WriteAllText(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0], identifiedCollection); if (_Configuration.LoadOrCreateThenSaveIndex) _Index.SetIndex(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0]); } + if (_Configuration.SaveRandomForOutputResolutions.Contains(outputResolution)) + _Random.Random(_Configuration.PropertyConfiguration, mapLogic, outputResolution, mappingCollection); if (!_IsEnvironment.Development) { _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(aResultsFullGroupDirectory, "{}")); diff --git a/Instance/Models/Binder/Configuration.cs b/Instance/Models/Binder/Configuration.cs index 529efdb..62a289c 100644 --- a/Instance/Models/Binder/Configuration.cs +++ b/Instance/Models/Binder/Configuration.cs @@ -70,6 +70,7 @@ public class Configuration [Display(Name = "Save Mapped"), Required] public string[] SaveMappedForOutputResolutions { get; set; } [Display(Name = "Save Not Mapped"), Required] public string[] SaveNotMappedForOutputResolutions { get; set; } [Display(Name = "Save Resized Images by Person Key Formatted"), Required] public string[] SaveFilteredOriginalImagesFromJLinksForOutputResolutions { get; set; } + [Display(Name = "Save Random For Output Resolutions"), Required] public string[] SaveRandomForOutputResolutions { get; set; } [Display(Name = "Save Resized Subfiles"), Required] public bool? SaveResizedSubfiles { get; set; } [Display(Name = "Save Shortcuts"), Required] public string[] SaveShortcutsForOutputResolutions { get; set; } [Display(Name = "Skip Search"), Required] public bool? SkipSearch { get; set; } @@ -191,6 +192,7 @@ public class Configuration configuration.SaveMappedForOutputResolutions ??= Array.Empty(); configuration.SaveNotMappedForOutputResolutions ??= Array.Empty(); configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions ??= Array.Empty(); + configuration.SaveRandomForOutputResolutions ??= Array.Empty(); if (configuration.SaveResizedSubfiles is null) throw new NullReferenceException(nameof(configuration.SaveResizedSubfiles)); configuration.SaveShortcutsForOutputResolutions ??= Array.Empty(); @@ -217,8 +219,8 @@ public class Configuration configuration.DistancePixelDistanceTolerance.Value, configuration.DistanceRenameToMatch.Value, configuration.FaceAreaPermille.Value, - configuration.FaceDistanceHiddenImageFactor.Value, configuration.FaceConfidencePercent.Value, + configuration.FaceDistanceHiddenImageFactor.Value, configuration.FaceDistancePermyriad.Value, configuration.ForceFaceLastWriteTimeToCreationTime.Value, configuration.ForceMetadataLastWriteTimeToCreationTime.Value, @@ -256,16 +258,17 @@ public class Configuration configuration.PropertiesChangedForMetadata.Value, configuration.PropertiesChangedForResize.Value, configuration.RangeDaysDeltaTolerance, + configuration.RangeDistanceTolerance, configuration.RangeFaceAreaPermilleTolerance, configuration.RangeFaceConfidence, - configuration.RangeDistanceTolerance, configuration.Reverse.Value, configuration.SaveFaceDistancesForOutputResolutions, configuration.SaveFaceLandmarkForOutputResolutions, + configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions, configuration.SaveFullYearOfRandomFiles.Value, configuration.SaveMappedForOutputResolutions, configuration.SaveNotMappedForOutputResolutions, - configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions, + configuration.SaveRandomForOutputResolutions, configuration.SaveResizedSubfiles.Value, configuration.SaveShortcutsForOutputResolutions, configuration.SkipSearch.Value, diff --git a/Instance/Models/Configuration.cs b/Instance/Models/Configuration.cs index 754ff19..2d38d72 100644 --- a/Instance/Models/Configuration.cs +++ b/Instance/Models/Configuration.cs @@ -66,6 +66,7 @@ public class Configuration public string[] SaveMappedForOutputResolutions { init; get; } public string[] SaveNotMappedForOutputResolutions { init; get; } public string[] SaveFilteredOriginalImagesFromJLinksForOutputResolutions { init; get; } + public string[] SaveRandomForOutputResolutions { init; get; } public bool SaveResizedSubfiles { init; get; } public string[] SaveShortcutsForOutputResolutions { init; get; } public bool SkipSearch { init; get; } @@ -85,8 +86,8 @@ public class Configuration int distancePixelDistanceTolerance, bool distanceRenameToMatch, int faceAreaPermille, - int faceDistanceHiddenImageFactor, int faceConfidencePercent, + int faceDistanceHiddenImageFactor, int faceDistancePermyriad, bool forceFaceLastWriteTimeToCreationTime, bool forceMetadataLastWriteTimeToCreationTime, @@ -124,16 +125,17 @@ public class Configuration bool propertiesChangedForMetadata, bool propertiesChangedForResize, int[] rangeDaysDeltaTolerance, + double[] rangeDistanceTolerance, int[] rangeFaceAreaPermilleTolerance, double[] rangeFaceConfidence, - double[] rangeDistanceTolerance, bool reverse, string[] saveFaceDistancesForOutputResolutions, string[] saveFaceLandmarkForOutputResolutions, + string[] saveFilteredOriginalImagesFromJLinksForOutputResolutions, bool saveFullYearOfRandomFiles, string[] saveMappedForOutputResolutions, string[] saveNotMappedForOutputResolutions, - string[] saveFilteredOriginalImagesFromJLinksForOutputResolutions, + string[] saveRandomForOutputResolutions, bool saveResizedSubfiles, string[] saveShortcutsForOutputResolutions, bool skipSearch, @@ -202,6 +204,7 @@ public class Configuration SaveNotMappedForOutputResolutions = saveNotMappedForOutputResolutions; SaveFilteredOriginalImagesFromJLinksForOutputResolutions = saveFilteredOriginalImagesFromJLinksForOutputResolutions; SaveResizedSubfiles = saveResizedSubfiles; + SaveRandomForOutputResolutions = saveRandomForOutputResolutions; SaveShortcutsForOutputResolutions = saveShortcutsForOutputResolutions; SkipSearch = skipSearch; SortingMaximumPerFaceShouldBeHigh = sortingMaximumPerFaceShouldBeHigh; diff --git a/Instance/Models/_F_Random.cs b/Instance/Models/_F_Random.cs index d69d00c..a003244 100644 --- a/Instance/Models/_F_Random.cs +++ b/Instance/Models/_F_Random.cs @@ -43,46 +43,74 @@ internal class F_Random return result; } - internal void Random(Property.Models.Configuration configuration, string outputResolution, List mappingFromItemCollection) + private static Dictionary> Get(Map.Models.MapLogic mapLogic, Shared.Models.Mapping[] mappingCollection, string dateFormat) { + Dictionary> results = new(); + string key; + DateTime dateTime; + List? personKeys; + List? relativePaths; + Dictionary> idToPersonKeys = mapLogic.GetIdToPeronKeys(); + foreach (Shared.Models.Mapping mapping in mappingCollection) + { + if (mapping.MappingFromItem.ImageFileHolder.DirectoryName is null || mapping.MappingFromPerson is null) + continue; + if (!idToPersonKeys.TryGetValue(mapping.MappingFromItem.Id, out personKeys)) + continue; + if (!personKeys.Contains(mapping.MappingFromPerson.PersonBirthday.Value.Ticks)) + continue; + dateTime = new(mapping.MappingFromPerson.PersonBirthday.Value.Ticks); + key = dateTime.ToString(dateFormat); + if (!results.TryGetValue(key, out relativePaths)) + { + results.Add(key, new()); + if (!results.TryGetValue(key, out relativePaths)) + throw new Exception(); + } + relativePaths.Add(mapping.MappingFromItem.RelativePath); + } + return results; + } + + internal void Random(Property.Models.Configuration configuration, Map.Models.MapLogic mapLogic, string outputResolution, Shared.Models.Mapping[] mappingCollection) + { + string key; string json; string jsonFile; Random random = new(); + List? collection; + string dateFormat = "MM-dd"; List relativePaths = new(); - List ignoreRelativePaths = new(); - DateTime dateTime = new(2024, 1, 1); //Leap year + DateTime dateTime = new(2024, 1, 1); //Leap year + Dictionary> dayToRelativePaths = Get(mapLogic, mappingCollection, dateFormat); string fRandomCollectionDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(F_Random), "[]"); string[] files = Directory.GetFiles(fRandomCollectionDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string file in files) File.Delete(file); - foreach (Shared.Models.MappingFromItem mappingFromItem in mappingFromItemCollection) + foreach (Shared.Models.Mapping mapping in mappingCollection) { - if (mappingFromItem.ImageFileHolder.DirectoryName is null) + if (mapping.MappingFromItem.ImageFileHolder.DirectoryName is null) continue; - if (!_Configuration.IgnoreRelativePaths.Any(l => mappingFromItem.ImageFileHolder.DirectoryName.Contains(l)) || !IsIgnoreRelativePath(mappingFromItem.ImageFileHolder.DirectoryName)) - relativePaths.Add(mappingFromItem.RelativePath); - else - ignoreRelativePaths.Add(mappingFromItem.RelativePath); + if (_Configuration.IgnoreRelativePaths.Any(l => mapping.MappingFromItem.ImageFileHolder.DirectoryName.Contains(l)) && IsIgnoreRelativePath(mapping.MappingFromItem.ImageFileHolder.DirectoryName)) + continue; + relativePaths.Add(mapping.MappingFromItem.RelativePath); } if (relativePaths.Any()) { for (int i = 0; i < 366; i++) { - relativePaths = (from l in relativePaths orderby random.NextDouble() select l).ToList(); - jsonFile = Path.Combine(fRandomCollectionDirectory, $"{dateTime.AddDays(i):MM-dd}.json"); - json = JsonSerializer.Serialize(relativePaths, _WriteIndentedJsonSerializerOptions); + key = dateTime.AddDays(i).ToString(dateFormat); + if (dayToRelativePaths.TryGetValue(key, out collection) && collection.Count > 10) + collection = (from l in collection orderby random.NextDouble() select l).ToList(); + else + collection = (from l in relativePaths orderby random.NextDouble() select l).ToList(); + jsonFile = Path.Combine(fRandomCollectionDirectory, $"{key}.json"); + json = JsonSerializer.Serialize(collection, _WriteIndentedJsonSerializerOptions); _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(jsonFile, json, updateDateWhenMatches: false, compareBeforeWrite: false); if (!_Configuration.SaveFullYearOfRandomFiles) break; } } - if (ignoreRelativePaths.Any()) - { - ignoreRelativePaths = (from l in ignoreRelativePaths orderby random.NextDouble() select l).ToList(); - jsonFile = Path.Combine(fRandomCollectionDirectory, "01-01.txt"); - json = JsonSerializer.Serialize(ignoreRelativePaths, _WriteIndentedJsonSerializerOptions); - _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(jsonFile, json, updateDateWhenMatches: false, compareBeforeWrite: false); - } } } \ No newline at end of file diff --git a/Instance/appsettings.Development.json b/Instance/appsettings.Development.json index a942641..8514fb1 100644 --- a/Instance/appsettings.Development.json +++ b/Instance/appsettings.Development.json @@ -13,6 +13,7 @@ "DistanceRenameToMatch": true, "DistanceMoveUnableToMatch": true, "PersonCharacters": "!#]^_`~+", + "PersonCharactersToCopyTo": "x", "xPersonCharactersToCopyTo": "!", "xxPersonCharactersToCopyTo": "#", "xxxPersonCharactersToCopyTo": "]", @@ -22,9 +23,9 @@ "xxxxxxxPersonCharactersToCopyTo": "~", "xxxxxxxxPersonCharactersToCopyTo": "+", "xRootDirectory": "D:/Tmp/phares/Pictures", - "xxRootDirectory": "D:/Tmp/Phares/Compare/Not-Copy-Copy-37c7b67", - "xxxRootDirectory": "D:/Tmp/Phares/Compare/Corrupt", - "RootDirectory": "D:/1) Images A/Images-37c7b67", + "xxRootDirectory": "D:/Tmp/Phares/Compare/Corrupt", + "RootDirectory": "D:/Tmp/Phares/Compare/Not-Copy-Copy-37c7b67", + "xxxxRootDirectory": "D:/1) Images A/Images-37c7b67", "xxxxxRootDirectory": "D:/1) Images A/Images-37c7b67/Facebook/=2022.3 Facebook", "JLinks": [ "Julie" @@ -63,9 +64,12 @@ "7680 x 4320" ], "SaveFaceLandmarkForOutputResolutions": [], + "SaveFilteredOriginalImagesFromJLinksForOutputResolutions": [], "SaveMappedForOutputResolutions": [], "SaveNotMappedForOutputResolutions": [], - "SaveFilteredOriginalImagesFromJLinksForOutputResolutions": [], + "SaveRandomForOutputResolutions": [ + "7680 x 4320" + ], "SaveShortcutsForOutputResolutions": [], "IgnoreRelativePaths": [ "3757 W Whitman 2017", diff --git a/Instance/appsettings.json b/Instance/appsettings.json index 26c4bf6..5b2aded 100644 --- a/Instance/appsettings.json +++ b/Instance/appsettings.json @@ -147,14 +147,15 @@ 0.2, 100 ], - "SaveFaceDistancesForOutputResolutions": [], - "SaveFaceLandmarkForOutputResolutions": [], - "SaveMappedForOutputResolutions": [], - "SaveNotMappedForOutputResolutions": [], - "SaveFilteredOriginalImagesFromJLinksForOutputResolutions": [], - "SaveShortcutsForOutputResolutions": [], "IgnoreRelativePaths": [], "MixedYearRelativePaths": [], + "SaveFaceDistancesForOutputResolutions": [], + "SaveFaceLandmarkForOutputResolutions": [], + "SaveFilteredOriginalImagesFromJLinksForOutputResolutions": [], + "SaveMappedForOutputResolutions": [], + "SaveNotMappedForOutputResolutions": [], + "SaveRandomForOutputResolutions": [], + "SaveShortcutsForOutputResolutions": [], "VerifyToSeason": [], "ValidImageFormatExtensions": [ ".bmp", diff --git a/Map/Models/MapLogic.cs b/Map/Models/MapLogic.cs index f8a03c5..b7f87e1 100644 --- a/Map/Models/MapLogic.cs +++ b/Map/Models/MapLogic.cs @@ -101,13 +101,71 @@ public class MapLogic : Shared.Models.Methods.IMapLogic return result; } + private static Dictionary> GetIdToPersonKeys(Dictionary> keyValuePairs) + { + Dictionary> results = new(); + List? collection; + foreach (KeyValuePair> keyValuePair in keyValuePairs) + { + foreach (int id in keyValuePair.Value) + { + if (!results.TryGetValue(id, out collection)) + { + results.Add(id, new()); + if (!results.TryGetValue(id, out collection)) + throw new Exception(); + } + if (collection.Contains(keyValuePair.Key)) + continue; + collection.Add(keyValuePair.Key); + } + } + return results; + } + + public Dictionary> GetIdToPeronKeys() + { + Dictionary> results; + long key; + const int zero = 0; + List? collection; + PersonBirthday personBirthday; + Dictionary> keyValuePairs = new(); + foreach (KeyValuePair> idToCollection in _IdThenNormalizedRectangleToPersonContainers) + { + foreach (KeyValuePair normalizedRectangleToPersonContainers in idToCollection.Value) + { + foreach (PersonContainer personContainer in normalizedRectangleToPersonContainers.Value) + { + if (personContainer.Key is null || personContainer.Birthdays is null || !personContainer.Birthdays.Any()) + continue; + personBirthday = personContainer.Birthdays[zero]; + if (IPersonBirthday.IsCounterPersonBirthday(personBirthday)) + continue; + key = personBirthday.Value.Ticks; + if (!keyValuePairs.TryGetValue(key, out collection)) + { + keyValuePairs.Add(key, new()); + if (!keyValuePairs.TryGetValue(key, out collection)) + throw new Exception(); + } + if (collection.Contains(idToCollection.Key)) + continue; + collection.Add(idToCollection.Key); + } + } + } + results = GetIdToPersonKeys(keyValuePairs); + return results; + } + (bool, Dictionary?) Shared.Models.Methods.IMapLogic.GetNormalizedRectangleToPersonContainers(int id) { bool result = _IdThenNormalizedRectangleToPersonContainers.TryGetValue(id, out Dictionary? normalizedRectangleToPersonContainers); return new(result, normalizedRectangleToPersonContainers); } - public (Dictionary, int) AddToMapping(Mapping[] mappingCollection) + public int UpdateMappingFromPerson(Mapping[] mappingCollection) { if (_Configuration is null) throw new NullReferenceException(nameof(_Configuration)); @@ -116,21 +174,19 @@ public class MapLogic : Shared.Models.Methods.IMapLogic const int zero = 0; string mappingSegmentB; PersonBirthday personBirthday; - PersonContainer[]? collection; - List personContainers = new(); - Dictionary personKeyToCount = new(); + PersonContainer[]? personContainers; Dictionary? normalizedRectangleToPersonContainers; foreach (Mapping mapping in mappingCollection) { - personContainers.Clear(); if (!_IdThenNormalizedRectangleToPersonContainers.TryGetValue(mapping.MappingFromItem.Id, out normalizedRectangleToPersonContainers)) - result += 1; - else { - if (!normalizedRectangleToPersonContainers.TryGetValue(mapping.MappingFromLocation.NormalizedRectangle, out collection)) - result += 1; - else - personContainers.AddRange(collection); + result += 1; + continue; + } + if (!normalizedRectangleToPersonContainers.TryGetValue(mapping.MappingFromLocation.NormalizedRectangle, out personContainers)) + { + result += 1; + continue; } foreach (PersonContainer personContainer in personContainers) { @@ -138,14 +194,41 @@ public class MapLogic : Shared.Models.Methods.IMapLogic continue; personBirthday = personContainer.Birthdays[zero]; personKey = personBirthday.Value.Ticks; - if (!personKeyToCount.ContainsKey(personKey)) - personKeyToCount.Add(personKey, 0); - personKeyToCount[personKey] += 1; mappingSegmentB = Stateless.MapLogic.GetMappingSegmentB(_Ticks, personBirthday, personContainer.ApproximateYears, mapping.MappingFromItem); mapping.UpdateMappingFromPerson(personContainer.ApproximateYears, personContainer.DisplayDirectoryName, personBirthday, mappingSegmentB); } } - return new(personKeyToCount, result); + return result; + } + + public Dictionary GetPersonKeyToCount(Mapping[] mappingCollection) + { + if (_Configuration is null) + throw new NullReferenceException(nameof(_Configuration)); + Dictionary results = new(); + long personKey; + const int zero = 0; + PersonBirthday personBirthday; + PersonContainer[]? personContainers; + Dictionary? normalizedRectangleToPersonContainers; + foreach (Mapping mapping in mappingCollection) + { + if (!_IdThenNormalizedRectangleToPersonContainers.TryGetValue(mapping.MappingFromItem.Id, out normalizedRectangleToPersonContainers)) + continue; + if (!normalizedRectangleToPersonContainers.TryGetValue(mapping.MappingFromLocation.NormalizedRectangle, out personContainers)) + continue; + foreach (PersonContainer personContainer in personContainers) + { + if (personContainer.Key is null || personContainer.Birthdays is null || !personContainer.Birthdays.Any()) + continue; + personBirthday = personContainer.Birthdays[zero]; + personKey = personBirthday.Value.Ticks; + if (!results.ContainsKey(personKey)) + results.Add(personKey, 0); + results[personKey] += 1; + } + } + return results; } public void SaveContainers(int totalNotMapped, int? updated, List saveContainers) @@ -1200,11 +1283,11 @@ public class MapLogic : Shared.Models.Methods.IMapLogic public Dictionary> GetMissing(Dictionary> idToNormalizedRectangleToMapping) { Dictionary> results = new(); - foreach (KeyValuePair> normalizedRectangleToPersonContainers in _IdThenNormalizedRectangleToPersonContainers) + foreach (KeyValuePair> idToCollection in _IdThenNormalizedRectangleToPersonContainers) { - if (idToNormalizedRectangleToMapping.ContainsKey(normalizedRectangleToPersonContainers.Key)) + if (idToNormalizedRectangleToMapping.ContainsKey(idToCollection.Key)) continue; - results.Add(normalizedRectangleToPersonContainers.Key, normalizedRectangleToPersonContainers.Value); + results.Add(idToCollection.Key, idToCollection.Value); } return results; } diff --git a/PhotoPrism/Models/_F_PhotoPrism.cs b/PhotoPrism/Models/_F_PhotoPrism.cs index 8b0a2b7..e4a119e 100644 --- a/PhotoPrism/Models/_F_PhotoPrism.cs +++ b/PhotoPrism/Models/_F_PhotoPrism.cs @@ -88,7 +88,7 @@ public class F_PhotoPrism return results; } - public static void WriteMatches(string fPhotoPrismContentDirectory, string personBirthdayFormat, long ticks, Face[] distinctFilteredFaces, Shared.Models.Methods.IMapLogic mapLogic) + public static void WriteMatches(string fPhotoPrismContentDirectory, string personBirthdayFormat, long ticks, List distinctFilteredFaces, Shared.Models.Methods.IMapLogic mapLogic) { long? personKey; const int zero = 0;