From cfeeb0eb9596eac5c79c8d0b95055d677f4a66c2 Mon Sep 17 00:00:00 2001 From: Mike Phares Date: Sun, 23 Oct 2022 00:50:36 -0700 Subject: [PATCH] Drag Drop Form and Copy Not Mapped Faces --- .vscode/launch.json | 1 + Drag-Drop/Form.cs | 2 +- Drag-Drop/Program.cs | 3 +- Instance/DlibDotNet.cs | 55 +++++++++++------ Instance/Models/Binder/Configuration.cs | 20 +++--- Instance/Models/Configuration.cs | 18 +++--- Instance/Models/_F_Random.cs | 12 ++-- Instance/Models/_G_Index.cs | 3 +- Instance/appsettings.Development.json | 8 ++- Instance/appsettings.Staging.json | 4 +- Instance/appsettings.json | 4 +- Map/Models/MapLogic.cs | 61 +++++++++++++++++++ Map/Models/Stateless/IMapLogic.cs | 1 + Tests/UnitTestCalculations.cs | 10 +-- Tests/UnitTestResize.cs | 10 ++- .../Models/Binder/Configuration.cs | 20 +++--- .../Models/Configuration.cs | 18 +++--- .../UnitTestFace.cs | 6 +- 18 files changed, 175 insertions(+), 81 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 92f3352..cac894c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,6 +12,7 @@ // If you have changed target frameworks, make sure to update the program path. "programC": "${workspaceFolder}/Compare/bin/Debug/net6.0/win-x64/Compare.dll", "programD": "${workspaceFolder}/Date-Group/bin/Debug/net6.0/win-x64/Date-Group.dll", + "programDD": "${workspaceFolder}/Drag-Drop/bin/Debug/net6.0-windows/win-x64/Drag-Drop.dll", "program": "${workspaceFolder}/Instance/bin/Debug/net6.0/win-x64/Instance.dll", "programN": "${workspaceFolder}/Not-Copy-Copy/bin/Debug/net6.0/win-x64/Not-Copy-Copy.dll", "programP": "${workspaceFolder}/PrepareForOld/bin/Debug/net6.0/win-x64/PrepareForOld.dll", diff --git a/Drag-Drop/Form.cs b/Drag-Drop/Form.cs index 68270e3..0bb5e0c 100644 --- a/Drag-Drop/Form.cs +++ b/Drag-Drop/Form.cs @@ -80,7 +80,7 @@ public partial class Form : System.Windows.Forms.Form DragEnter += new DragEventHandler(Form1_DragEnter); Property.Models.A_Property propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _ResizeFileNameExtension, _Configuration.Reverse); (j, f, t, containers) = Property.Models.Stateless.Container.GetContainers(_Configuration.PropertyConfiguration, firstRun, propertyLogic); - List collection = Program.GetCollection(_Configuration, containers); + List collection = Program.GetMappingFromItemCollection(_Configuration, containers); foreach (MappingFromItem mappingFromItem in collection) { if (_IdToMappingFromItem.ContainsKey(mappingFromItem.Id)) diff --git a/Drag-Drop/Program.cs b/Drag-Drop/Program.cs index d5c8449..f450da3 100644 --- a/Drag-Drop/Program.cs +++ b/Drag-Drop/Program.cs @@ -47,7 +47,7 @@ static class Program return result; } - public static List GetCollection(Models.Configuration configuration, Container[] containers) + public static List GetMappingFromItemCollection(Models.Configuration configuration, Container[] containers) { List results = new(); bool? isWrongYear; @@ -75,4 +75,5 @@ static class Program } return results; } + } \ No newline at end of file diff --git a/Instance/DlibDotNet.cs b/Instance/DlibDotNet.cs index 7b44398..afcb7d2 100644 --- a/Instance/DlibDotNet.cs +++ b/Instance/DlibDotNet.cs @@ -36,8 +36,6 @@ public partial class DlibDotNet private readonly Models.Configuration _Configuration; private readonly bool _ArgZeroIsConfigurationRootDirectory; private readonly Map.Models.Configuration _MapConfiguration; - private readonly List> _FileKeyValuePairs; - private readonly Dictionary>> _FilePropertiesKeyValuePairs; public DlibDotNet( List args, @@ -57,8 +55,6 @@ public partial class DlibDotNet _Exceptions = new List(); PersonContainer[] personContainers; _Log = Serilog.Log.ForContext(); - _FileKeyValuePairs = new List>(); - _FilePropertiesKeyValuePairs = new Dictionary>>(); Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot); _Log.Information(propertyConfiguration.RootDirectory); Property.Models.Configuration.Verify(propertyConfiguration, requireExist: false); @@ -504,8 +500,6 @@ public partial class DlibDotNet string json; string checkFile; int sourceDirectoryLength = container.SourceDirectory.Length; - if (_ArgZeroIsConfigurationRootDirectory && outputResolution == _Configuration.OutputResolutions[0]) - _FilePropertiesKeyValuePairs.Add(container.SourceDirectory, new List>()); JsonSerializerOptions writeIndentedJsonSerializerOptions = new() { WriteIndented = false }; if (!(from l in propertyCollection where l?.Width is null select true).Any()) { @@ -526,11 +520,6 @@ public partial class DlibDotNet if (item.ImageFileHolder is null) continue; key = Shared.Models.Stateless.Methods.IPath.GetRelativePath(item.ImageFileHolder.FullName, sourceDirectoryLength); - if (_ArgZeroIsConfigurationRootDirectory && outputResolution == _Configuration.OutputResolutions[0]) - { - _FileKeyValuePairs.Add(new KeyValuePair(container.SourceDirectory, key)); - _FilePropertiesKeyValuePairs[container.SourceDirectory].Add(new Tuple(key, propertyCollection[i])); - } propertyCollectionKeyValuePairs.Add(new KeyValuePair(key, propertyCollection[i])); resizeKeyValuePairsCollections.Add(new KeyValuePair>(key, resizeKeyValuePairs[i])); imageFaceCollectionsKeyValuePairs.Add(new KeyValuePair?>(key, imageFaceCollections[i])); @@ -693,8 +682,6 @@ public partial class DlibDotNet foreach (string outputResolution in _Configuration.OutputResolutions) { total = 0; - _FileKeyValuePairs.Clear(); - _FilePropertiesKeyValuePairs.Clear(); (aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); for (int i = 0; i < containers.Length; i++) { @@ -872,10 +859,11 @@ public partial class DlibDotNet mapLogicSupport); Dictionary> idToNormalizedPixelPercentageToMapping = MapLogicSupport.GetIdToNormalizedPixelPercentageToFace(mappingCollection); mapLogic.CopyManualFiles(dFacesContentDirectory, idToNormalizedPixelPercentageToMapping); + mapLogic.CopyNotMappedFaces(_Configuration.FaceDistanceAreaPermilleTolerance, dFacesContentDirectory, idToNormalizedPixelPercentageToMapping); (Dictionary personKeyToCount, int totalNotMapped) = mapLogic.AddToMapping(mappingCollection); - if (_Configuration.MappingSaveMapped) + if (_Configuration.SaveMappedForOutputResolutions.Contains(outputResolution)) mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, mappingCollection, idToNormalizedPixelPercentageToMapping, personKeyToCount, totalNotMapped); - if (_Configuration.MappingSaveNotMapped) + if (_Configuration.SaveNotMappedForOutputResolutions.Contains(outputResolution)) mapLogic.SaveNotMappedTicks(); if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution)) mapLogic.SaveShortcuts(a2PeopleSingletonDirectory, personContainers, filteredItems, mappingCollection, personKeyToCount); @@ -963,6 +951,35 @@ public partial class DlibDotNet return result; } + private List GetMappingFromItemCollection(Container[] containers) + { + List results = new(); + bool? isWrongYear; + Item[] filteredItems; + DateTime minimumDateTime; + MappingFromItem mappingFromItem; + foreach (Container container in containers) + { + if (!container.Items.Any()) + continue; + if (_Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && IsIgnoreRelativePath(container.SourceDirectory)) + continue; + filteredItems = GetFilterItems(container); + if (!filteredItems.Any()) + continue; + foreach (Item item in filteredItems) + { + if (item.Property?.Id is null || item.ImageFileHolder is null) + continue; + minimumDateTime = Shared.Models.Stateless.Methods.IProperty.GetMinimumDateTime(item.Property); + (isWrongYear, _) = item.Property.IsWrongYear(item.ImageFileHolder.FullName, minimumDateTime); + mappingFromItem = new(item.Property.Id.Value, item.ImageFileHolder, isWrongYear, minimumDateTime, item.RelativePath, item.ImageFileHolder); + results.Add(mappingFromItem); + } + } + return results; + } + private void Search(long ticks, string argZero, string propertyRoot, PersonContainer[] personContainers) { int j; @@ -1024,13 +1041,13 @@ public partial class DlibDotNet MapLogic(argZero, ticks, personContainers, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2FacePartsContentDirectory, eDistanceContentDirectory, outputResolution); if (_IsEnvironment.Development) continue; - if (_FileKeyValuePairs.Any()) - _Random.Random(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0], _FileKeyValuePairs); + List mappingFromItemCollection = GetMappingFromItemCollection(containers); + _Random.Random(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0], mappingFromItemCollection); G2_Identify identify = new(_Configuration); List identifiedCollection = identify.GetIdentifiedCollection(_IsEnvironment, _Configuration.PropertyConfiguration, _Faces.FileNameExtension); identify.WriteAllText(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0], identifiedCollection); - if (_Configuration.LoadOrCreateThenSaveIndex && _FilePropertiesKeyValuePairs.Any()) - _Index.SetIndex(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0], _FilePropertiesKeyValuePairs); + if (_Configuration.LoadOrCreateThenSaveIndex) + _Index.SetIndex(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0]); } if (!_IsEnvironment.Development) { diff --git a/Instance/Models/Binder/Configuration.cs b/Instance/Models/Binder/Configuration.cs index d667c12..235af5a 100644 --- a/Instance/Models/Binder/Configuration.cs +++ b/Instance/Models/Binder/Configuration.cs @@ -36,8 +36,6 @@ public class Configuration [Display(Name = "Location Factor"), Required] public int? LocationFactor { get; set; } [Display(Name = "Mapped Max Index"), Required] public int? MappedMaxIndex { get; set; } [Display(Name = "Mapping Default Name"), Required] public string MappingDefaultName { get; set; } - [Display(Name = "Mapping Save Mapped"), Required] public bool? MappingSaveMapped { get; set; } - [Display(Name = "Mapping Save Not Mapped"), Required] public bool? MappingSaveNotMapped { get; set; } [Display(Name = "Mapping Use Deterministic Hash Code Unknown Face Key Value Pairs for Add to Mapping"), Required] public bool? MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping { get; set; } [Display(Name = "Mapping Use Deterministic Hash Code Unknown Face Key Value Pairs for Save Mapping"), Required] public bool? MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping { get; set; } [Display(Name = "Max Items In Distance Collection"), Required] public int? MaxItemsInDistanceCollection { get; set; } @@ -67,9 +65,11 @@ public class Configuration [Display(Name = "Reverse"), Required] public bool? Reverse { get; set; } [Display(Name = "Save Face Landmark For Output Resolutions"), Required] public string[] SaveFaceLandmarkForOutputResolutions { get; set; } [Display(Name = "Save Full Year Of Random Files"), Required] public bool? SaveFullYearOfRandomFiles { get; set; } + [Display(Name = "Save Mapped"), Required] public string[] SaveMappedForOutputResolutions { get; set; } + [Display(Name = "Save Not Mapped"), Required] public string[] SaveNotMappedForOutputResolutions { get; set; } [Display(Name = "Save Resized Images by Person Key Formatted"), Required] public string[] SaveResizedImagesByPersonKeyFormattedForOutputResolutions { get; set; } - [Display(Name = "Save Shortcuts"), Required] public string[] SaveShortcutsForOutputResolutions { get; set; } [Display(Name = "Save Resized Subfiles"), Required] public bool? SaveResizedSubfiles { get; set; } + [Display(Name = "Save Shortcuts"), Required] public string[] SaveShortcutsForOutputResolutions { get; set; } [Display(Name = "Skip Search"), Required] public bool? SkipSearch { get; set; } [Display(Name = "Sorting Days Delta Tolerance"), Required] public int? SortingDaysDeltaTolerance { get; set; } [Display(Name = "SortingMaximumPerFaceShould be High"), Required] public int? SortingMaximumPerFaceShouldBeHigh { get; set; } @@ -137,10 +137,6 @@ public class Configuration throw new NullReferenceException(nameof(configuration.LocationFactor)); if (configuration.MappingDefaultName is null) throw new NullReferenceException(nameof(configuration.MappingDefaultName)); - if (configuration.MappingSaveMapped is null) - throw new NullReferenceException(nameof(configuration.MappingSaveMapped)); - if (configuration.MappingSaveNotMapped is null) - throw new NullReferenceException(nameof(configuration.MappingSaveNotMapped)); if (configuration.MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping is null) throw new NullReferenceException(nameof(configuration.MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping)); if (configuration.MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping is null) @@ -187,6 +183,10 @@ public class Configuration throw new NullReferenceException(nameof(configuration.Reverse)); if (configuration.SaveFaceLandmarkForOutputResolutions is null) configuration.SaveFaceLandmarkForOutputResolutions = Array.Empty(); + if (configuration.SaveMappedForOutputResolutions is null) + configuration.SaveMappedForOutputResolutions = Array.Empty(); + if (configuration.SaveNotMappedForOutputResolutions is null) + configuration.SaveNotMappedForOutputResolutions = Array.Empty(); if (configuration.SaveFullYearOfRandomFiles is null) throw new NullReferenceException(nameof(configuration.SaveFullYearOfRandomFiles)); if (configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions is null) @@ -239,8 +239,6 @@ public class Configuration configuration.LocationFactor.Value, configuration.MappedMaxIndex, configuration.MappingDefaultName, - configuration.MappingSaveNotMapped.Value, - configuration.MappingSaveMapped.Value, configuration.MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping.Value, configuration.MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping.Value, configuration.MaxItemsInDistanceCollection.Value, @@ -269,9 +267,11 @@ public class Configuration configuration.Reverse.Value, configuration.SaveFaceLandmarkForOutputResolutions, configuration.SaveFullYearOfRandomFiles.Value, + configuration.SaveMappedForOutputResolutions, + configuration.SaveNotMappedForOutputResolutions, configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions, - configuration.SaveShortcutsForOutputResolutions, configuration.SaveResizedSubfiles.Value, + configuration.SaveShortcutsForOutputResolutions, configuration.SkipSearch.Value, configuration.SortingDaysDeltaTolerance.Value, configuration.SortingMaximumPerFaceShouldBeHigh.Value, diff --git a/Instance/Models/Configuration.cs b/Instance/Models/Configuration.cs index 07a38b1..a3eb4f9 100644 --- a/Instance/Models/Configuration.cs +++ b/Instance/Models/Configuration.cs @@ -35,8 +35,6 @@ public class Configuration public int LocationFactor { init; get; } public int? MappedMaxIndex { init; get; } public string MappingDefaultName { init; get; } - public bool MappingSaveNotMapped { init; get; } - public bool MappingSaveMapped { init; get; } public bool MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping { init; get; } public bool MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping { init; get; } public int MaxItemsInDistanceCollection { init; get; } @@ -65,9 +63,11 @@ public class Configuration public bool Reverse { init; get; } public string[] SaveFaceLandmarkForOutputResolutions { init; get; } public bool SaveFullYearOfRandomFiles { init; get; } + public string[] SaveMappedForOutputResolutions { init; get; } + public string[] SaveNotMappedForOutputResolutions { init; get; } public string[] SaveResizedImagesByPersonKeyFormattedForOutputResolutions { init; get; } - public string[] SaveShortcutsForOutputResolutions { init; get; } public bool SaveResizedSubfiles { init; get; } + public string[] SaveShortcutsForOutputResolutions { init; get; } public bool SkipSearch { init; get; } public int SortingDaysDeltaTolerance { init; get; } public int SortingMaximumPerFaceShouldBeHigh { init; get; } @@ -104,8 +104,6 @@ public class Configuration int locationFactor, int? mappedMaxIndex, string mappingDefaultName, - bool mappingSaveNotMapped, - bool mappingSaveMapped, bool mappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping, bool mappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping, int maxItemsInDistanceCollection, @@ -134,9 +132,11 @@ public class Configuration bool reverse, string[] saveFaceLandmarkForOutputResolutions, bool saveFullYearOfRandomFiles, + string[] saveMappedForOutputResolutions, + string[] saveNotMappedForOutputResolutions, string[] saveResizedImagesByPersonKeyFormattedForOutputResolutions, - string[] saveShortcutsForOutputResolutions, bool saveResizedSubfiles, + string[] saveShortcutsForOutputResolutions, bool skipSearch, int sortingDaysDeltaTolerance, int sortingMaximumPerFaceShouldBeHigh, @@ -172,8 +172,6 @@ public class Configuration LocationFactor = locationFactor; MappedMaxIndex = mappedMaxIndex; MappingDefaultName = mappingDefaultName; - MappingSaveNotMapped = mappingSaveNotMapped; - MappingSaveMapped = mappingSaveMapped; MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping = mappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping; MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping = mappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping; MaxItemsInDistanceCollection = maxItemsInDistanceCollection; @@ -202,9 +200,11 @@ public class Configuration Reverse = reverse; SaveFaceLandmarkForOutputResolutions = saveFaceLandmarkForOutputResolutions; SaveFullYearOfRandomFiles = saveFullYearOfRandomFiles; + SaveMappedForOutputResolutions = saveMappedForOutputResolutions; + SaveNotMappedForOutputResolutions = saveNotMappedForOutputResolutions; SaveResizedImagesByPersonKeyFormattedForOutputResolutions = saveResizedImagesByPersonKeyFormattedForOutputResolutions; - SaveShortcutsForOutputResolutions = saveShortcutsForOutputResolutions; SaveResizedSubfiles = saveResizedSubfiles; + SaveShortcutsForOutputResolutions = saveShortcutsForOutputResolutions; SkipSearch = skipSearch; SortingDaysDeltaTolerance = sortingDaysDeltaTolerance; SortingMaximumPerFaceShouldBeHigh = sortingMaximumPerFaceShouldBeHigh; diff --git a/Instance/Models/_F_Random.cs b/Instance/Models/_F_Random.cs index a96de2c..d69d00c 100644 --- a/Instance/Models/_F_Random.cs +++ b/Instance/Models/_F_Random.cs @@ -43,7 +43,7 @@ internal class F_Random return result; } - internal void Random(Property.Models.Configuration configuration, string outputResolution, List> fileKeyValuePairs) + internal void Random(Property.Models.Configuration configuration, string outputResolution, List mappingFromItemCollection) { string json; string jsonFile; @@ -55,12 +55,14 @@ internal class F_Random string[] files = Directory.GetFiles(fRandomCollectionDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string file in files) File.Delete(file); - foreach (KeyValuePair keyValuePair in fileKeyValuePairs) + foreach (Shared.Models.MappingFromItem mappingFromItem in mappingFromItemCollection) { - if (!_Configuration.IgnoreRelativePaths.Any(l => keyValuePair.Key.Contains(l)) || !IsIgnoreRelativePath(keyValuePair.Key)) - relativePaths.Add(keyValuePair.Value); + if (mappingFromItem.ImageFileHolder.DirectoryName is null) + continue; + if (!_Configuration.IgnoreRelativePaths.Any(l => mappingFromItem.ImageFileHolder.DirectoryName.Contains(l)) || !IsIgnoreRelativePath(mappingFromItem.ImageFileHolder.DirectoryName)) + relativePaths.Add(mappingFromItem.RelativePath); else - ignoreRelativePaths.Add(keyValuePair.Value); + ignoreRelativePaths.Add(mappingFromItem.RelativePath); } if (relativePaths.Any()) { diff --git a/Instance/Models/_G_Index.cs b/Instance/Models/_G_Index.cs index d0ef541..e5d039d 100644 --- a/Instance/Models/_G_Index.cs +++ b/Instance/Models/_G_Index.cs @@ -140,8 +140,9 @@ public class G_Index : Shared.Models.Properties.IIndex, IIndex } } - internal void SetIndex(Property.Models.Configuration configuration, string outputResolution, Dictionary>> filePropertiesKeyValuePairs) + internal void SetIndex(Property.Models.Configuration configuration, string outputResolution) { + Dictionary>> filePropertiesKeyValuePairs = new(); FileInfo fileInfo; G_Index indexInfo; string parentCheck; diff --git a/Instance/appsettings.Development.json b/Instance/appsettings.Development.json index 6ac0c87..48ad23e 100644 --- a/Instance/appsettings.Development.json +++ b/Instance/appsettings.Development.json @@ -75,8 +75,6 @@ "MappedMaxIndex": 1034720, "MappingDefaultName": "John Doe~25", "MappingSaveFaceEncoding": false, - "MappingSaveMapped": false, - "MappingSaveNotMapped": false, "MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping": false, "MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping": false, "MaxImagesInDirectoryForTopLevelFirstPass": 10, @@ -146,8 +144,12 @@ "176 x 176", "256 x 256" ], + "SaveMappedForOutputResolutions": [], + "SaveNotMappedForOutputResolutions": [], "SaveResizedImagesByPersonKeyFormattedForOutputResolutions": [], - "SaveShortcutsForOutputResolutions": [], + "SaveShortcutsForOutputResolutions": [ + "7680 x 4320" + ], "ValidImageFormatExtensions": [ ".bmp", ".BMP", diff --git a/Instance/appsettings.Staging.json b/Instance/appsettings.Staging.json index 086414f..b823453 100644 --- a/Instance/appsettings.Staging.json +++ b/Instance/appsettings.Staging.json @@ -75,8 +75,6 @@ "MappedMaxIndex": 1034720, "MappingDefaultName": "John Doe~25", "MappingSaveFaceEncoding": false, - "MappingSaveMapped": false, - "MappingSaveNotMapped": false, "MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping": false, "MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping": false, "MaxImagesInDirectoryForTopLevelFirstPass": 10, @@ -136,6 +134,8 @@ "176 x 176", "256 x 256" ], + "SaveMappedForOutputResolutions": [], + "SaveNotMappedForOutputResolutions": [], "SaveResizedImagesByPersonKeyFormattedForOutputResolutions": [ "1920 x 1080" ], diff --git a/Instance/appsettings.json b/Instance/appsettings.json index ff0840c..2fcb271 100644 --- a/Instance/appsettings.json +++ b/Instance/appsettings.json @@ -75,8 +75,6 @@ "MappedMaxIndex": 1034720, "MappingDefaultName": "John Doe~25", "MappingSaveFaceEncoding": false, - "MappingSaveMapped": false, - "MappingSaveNotMapped": false, "MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping": false, "MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping": false, "MaxImagesInDirectoryForTopLevelFirstPass": 10, @@ -145,6 +143,8 @@ "176 x 176", "256 x 256" ], + "SaveMappedForOutputResolutions": [], + "SaveNotMappedForOutputResolutions": [], "SaveResizedImagesByPersonKeyFormattedForOutputResolutions": [ "1920 x 1080" ], diff --git a/Map/Models/MapLogic.cs b/Map/Models/MapLogic.cs index 011efb0..0c895dc 100644 --- a/Map/Models/MapLogic.cs +++ b/Map/Models/MapLogic.cs @@ -968,4 +968,65 @@ public class MapLogic } } + private List<(string, FileHolder, string)> GetCollection(int faceDistanceAreaPermilleTolerance, string dFacesContentDirectory, Dictionary> idToNormalizedPixelPercentageToMapping) + { + if (_Configuration is null) + throw new NullReferenceException(nameof(_Configuration)); + List<(string, FileHolder, string)> results = new(); + Mapping mapping; + string checkFile; + string directory; + string facesDirectory; + string? directoryName; + FileHolder faceFileHolder; + List? normalizedPixelPercentages; + string by = nameof(IMapLogic.CopyNotMappedFaces); + Dictionary? normalizedPixelPercentageToPersonContainers; + foreach (KeyValuePair> keyValuePair in idToNormalizedPixelPercentageToMapping) + { + _ = _IdThenNormalizedPixelPercentageToPersonContainers.TryGetValue(keyValuePair.Key, out normalizedPixelPercentageToPersonContainers); + foreach (KeyValuePair normalizedPixelPercentageAndMapping in keyValuePair.Value) + { + mapping = normalizedPixelPercentageAndMapping.Value; + if (mapping.MappingFromLocation.AreaPermille < faceDistanceAreaPermilleTolerance) + continue; + if (normalizedPixelPercentageToPersonContainers is not null && normalizedPixelPercentageToPersonContainers.ContainsKey(mapping.MappingFromLocation.NormalizedPixelPercentage)) + continue; + _ = _SkipCollection.TryGetValue(keyValuePair.Key, out normalizedPixelPercentages); + if (normalizedPixelPercentages is not null && normalizedPixelPercentages.Contains(mapping.MappingFromLocation.NormalizedPixelPercentage)) + continue; + directoryName = Path.GetDirectoryName(mapping.MappingFromItem.RelativePath); + facesDirectory = Path.Combine($"{dFacesContentDirectory}{directoryName}", mapping.MappingFromItem.ImageFileHolder.NameWithoutExtension); + faceFileHolder = new(Path.Combine(facesDirectory, $"{mapping.MappingFromLocation.DeterministicHashCodeKey}{mapping.MappingFromItem.ImageFileHolder.ExtensionLowered}{_Configuration.FacesFileNameExtension}")); + if (directoryName is null || !faceFileHolder.Exists) + continue; + directory = Path.Combine(_EDistanceContentTicksDirectory, by, $"{mapping.MappingFromItem.MinimumDateTime.ToString("yyyy")[..3]}#A{mapping.MappingFromLocation.AreaPermille:0000}"); + checkFile = Path.Combine(directory, $"{mapping.MappingFromLocation.DeterministicHashCodeKey}{mapping.MappingFromItem.ImageFileHolder.ExtensionLowered}{_Configuration.FacesFileNameExtension}"); + results.Add(new(directory, faceFileHolder, checkFile)); + } + } + return results; + } + + public void CopyNotMappedFaces(int faceDistanceAreaPermilleTolerance, string dFacesContentDirectory, Dictionary> idToNormalizedPixelPercentageToMapping) + { + if (_Configuration is null) + throw new NullReferenceException(nameof(_Configuration)); + List<(string Directory, FileHolder FaceFileHolder, string CheckFile)> collection = GetCollection(faceDistanceAreaPermilleTolerance, dFacesContentDirectory, idToNormalizedPixelPercentageToMapping); + string[] directories = (from l in collection select l.Directory).Distinct().ToArray(); + foreach (string directory in directories) + { + if (string.IsNullOrEmpty(directory)) + continue; + if (!Directory.Exists(directory)) + _ = Directory.CreateDirectory(directory); + } + foreach ((string directory, FileHolder faceFileHolder, string checkFile) in collection) + { + if (File.Exists(checkFile)) + continue; + File.Copy(faceFileHolder.FullName, checkFile); + } + } + } \ No newline at end of file diff --git a/Map/Models/Stateless/IMapLogic.cs b/Map/Models/Stateless/IMapLogic.cs index 8ec8526..57869c4 100644 --- a/Map/Models/Stateless/IMapLogic.cs +++ b/Map/Models/Stateless/IMapLogic.cs @@ -3,6 +3,7 @@ namespace View_by_Distance.Map.Models.Stateless; public interface IMapLogic { // ... + const int CopyNotMappedFaces = 5; const int ForceSingleImage = 3; const int ManualCopy = 4; const int Mapping = 1; diff --git a/Tests/UnitTestCalculations.cs b/Tests/UnitTestCalculations.cs index 7c121ab..5bca29a 100644 --- a/Tests/UnitTestCalculations.cs +++ b/Tests/UnitTestCalculations.cs @@ -461,7 +461,7 @@ public class UnitTestCalculations x2 = x.Value; y2 = y.Value; double distance = Math.Sqrt(Math.Pow(x1 - x2, 2) + Math.Pow(y1 - y2, 2)); - Assert.IsTrue(distance == 3); + Assert.IsTrue(distance == 2163.2958651095323); } [TestMethod] @@ -490,6 +490,7 @@ public class UnitTestCalculations public void TestMoveToVerify() { string http; + string[] lines; string? minusOne; string? minusTwo; string[] segments; @@ -503,8 +504,9 @@ public class UnitTestCalculations string sourceDirectory = @""; string sourceFile = @"\People - C.tsv"; if (!File.Exists(sourceFile)) - throw new Exception(); - string[] lines = File.ReadAllLines(sourceFile); + lines = Array.Empty(); + else + lines = File.ReadAllLines(sourceFile); for (int i = 0; i < lines.Length; i++) { if (!lines[i].Contains("https://")) @@ -536,7 +538,7 @@ public class UnitTestCalculations _ = Directory.CreateDirectory(personKeyFormattedDirectoryNew); File.WriteAllText(Path.Combine(personKeyFormattedDirectoryNew, "Facebook.txt"), http); } - Assert.IsTrue(lines.Any()); + Assert.IsNotNull(lines); } } \ No newline at end of file diff --git a/Tests/UnitTestResize.cs b/Tests/UnitTestResize.cs index 1f1a3b3..958c513 100644 --- a/Tests/UnitTestResize.cs +++ b/Tests/UnitTestResize.cs @@ -88,8 +88,14 @@ public class UnitTestResize // string sourceDirectoryName = "Mackenzie Prom 2017"; // string sourceFileName = "Fall 2005 (113).jpg"; // string sourceDirectoryName = "=2005.3 Fall"; - string sourceFileName = "12314542_10208270578946392_3555034423896018896_o.jpg"; - string sourceDirectoryName = "Other"; + // string sourceFileName = "DSCN0534.jpg"; + // string sourceDirectoryName = "Logan Swimming Lessons 2013"; + // string sourceFileName = "DSC_4913.jpg"; + // string sourceDirectoryName = "Disneyland 2014"; + // string sourceFileName = "Logan Michael Sept 08 (193).jpg"; + // string sourceDirectoryName = "=2008.2 Summer Logan Michael"; + string sourceFileName = "Halloween 2006 (112).jpg"; + string sourceDirectoryName = "Halloween 2006"; Item item; bool reverse = false; string original = "Original"; diff --git a/TestsWithFaceRecognitionDotNet/Models/Binder/Configuration.cs b/TestsWithFaceRecognitionDotNet/Models/Binder/Configuration.cs index 6e8d359..643f33b 100644 --- a/TestsWithFaceRecognitionDotNet/Models/Binder/Configuration.cs +++ b/TestsWithFaceRecognitionDotNet/Models/Binder/Configuration.cs @@ -36,8 +36,6 @@ public class Configuration [Display(Name = "Location Factor"), Required] public int? LocationFactor { get; set; } [Display(Name = "Mapped Max Index"), Required] public int? MappedMaxIndex { get; set; } [Display(Name = "Mapping Default Name"), Required] public string MappingDefaultName { get; set; } - [Display(Name = "Mapping Save Mapped"), Required] public bool? MappingSaveMapped { get; set; } - [Display(Name = "Mapping Save Not Mapped"), Required] public bool? MappingSaveNotMapped { get; set; } [Display(Name = "Mapping Use Deterministic Hash Code Unknown Face Key Value Pairs for Add to Mapping"), Required] public bool? MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping { get; set; } [Display(Name = "Mapping Use Deterministic Hash Code Unknown Face Key Value Pairs for Save Mapping"), Required] public bool? MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping { get; set; } [Display(Name = "Max Items In Distance Collection"), Required] public int? MaxItemsInDistanceCollection { get; set; } @@ -67,9 +65,11 @@ public class Configuration [Display(Name = "Reverse"), Required] public bool? Reverse { get; set; } [Display(Name = "Save Face Landmark For Output Resolutions"), Required] public string[] SaveFaceLandmarkForOutputResolutions { get; set; } [Display(Name = "Save Full Year Of Random Files"), Required] public bool? SaveFullYearOfRandomFiles { get; set; } + [Display(Name = "Save Mapped"), Required] public string[] SaveMappedForOutputResolutions { get; set; } + [Display(Name = "Save Not Mapped"), Required] public string[] SaveNotMappedForOutputResolutions { get; set; } [Display(Name = "Save Resized Images by Person Key Formatted"), Required] public string[] SaveResizedImagesByPersonKeyFormattedForOutputResolutions { get; set; } - [Display(Name = "Save Shortcuts"), Required] public string[] SaveShortcutsForOutputResolutions { get; set; } [Display(Name = "Save Resized Subfiles"), Required] public bool? SaveResizedSubfiles { get; set; } + [Display(Name = "Save Shortcuts"), Required] public string[] SaveShortcutsForOutputResolutions { get; set; } [Display(Name = "Skip Search"), Required] public bool? SkipSearch { get; set; } [Display(Name = "Sorting Days Delta Tolerance"), Required] public int? SortingDaysDeltaTolerance { get; set; } [Display(Name = "SortingMaximumPerFaceShould be High"), Required] public int? SortingMaximumPerFaceShouldBeHigh { get; set; } @@ -137,10 +137,6 @@ public class Configuration throw new NullReferenceException(nameof(configuration.LocationFactor)); if (configuration.MappingDefaultName is null) throw new NullReferenceException(nameof(configuration.MappingDefaultName)); - if (configuration.MappingSaveMapped is null) - throw new NullReferenceException(nameof(configuration.MappingSaveMapped)); - if (configuration.MappingSaveNotMapped is null) - throw new NullReferenceException(nameof(configuration.MappingSaveNotMapped)); if (configuration.MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping is null) throw new NullReferenceException(nameof(configuration.MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping)); if (configuration.MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping is null) @@ -187,6 +183,10 @@ public class Configuration throw new NullReferenceException(nameof(configuration.Reverse)); if (configuration.SaveFaceLandmarkForOutputResolutions is null) configuration.SaveFaceLandmarkForOutputResolutions = Array.Empty(); + if (configuration.SaveMappedForOutputResolutions is null) + configuration.SaveMappedForOutputResolutions = Array.Empty(); + if (configuration.SaveNotMappedForOutputResolutions is null) + configuration.SaveNotMappedForOutputResolutions = Array.Empty(); if (configuration.SaveFullYearOfRandomFiles is null) throw new NullReferenceException(nameof(configuration.SaveFullYearOfRandomFiles)); if (configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions is null) @@ -237,8 +237,6 @@ public class Configuration configuration.LocationFactor.Value, configuration.MappedMaxIndex, configuration.MappingDefaultName, - configuration.MappingSaveNotMapped.Value, - configuration.MappingSaveMapped.Value, configuration.MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping.Value, configuration.MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping.Value, configuration.MaxItemsInDistanceCollection.Value, @@ -267,9 +265,11 @@ public class Configuration configuration.Reverse.Value, configuration.SaveFaceLandmarkForOutputResolutions, configuration.SaveFullYearOfRandomFiles.Value, + configuration.SaveMappedForOutputResolutions, + configuration.SaveNotMappedForOutputResolutions, configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions, - configuration.SaveShortcutsForOutputResolutions, configuration.SaveResizedSubfiles.Value, + configuration.SaveShortcutsForOutputResolutions, configuration.SkipSearch.Value, configuration.SortingDaysDeltaTolerance.Value, configuration.SortingMaximumPerFaceShouldBeHigh.Value, diff --git a/TestsWithFaceRecognitionDotNet/Models/Configuration.cs b/TestsWithFaceRecognitionDotNet/Models/Configuration.cs index 48dffa1..679ea76 100644 --- a/TestsWithFaceRecognitionDotNet/Models/Configuration.cs +++ b/TestsWithFaceRecognitionDotNet/Models/Configuration.cs @@ -35,8 +35,6 @@ public class Configuration public int LocationFactor { init; get; } public int? MappedMaxIndex { init; get; } public string MappingDefaultName { init; get; } - public bool MappingSaveNotMapped { init; get; } - public bool MappingSaveMapped { init; get; } public bool MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping { init; get; } public bool MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping { init; get; } public int MaxItemsInDistanceCollection { init; get; } @@ -65,9 +63,11 @@ public class Configuration public bool Reverse { init; get; } public string[] SaveFaceLandmarkForOutputResolutions { init; get; } public bool SaveFullYearOfRandomFiles { init; get; } + public string[] SaveMappedForOutputResolutions { init; get; } + public string[] SaveNotMappedForOutputResolutions { init; get; } public string[] SaveResizedImagesByPersonKeyFormattedForOutputResolutions { init; get; } - public string[] SaveShortcutsForOutputResolutions { init; get; } public bool SaveResizedSubfiles { init; get; } + public string[] SaveShortcutsForOutputResolutions { init; get; } public bool SkipSearch { init; get; } public int SortingDaysDeltaTolerance { init; get; } public int SortingMaximumPerFaceShouldBeHigh { init; get; } @@ -104,8 +104,6 @@ public class Configuration int locationFactor, int? mappedMaxIndex, string mappingDefaultName, - bool mappingSaveNotMapped, - bool mappingSaveMapped, bool mappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping, bool mappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping, int maxItemsInDistanceCollection, @@ -134,9 +132,11 @@ public class Configuration bool reverse, string[] saveFaceLandmarkForOutputResolutions, bool saveFullYearOfRandomFiles, + string[] saveMappedForOutputResolutions, + string[] saveNotMappedForOutputResolutions, string[] saveResizedImagesByPersonKeyFormattedForOutputResolutions, - string[] saveShortcutsForOutputResolutions, bool saveResizedSubfiles, + string[] saveShortcutsForOutputResolutions, bool skipSearch, int sortingDaysDeltaTolerance, int sortingMaximumPerFaceShouldBeHigh, @@ -172,8 +172,6 @@ public class Configuration LocationFactor = locationFactor; MappedMaxIndex = mappedMaxIndex; MappingDefaultName = mappingDefaultName; - MappingSaveNotMapped = mappingSaveNotMapped; - MappingSaveMapped = mappingSaveMapped; MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping = mappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping; MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping = mappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping; MaxItemsInDistanceCollection = maxItemsInDistanceCollection; @@ -202,9 +200,11 @@ public class Configuration Reverse = reverse; SaveFaceLandmarkForOutputResolutions = saveFaceLandmarkForOutputResolutions; SaveFullYearOfRandomFiles = saveFullYearOfRandomFiles; + SaveMappedForOutputResolutions = saveMappedForOutputResolutions; + SaveNotMappedForOutputResolutions = saveNotMappedForOutputResolutions; SaveResizedImagesByPersonKeyFormattedForOutputResolutions = saveResizedImagesByPersonKeyFormattedForOutputResolutions; - SaveShortcutsForOutputResolutions = saveShortcutsForOutputResolutions; SaveResizedSubfiles = saveResizedSubfiles; + SaveShortcutsForOutputResolutions = saveShortcutsForOutputResolutions; SkipSearch = skipSearch; SortingDaysDeltaTolerance = sortingDaysDeltaTolerance; SortingMaximumPerFaceShouldBeHigh = sortingMaximumPerFaceShouldBeHigh; diff --git a/TestsWithFaceRecognitionDotNet/UnitTestFace.cs b/TestsWithFaceRecognitionDotNet/UnitTestFace.cs index dfc4cd9..f90cbb7 100644 --- a/TestsWithFaceRecognitionDotNet/UnitTestFace.cs +++ b/TestsWithFaceRecognitionDotNet/UnitTestFace.cs @@ -145,10 +145,10 @@ public class UnitTestFace Assert.IsTrue(ToleranceAfterFactor == 600); double valueA = 0.00001d; int checkA = (int)(Math.Round(valueA, _Configuration.LocationDigits) * _Configuration.LocationFactor); - Assert.IsTrue(checkA == 10); + Assert.IsTrue(checkA == 0); double valueB = 0.01d; int checkB = (int)(Math.Round(valueB, _Configuration.LocationDigits) * _Configuration.LocationFactor); - Assert.IsTrue(checkB == 10000); + Assert.IsTrue(checkB == 100); Assert.IsTrue(checkB > checkA); int checkC = (int)(_Configuration.FaceDistanceTolerance * DistanceFactor); Assert.IsTrue(checkC == ToleranceAfterFactor); @@ -185,7 +185,6 @@ public class UnitTestFace string outputResolution = _Configuration.OutputResolutions[0]; Property.Models.A_Property propertyLogic = GetPropertyLogic(reverse); string sourceDirectory = Path.Combine(_PropertyConfiguration.RootDirectory, sourceDirectoryName); - (Model model, PredictorModel predictorModel, ModelParameter modelParameter) = GetModel(_Configuration); _Logger.Information(_Configuration.ModelDirectory); aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _PropertyConfiguration, nameof(Property.Models.A_Property), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false); @@ -240,6 +239,7 @@ public class UnitTestFace resize.SaveResizedSubfile(outputResolution, cResultsFullGroupDirectory, subFileTuples, item, original, imageResizeKeyValuePairs); Image image = FaceRecognition.LoadImageFile(item.ResizedFileHolder.FullName); Assert.IsNotNull(image); + (Model model, PredictorModel predictorModel, ModelParameter modelParameter) = GetModel(_Configuration); FaceRecognition faceRecognition = new(_Configuration.NumberOfTimesToUpsample, _Configuration.NumberOfJitters, predictorModel, model, modelParameter); List<(Location Location, FaceRecognitionDotNet.FaceEncoding? FaceEncoding, Dictionary? FaceParts)> collection; collection = faceRecognition.GetCollection(image, includeFaceEncoding: true, includeFaceParts: true);