From b756859b616424dc98b7742a64c15a8951632473 Mon Sep 17 00:00:00 2001 From: Mike Phares Date: Sun, 14 Aug 2022 10:42:28 -0700 Subject: [PATCH] Simplified GetDirectoryInfoCollection --- Instance/DlibDotNet.cs | 185 +++++----- Instance/Models/_D2_FaceLandmark.cs | 17 +- Instance/Models/_D_Face.cs | 49 ++- Instance/Models/_E_Distance.cs | 347 ++++++++++++------ Metadata/Models/B_Metadata.cs | 34 +- Not-Copy-Copy/Not-Copy-Copy.cs | 17 - Property/Models/Closest.cs | 21 +- Property/Models/Stateless/IResult.cs | 5 +- Property/Models/Stateless/Result.cs | 54 ++- Resize/Models/_C_Resize.cs | 42 ++- Tests/UnitTestResize.cs | 73 ++-- .../UnitTestFace.cs | 72 ++-- 12 files changed, 556 insertions(+), 360 deletions(-) diff --git a/Instance/DlibDotNet.cs b/Instance/DlibDotNet.cs index c425967..8ddae9c 100644 --- a/Instance/DlibDotNet.cs +++ b/Instance/DlibDotNet.cs @@ -100,7 +100,7 @@ public class DlibDotNet _Log.Information(propertyConfiguration.RootDirectory); } if (!configuration.SkipSearch.Value) - Search(propertyConfiguration, configuration.Reverse.Value, model, predictorModel, argZero, people); + Search(propertyConfiguration, configuration.Reverse.Value, model, predictorModel, argZero, people, isSilent); if (_Exceptions.Count == 0 && _ArgZeroIsConfigurationRootDirectory) { long ticks = DateTime.Now.Ticks; @@ -305,7 +305,7 @@ public class DlibDotNet throw new Exception("Input directory should be the source and not a resized directory!"); } - private void FullParallelForWork(PropertyLogic propertyLogic, string outputResolution, List> sourceDirectoryChanges, List propertyFileHolderCollection, List propertyCollection, List>> metadataCollections, List> resizeKeyValuePairs, List> imageFaceCollections, string sourceDirectory, int index, Item item) + private void FullParallelForWork(PropertyLogic propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List> sourceDirectoryChanges, List propertyFileHolderCollection, List propertyCollection, List>> metadataCollections, List> resizeKeyValuePairs, List> imageFaceCollections, string sourceDirectory, int index, Item item) { if (item.ImageFileHolder is null) throw new NullReferenceException(nameof(item.ImageFileHolder)); @@ -326,7 +326,13 @@ public class DlibDotNet Dictionary imageResizeKeyValuePairs; List> subFileTuples = new(); List> metadataCollection; - if (item.Property is null) + if (item.Property is not null) + { + property = item.Property; + if ((item.Changed.HasValue && item.Changed.Value) || (item.Moved.HasValue && item.Moved.Value) || (item.Abandoned.HasValue && item.Abandoned.Value)) + throw new NotImplementedException(); + } + else { property = propertyLogic.GetProperty(item, subFileTuples, parseExceptions); item.Update(property); @@ -336,34 +342,24 @@ public class DlibDotNet sourceDirectoryChanges.Add(new Tuple(nameof(A_Property), (from l in subFileTuples select l.Item2).Max())); } } - else - { - property = item.Property; - if (item.Changed.HasValue && item.Changed.Value) - { - // lock (sourceDirectoryChanges) - // sourceDirectoryChanges.Add(new Tuple(nameof(A_Property), DateTime.Now)); - throw new NotImplementedException(); - } - } - (int metadataGroups, metadataCollection) = _Metadata.GetMetadataCollection(subFileTuples, parseExceptions, item); + (int metadataGroups, metadataCollection) = _Metadata.GetMetadataCollection(bResultsFullGroupDirectory, subFileTuples, parseExceptions, item); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection)); FileHolder resizedFileHolder = new(Path.Combine(_Resize.AngleBracketCollection[0].Replace("<>", "()"), Path.GetFileName(item.ImageFileHolder.FullName))); item.SetResizedFileHolder(resizedFileHolder); - imageResizeKeyValuePairs = _Resize.GetResizeKeyValuePairs(subFileTuples, parseExceptions, original, metadataCollection, property, item); + imageResizeKeyValuePairs = _Resize.GetResizeKeyValuePairs(cResultsFullGroupDirectory, subFileTuples, parseExceptions, original, metadataCollection, property, item); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(C_Resize.GetResizeKeyValuePairs)); if (_Configuration.SaveResizedSubfiles.Value) { - _Resize.SaveResizedSubfile(outputResolution, subFileTuples, item, original, property, imageResizeKeyValuePairs); + _Resize.SaveResizedSubfile(outputResolution, cResultsFullGroupDirectory, subFileTuples, item, original, property, imageResizeKeyValuePairs); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(C_Resize.SaveResizedSubfile)); item.SetResizedFileHolder(FileHolder.Refresh(resizedFileHolder)); } else if (outputResolution == _Configuration.OutputResolutions[0] && false) { - byte[] bytes = _Resize.GetResizedBytes(outputResolution, subFileTuples, item, property, imageResizeKeyValuePairs); + byte[] bytes = _Resize.GetResizedBytes(outputResolution, cResultsFullGroupDirectory, subFileTuples, item, property, imageResizeKeyValuePairs); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(C_Resize.GetResizedBytes)); string path = Path.Combine(resizedFileHolder.DirectoryName, resizedFileHolder.NameWithoutExtension); @@ -377,15 +373,15 @@ public class DlibDotNet int outputResolutionWidth = outputResolutionCollection[0]; int outputResolutionHeight = outputResolutionCollection[1]; int outputResolutionOrientation = outputResolutionCollection[2]; - faceCollection = _Faces.GetFaces(_Configuration.PropertyConfiguration, outputResolution, subFileTuples, parseExceptions, item, property, resizedFileHolder, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation); + faceCollection = _Faces.GetFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, item, property, resizedFileHolder, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(D_Face.GetFaces)); - _Faces.SaveFaces(_Configuration.PropertyConfiguration, subFileTuples, parseExceptions, item, faceCollection); + _Faces.SaveFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, item, faceCollection); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(D_Face.SaveFaces)); if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution)) { - _FaceLandmarks.SaveFaceLandmarkImages(subFileTuples, parseExceptions, item, faceCollection); + _FaceLandmarks.SaveFaceLandmarkImages(d2ResultsFullGroupDirectory, sourceDirectory, subFileTuples, parseExceptions, item, faceCollection); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(D2_FaceLandmarks.SaveFaceLandmarkImages)); } @@ -401,7 +397,7 @@ public class DlibDotNet } } - private int FullParallelWork(long ticks, PropertyLogic propertyLogic, string outputResolution, List> sourceDirectoryChanges, List propertyFileHolderCollection, List propertyCollection, List>> metadataCollection, List> resizeKeyValuePairs, List> faceCollections, int containersCount, Container container, Item[] filteredItems) + private int FullParallelWork(long ticks, PropertyLogic propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List> sourceDirectoryChanges, List propertyFileHolderCollection, List propertyCollection, List>> metadataCollection, List> resizeKeyValuePairs, List> faceCollections, int containersCount, Container container, Item[] filteredItems) { int result = 0; if (_Log is null) @@ -429,7 +425,7 @@ public class DlibDotNet { try { - FullParallelForWork(propertyLogic, outputResolution, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, container.SourceDirectory, index: i, filteredItems[i]); + FullParallelForWork(propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, container.SourceDirectory, index: i, filteredItems[i]); if (i == 0 || sourceDirectoryChanges.Any()) progressBar.Tick(); } @@ -442,6 +438,15 @@ public class DlibDotNet } }); } +#pragma warning disable + int[] ids = (from l in filteredItems where l.Property?.Id is not null select l.Property.Id.Value).ToArray(); +#pragma warning restore + bool hasDuplicateId = ids.Length != ids.Distinct().Count(); + if (hasDuplicateId) + { + if (hasDuplicateId) + { } + } return result; } @@ -564,7 +569,7 @@ public class DlibDotNet } } - private void FullDoWork(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string argZero, Dictionary> peopleCollection, PropertyLogic propertyLogic, List containers) + private void FullDoWork(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string argZero, Dictionary> peopleCollection, PropertyLogic propertyLogic, List containers, bool isSilent) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); @@ -573,6 +578,13 @@ public class DlibDotNet int exceptionCount; Item[] filteredItems; long ticks = DateTime.Now.Ticks; + string aResultsFullGroupDirectory; + string bResultsFullGroupDirectory; + string cResultsFullGroupDirectory; + string dResultsFullGroupDirectory; + string eResultsFullGroupDirectory; + string zResultsFullGroupDirectory; + string d2ResultsFullGroupDirectory; List> faceCollections = new(); List propertyCollection = new(); List propertyFileHolderCollection = new(); @@ -585,6 +597,20 @@ public class DlibDotNet { _FileKeyValuePairs.Clear(); _FilePropertiesKeyValuePairs.Clear(); + aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + configuration, model, predictorModel, nameof(A_Property), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false); + bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + configuration, model, predictorModel, nameof(B_Metadata), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false); + cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + configuration, model, predictorModel, nameof(C_Resize), outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false); + d2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + configuration, model, predictorModel, nameof(D2_FaceLandmarks), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true); + dResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + configuration, model, predictorModel, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true); + eResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true); + zResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + configuration, model, predictorModel, $"Z_{nameof(Item)}", outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true); foreach (Container container in containers) { if (!container.Items.Any()) @@ -604,70 +630,32 @@ public class DlibDotNet _Resize.AngleBracketCollection.Clear(); _Metadata.AngleBracketCollection.Clear(); propertyLogic.AngleBracketCollection.Clear(); - _FaceLandmarks.AngleBracketCollection.Clear(); propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration, - model, - predictorModel, - container.SourceDirectory, - nameof(A_Property), - outputResolution, - includeResizeGroup: false, - includeModel: false, - includePredictorModel: false, - contentDescription: string.Empty, - singletonDescription: "Properties for each image", - collectionDescription: string.Empty)); + container.SourceDirectory, + aResultsFullGroupDirectory, + contentDescription: string.Empty, + singletonDescription: "Properties for each image", + collectionDescription: string.Empty)); _Metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration, - model, - predictorModel, - container.SourceDirectory, - nameof(B_Metadata), - outputResolution, - includeResizeGroup: false, - includeModel: false, - includePredictorModel: false, - contentDescription: string.Empty, - singletonDescription: "Metadata as key value pairs", - collectionDescription: string.Empty)); + container.SourceDirectory, + bResultsFullGroupDirectory, + contentDescription: string.Empty, + singletonDescription: "Metadata as key value pairs", + collectionDescription: string.Empty)); _Resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration, - model, - predictorModel, - container.SourceDirectory, - nameof(C_Resize), - outputResolution, - includeResizeGroup: true, - includeModel: false, - includePredictorModel: false, - contentDescription: "Resized image", - singletonDescription: "Resize dimensions for each resolution", - collectionDescription: string.Empty)); + container.SourceDirectory, + cResultsFullGroupDirectory, + contentDescription: "Resized image", + singletonDescription: "Resize dimensions for each resolution", + collectionDescription: string.Empty)); if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution)) _Faces.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration, - model, - predictorModel, - container.SourceDirectory, - nameof(D_Face), - outputResolution, - includeResizeGroup: true, - includeModel: true, - includePredictorModel: true, - contentDescription: "n png file(s) for each face found", - singletonDescription: string.Empty, - collectionDescription: "For each image a json file with all faces found")); - if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution)) - _FaceLandmarks.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration, - model, - predictorModel, - container.SourceDirectory, - nameof(D2_FaceLandmarks), - outputResolution, - includeResizeGroup: true, - includeModel: true, - includePredictorModel: true, - contentDescription: "n x 2 png file(s) for each face found", - singletonDescription: string.Empty, - collectionDescription: string.Empty)); - exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, containers.Count, container, filteredItems); + container.SourceDirectory, + dResultsFullGroupDirectory, + contentDescription: "n png file(s) for each face found", + singletonDescription: string.Empty, + collectionDescription: "For each image a json file with all faces found")); + exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, containers.Count, container, filteredItems); if (metadataCollection.Count != filteredItems.Length || propertyCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || faceCollections.Count != filteredItems.Length) throw new Exception("Counts don't match!"); if (exceptionCount != 0) @@ -680,21 +668,27 @@ public class DlibDotNet { if (outputResolution == _Configuration.OutputResolutions[0] || _Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution)) { + if (isSilent && container.SourceDirectory.EndsWith("Bohdi Ray 2016")) // 7#.1 + break; + // if (isSilent && container.SourceDirectory.EndsWith("Halloween 2013")) // 12#.1 + // break; + // if (isSilent && container.SourceDirectory.EndsWith("zzz =2014.4 Winter Tracy Pictures")) // 30#.2 + // break; + // if (isSilent && container.SourceDirectory.EndsWith("Texas 2015")) //46#.4 + // break; for (int i = 0; i < faceCollections.Count; i++) filteredItems[i].Faces.AddRange(from l in faceCollections[i] select l); Item.AddToNamed(propertyLogic, filteredItems); if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution)) - D_Face.SaveShortcuts(configuration, model, predictorModel, _Configuration.JuliePhares, ticks, peopleCollection, propertyLogic, outputResolution, filteredItems); + D_Face.SaveShortcuts(_Configuration.JuliePhares, dResultsFullGroupDirectory, ticks, peopleCollection, propertyLogic, filteredItems); } } if (exceptionCount == 0 && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)) - _Distance.LoadOrCreateThenSaveDistanceResultsForOutputResolutions(configuration, model, predictorModel, container.SourceDirectory, outputResolution, sourceDirectoryChanges, filteredItems, faceCollections); + _Distance.LoadOrCreateThenSaveDistanceResultsForOutputResolutions(configuration, eResultsFullGroupDirectory, container.SourceDirectory, outputResolution, sourceDirectoryChanges, filteredItems, faceCollections); if (_Resize.AngleBracketCollection.Any()) _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(_Resize.AngleBracketCollection[0].Replace("<>", "()")); if (_Faces.AngleBracketCollection.Any()) _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(_Faces.AngleBracketCollection[0].Replace("<>", "()")); - if (_FaceLandmarks.AngleBracketCollection.Any()) - _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(_FaceLandmarks.AngleBracketCollection[0].Replace("<>", "()")); if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any()) { for (int y = 0; y < int.MaxValue; y++) @@ -711,22 +705,19 @@ public class DlibDotNet propertyLogic.SaveAllCollection(); if (propertyLogic.NamedFaceInfoDeterministicHashCodeIndices.Any()) { - const int maxPer = 5; - const bool skipIsWrongYear = true; string dFacesContentDirectory; string eDistanceContentDirectory; string eDistanceCollectionDirectory; string zPropertyHolderSingletonDirectory; + dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()"); + eDistanceContentDirectory = Path.Combine(eResultsFullGroupDirectory, $"({ticks})"); + zPropertyHolderSingletonDirectory = Path.Combine(zResultsFullGroupDirectory, "{}"); + eDistanceCollectionDirectory = Path.Combine(eResultsFullGroupDirectory, $"[{ticks}]"); List<(DateTime, bool?, PersonBirthday, FaceRecognitionDotNet.FaceEncoding[])> collection; - dFacesContentDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), "()"); - eDistanceContentDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), $"({ticks})"); - eDistanceCollectionDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), $"[{ticks}]"); - zPropertyHolderSingletonDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, $"Z_{nameof(Item)}", outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), "{}"); - collection = E_Distance.GetThreeSigmaFaceEncodings(argZero, containers); - E_Distance.AddClosest(argZero, containers, collection, skipIsWrongYear, maxPer); + collection = E_Distance.ParallelWork(_AppSettings.MaxDegreeOfParallelism.Value, argZero, containers); E_Distance.SavePropertyHolders(argZero, containers, zPropertyHolderSingletonDirectory); - E_Distance.SaveClosest(argZero, containers, eDistanceContentDirectory, dFacesContentDirectory); E_Distance.SaveThreeSigmaFaceEncodings(collection, peopleCollection, eDistanceCollectionDirectory); + E_Distance.SaveClosest(argZero, containers, peopleCollection, eDistanceContentDirectory, dFacesContentDirectory); } if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any()) break; @@ -758,7 +749,7 @@ public class DlibDotNet return result; } - private void Search(Property.Models.Configuration configuration, bool reverse, Model? model, PredictorModel? predictorModel, string argZero, Person[] people) + private void Search(Property.Models.Configuration configuration, bool reverse, Model? model, PredictorModel? predictorModel, string argZero, Person[] people, bool isSilent) { List containers; Dictionary> peopleCollection = A2_People.Convert(people); @@ -767,7 +758,7 @@ public class DlibDotNet containers = Property.Models.Stateless.A_Property.Get(configuration, propertyLogic); else containers = Property.Models.Stateless.Container.GetContainers(configuration, propertyLogic); - FullDoWork(configuration, model, predictorModel, argZero, peopleCollection, propertyLogic, containers); + FullDoWork(configuration, model, predictorModel, argZero, peopleCollection, propertyLogic, containers, isSilent); } internal void RenameQueue(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel) => _Rename.RenameQueue(configuration, model, predictorModel); diff --git a/Instance/Models/_D2_FaceLandmark.cs b/Instance/Models/_D2_FaceLandmark.cs index 25f1b86..db93dce 100644 --- a/Instance/Models/_D2_FaceLandmark.cs +++ b/Instance/Models/_D2_FaceLandmark.cs @@ -13,15 +13,12 @@ namespace View_by_Distance.Instance.Models; internal class D2_FaceLandmarks { - internal List AngleBracketCollection { get; } - private readonly Serilog.ILogger? _Log; private readonly Configuration _Configuration; internal D2_FaceLandmarks(Configuration configuration) { _Configuration = configuration; - AngleBracketCollection = new List(); _Log = Serilog.Log.ForContext(); } @@ -107,8 +104,10 @@ internal class D2_FaceLandmarks #pragma warning restore CA1416 - internal void SaveFaceLandmarkImages(List> subFileTuples, List parseExceptions, Item item, List faceCollections) + internal void SaveFaceLandmarkImages(string d2ResultsFullGroupDirectory, string sourceDirectory, List> subFileTuples, List parseExceptions, Item item, List faceCollections) { + if (_Configuration.PropertyConfiguration is null) + throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration)); if (item.ImageFileHolder is null) throw new NullReferenceException(nameof(item.ImageFileHolder)); if (item.ResizedFileHolder is null) @@ -122,7 +121,15 @@ internal class D2_FaceLandmarks long ticks = DateTime.Now.Ticks; List imageFiles = new(); bool updateDateWhenMatches = false; - string facesDirectory = Path.Combine(AngleBracketCollection[0].Replace("<>", "()"), item.ImageFileHolder.NameWithoutExtension); + List angleBracketCollection = new(); + angleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection( + _Configuration.PropertyConfiguration, + sourceDirectory, + d2ResultsFullGroupDirectory, + contentDescription: "n x 2 png file(s) for each face found", + singletonDescription: string.Empty, + collectionDescription: string.Empty)); + string facesDirectory = Path.Combine(angleBracketCollection[0].Replace("<>", "()"), item.ImageFileHolder.NameWithoutExtension); string[] changesFrom = new string[] { nameof(A_Property), nameof(B_Metadata), nameof(C_Resize), nameof(D_Face) }; List dateTimes = (from l in subFileTuples where changesFrom.Contains(l.Item1) select l.Item2).ToList(); if (!Directory.Exists(facesDirectory)) diff --git a/Instance/Models/_D_Face.cs b/Instance/Models/_D_Face.cs index fbed239..79debb3 100644 --- a/Instance/Models/_D_Face.cs +++ b/Instance/Models/_D_Face.cs @@ -255,7 +255,7 @@ public class D_Face : Shared.Models.Properties.IFace, IFace { List results = new(); if (_Configuration.PaddingLoops is null) - throw new Exception(); + throw new NullReferenceException(nameof(_Configuration.PaddingLoops)); if (item.ImageFileHolder is null) throw new NullReferenceException(nameof(item.ImageFileHolder)); if (_Configuration.NumberOfJitters is null) @@ -412,13 +412,17 @@ public class D_Face : Shared.Models.Properties.IFace, IFace _Populated = populated; } - internal List GetFaces(Property.Models.Configuration configuration, string outputResolution, List> subFileTuples, List parseExceptions, Item item, A_Property property, FileHolder resizedFileHolder, int outputResolutionWidth, int outputResolutionHeight, int outputResolutionOrientation) + internal List GetFaces(string dResultsFullGroupDirectory, List> subFileTuples, List parseExceptions, Item item, A_Property property, FileHolder resizedFileHolder, int outputResolutionWidth, int outputResolutionHeight, int outputResolutionOrientation) { List? results; + if (item.Property?.Id is null) + throw new NullReferenceException(nameof(item.Property.Id)); if (_Configuration.PropertiesChangedForFaces is null) - throw new Exception(); + throw new NullReferenceException(nameof(_Configuration.PropertiesChangedForFaces)); if (item.ImageFileHolder is null) throw new NullReferenceException(nameof(item.ImageFileHolder)); + if (string.IsNullOrEmpty(dResultsFullGroupDirectory)) + throw new NullReferenceException(nameof(dResultsFullGroupDirectory)); string json; D_Face face; bool checkForOutputResolutionChange = false; @@ -426,13 +430,30 @@ public class D_Face : Shared.Models.Properties.IFace, IFace List dateTimes = (from l in subFileTuples where changesFrom.Contains(l.Item1) select l.Item2).ToList(); string facesDirectory = Path.Combine(AngleBracketCollection[0].Replace("<>", "()"), item.ImageFileHolder.NameWithoutExtension); FileInfo fileInfo = new(Path.Combine(AngleBracketCollection[0].Replace("<>", "[]"), $"{item.ImageFileHolder.NameWithoutExtension}.json")); - if (!fileInfo.Exists) + string dCollectionFile = Path.Combine(dResultsFullGroupDirectory, "[]", "_ _ _", $"{item.Property.Id.Value}{item.ImageFileHolder.ExtensionLowered}.json"); + if (fileInfo.Exists) { - if (fileInfo.Directory?.Parent is null) - throw new Exception(); - string parentCheck = Path.Combine(fileInfo.Directory.Parent.FullName, fileInfo.Name); - if (File.Exists(parentCheck)) - File.Delete(parentCheck); + if (!File.Exists(dCollectionFile)) + { + fileInfo.MoveTo(dCollectionFile); + fileInfo = new(dCollectionFile); + } + } + else + { + if (File.Exists(dCollectionFile)) + fileInfo = new(dCollectionFile); + else + { + if (fileInfo.Directory?.Parent is null) + throw new Exception(); + string parentCheck = Path.Combine(fileInfo.Directory.Parent.FullName, fileInfo.Name); + if (File.Exists(parentCheck)) + { + File.Move(parentCheck, fileInfo.FullName); + fileInfo.Refresh(); + } + } } if (!Directory.Exists(facesDirectory)) _ = Directory.CreateDirectory(facesDirectory); @@ -487,14 +508,16 @@ public class D_Face : Shared.Models.Properties.IFace, IFace return results; } - internal void SaveFaces(Property.Models.Configuration configuration, List> subFileTuples, List parseExceptions, Item item, List faceCollection) + internal void SaveFaces(string dResultsFullGroupDirectory, List> subFileTuples, List parseExceptions, Item item, List faceCollection) { if (_Configuration.OverrideForFaceImages is null) - throw new Exception(); + throw new NullReferenceException(nameof(_Configuration.OverrideForFaceImages)); if (item.ImageFileHolder is null) throw new NullReferenceException(nameof(item.ImageFileHolder)); if (item.ResizedFileHolder is null) throw new NullReferenceException(nameof(item.ResizedFileHolder)); + if (string.IsNullOrEmpty(dResultsFullGroupDirectory)) + throw new NullReferenceException(nameof(dResultsFullGroupDirectory)); FileInfo fileInfo; bool check = false; string parentCheck; @@ -533,7 +556,7 @@ public class D_Face : Shared.Models.Properties.IFace, IFace SaveFaces(faceCollection, item.ResizedFileHolder, imageFiles); } - internal static void SaveShortcuts(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string[] juliePhares, long ticks, Dictionary> peopleCollection, PropertyLogic propertyLogic, string outputResolution, Item[] filteredItems) + internal static void SaveShortcuts(string[] juliePhares, string dResultsFullGroupDirectory, long ticks, Dictionary> peopleCollection, PropertyLogic propertyLogic, Item[] filteredItems) { Person person; string fileName; @@ -541,7 +564,7 @@ public class D_Face : Shared.Models.Properties.IFace, IFace DateTime? minimumDateTime; WindowsShortcut windowsShortcut; const string pattern = @"[\\,\/,\:,\*,\?,\"",\<,\>,\|]"; - string dFacesContentDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), $"({ticks})"); + string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, $"({ticks})"); List<(Item, (string, Shared.Models.Properties.IFace?, (string, string, string, string))[])> collections = Item.GetCollection(propertyLogic, filteredItems, dFacesContentDirectory); foreach ((Item item, (string personKey, Shared.Models.Properties.IFace? _, (string, string, string, string))[] collection) in collections) { diff --git a/Instance/Models/_E_Distance.cs b/Instance/Models/_E_Distance.cs index e9bb776..5b04b3e 100644 --- a/Instance/Models/_E_Distance.cs +++ b/Instance/Models/_E_Distance.cs @@ -59,7 +59,7 @@ internal class E_Distance { List> results = new() { new(face, string.Empty) }; if (_Configuration.MaxItemsInDistanceCollection is null) - throw new Exception(); + throw new NullReferenceException(nameof(_Configuration.MaxItemsInDistanceCollection)); for (int n = 0; n < faceCollections.Count; n++) { if (i == n) @@ -76,9 +76,9 @@ internal class E_Distance { List results = new(); if (_Configuration.LocationConfidenceFactor is null) - throw new Exception(); + throw new NullReferenceException(nameof(_Configuration.LocationConfidenceFactor)); if (_Configuration.DistanceFactor is null) - throw new Exception(); + throw new NullReferenceException(nameof(_Configuration.DistanceFactor)); D_Face face; int[] locationIndices; for (int d = 0; d < faceDistances.Count; d++) @@ -97,7 +97,7 @@ internal class E_Distance { List> results = new(); if (_Configuration.MaxItemsInDistanceCollection is null) - throw new Exception(); + throw new NullReferenceException(nameof(_Configuration.MaxItemsInDistanceCollection)); int[] locationIndices; for (int t = 0; t < indicesAndValues.Count; t++) { @@ -128,7 +128,7 @@ internal class E_Distance return result; } - private void LoadOrCreateThenSaveDistanceResultsForOutputResolutionsLoop(Property.Models.Configuration configuration, List> faceCollections, int subFilesCount, int i, List faceCollection, List locationIndicesCollection, List> subFileTuples, List faceEncodingCollection, List faceEncodingCollections, string fileNameWithoutExtension, string jsonDirectory, string tvsDirectory, bool updateDateWhenMatches, DateTime? updateToWhenMatches) + private void LoadOrCreateThenSaveDistanceResultsForOutputResolutionsLoop(List> faceCollections, int subFilesCount, int i, List faceCollection, List locationIndicesCollection, List> subFileTuples, List faceEncodingCollection, List faceEncodingCollections, string fileNameWithoutExtension, string jsonDirectory, string tvsDirectory, bool updateDateWhenMatches, DateTime? updateToWhenMatches) { string text; string json; @@ -171,7 +171,7 @@ internal class E_Distance } } - private void LoadOrCreateThenSaveDistanceResultsForOutputResolutions(Property.Models.Configuration configuration, Item[] filteredItems, List> faceCollections, List directories, bool updateDateWhenMatches, DateTime? updateToWhenMatches) + private void LoadOrCreateThenSaveDistanceResultsForOutputResolutions(Item[] filteredItems, List> faceCollections, List directories, bool updateDateWhenMatches, DateTime? updateToWhenMatches) { FileHolder? fileHolder; List locationIndicesCollection = new(); @@ -188,16 +188,16 @@ internal class E_Distance fileHolder = filteredItems[i].ImageFileHolder; if (fileHolder is null) continue; - LoadOrCreateThenSaveDistanceResultsForOutputResolutionsLoop(configuration, faceCollections, filteredItems.Length, i, faceCollections[i], locationIndicesCollection, subFileTuples, faceEncodingCollection, faceEncodingCollections[i], fileHolder.NameWithoutExtension, directories[i][0], directories[i][1], updateDateWhenMatches, updateToWhenMatches); + LoadOrCreateThenSaveDistanceResultsForOutputResolutionsLoop(faceCollections, filteredItems.Length, i, faceCollections[i], locationIndicesCollection, subFileTuples, faceEncodingCollection, faceEncodingCollections[i], fileHolder.NameWithoutExtension, directories[i][0], directories[i][1], updateDateWhenMatches, updateToWhenMatches); } } - internal void LoadOrCreateThenSaveDistanceResultsForOutputResolutions(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string sourceDirectory, string outputResolution, List> sourceDirectoryChanges, Item[] filteredItems, List> faceCollections) + internal void LoadOrCreateThenSaveDistanceResultsForOutputResolutions(Property.Models.Configuration configuration, string eResultsFullGroupDirectory, string sourceDirectory, string outputResolution, List> sourceDirectoryChanges, Item[] filteredItems, List> faceCollections) { if (_Configuration.CheckJsonForDistanceResults is null) - throw new Exception(); + throw new NullReferenceException(nameof(_Configuration.CheckJsonForDistanceResults)); if (_Configuration.PropertiesChangedForDistance is null) - throw new Exception(); + throw new NullReferenceException(nameof(_Configuration.PropertiesChangedForDistance)); string json; bool check = false; string parentCheck; @@ -210,18 +210,13 @@ internal class E_Distance System.IO.DirectoryInfo tvsDirectoryInfo; string[] changesFrom = new string[] { nameof(A_Property), nameof(B_Metadata), nameof(C_Resize), nameof(D_Face) }; List dateTimes = (from l in sourceDirectoryChanges where changesFrom.Contains(l.Item1) select l.Item2).ToList(); - List directoryInfoCollection = Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration, - model, - predictorModel, - sourceDirectory, - nameof(E_Distance), - outputResolution, - includeResizeGroup: true, - includeModel: true, - includePredictorModel: true, - contentDescription: ".tvs File", - singletonDescription: string.Empty, - collectionDescription: "n json file(s) for each face found (one to many)"); + List directoryInfoCollection = Property.Models.Stateless.IResult.GetDirectoryInfoCollection( + configuration, + sourceDirectory, + eResultsFullGroupDirectory, + contentDescription: ".tvs File", + singletonDescription: string.Empty, + collectionDescription: "n json file(s) for each face found (one to many)"); for (int i = 0; i < filteredItems.Length; i++) { fileHolder = filteredItems[i].ImageFileHolder; @@ -269,7 +264,7 @@ internal class E_Distance } } if (check) - LoadOrCreateThenSaveDistanceResultsForOutputResolutions(configuration, filteredItems, faceCollections, directories, updateDateWhenMatches, updateToWhenMatches: dateTime); + LoadOrCreateThenSaveDistanceResultsForOutputResolutions(filteredItems, faceCollections, directories, updateDateWhenMatches, updateToWhenMatches: dateTime); _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(directoryInfoCollection[0].Replace("<>", "()")); } @@ -414,113 +409,177 @@ internal class E_Distance return result; } - internal static List<(DateTime, bool?, Shared.Models.PersonBirthday, FaceEncoding[])> GetThreeSigmaFaceEncodings(string argZero, List containers) + private static FaceEncoding? GetFaceEncoding(Shared.Models.Properties.IFace face) { - List<(DateTime, bool?, Shared.Models.PersonBirthday, FaceEncoding[])> results = new(); - double lcl; - double ucl; - double sum; - double average; - int lowestIndex; - double lowestSum; - const int zero = 0; - double standardDeviation; - FaceEncoding faceEncoding; - List faceDistances; - List faceEncodings; - Dictionary> keyValuePairs = Item.GetKeyValuePairs(argZero, containers); - foreach (KeyValuePair> keyValuePair in keyValuePairs) + FaceEncoding? result; + if (!face.Populated) + result = null; + else + result = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding); + return result; + } + + private static (int index, double sum) GetIndexAndSum(int i, List results) + { + List faceDistances = FaceRecognition.FaceDistances(results, results[i]); + return new(i, faceDistances.Sum()); + } + + private static List GetFaceEncodings(int maxDegreeOfParallelism, List<(DateTime MinimumDateTime, bool? IsWrongYear, Shared.Models.PersonBirthday PersonBirthday, Shared.Models.Properties.IFace Face)> collection) + { + List results; + if (maxDegreeOfParallelism == 1) { - lowestIndex = 0; - faceEncodings = new(); - lowestSum = double.MaxValue; - foreach ((DateTime minimumDateTime, bool? isWrongYear, Shared.Models.PersonBirthday personBirthday, Shared.Models.Properties.IFace face) in keyValuePair.Value) + results = new(); + FaceEncoding faceEncoding; + foreach ((DateTime _, bool? _, Shared.Models.PersonBirthday _, Shared.Models.Properties.IFace face) in collection) { if (!face.Populated) continue; faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding); - faceEncodings.Add(faceEncoding); + results.Add(faceEncoding); } - if (keyValuePair.Value.Count != faceEncodings.Count) - continue; - if (faceEncodings.Count < 2) - continue; - for (int i = 0; i < faceEncodings.Count; i++) + } + else + { + results = new(); + ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; + _ = Parallel.For(0, collection.Count, parallelOptions, i => { - faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncodings[i]); - sum = faceDistances.Sum(); - if (sum > lowestSum) - continue; - lowestIndex = i; - lowestSum = sum; + FaceEncoding? faceEncoding = GetFaceEncoding(collection[i].Face); + if (faceEncoding is not null) + { + lock (results) + results.Add(faceEncoding); + } + }); + } + if (collection.Count == results.Count && results.Count > 1) + { + double sum; + int lowestIndex; + double lowestSum; + List faceDistances; + if (maxDegreeOfParallelism == 1) + { + lowestIndex = 0; + lowestSum = double.MaxValue; + for (int i = 0; i < results.Count; i++) + { + faceDistances = FaceRecognition.FaceDistances(results, results[i]); + sum = faceDistances.Sum(); + if (sum >= lowestSum) + continue; + lowestIndex = i; + lowestSum = sum; + } } - faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncodings[lowestIndex]); + else + { + List<(int Index, double Sum)> indicesAndSums = new(); + ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; + _ = Parallel.For(0, results.Count, parallelOptions, i => + { + (int index, double sum) = GetIndexAndSum(i, results); + lock (indicesAndSums) + indicesAndSums.Add(new(index, sum)); + }); + (lowestIndex, lowestSum) = (from l in indicesAndSums orderby l.Sum select l).First(); + } + faceDistances = FaceRecognition.FaceDistances(results, results[lowestIndex]); sum = faceDistances.Sum(); - if (sum != lowestSum) - continue; - average = faceDistances.Average(); - standardDeviation = GetStandardDeviation(faceDistances, average); - lcl = average - (standardDeviation * 3); - ucl = average + (standardDeviation * 3); - for (int i = faceEncodings.Count - 1; i > -1; i--) + if (sum == lowestSum) { - if (faceDistances[i] < lcl || faceDistances[i] > ucl) - faceEncodings.RemoveAt(i); + double average = faceDistances.Average(); + double standardDeviation = GetStandardDeviation(faceDistances, average); + double lcl = average - (standardDeviation * 3); + double ucl = average + (standardDeviation * 3); + for (int i = results.Count - 1; i > -1; i--) + { + if (faceDistances[i] < lcl || faceDistances[i] > ucl) + results.RemoveAt(i); + } } + } + return results; + } + + private static List<(DateTime, bool?, Shared.Models.PersonBirthday, FaceEncoding[])> GetThreeSigmaFaceEncodings(int maxDegreeOfParallelism, Dictionary> keyValuePairs) + { + List<(DateTime, bool?, Shared.Models.PersonBirthday, FaceEncoding[])> results = new(); + const int zero = 0; + List faceEncodings; + foreach (KeyValuePair> keyValuePair in keyValuePairs) + { + faceEncodings = GetFaceEncodings(maxDegreeOfParallelism, keyValuePair.Value); results.Add(new(keyValuePair.Value[zero].MinimumDateTime, keyValuePair.Value[zero].IsWrongYear, keyValuePair.Value[zero].PersonBirthday, faceEncodings.ToArray())); } return results; } - internal static void SaveThreeSigmaFaceEncodings(List<(DateTime, bool?, Shared.Models.PersonBirthday, FaceEncoding[])> collection, Dictionary> peopleCollection, string eDistanceCollectionDirectory) + private static Closest? GetClosestParallelFor(float maxMinimum, DateTime minimumDateTime, bool? isWrongYear, Shared.Models.Properties.IFace face, FaceEncoding faceEncoding, (DateTime MinimumDateTime, bool? IsWrongYear, Shared.Models.PersonBirthday PersonBirthday, FaceEncoding[] FaceEncodings) tuple) { - string json; - string checkFile; - string personKey; - string directory; - List rawEncodings; - Shared.Models.Person person; - const string facePopulatedKey = "ThreeSigma"; - const string pattern = @"[\\,\/,\:,\*,\?,\"",\<,\>,\|]"; - foreach ((DateTime minimumDateTime, bool? isWrongYear, Shared.Models.PersonBirthday personBirthday, FaceEncoding[] faceEncodings) in collection) + Closest? result; + if (isWrongYear.HasValue && !isWrongYear.Value && minimumDateTime < tuple.PersonBirthday.Value) + result = null; + else { - rawEncodings = new(); - checkFile = string.Empty; - personKey = Shared.Models.Stateless.Methods.IPersonBirthday.GetFormatted(personBirthday); - directory = Item.GetDirectory(eDistanceCollectionDirectory, facePopulatedKey, minimumDateTime, isWrongYear, personBirthday, personKey); - person = peopleCollection[personKey][0]; - checkFile = string.Concat(directory, " - ", Regex.Replace(Shared.Models.Stateless.Methods.IPersonName.GetFullName(person.Name), pattern, string.Empty), ".json"); - if (string.IsNullOrEmpty(checkFile)) - continue; - if (!Directory.Exists(directory)) - _ = Directory.CreateDirectory(directory); - for (int i = 0; i < faceEncodings.Length; i++) - rawEncodings.Add(faceEncodings[i].GetRawEncoding()); - json = JsonSerializer.Serialize(rawEncodings, new JsonSerializerOptions { WriteIndented = true }); - _ = Property.Models.Stateless.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: true, compareBeforeWrite: true); + List faceDistances = FaceRecognition.FaceDistances(tuple.FaceEncodings, faceEncoding); + result = new(face.LocationIndex, tuple.MinimumDateTime, tuple.IsWrongYear, tuple.PersonBirthday, faceDistances); + if (result.Minimum > maxMinimum) + result = null; } + return result; } - public static void AddClosest(string argZero, List containers, List<(DateTime, bool?, Shared.Models.PersonBirthday, FaceEncoding[])> collection, bool skipIsWrongYear, int maxPer) + private static List GetClosestCollection(int maxDegreeOfParallelism, List<(DateTime, bool?, Shared.Models.PersonBirthday, FaceEncoding[])> collection, float maxMinimum, DateTime itemMinimumDateTime, bool? itemIsWrongYear, Shared.Models.Properties.IFace face) + { + List results; + FaceEncoding faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding); + if (maxDegreeOfParallelism == 1) + { + results = new(); + Closest closest; + List faceDistances; + foreach ((DateTime minimumDateTime, bool? isWrongYear, Shared.Models.PersonBirthday personBirthday, FaceEncoding[] faceEncodings) in collection) + { + if (itemIsWrongYear.HasValue && !itemIsWrongYear.Value && itemMinimumDateTime < personBirthday.Value) + continue; + faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncoding); + closest = new(face.LocationIndex, minimumDateTime, isWrongYear, personBirthday, faceDistances); + if (closest.Minimum > maxMinimum) + continue; + results.Add(closest); + } + } + else + { + results = new(); + ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; + _ = Parallel.For(0, collection.Count, parallelOptions, i => + { + Closest? closest = GetClosestParallelFor(maxMinimum, itemMinimumDateTime, itemIsWrongYear, face, faceEncoding, collection[i]); + if (closest is not null) + { + lock (results) + results.Add(closest); + } + }); + } + return results; + } + + private static void AddClosest(int maxDegreeOfParallelism, string argZero, List containers, List<(DateTime, bool?, Shared.Models.PersonBirthday, FaceEncoding[])> collection, bool skipIsWrongYear, int maxPer, float maxMinimum) { string key; - double average; - double lowestAverage; + Closest closest; bool? itemIsWrongYear; - bool? lowestIsWrongYear; - FaceEncoding faceEncoding; - List faceDistances; DateTime? itemMinimumDateTime; - DateTime? lowestMinimumDateTime; + List closestCollection; Shared.Models.Properties.IFace face; Dictionary results = new(); - Shared.Models.PersonBirthday? lowestPersonBirthday; foreach (Container container in containers) { - lowestIsWrongYear = null; - lowestPersonBirthday = null; - lowestMinimumDateTime = null; - lowestAverage = double.MaxValue; if (!container.Items.Any()) continue; if (!container.SourceDirectory.StartsWith(argZero)) @@ -539,37 +598,40 @@ internal class E_Distance for (int i = 0; i < item.Faces.Count; i++) { face = item.Faces[i]; - item.Closest.Add(new(face.LocationIndex, itemMinimumDateTime.Value, itemIsWrongYear, null, null)); + closest = new(face.LocationIndex, itemMinimumDateTime.Value, itemIsWrongYear); + item.Closest.Add(closest); if (!face.Populated) continue; - faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding); - foreach ((DateTime minimumDateTime, bool? isWrongYear, Shared.Models.PersonBirthday personBirthday, FaceEncoding[] faceEncodings) in collection) - { - if (itemIsWrongYear.HasValue && !itemIsWrongYear.Value && itemMinimumDateTime.Value < personBirthday.Value) - continue; - faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncoding); - average = faceDistances.Average(); - if (average > lowestAverage) - continue; - lowestAverage = average; - lowestIsWrongYear = isWrongYear; - lowestPersonBirthday = personBirthday; - lowestMinimumDateTime = minimumDateTime; - } - if (lowestPersonBirthday is null || lowestMinimumDateTime is null) + closestCollection = GetClosestCollection(maxDegreeOfParallelism, collection, maxMinimum, itemMinimumDateTime.Value, itemIsWrongYear, face); + if (!closestCollection.Any()) continue; - key = Item.GetKey(lowestMinimumDateTime.Value, lowestIsWrongYear, lowestPersonBirthday); + closest = Closest.Get(closestCollection); + if (closest.PersonBirthday is null) + continue; + key = Item.GetKey(closest.MinimumDateTime, closest.IsWrongYear, closest.PersonBirthday); if (!results.ContainsKey(key)) results.Add(key, 0); else if (results[key] > maxPer) continue; results[key] += 1; - item.Closest[0] = new(face.LocationIndex, lowestMinimumDateTime.Value, lowestIsWrongYear, lowestPersonBirthday, lowestAverage); + item.Closest[0] = closest; } } } } + internal static List<(DateTime, bool?, Shared.Models.PersonBirthday, FaceEncoding[])> ParallelWork(int maxDegreeOfParallelism, string argZero, List containers) + { + List<(DateTime, bool?, Shared.Models.PersonBirthday, FaceEncoding[])> results; + const int maxPer = 5; + const float maxMinimum = 0.50f; + const bool skipIsWrongYear = true; + Dictionary> keyValuePairs = Item.GetKeyValuePairs(argZero, containers); + results = GetThreeSigmaFaceEncodings(maxDegreeOfParallelism, keyValuePairs); + AddClosest(maxDegreeOfParallelism, argZero, containers, results, skipIsWrongYear, maxPer, maxMinimum); + return results; + } + public static void SavePropertyHolders(string argZero, List containers, string zPropertyHolderSingletonDirectory) { string json; @@ -599,7 +661,38 @@ internal class E_Distance } } - internal static void SaveClosest(string argZero, List containers, string eDistanceContentDirectory, string dFacesContentDirectory) + internal static void SaveThreeSigmaFaceEncodings(List<(DateTime, bool?, Shared.Models.PersonBirthday, FaceEncoding[])> collection, Dictionary> peopleCollection, string eDistanceCollectionDirectory) + { + string json; + string checkFile; + string personKey; + string directory; + List rawEncodings; + Shared.Models.Person person; + const string facePopulatedKey = "ThreeSigma"; + const string pattern = @"[\\,\/,\:,\*,\?,\"",\<,\>,\|]"; + foreach ((DateTime minimumDateTime, bool? isWrongYear, Shared.Models.PersonBirthday personBirthday, FaceEncoding[] faceEncodings) in collection) + { + rawEncodings = new(); + checkFile = string.Empty; + personKey = Shared.Models.Stateless.Methods.IPersonBirthday.GetFormatted(personBirthday); + directory = Item.GetDirectory(eDistanceCollectionDirectory, facePopulatedKey, minimumDateTime, isWrongYear, personBirthday, personKey); + if (!peopleCollection.ContainsKey(personKey)) + continue; + person = peopleCollection[personKey][0]; + checkFile = string.Concat(directory, " - ", Regex.Replace(Shared.Models.Stateless.Methods.IPersonName.GetFullName(person.Name), pattern, string.Empty), ".json"); + if (string.IsNullOrEmpty(checkFile)) + continue; + if (!Directory.Exists(directory)) + _ = Directory.CreateDirectory(directory); + for (int i = 0; i < faceEncodings.Length; i++) + rawEncodings.Add(faceEncodings[i].GetRawEncoding()); + json = JsonSerializer.Serialize(rawEncodings, new JsonSerializerOptions { WriteIndented = true }); + _ = Property.Models.Stateless.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: true, compareBeforeWrite: true); + } + } + + internal static void SaveClosest(string argZero, List containers, Dictionary> peopleCollection, string eDistanceContentDirectory, string dFacesContentDirectory) { string copyFile; string checkFile; @@ -608,7 +701,9 @@ internal class E_Distance string? directoryName; string facesDirectory; string faceFullFileName; - const string facePopulatedKey = "Closet"; + Shared.Models.Person person; + const string facePopulatedKey = "Closest"; + const string pattern = @"[\\,\/,\:,\*,\?,\"",\<,\>,\|]"; foreach (Container container in containers) { if (!container.Items.Any()) @@ -617,7 +712,7 @@ internal class E_Distance continue; foreach (Item item in container.Items) { - if (item.ImageFileHolder is null || item.Property is null || item.ResizedFileHolder is null || item.Named.Any()) + if (item.ImageFileHolder is null || item.Property?.Id is null || item.ResizedFileHolder is null || item.Named.Any()) continue; if (!item.Closest.Any()) continue; @@ -630,9 +725,17 @@ internal class E_Distance continue; personKey = Shared.Models.Stateless.Methods.IPersonBirthday.GetFormatted(closest.PersonBirthday); directory = Item.GetDirectory(eDistanceContentDirectory, facePopulatedKey, closest.MinimumDateTime, closest.IsWrongYear, closest.PersonBirthday, personKey); - checkFile = Path.Combine(directory, item.ImageFileHolder.Name); if (!Directory.Exists(directory)) _ = Directory.CreateDirectory(directory); + // checkFile = Path.Combine(directory, item.ImageFileHolder.Name); + checkFile = Path.Combine(directory, $"{closest.FaceLocationIndex.Value} - {item.Property.Id.Value}{item.ImageFileHolder.ExtensionLowered}"); + if (peopleCollection.ContainsKey(personKey)) + { + person = peopleCollection[personKey][0]; + directory = Path.Combine(directory, Regex.Replace(Shared.Models.Stateless.Methods.IPersonName.GetFullName(person.Name), pattern, string.Empty)); + if (!Directory.Exists(directory)) + _ = Directory.CreateDirectory(directory); + } facesDirectory = string.Concat(dFacesContentDirectory, Path.Combine(directoryName, item.ImageFileHolder.NameWithoutExtension)); faceFullFileName = Path.Combine(facesDirectory, $"{closest.FaceLocationIndex.Value} - {item.ImageFileHolder.NameWithoutExtension}.png"); if (Directory.Exists(facesDirectory) && File.Exists(faceFullFileName)) diff --git a/Metadata/Models/B_Metadata.cs b/Metadata/Models/B_Metadata.cs index 399aef6..44db077 100644 --- a/Metadata/Models/B_Metadata.cs +++ b/Metadata/Models/B_Metadata.cs @@ -78,25 +78,43 @@ public class B_Metadata return results; } - public (int, List>) GetMetadataCollection(List> subFileTuples, List parseExceptions, Property.Models.Item item) + public (int, List>) GetMetadataCollection(string bResultsFullGroupDirectory, List> subFileTuples, List parseExceptions, Property.Models.Item item) { List> results = new(); + if (item.Property?.Id is null) + throw new NullReferenceException(nameof(item.Property.Id)); if (item.ImageFileHolder is null) throw new NullReferenceException(nameof(item.ImageFileHolder)); + if (string.IsNullOrEmpty(bResultsFullGroupDirectory)) + throw new NullReferenceException(nameof(bResultsFullGroupDirectory)); Dictionary>>? dictionary; string json = string.Empty; string[] changesFrom = Array.Empty(); List dateTimes = (from l in subFileTuples where changesFrom.Contains(l.Item1) select l.Item2).ToList(); FileInfo fileInfo = new(Path.Combine(AngleBracketCollection[0].Replace("<>", "{}"), string.Concat(item.ImageFileHolder.NameWithoutExtension, ".json"))); - if (!fileInfo.Exists) + string bMetadataSingletonFile = Path.Combine(bResultsFullGroupDirectory, "{}", "_ _ _", $"{item.Property.Id.Value}{item.ImageFileHolder.ExtensionLowered}.json"); + if (fileInfo.Exists) { - if (fileInfo.Directory?.Parent is null) - throw new Exception(); - string parentCheck = Path.Combine(fileInfo.Directory.Parent.FullName, fileInfo.Name); - if (File.Exists(parentCheck)) + if (!File.Exists(bMetadataSingletonFile)) { - File.Move(parentCheck, fileInfo.FullName); - fileInfo.Refresh(); + fileInfo.MoveTo(bMetadataSingletonFile); + fileInfo = new(bMetadataSingletonFile); + } + } + else + { + if (File.Exists(bMetadataSingletonFile)) + fileInfo = new(bMetadataSingletonFile); + else + { + if (fileInfo.Directory?.Parent is null) + throw new Exception(); + string parentCheck = Path.Combine(fileInfo.Directory.Parent.FullName, fileInfo.Name); + if (File.Exists(parentCheck)) + { + File.Move(parentCheck, fileInfo.FullName); + fileInfo.Refresh(); + } } } if (_ForceMetadataLastWriteTimeToCreationTime && !fileInfo.Exists && File.Exists(Path.ChangeExtension(fileInfo.FullName, ".delete"))) diff --git a/Not-Copy-Copy/Not-Copy-Copy.cs b/Not-Copy-Copy/Not-Copy-Copy.cs index 6f614d3..4d321e0 100644 --- a/Not-Copy-Copy/Not-Copy-Copy.cs +++ b/Not-Copy-Copy/Not-Copy-Copy.cs @@ -12,11 +12,8 @@ public class NotCopyCopy private readonly Serilog.ILogger? _Log; private readonly AppSettings _AppSettings; - private readonly List _Exceptions; private readonly IsEnvironment _IsEnvironment; private readonly Models.Configuration _Configuration; - private readonly List> _FileKeyValuePairs; - private readonly Dictionary>> _FilePropertiesKeyValuePairs; public NotCopyCopy(List args, IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, AppSettings appSettings, string workingDirectory, bool isSilent, IConsole console) { @@ -30,10 +27,7 @@ public class NotCopyCopy if (appSettings.MaxDegreeOfParallelism is null) throw new NullReferenceException(nameof(appSettings.MaxDegreeOfParallelism)); _IsEnvironment = isEnvironment; - _Exceptions = new List(); _Log = Serilog.Log.ForContext(); - _FileKeyValuePairs = new List>(); - _FilePropertiesKeyValuePairs = new Dictionary>>(); Property.Models.Configuration propertyConfiguration = Property.Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory); Property.Models.Configuration.Verify(propertyConfiguration); Models.Configuration configuration = Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory, propertyConfiguration); @@ -111,17 +105,6 @@ public class NotCopyCopy throw new NullReferenceException(nameof(configuration.SelectedSource)); } - private long LogDelta(long ticks, string methodName) - { - long result; - if (_Log is null) - throw new NullReferenceException(nameof(_Log)); - double delta = new TimeSpan(DateTime.Now.Ticks - ticks).TotalMilliseconds; - _Log.Debug($"{methodName} took {Math.Floor(delta)} millisecond(s)"); - result = DateTime.Now.Ticks; - return result; - } - private PropertyLogic GetPropertyLogic(bool reverse, Model? model, PredictorModel? predictorModel) { PropertyLogic result; diff --git a/Property/Models/Closest.cs b/Property/Models/Closest.cs index e58370a..b6b22f5 100644 --- a/Property/Models/Closest.cs +++ b/Property/Models/Closest.cs @@ -9,32 +9,37 @@ public class Closest protected readonly double? _Average; protected readonly int? _FaceLocationIndex; protected readonly bool? _IsWrongYear; + protected readonly double? _Minimum; protected readonly DateTime _MinimumDateTime; protected readonly PersonBirthday? _PersonBirthday; - public double? Average => _Average; public int? FaceLocationIndex => _FaceLocationIndex; public bool? IsWrongYear => _IsWrongYear; + public double? Minimum => _Minimum; public DateTime MinimumDateTime => _MinimumDateTime; public PersonBirthday? PersonBirthday => _PersonBirthday; [JsonConstructor] - public Closest(double? average, int? faceLocationIndex, bool? isWrongYear, DateTime minimumDateTime, PersonBirthday? personBirthday) + public Closest(double? average, int? faceLocationIndex, bool? isWrongYear, double? minimum, DateTime minimumDateTime, PersonBirthday? personBirthday) { _Average = average; _FaceLocationIndex = faceLocationIndex; _IsWrongYear = isWrongYear; + _Minimum = minimum; _MinimumDateTime = minimumDateTime; _PersonBirthday = personBirthday; } - public Closest(int? faceLocationIndex, DateTime minimumDateTime, bool? isWrongYear, PersonBirthday? personBirthday, double? average) + public Closest(int? faceLocationIndex, DateTime minimumDateTime, bool? isWrongYear) : + this(null, faceLocationIndex, isWrongYear, null, minimumDateTime, null) { - _Average = average; - _FaceLocationIndex = faceLocationIndex; - _IsWrongYear = isWrongYear; - _MinimumDateTime = minimumDateTime; - _PersonBirthday = personBirthday; } + public Closest(int? faceLocationIndex, DateTime minimumDateTime, bool? isWrongYear, PersonBirthday? personBirthday, List faceDistances) : + this(faceDistances.Average(), faceLocationIndex, isWrongYear, faceDistances.Min(), minimumDateTime, personBirthday) + { + } + + public static Closest Get(List collection) => (from l in collection orderby l.Minimum < 0.05, l.Average select l).First(); + } \ No newline at end of file diff --git a/Property/Models/Stateless/IResult.cs b/Property/Models/Stateless/IResult.cs index 6ca907a..0590926 100644 --- a/Property/Models/Stateless/IResult.cs +++ b/Property/Models/Stateless/IResult.cs @@ -17,10 +17,13 @@ public interface IResult string TestStatic_GetResultsDateGroupDirectory(Models.Configuration configuration, string description, string jsonGroup); static string GetResultsDateGroupDirectory(Models.Configuration configuration, string description, string jsonGroup) => Result.GetResultsDateGroupDirectory(configuration, description, jsonGroup); + List TestStatic_GetDirectoryInfoCollection(Models.Configuration configuration, string sourceDirectory, string dateGroupDirectory, string contentDescription, string singletonDescription, string collectionDescription); + static List GetDirectoryInfoCollection(Models.Configuration configuration, string sourceDirectory, string dateGroupDirectory, string contentDescription, string singletonDescription, string collectionDescription) => Result.GetDirectoryInfoCollection(configuration, sourceDirectory, dateGroupDirectory, contentDescription, singletonDescription, collectionDescription); + string TestStatic_GetResultsFullGroupDirectory(Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string description, string outputResolution, bool includeResizeGroup, bool includeModel, bool includePredictorModel); static string GetResultsFullGroupDirectory(Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string description, string outputResolution, bool includeResizeGroup, bool includeModel, bool includePredictorModel) => Result.GetResultsFullGroupDirectory(configuration, model, predictorModel, description, outputResolution, includeResizeGroup, includeModel, includePredictorModel); - List TestStatic_GetDirectoryInfoCollection(Models.Configuration configuration, string sourceDirectory, string description, string outputResolution, bool includeResizeGroup, bool includeModel, bool includePredictorModel, string contentDescription, string singletonDescription, string collectionDescription); + List TestStatic_GetDirectoryInfoCollection(Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string sourceDirectory, string description, string outputResolution, bool includeResizeGroup, bool includeModel, bool includePredictorModel, string contentDescription, string singletonDescription, string collectionDescription); static List GetDirectoryInfoCollection(Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string sourceDirectory, string description, string outputResolution, bool includeResizeGroup, bool includeModel, bool includePredictorModel, string contentDescription, string singletonDescription, string collectionDescription) => Result.GetDirectoryInfoCollection(configuration, model, predictorModel, sourceDirectory, description, outputResolution, includeResizeGroup, includeModel, includePredictorModel, contentDescription, singletonDescription, collectionDescription); } \ No newline at end of file diff --git a/Property/Models/Stateless/Result.cs b/Property/Models/Stateless/Result.cs index abae4d6..692a4fa 100644 --- a/Property/Models/Stateless/Result.cs +++ b/Property/Models/Stateless/Result.cs @@ -64,13 +64,65 @@ internal class Result return result; } + internal static List GetDirectoryInfoCollection(Models.Configuration configuration, string sourceDirectory, string dateGroupDirectory, string contentDescription, string singletonDescription, string collectionDescription) + { + List results = new(); + string result = string.Empty; + string checkDirectory; + const string allInOne = "_ _ _"; + string sourceDirectorySegment = GetRelativePath(configuration, sourceDirectory); + if (!string.IsNullOrEmpty(contentDescription)) + { + checkDirectory = Path.Combine(dateGroupDirectory, "()", allInOne); + if (!Directory.Exists(checkDirectory)) + _ = Directory.CreateDirectory(checkDirectory); + result = string.Concat(Path.Combine(dateGroupDirectory, "<>"), sourceDirectorySegment); + string contentDirectory = new(result.Replace("<>", "()")); + if (!Directory.Exists(contentDirectory)) + _ = Directory.CreateDirectory(contentDirectory); + checkDirectory = Path.Combine(dateGroupDirectory, string.Concat("() - ", contentDescription)); + if (!Directory.Exists(checkDirectory)) + _ = Directory.CreateDirectory(checkDirectory); + } + if (!string.IsNullOrEmpty(singletonDescription)) + { + checkDirectory = Path.Combine(dateGroupDirectory, "{}", allInOne); + if (!Directory.Exists(checkDirectory)) + _ = Directory.CreateDirectory(checkDirectory); + result = string.Concat(Path.Combine(dateGroupDirectory, "<>"), sourceDirectorySegment); + string singletonDirectory = new(result.Replace("<>", "{}")); + if (!Directory.Exists(singletonDirectory)) + _ = Directory.CreateDirectory(singletonDirectory); + checkDirectory = Path.Combine(dateGroupDirectory, string.Concat("{} - ", singletonDescription)); + if (!Directory.Exists(checkDirectory)) + _ = Directory.CreateDirectory(checkDirectory); + } + if (!string.IsNullOrEmpty(collectionDescription)) + { + checkDirectory = Path.Combine(dateGroupDirectory, "[]", allInOne); + if (!Directory.Exists(checkDirectory)) + _ = Directory.CreateDirectory(checkDirectory); + result = string.Concat(Path.Combine(dateGroupDirectory, "<>"), sourceDirectorySegment); + string collectionDirectory = new(result.Replace("<>", "[]")); + if (!Directory.Exists(collectionDirectory)) + _ = Directory.CreateDirectory(collectionDirectory); + checkDirectory = Path.Combine(dateGroupDirectory, string.Concat("[] - ", collectionDescription)); + if (!Directory.Exists(checkDirectory)) + _ = Directory.CreateDirectory(checkDirectory); + } + if (string.IsNullOrEmpty(result)) + throw new Exception(); + results.Add(result); + return results; + } + internal static List GetDirectoryInfoCollection(Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string sourceDirectory, string description, string outputResolution, bool includeResizeGroup, bool includeModel, bool includePredictorModel, string contentDescription, string singletonDescription, string collectionDescription) { List results = new(); string result = string.Empty; string checkDirectory; string sourceDirectorySegment = GetRelativePath(configuration, sourceDirectory); - string dateGroupDirectory = IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, description, outputResolution, includeResizeGroup, includeModel, includePredictorModel); + string dateGroupDirectory = GetResultsFullGroupDirectory(configuration, model, predictorModel, description, outputResolution, includeResizeGroup, includeModel, includePredictorModel); if (!string.IsNullOrEmpty(contentDescription)) { result = string.Concat(Path.Combine(dateGroupDirectory, "<>"), sourceDirectorySegment); diff --git a/Resize/Models/_C_Resize.cs b/Resize/Models/_C_Resize.cs index d6e711f..c9bc61c 100644 --- a/Resize/Models/_C_Resize.cs +++ b/Resize/Models/_C_Resize.cs @@ -284,13 +284,15 @@ public class C_Resize return results; } - public byte[] GetResizedBytes(string outputResolution, List> subFileTuples, Item item, A_Property property, Dictionary imageResizes) + public byte[] GetResizedBytes(string outputResolution, string cResultsFullGroupDirectory, List> subFileTuples, Item item, A_Property property, Dictionary imageResizes) { byte[] results; if (item.ImageFileHolder is null) throw new NullReferenceException(nameof(item.ImageFileHolder)); if (!imageResizes.ContainsKey(outputResolution)) throw new Exception(); + if (string.IsNullOrEmpty(cResultsFullGroupDirectory)) + throw new NullReferenceException(nameof(cResultsFullGroupDirectory)); int[] resize = imageResizes[outputResolution]; int outputResolutionWidth = resize[_OutputResolutionWidthIndex]; int outputResolutionHeight = resize[_OutputResolutionHeightIndex]; @@ -300,12 +302,14 @@ public class C_Resize return results; } - public void SaveResizedSubfile(string outputResolution, List> subFileTuples, Item item, string original, A_Property property, Dictionary imageResizes) + public void SaveResizedSubfile(string outputResolution, string cResultsFullGroupDirectory, List> subFileTuples, Item item, string original, A_Property property, Dictionary imageResizes) { if (item.ImageFileHolder is null) throw new NullReferenceException(nameof(item.ImageFileHolder)); if (item.ResizedFileHolder is null) throw new NullReferenceException(nameof(item.ResizedFileHolder)); + if (string.IsNullOrEmpty(cResultsFullGroupDirectory)) + throw new NullReferenceException(nameof(cResultsFullGroupDirectory)); FileHolder fileHolder = item.ResizedFileHolder; if (!imageResizes.ContainsKey(outputResolution)) throw new Exception(); @@ -422,24 +426,42 @@ public class C_Resize return results; } - public Dictionary GetResizeKeyValuePairs(List> subFileTuples, List parseExceptions, string original, List> metadataCollection, A_Property property, Item item) + public Dictionary GetResizeKeyValuePairs(string cResultsFullGroupDirectory, List> subFileTuples, List parseExceptions, string original, List> metadataCollection, A_Property property, Item item) { Dictionary results; + if (item.Property?.Id is null) + throw new NullReferenceException(nameof(item.Property.Id)); if (item.ImageFileHolder is null) throw new NullReferenceException(nameof(item.ImageFileHolder)); + if (string.IsNullOrEmpty(cResultsFullGroupDirectory)) + throw new NullReferenceException(nameof(cResultsFullGroupDirectory)); string json; string[] changesFrom = new string[] { nameof(A_Property), nameof(B_Metadata) }; List dateTimes = (from l in subFileTuples where changesFrom.Contains(l.Item1) select l.Item2).ToList(); FileInfo fileInfo = new(Path.Combine(AngleBracketCollection[0].Replace("<>", "{}"), string.Concat(item.ImageFileHolder.NameWithoutExtension, ".json"))); - if (!fileInfo.Exists) + string cResizeSingletonFile = Path.Combine(cResultsFullGroupDirectory, "{}", "_ _ _", $"{item.Property.Id.Value}{item.ImageFileHolder.ExtensionLowered}.json"); + if (fileInfo.Exists) { - if (fileInfo.Directory?.Parent is null) - throw new Exception(); - string parentCheck = Path.Combine(fileInfo.Directory.Parent.FullName, fileInfo.Name); - if (File.Exists(parentCheck)) + if (!File.Exists(cResizeSingletonFile)) { - File.Move(parentCheck, fileInfo.FullName); - fileInfo.Refresh(); + fileInfo.MoveTo(cResizeSingletonFile); + fileInfo = new(cResizeSingletonFile); + } + } + else + { + if (File.Exists(cResizeSingletonFile)) + fileInfo = new(cResizeSingletonFile); + else + { + if (fileInfo.Directory?.Parent is null) + throw new Exception(); + string parentCheck = Path.Combine(fileInfo.Directory.Parent.FullName, fileInfo.Name); + if (File.Exists(parentCheck)) + { + File.Move(parentCheck, fileInfo.FullName); + fileInfo.Refresh(); + } } } if (_ForceResizeLastWriteTimeToCreationTime && !fileInfo.Exists && File.Exists(Path.ChangeExtension(fileInfo.FullName, ".delete"))) diff --git a/Tests/UnitTestResize.cs b/Tests/UnitTestResize.cs index 304021f..4089ef5 100644 --- a/Tests/UnitTestResize.cs +++ b/Tests/UnitTestResize.cs @@ -106,6 +106,9 @@ public class UnitTestResize bool reverse = false; Property.Models.Item item; string original = "Original"; + string aResultsFullGroupDirectory; + string bResultsFullGroupDirectory; + string cResultsFullGroupDirectory; List parseExceptions = new(); PredictorModel? predictorModel = null; bool isValidImageFormatExtension = true; @@ -117,46 +120,38 @@ public class UnitTestResize string outputResolution = _Configuration.OutputResolutions[0]; Property.Models.PropertyLogic propertyLogic = GetPropertyLogic(reverse, model, predictorModel); string sourceDirectory = Path.Combine(_PropertyConfiguration.RootDirectory, sourceDirectoryName); + _Logger.Information(_Configuration.ModelDirectory); + aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + _PropertyConfiguration, model, predictorModel, nameof(Property.Models.A_Property), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false); + bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + _PropertyConfiguration, model, predictorModel, nameof(B_Metadata), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false); + cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + _PropertyConfiguration, model, predictorModel, nameof(C_Resize), outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false); string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_PropertyConfiguration, nameof(Property.Models.A_Property), "{}"); B_Metadata metadata = new(_Configuration.ForceMetadataLastWriteTimeToCreationTime.Value, _Configuration.PropertiesChangedForMetadata.Value); (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters) = C_Resize.GetTuple(_Configuration.OutputExtension, _Configuration.OutputQuality.Value); C_Resize resize = new(_Configuration.ForceResizeLastWriteTimeToCreationTime.Value, _Configuration.OverrideForResizeImages.Value, _Configuration.PropertiesChangedForResize.Value, _Configuration.ValidResolutions, imageCodecInfo, encoderParameters); - propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(_PropertyConfiguration, - model, - predictorModel, - sourceDirectory, - nameof(Property.Models.A_Property), - outputResolution, - includeResizeGroup: false, - includeModel: false, - includePredictorModel: false, - contentDescription: string.Empty, - singletonDescription: "Properties for each image", - collectionDescription: string.Empty)); - metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(_PropertyConfiguration, - model, - predictorModel, - sourceDirectory, - nameof(B_Metadata), - outputResolution, - includeResizeGroup: false, - includeModel: false, - includePredictorModel: false, - contentDescription: string.Empty, - singletonDescription: "Metadata as key value pairs", - collectionDescription: string.Empty)); - resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(_PropertyConfiguration, - model, - predictorModel, - sourceDirectory, - nameof(C_Resize), - outputResolution, - includeResizeGroup: true, - includeModel: false, - includePredictorModel: false, - contentDescription: "Resized image", - singletonDescription: "Resize dimensions for each resolution", - collectionDescription: string.Empty)); + propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection( + _PropertyConfiguration, + sourceDirectory, + aResultsFullGroupDirectory, + contentDescription: string.Empty, + singletonDescription: "Properties for each image", + collectionDescription: string.Empty)); + metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection( + _PropertyConfiguration, + sourceDirectory, + bResultsFullGroupDirectory, + contentDescription: string.Empty, + singletonDescription: "Metadata as key value pairs", + collectionDescription: string.Empty)); + resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection( + _PropertyConfiguration, + sourceDirectory, + cResultsFullGroupDirectory, + contentDescription: "Resized image", + singletonDescription: "Resize dimensions for each resolution", + collectionDescription: string.Empty)); string sourceDirectoryFile = ".json"; Property.Models.FileHolder fileHolder = new(Path.Combine(sourceDirectory, sourceFileName)); string relativePath = Property.Models.Stateless.IPath.GetRelativePath(fileHolder.FullName, length); @@ -164,11 +159,11 @@ public class UnitTestResize item = new(sourceDirectoryFile, relativePath, fileHolder, isValidImageFormatExtension, property, false, false); Assert.IsNotNull(item.ImageFileHolder); property = propertyLogic.GetProperty(item, subFileTuples, parseExceptions); - (int _, metadataCollection) = metadata.GetMetadataCollection(subFileTuples, parseExceptions, item); - imageResizeKeyValuePairs = resize.GetResizeKeyValuePairs(subFileTuples, parseExceptions, original, metadataCollection, property, item); + (int _, metadataCollection) = metadata.GetMetadataCollection(bResultsFullGroupDirectory, subFileTuples, parseExceptions, item); + imageResizeKeyValuePairs = resize.GetResizeKeyValuePairs(cResultsFullGroupDirectory, subFileTuples, parseExceptions, original, metadataCollection, property, item); Property.Models.FileHolder resizedFileHolder = new(Path.Combine(resize.AngleBracketCollection[0].Replace("<>", "()"), Path.GetFileName(item.ImageFileHolder.FullName))); item.SetResizedFileHolder(resizedFileHolder); - resize.SaveResizedSubfile(outputResolution, subFileTuples, item, original, property, imageResizeKeyValuePairs); + resize.SaveResizedSubfile(outputResolution, cResultsFullGroupDirectory, subFileTuples, item, original, property, imageResizeKeyValuePairs); item.SetResizedFileHolder(Property.Models.FileHolder.Refresh(resizedFileHolder)); } diff --git a/TestsWithFaceRecognitionDotNet/UnitTestFace.cs b/TestsWithFaceRecognitionDotNet/UnitTestFace.cs index 8891fa4..b87a568 100644 --- a/TestsWithFaceRecognitionDotNet/UnitTestFace.cs +++ b/TestsWithFaceRecognitionDotNet/UnitTestFace.cs @@ -168,6 +168,9 @@ public class UnitTestFace bool reverse = false; Property.Models.Item item; string original = "Original"; + string aResultsFullGroupDirectory; + string bResultsFullGroupDirectory; + string cResultsFullGroupDirectory; List parseExceptions = new(); bool isValidImageFormatExtension = true; Property.Models.A_Property? property = null; @@ -180,46 +183,37 @@ public class UnitTestFace Property.Models.PropertyLogic propertyLogic = GetPropertyLogic(reverse, model, predictorModel); string sourceDirectory = Path.Combine(_PropertyConfiguration.RootDirectory, sourceDirectoryName); _Logger.Information(_Configuration.ModelDirectory); + aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + _PropertyConfiguration, model, predictorModel, nameof(Property.Models.A_Property), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false); + bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + _PropertyConfiguration, model, predictorModel, nameof(B_Metadata), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false); + cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + _PropertyConfiguration, model, predictorModel, nameof(C_Resize), outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false); string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_PropertyConfiguration, nameof(Property.Models.A_Property), "{}"); B_Metadata metadata = new(_Configuration.ForceMetadataLastWriteTimeToCreationTime.Value, _Configuration.PropertiesChangedForMetadata.Value); (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters) = C_Resize.GetTuple(_Configuration.OutputExtension, _Configuration.OutputQuality.Value); C_Resize resize = new(_Configuration.ForceResizeLastWriteTimeToCreationTime.Value, _Configuration.OverrideForResizeImages.Value, _Configuration.PropertiesChangedForResize.Value, _Configuration.ValidResolutions, imageCodecInfo, encoderParameters); - propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(_PropertyConfiguration, - model, - predictorModel, - sourceDirectory, - nameof(Property.Models.A_Property), - outputResolution, - includeResizeGroup: false, - includeModel: false, - includePredictorModel: false, - contentDescription: string.Empty, - singletonDescription: "Properties for each image", - collectionDescription: string.Empty)); - metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(_PropertyConfiguration, - model, - predictorModel, - sourceDirectory, - nameof(B_Metadata), - outputResolution, - includeResizeGroup: false, - includeModel: false, - includePredictorModel: false, - contentDescription: string.Empty, - singletonDescription: "Metadata as key value pairs", - collectionDescription: string.Empty)); - resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(_PropertyConfiguration, - model, - predictorModel, - sourceDirectory, - nameof(C_Resize), - outputResolution, - includeResizeGroup: true, - includeModel: false, - includePredictorModel: false, - contentDescription: "Resized image", - singletonDescription: "Resize dimensions for each resolution", - collectionDescription: string.Empty)); + propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection( + _PropertyConfiguration, + sourceDirectory, + aResultsFullGroupDirectory, + contentDescription: string.Empty, + singletonDescription: "Properties for each image", + collectionDescription: string.Empty)); + metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection( + _PropertyConfiguration, + sourceDirectory, + bResultsFullGroupDirectory, + contentDescription: string.Empty, + singletonDescription: "Metadata as key value pairs", + collectionDescription: string.Empty)); + resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection( + _PropertyConfiguration, + sourceDirectory, + cResultsFullGroupDirectory, + contentDescription: "Resized image", + singletonDescription: "Resize dimensions for each resolution", + collectionDescription: string.Empty)); string sourceDirectoryFile = ".json"; Property.Models.FileHolder fileHolder = new(Path.Combine(sourceDirectory, sourceFileName)); string relativePath = Property.Models.Stateless.IPath.GetRelativePath(fileHolder.FullName, length); @@ -227,11 +221,11 @@ public class UnitTestFace item = new(sourceDirectoryFile, relativePath, fileHolder, isValidImageFormatExtension, property, false, false); Assert.IsNotNull(item.ImageFileHolder); property = propertyLogic.GetProperty(item, subFileTuples, parseExceptions); - (int _, metadataCollection) = metadata.GetMetadataCollection(subFileTuples, parseExceptions, item); - imageResizeKeyValuePairs = resize.GetResizeKeyValuePairs(subFileTuples, parseExceptions, original, metadataCollection, property, item); + (int _, metadataCollection) = metadata.GetMetadataCollection(bResultsFullGroupDirectory, subFileTuples, parseExceptions, item); + imageResizeKeyValuePairs = resize.GetResizeKeyValuePairs(cResultsFullGroupDirectory, subFileTuples, parseExceptions, original, metadataCollection, property, item); Property.Models.FileHolder resizedFileHolder = new(Path.Combine(resize.AngleBracketCollection[0].Replace("<>", "()"), Path.GetFileName(item.ImageFileHolder.FullName))); item.SetResizedFileHolder(resizedFileHolder); - resize.SaveResizedSubfile(outputResolution, subFileTuples, item, original, property, imageResizeKeyValuePairs); + resize.SaveResizedSubfile(outputResolution, cResultsFullGroupDirectory, subFileTuples, item, original, property, imageResizeKeyValuePairs); item.SetResizedFileHolder(Property.Models.FileHolder.Refresh(resizedFileHolder)); Assert.IsNotNull(item.ResizedFileHolder); Image? image = FaceRecognition.LoadImageFile(item.ResizedFileHolder.FullName);