using Microsoft.Extensions.Configuration; using Phares.Shared; using ShellProgressBar; using System.Drawing.Imaging; using System.Text.Json; using View_by_Distance.FaceRecognitionDotNet; using View_by_Distance.Instance.Models; using View_by_Distance.Metadata.Models; using View_by_Distance.Property.Models; using View_by_Distance.Resize.Models; using View_by_Distance.Shared.Models; using View_by_Distance.Shared.Models.Methods; using View_by_Distance.Shared.Models.Stateless; namespace View_by_Distance.Instance; public class DlibDotNet { private readonly D_Face _Faces; private readonly G_Index _Index; private readonly C_Resize _Resize; private readonly F_Random _Random; private readonly A2_People _People; private readonly E3_Rename _Rename; private readonly B_Metadata _Metadata; private readonly E_Distance _Distance; private readonly Serilog.ILogger? _Log; private readonly AppSettings _AppSettings; private readonly List _Exceptions; private readonly IsEnvironment _IsEnvironment; private readonly D2_FaceLandmarks _FaceLandmarks; private readonly Models.Configuration _Configuration; private readonly bool _ArgZeroIsConfigurationRootDirectory; private readonly List> _FileKeyValuePairs; private readonly Dictionary>> _FilePropertiesKeyValuePairs; public DlibDotNet(List args, IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, AppSettings appSettings, string workingDirectory, bool isSilent, IConsole console) { string argZero; Person[] people; _AppSettings = appSettings; _IsEnvironment = isEnvironment; _Exceptions = new List(); _Log = Serilog.Log.ForContext(); _FileKeyValuePairs = new List>(); _FilePropertiesKeyValuePairs = new Dictionary>>(); Property.Models.Configuration propertyConfiguration = Property.Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory); _Log.Information(propertyConfiguration.RootDirectory); Property.Models.Configuration.Verify(propertyConfiguration); Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration); Verify(configuration); VerifyExtra(args, propertyConfiguration, configuration); _Configuration = configuration; _Index = new G_Index(configuration); _Random = new F_Random(configuration); _People = new A2_People(configuration); _Rename = new E3_Rename(configuration); _Distance = new E_Distance(configuration); _FaceLandmarks = new D2_FaceLandmarks(configuration); if (configuration.IgnoreExtensions is null) throw new NullReferenceException(nameof(configuration.IgnoreExtensions)); _Metadata = new(configuration.ForceMetadataLastWriteTimeToCreationTime, configuration.PropertiesChangedForMetadata); if (args.Count > 0) argZero = Path.GetFullPath(args[0]); else argZero = Path.GetFullPath(propertyConfiguration.RootDirectory); _ArgZeroIsConfigurationRootDirectory = propertyConfiguration.RootDirectory == argZero; (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters) = C_Resize.GetTuple(configuration.OutputExtension, configuration.OutputQuality); _Resize = new C_Resize(configuration.ForceResizeLastWriteTimeToCreationTime, configuration.OverrideForResizeImages, configuration.PropertiesChangedForResize, configuration.ValidResolutions, imageCodecInfo, encoderParameters); _Log.Information(configuration.ModelDirectory); (Model model, PredictorModel predictorModel, ModelParameter modelParameter) = GetModel(configuration); _Faces = new D_Face(configuration, argZero, model, modelParameter, predictorModel); if (!_ArgZeroIsConfigurationRootDirectory) people = Array.Empty(); else people = _People.GetPeople(propertyConfiguration); if (!isSilent && configuration.TestDistanceResults) { E2_Navigate e2Navigate = new(console, configuration, argZero); e2Navigate.Navigate(propertyConfiguration, model, predictorModel, configuration.OutputResolutions[0]); _Log.Information(propertyConfiguration.RootDirectory); } if (!configuration.SkipSearch) Search(propertyConfiguration, configuration.Reverse, model, predictorModel, argZero, people, isSilent); if (_Exceptions.Count == 0 && _ArgZeroIsConfigurationRootDirectory) { long ticks = DateTime.Now.Ticks; List directoryCollections = _Rename.GetDirectoryRenameCollections(propertyConfiguration, model, predictorModel, relativePath: string.Empty, newDirectoryName: string.Empty, jsonFiles4InfoAny: false); foreach (string[] directoryCollection in directoryCollections) { _Log.Information(string.Concat("Cleaning <", directoryCollection[0], ">")); _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(directoryCollection[0]); } string d2FaceLandmarksRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(D2_FaceLandmarks)); _Log.Information(string.Concat("Cleaning <", d2FaceLandmarksRootDirectory, ">")); _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(d2FaceLandmarksRootDirectory); if (appSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Property.Models.Stateless.IPath.DeleteEmptyDirectories)); } string message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}"; _Log.Information(message); if (_Exceptions.Count != 0) throw new Exception(message); if (configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions.Any()) { long ticks = DateTime.Now.Ticks; foreach (string outputResolution in configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions) _Distance.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions(propertyConfiguration, model, predictorModel, outputResolution); if (appSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(E_Distance.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions)); } } private long LogDelta(long ticks, string methodName) { long result; if (_Log is null) throw new NullReferenceException(nameof(_Log)); double delta = new TimeSpan(DateTime.Now.Ticks - ticks).TotalMilliseconds; _Log.Debug($"{methodName} took {Math.Floor(delta)} millisecond(s)"); result = DateTime.Now.Ticks; return result; } private long LogDeltaInMinutes(long ticks, string methodName) { long result; if (_Log is null) throw new NullReferenceException(nameof(_Log)); double delta = new TimeSpan(DateTime.Now.Ticks - ticks).Minutes; _Log.Debug($"{methodName} took {Math.Floor(delta)} minutes(s)"); result = DateTime.Now.Ticks; return result; } #pragma warning disable CA1416 #pragma warning restore CA1416 private static (Model model, PredictorModel predictorModel, ModelParameter modelParameter) GetModel(Models.Configuration configuration) { (Model, PredictorModel, ModelParameter) result; Array array; Model? model = null; PredictorModel? predictorModel = null; array = Enum.GetValues(typeof(Model)); foreach (Model check in array) { if (configuration.ModelName.Contains(check.ToString())) { model = check; break; } } if (model is null) throw new Exception("Destination directory must have Model name!"); model = model.Value; array = Enum.GetValues(typeof(PredictorModel)); foreach (PredictorModel check in array) { if (configuration.PredictorModelName.Contains(check.ToString())) { predictorModel = check; break; } } if (predictorModel is null) throw new Exception("Destination directory must have Predictor Model name!"); predictorModel = predictorModel.Value; ModelParameter modelParameter = new() { CnnFaceDetectorModel = File.ReadAllBytes(Path.Combine(configuration.ModelDirectory, "mmod_human_face_detector.dat")), FaceRecognitionModel = File.ReadAllBytes(Path.Combine(configuration.ModelDirectory, "dlib_face_recognition_resnet_model_v1.dat")), PosePredictor5FaceLandmarksModel = File.ReadAllBytes(Path.Combine(configuration.ModelDirectory, "shape_predictor_5_face_landmarks.dat")), PosePredictor68FaceLandmarksModel = File.ReadAllBytes(Path.Combine(configuration.ModelDirectory, "shape_predictor_68_face_landmarks.dat")) }; result = new(model.Value, predictorModel.Value, modelParameter); return result; } private void Verify(Models.Configuration configuration) { if (!configuration.OutputResolutions.Any() || string.IsNullOrEmpty(configuration.OutputResolutions[0]) || !configuration.ValidResolutions.Contains(configuration.OutputResolutions[0])) throw new NullReferenceException($"{nameof(configuration.OutputResolutions)} must be a valid outputResolution!"); if ((from l in configuration.OutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.OutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.SaveShortcutsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.SaveShortcutsForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.SaveFaceLandmarkForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.SaveFaceLandmarkForOutputResolutions)} are not in the ValidResolutions list!"); if (configuration.DistanceFactor + configuration.LocationConfidenceFactor != 10) throw new NullReferenceException(nameof(configuration.DistanceFactor)); if (string.IsNullOrEmpty(configuration.ModelName)) throw new NullReferenceException(nameof(configuration.ModelName)); if (string.IsNullOrEmpty(configuration.OutputExtension)) throw new NullReferenceException(nameof(configuration.OutputExtension)); if (string.IsNullOrEmpty(configuration.PredictorModelName)) throw new NullReferenceException(nameof(configuration.PredictorModelName)); if (string.IsNullOrEmpty(configuration.ModelDirectory) || !Directory.Exists(configuration.ModelDirectory)) throw new NullReferenceException(nameof(configuration.ModelDirectory)); if (configuration.MappedMaxIndex.HasValue && configuration.LoadOrCreateThenSaveIndex && !_IsEnvironment.DebuggerWasAttachedDuringConstructor) throw new NullReferenceException(nameof(configuration.MappedMaxIndex)); } private void VerifyExtra(List args, Property.Models.Configuration propertyConfiguration, Models.Configuration configuration) { string[] sourceDirectoryNames; if (!args.Any()) sourceDirectoryNames = Array.Empty(); else { string argZero = Path.GetFullPath(args[0]); sourceDirectoryNames = argZero.Split(Path.DirectorySeparatorChar); if (!argZero.StartsWith(propertyConfiguration.RootDirectory)) throw new Exception($"Source directory must be inside root directory! <{argZero}> <{propertyConfiguration.RootDirectory}>"); if (_ArgZeroIsConfigurationRootDirectory && propertyConfiguration.RootDirectory != argZero) { if (!configuration.MixedYearRelativePaths.Contains(sourceDirectoryNames[0])) { string[] segments = sourceDirectoryNames[0].Split(' '); if (segments.Length < 2 || segments[^1].Length != 4 || (segments[^1][..2] != "19" && segments[^1][..2] != "20")) throw new Exception("root subdirectory must have a year at the end or directory name needs to be added to the exclude list!"); } } } string[] resizeMatch = (from l in sourceDirectoryNames where configuration.ValidResolutions.Contains(l) select l).ToArray(); if (resizeMatch.Any()) throw new Exception("Input directory should be the source and not a resized directory!"); } private void FullParallelForWork(PropertyLogic propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List> sourceDirectoryChanges, List propertyFileHolderCollection, List propertyCollection, List>> metadataCollections, List> resizeKeyValuePairs, List> imageFaceCollections, string sourceDirectory, int index, Item item) { if (item.ImageFileHolder is null) throw new NullReferenceException(nameof(item.ImageFileHolder)); if (_Configuration?.PropertyConfiguration is null) throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration)); if (_Configuration.PropertyConfiguration.WriteBitmapDataBytes is null) throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration.WriteBitmapDataBytes)); A_Property property; List faceCollection; string original = "Original"; long ticks = DateTime.Now.Ticks; DateTime dateTime = DateTime.Now; List parseExceptions = new(); Dictionary imageResizeKeyValuePairs; List> subFileTuples = new(); List> metadataCollection; if (item.Property is not null) { property = item.Property; if ((item.Changed.HasValue && item.Changed.Value) || (item.Moved.HasValue && item.Moved.Value) || (item.Abandoned.HasValue && item.Abandoned.Value)) throw new NotImplementedException(); } else { property = propertyLogic.GetProperty(item, subFileTuples, parseExceptions); item.Update(property); if (subFileTuples.Any()) { lock (sourceDirectoryChanges) sourceDirectoryChanges.Add(new Tuple(nameof(A_Property), (from l in subFileTuples select l.Item2).Max())); } } (int metadataGroups, metadataCollection) = _Metadata.GetMetadataCollection(bResultsFullGroupDirectory, subFileTuples, parseExceptions, item); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection)); FileHolder resizedFileHolder = new(Path.Combine(_Resize.AngleBracketCollection[0].Replace("<>", "()"), Path.GetFileName(item.ImageFileHolder.FullName))); item.SetResizedFileHolder(resizedFileHolder); imageResizeKeyValuePairs = _Resize.GetResizeKeyValuePairs(cResultsFullGroupDirectory, subFileTuples, parseExceptions, original, metadataCollection, item); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(C_Resize.GetResizeKeyValuePairs)); if (_Configuration.SaveResizedSubfiles) { _Resize.SaveResizedSubfile(outputResolution, cResultsFullGroupDirectory, subFileTuples, item, original, imageResizeKeyValuePairs); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(C_Resize.SaveResizedSubfile)); item.SetResizedFileHolder(FileHolder.Refresh(resizedFileHolder)); } else if (outputResolution == _Configuration.OutputResolutions[0] && false) { byte[] bytes = _Resize.GetResizedBytes(outputResolution, cResultsFullGroupDirectory, subFileTuples, item, property, imageResizeKeyValuePairs); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(C_Resize.GetResizedBytes)); string path = Path.Combine(resizedFileHolder.DirectoryName, resizedFileHolder.NameWithoutExtension); File.WriteAllBytes(path, bytes); } if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution)) faceCollection = new(); else { int[] outputResolutionCollection = imageResizeKeyValuePairs[outputResolution]; int outputResolutionWidth = outputResolutionCollection[0]; int outputResolutionHeight = outputResolutionCollection[1]; int outputResolutionOrientation = outputResolutionCollection[2]; faceCollection = _Faces.GetFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, item, property, resizedFileHolder, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(D_Face.GetFaces)); _Faces.SaveFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, item, faceCollection); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(D_Face.SaveFaces)); if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution)) { _FaceLandmarks.SaveFaceLandmarkImages(d2ResultsFullGroupDirectory, sourceDirectory, subFileTuples, parseExceptions, item, faceCollection); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(D2_FaceLandmarks.SaveFaceLandmarkImages)); } } lock (sourceDirectoryChanges) { propertyCollection[index] = property; imageFaceCollections[index] = faceCollection; metadataCollections[index] = metadataCollection; resizeKeyValuePairs[index] = imageResizeKeyValuePairs; propertyFileHolderCollection[index] = item.ImageFileHolder; sourceDirectoryChanges.AddRange(from l in subFileTuples where l.Item2 > dateTime select l); } } private int FullParallelWork(long ticks, PropertyLogic propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List> sourceDirectoryChanges, List propertyFileHolderCollection, List propertyCollection, List>> metadataCollection, List> resizeKeyValuePairs, List> faceCollections, int containersCount, Container container, Item[] filteredItems) { int result = 0; if (_Log is null) throw new NullReferenceException(nameof(_Log)); ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism }; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; if (faceCollections.Count != filteredItems.Length || metadataCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || propertyCollection.Count != filteredItems.Length) { for (int i = 0; i < filteredItems.Length; i++) { faceCollections.Add(new()); metadataCollection.Add(new()); resizeKeyValuePairs.Add(new()); propertyCollection.Add(new()); propertyFileHolderCollection.Add(null); } } int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); string message = $"{container.R:000}.{container.G} / {containersCount:000}) {filteredItems.Length:000} file(s) - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}"; using (ProgressBar progressBar = new(filteredItems.Length, message, options)) { _ = Parallel.For(0, filteredItems.Length, parallelOptions, i => { try { FullParallelForWork(propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, container.SourceDirectory, index: i, filteredItems[i]); if (i == 0 || sourceDirectoryChanges.Any()) progressBar.Tick(); } catch (Exception ex) { result += 1; _Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, ex.Message, Environment.NewLine, ex.StackTrace), ex); if (result == filteredItems.Length) throw new Exception(string.Concat("All in [", container.SourceDirectory, "] failed!")); } }); } return result; } private static void WriteTab(string checkDirectory, List<(string Id, string Line)> metadataIdLines, string fileName) { string text; FileInfo fileInfo; List duplicates = new(); List metadataIds = new(); fileInfo = new(Path.Combine(checkDirectory, "[()]", Path.ChangeExtension(fileName, "tsv"))); if (fileInfo?.Directory is null) throw new Exception(); if (!fileInfo.Directory.Exists) fileInfo.Directory.Create(); foreach ((string Id, string Line) metadataIdLine in metadataIdLines) { if (metadataIds.Contains(metadataIdLine.Id)) duplicates.Add(metadataIdLine.Id); else metadataIds.Add(metadataIdLine.Id); } for (int i = metadataIdLines.Count - 1; i > -1; i--) { if (duplicates.Contains(metadataIdLines[i].Id)) metadataIdLines.RemoveAt(i); } if (metadataIdLines.Any()) { text = string.Join(Environment.NewLine, from l in metadataIdLines orderby l.Id select l.Line); _ = Property.Models.Stateless.IPath.WriteAllText(fileInfo.FullName, text, updateDateWhenMatches: true, compareBeforeWrite: true); } else { if (fileInfo.Exists) File.Delete(fileInfo.FullName); } } private void WriteGroup(Property.Models.Configuration configuration, PropertyLogic propertyLogic, List propertyCollection, List>> metadataCollection, List> faceCollections, List> resizeKeyValuePairs, string sourceDirectory, string outputResolution, Item[] filteredItems) { if (configuration.PropertiesChangedForProperty is null) throw new NullReferenceException(nameof(configuration.PropertiesChangedForProperty)); Item item; string key; string json; string checkFile; int sourceDirectoryLength = sourceDirectory.Length; _FilePropertiesKeyValuePairs.Add(sourceDirectory, new List>()); JsonSerializerOptions writeIndentedJsonSerializerOptions = new() { WriteIndented = false }; if (!(from l in propertyCollection where l?.Width is null select true).Any()) { string checkDirectory; List>> faceCollectionsKeyValuePairs = new(); List> propertyCollectionKeyValuePairs = new(); List>> resizeKeyValuePairsCollections = new(); List>>> metadataCollectionKeyValuePairs = new(); (int level, List directories) = Property.Models.Stateless.IPath.Get(configuration.RootDirectory, sourceDirectory); string fileName = string.Concat(string.Join(configuration.FileNameDirectorySeparator, directories), ".json"); for (int i = 0; i < filteredItems.Length; i++) { item = filteredItems[i]; if (item.Property is null) continue; if (item.ImageFileHolder is null) continue; key = Property.Models.Stateless.IPath.GetRelativePath(item.ImageFileHolder.FullName, sourceDirectoryLength); _FileKeyValuePairs.Add(new KeyValuePair(sourceDirectory, key)); _FilePropertiesKeyValuePairs[sourceDirectory].Add(new Tuple(key, propertyCollection[i])); faceCollectionsKeyValuePairs.Add(new KeyValuePair>(key, faceCollections[i])); propertyCollectionKeyValuePairs.Add(new KeyValuePair(key, propertyCollection[i])); resizeKeyValuePairsCollections.Add(new KeyValuePair>(key, resizeKeyValuePairs[i])); metadataCollectionKeyValuePairs.Add(new KeyValuePair>>(key, metadataCollection[i])); } if (propertyLogic.AngleBracketCollection.Any()) { checkDirectory = Property.Models.Stateless.IPath.GetDirectory(propertyLogic.AngleBracketCollection[0], level, "[{}]"); checkFile = Path.Combine(checkDirectory, fileName); if (File.Exists(checkFile)) File.Move(checkFile, Path.Combine(checkDirectory, fileName)); checkFile = Path.Combine(checkDirectory, fileName); json = JsonSerializer.Serialize(propertyCollectionKeyValuePairs, writeIndentedJsonSerializerOptions); _ = Property.Models.Stateless.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true); } if (_Metadata.AngleBracketCollection.Any()) { checkDirectory = Property.Models.Stateless.IPath.GetDirectory(_Metadata.AngleBracketCollection[0], level, "[{}]"); checkFile = Path.Combine(checkDirectory, fileName); if (File.Exists(checkFile)) File.Move(checkFile, Path.Combine(checkDirectory, fileName)); checkFile = Path.Combine(checkDirectory, fileName); json = JsonSerializer.Serialize(metadataCollectionKeyValuePairs, writeIndentedJsonSerializerOptions); _ = Property.Models.Stateless.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true); } if (_Resize.AngleBracketCollection.Any()) { checkDirectory = Property.Models.Stateless.IPath.GetDirectory(_Resize.AngleBracketCollection[0], level, "[{}]"); checkFile = Path.Combine(checkDirectory, fileName); if (File.Exists(checkFile)) File.Move(checkFile, Path.Combine(checkDirectory, fileName)); checkFile = Path.Combine(checkDirectory, fileName); json = JsonSerializer.Serialize(resizeKeyValuePairsCollections, writeIndentedJsonSerializerOptions); _ = Property.Models.Stateless.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true); } if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution) && _Faces.AngleBracketCollection.Any()) { checkDirectory = Property.Models.Stateless.IPath.GetDirectory(_Faces.AngleBracketCollection[0], level, "[{}]"); checkFile = Path.Combine(checkDirectory, fileName); if (File.Exists(checkFile)) File.Move(checkFile, Path.Combine(checkDirectory, fileName)); checkFile = Path.Combine(checkDirectory, fileName); json = JsonSerializer.Serialize(faceCollectionsKeyValuePairs, writeIndentedJsonSerializerOptions); _ = Property.Models.Stateless.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true); } } } private (string, string, string, string, string, string, string) GetResultsFullGroupDirectories(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string outputResolution) { string aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( configuration, model, predictorModel, nameof(A_Property), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false); string bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( configuration, model, predictorModel, nameof(B_Metadata), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false); string cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( configuration, model, predictorModel, nameof(C_Resize), outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false); string dResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( configuration, model, predictorModel, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true); string d2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( configuration, model, predictorModel, nameof(D2_FaceLandmarks), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true); string eResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true); string zResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( configuration, model, predictorModel, $"Z_{nameof(Item)}", outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true); return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory, zResultsFullGroupDirectory); } private void SetAngleBracketCollections(Property.Models.Configuration configuration, PropertyLogic propertyLogic, string outputResolution, Container container, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory) { _Faces.AngleBracketCollection.Clear(); _Resize.AngleBracketCollection.Clear(); _Metadata.AngleBracketCollection.Clear(); propertyLogic.AngleBracketCollection.Clear(); propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration, container.SourceDirectory, aResultsFullGroupDirectory, contentDescription: string.Empty, singletonDescription: "Properties for each image", collectionDescription: string.Empty, converted: false)); _Metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration, container.SourceDirectory, bResultsFullGroupDirectory, contentDescription: string.Empty, singletonDescription: "Metadata as key value pairs", collectionDescription: string.Empty, converted: true)); _Resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration, container.SourceDirectory, cResultsFullGroupDirectory, contentDescription: "Resized image", singletonDescription: "Resize dimensions for each resolution", collectionDescription: string.Empty, converted: true)); if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution)) _Faces.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration, container.SourceDirectory, dResultsFullGroupDirectory, contentDescription: "n png file(s) for each face found", singletonDescription: string.Empty, collectionDescription: "For each image a json file with all faces found", converted: true)); } private void FullDoWork(bool isSilent, string argZero, Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, long ticks, Dictionary> peopleCollection, PropertyLogic propertyLogic, List containers) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); int[] ids; int distinctCount; int exceptionCount; Item[] filteredItems; string aResultsFullGroupDirectory; string bResultsFullGroupDirectory; string cResultsFullGroupDirectory; string dResultsFullGroupDirectory; string eResultsFullGroupDirectory; string zResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; List> faceCollections = new(); List propertyCollection = new(); List propertyFileHolderCollection = new(); List> resizeKeyValuePairs = new(); List> sourceDirectoryChanges = new(); List>> metadataCollection = new(); string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(A_Property), "{}"); string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(configuration, nameof(A_Property)); foreach (string outputResolution in _Configuration.OutputResolutions) { _FileKeyValuePairs.Clear(); _FilePropertiesKeyValuePairs.Clear(); (aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory, zResultsFullGroupDirectory) = GetResultsFullGroupDirectories(configuration, model, predictorModel, outputResolution); foreach (Container container in containers) { if (!container.Items.Any()) continue; if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero)) continue; filteredItems = (from l in container.Items where l.ImageFileHolder is not null && (l.Abandoned is null || !l.Abandoned.Value) && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l).ToArray(); if (!filteredItems.Any()) continue; faceCollections.Clear(); metadataCollection.Clear(); propertyCollection.Clear(); resizeKeyValuePairs.Clear(); sourceDirectoryChanges.Clear(); propertyFileHolderCollection.Clear(); SetAngleBracketCollections(configuration, propertyLogic, outputResolution, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory); exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, containers.Count, container, filteredItems); #pragma warning disable ids = (from l in filteredItems where l.Property?.Id is not null select l.Property.Id.Value).ToArray(); #pragma warning restore distinctCount = ids.Distinct().Count(); if (ids.Length != distinctCount) _Log.Information($"{ids.Length} != {distinctCount} <{container.SourceDirectory}>"); if (metadataCollection.Count != filteredItems.Length || propertyCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || faceCollections.Count != filteredItems.Length) throw new Exception("Counts don't match!"); if (exceptionCount != 0) _Exceptions.Add(container.SourceDirectory); for (int i = 0; i < faceCollections.Count; i++) filteredItems[i].Faces.AddRange(from l in faceCollections[i] select l); if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0) WriteGroup(configuration, propertyLogic, propertyCollection, metadataCollection, faceCollections, resizeKeyValuePairs, container.SourceDirectory, outputResolution, filteredItems); if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0 && outputResolution == _Configuration.OutputResolutions[0]) propertyLogic.AddToPropertyLogicAllCollection(filteredItems); if (exceptionCount == 0 && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)) _Distance.LoadOrCreateThenSaveDistanceResults(configuration, eResultsFullGroupDirectory, container.SourceDirectory, outputResolution, sourceDirectoryChanges, filteredItems); if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any()) { for (int y = 0; y < int.MaxValue; y++) { _Log.Information("Press \"Y\" key when ready to continue or close console"); if (System.Console.ReadKey().Key == ConsoleKey.Y) break; } _Log.Information(". . ."); } if (isSilent && container.SourceDirectory.EndsWith("Bohdi Ray 2016")) // 7#.1 break; // if (isSilent && container.SourceDirectory.EndsWith("Halloween 2013")) // 12#.1 // break; // if (isSilent && container.SourceDirectory.EndsWith("zzz =2014.4 Winter Tracy Pictures")) // 30#.2 // break; // if (isSilent && container.SourceDirectory.EndsWith("=2009.0 Winter Logan Michael")) //34#.2 // break; // if (isSilent && container.SourceDirectory.EndsWith("Animal Kingdom")) //35#.3 // break; // if (isSilent && container.SourceDirectory.EndsWith("Texas 2015")) //46#.4 // break; } } } private PropertyLogic GetPropertyLogic(bool reverse, Model? model, PredictorModel? predictorModel) { PropertyLogic result; if (_Configuration?.PropertyConfiguration is null) throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration)); result = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, reverse, model, predictorModel); return result; } private void Search(Property.Models.Configuration configuration, bool reverse, Model? model, PredictorModel? predictorModel, string argZero, Person[] people, bool isSilent) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); List containers; long ticks = DateTime.Now.Ticks; string aResultsFullGroupDirectory; string bResultsFullGroupDirectory; string cResultsFullGroupDirectory; string dResultsFullGroupDirectory; string eResultsFullGroupDirectory; string zResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; Dictionary> peopleCollection = A2_People.Convert(people); PropertyLogic propertyLogic = GetPropertyLogic(reverse, model, predictorModel); if (string.IsNullOrEmpty(configuration.RootDirectory)) containers = Property.Models.Stateless.A_Property.Get(configuration, propertyLogic); else containers = Property.Models.Stateless.Container.GetContainers(configuration, propertyLogic); FullDoWork(isSilent, argZero, configuration, model, predictorModel, ticks, peopleCollection, propertyLogic, containers); foreach (string outputResolution in _Configuration.OutputResolutions) { (aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory, zResultsFullGroupDirectory) = GetResultsFullGroupDirectories(configuration, model, predictorModel, outputResolution); _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(aResultsFullGroupDirectory, "{}")); _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(bResultsFullGroupDirectory, "{}")); _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(cResultsFullGroupDirectory, "{}")); if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution)) _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(dResultsFullGroupDirectory, "[]")); if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)) _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(eResultsFullGroupDirectory, "[]")); if (_Configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions.Contains(outputResolution)) _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(eResultsFullGroupDirectory, "[]")); if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution)) _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(d2ResultsFullGroupDirectory, "[]")); if (_ArgZeroIsConfigurationRootDirectory && _Exceptions.Count == 0 && outputResolution == _Configuration.OutputResolutions[0] && (propertyLogic.NamedFaceInfoDeterministicHashCodeKeyValuePairs.Any() || propertyLogic.NamedDeterministicHashCodeKeyValuePairs.Any())) { if (!string.IsNullOrEmpty(propertyLogic.DeterministicHashCodeRootDirectory) && !propertyLogic.IncorrectDeterministicHashCodeKeyValuePairs.Any()) propertyLogic.UpdateKeyValuePairs(containers); foreach (Container container in containers) { Item.AddToNamed(propertyLogic, container.Items); if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution)) D_Face.SaveShortcuts(_Configuration.JuliePhares, dResultsFullGroupDirectory, ticks, peopleCollection, propertyLogic, container.Items); } propertyLogic.SaveAllCollection(); if (_Configuration.SaveResizedSubfiles) { string dFacesContentDirectory; string eDistanceContentDirectory; string eDistanceCollectionDirectory; string zPropertyHolderSingletonDirectory; dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()"); eDistanceContentDirectory = Path.Combine(eResultsFullGroupDirectory, $"({ticks})"); zPropertyHolderSingletonDirectory = Path.Combine(zResultsFullGroupDirectory, "{}"); eDistanceCollectionDirectory = Path.Combine(eResultsFullGroupDirectory, $"[{ticks}]"); List<(DateTime, bool?, PersonBirthday, FaceRecognitionDotNet.FaceEncoding[])> collection; collection = E_Distance.ParallelWork(_AppSettings.MaxDegreeOfParallelism, argZero, propertyLogic, containers); _ = LogDeltaInMinutes(ticks, nameof(E_Distance.ParallelWork)); E_Distance.SavePropertyHolders(argZero, containers, zPropertyHolderSingletonDirectory); _ = LogDeltaInMinutes(ticks, nameof(E_Distance.SavePropertyHolders)); E_Distance.SaveThreeSigmaFaceEncodings(collection, peopleCollection, eDistanceCollectionDirectory); _ = LogDeltaInMinutes(ticks, nameof(E_Distance.SaveThreeSigmaFaceEncodings)); E_Distance.SaveClosest(argZero, containers, peopleCollection, eDistanceContentDirectory, dFacesContentDirectory); _ = LogDeltaInMinutes(ticks, nameof(E_Distance.SaveClosest)); } if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any()) break; if (_FileKeyValuePairs.Any()) _Random.Random(configuration, _Configuration.OutputResolutions[0], _FileKeyValuePairs); if (_IsEnvironment.Development) continue; G2_Identify identify = new(_Configuration); List identifiedCollection = identify.GetIdentifiedCollection(configuration, _IsEnvironment, _People); _People.WriteAllText(configuration, _Configuration.OutputResolutions[0], identifiedCollection); identify.WriteAllText(configuration, _Configuration.OutputResolutions[0], identifiedCollection); if (_Configuration.LoadOrCreateThenSaveIndex && _FilePropertiesKeyValuePairs.Any()) _Index.SetIndex(configuration, model, predictorModel, _Configuration.OutputResolutions[0], _FilePropertiesKeyValuePairs); } } } internal void RenameQueue(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel) => _Rename.RenameQueue(configuration, model, predictorModel); }