using FaceRecognitionDotNet; using Microsoft.Extensions.Configuration; using Phares.Shared; using ShellProgressBar; using System.Drawing.Imaging; using System.Text.Json; using View_by_Distance.Instance.Models; using View_by_Distance.Metadata.Models; using View_by_Distance.Property.Models; using View_by_Distance.Resize.Models; using View_by_Distance.Shared.Models.Methods; namespace View_by_Distance.Instance; public class DlibDotNet { private readonly D_Face _Faces; private readonly G_Index _Index; private readonly C_Resize _Resize; private readonly F_Random _Random; private readonly A2_People _People; private readonly E3_Rename _Rename; private readonly B_Metadata _Metadata; private readonly E_Distance _Distance; private readonly Serilog.ILogger? _Log; private readonly AppSettings _AppSettings; private readonly List _Exceptions; private readonly IsEnvironment _IsEnvironment; private readonly D2_FaceLandmarks _FaceLandmarks; private readonly Models.Configuration _Configuration; private readonly bool _ArgZeroIsConfigurationRootDirectory; private readonly List> _FileKeyValuePairs; private readonly Dictionary>> _FilePropertiesKeyValuePairs; public DlibDotNet(List args, IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, AppSettings appSettings, string workingDirectory, bool isSilent, IConsole console) { string argZero; _AppSettings = appSettings; if (appSettings.MaxDegreeOfParallelism is null) throw new Exception($"{nameof(appSettings.MaxDegreeOfParallelism)} is null!"); _IsEnvironment = isEnvironment; _Exceptions = new List(); _Log = Serilog.Log.ForContext(); _FileKeyValuePairs = new List>(); _FilePropertiesKeyValuePairs = new Dictionary>>(); Property.Models.Configuration propertyConfiguration = Property.Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory); Property.Models.Configuration.Verify(propertyConfiguration); Models.Configuration configuration = Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory, propertyConfiguration); Verify(configuration); (Model model, PredictorModel predictorModel) = GetTuple(args, propertyConfiguration, configuration); if (configuration.SearchForAbandonedFilesFull is null) throw new Exception($"{nameof(configuration.SearchForAbandonedFilesFull)} is null!"); _Configuration = configuration; _Index = new G_Index(configuration); _Random = new F_Random(configuration); _People = new A2_People(configuration); _Rename = new E3_Rename(configuration); _Distance = new E_Distance(configuration); _FaceLandmarks = new D2_FaceLandmarks(configuration); if (configuration.ForceMetadataLastWriteTimeToCreationTime is null) throw new Exception($"{nameof(configuration.ForceMetadataLastWriteTimeToCreationTime)} is null!"); if (configuration.ForceResizeLastWriteTimeToCreationTime is null) throw new Exception($"{nameof(configuration.ForceResizeLastWriteTimeToCreationTime)} is null!"); if (configuration.IgnoreExtensions is null) throw new Exception($"{nameof(configuration.IgnoreExtensions)} is null!"); if (configuration.OutputQuality is null) throw new Exception($"{nameof(configuration.OutputQuality)} is null!"); if (configuration.OverrideForResizeImages is null) throw new Exception($"{nameof(configuration.OverrideForResizeImages)} is null!"); if (configuration.PropertiesChangedForMetadata is null) throw new Exception($"{nameof(configuration.PropertiesChangedForMetadata)} is null!"); if (configuration.PropertiesChangedForResize is null) throw new Exception($"{nameof(configuration.PropertiesChangedForResize)} is null!"); _Metadata = new(configuration.ForceMetadataLastWriteTimeToCreationTime.Value, configuration.PropertiesChangedForMetadata.Value); if (args.Count > 0) argZero = Path.GetFullPath(args[0]); else argZero = Path.GetFullPath(propertyConfiguration.RootDirectory); _ArgZeroIsConfigurationRootDirectory = propertyConfiguration.RootDirectory == argZero; (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters) = C_Resize.GetTuple(configuration.OutputExtension, configuration.OutputQuality.Value); _Resize = new C_Resize(configuration.ForceResizeLastWriteTimeToCreationTime.Value, configuration.OverrideForResizeImages.Value, configuration.PropertiesChangedForResize.Value, configuration.ValidResolutions, imageCodecInfo, encoderParameters); ModelParameter modelParameter = new() { CnnFaceDetectorModel = File.ReadAllBytes(Path.Combine(configuration.ModelDirectory, "mmod_human_face_detector.dat")), FaceRecognitionModel = File.ReadAllBytes(Path.Combine(configuration.ModelDirectory, "dlib_face_recognition_resnet_model_v1.dat")), PosePredictor5FaceLandmarksModel = File.ReadAllBytes(Path.Combine(configuration.ModelDirectory, "shape_predictor_5_face_landmarks.dat")), PosePredictor68FaceLandmarksModel = File.ReadAllBytes(Path.Combine(configuration.ModelDirectory, "shape_predictor_68_face_landmarks.dat")) }; _Faces = new D_Face(configuration, argZero, model, modelParameter, predictorModel); if (configuration.SkipSearch is null) throw new Exception($"{nameof(configuration.SkipSearch)} is null!"); if (_ArgZeroIsConfigurationRootDirectory) _ = _People.GetPeople(propertyConfiguration); if (!isSilent && configuration.TestDistanceResults.HasValue && configuration.TestDistanceResults.Value) { E2_Navigate e2Navigate = new(console, configuration, argZero); e2Navigate.Navigate(propertyConfiguration, configuration.OutputResolutions[0]); } if (_ArgZeroIsConfigurationRootDirectory) { long ticks = DateTime.Now.Ticks; string[] directories = Property.Models.Stateless.A_Property.GetDirectoryRenameCollection(propertyConfiguration, configuration.OutputResolutions[0], nameof(B_Metadata), nameof(C_Resize)); foreach (string directory in directories) { if (!Directory.Exists(directory)) continue; Property.Models.Stateless.A_Property.SearchForAbandonedFilesFull(argZero, directory, onlyJson: false); } if (appSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(Property.Models.Stateless.A_Property.SearchForAbandonedFilesFull)); } if (!configuration.SkipSearch.Value) Search(argZero); if (_Exceptions.Count == 0 && _ArgZeroIsConfigurationRootDirectory) { long ticks = DateTime.Now.Ticks; if (configuration.SearchForAbandonedFilesFull.Value) { string[] directories = _Rename.GetDirectoryRenameCollection(propertyConfiguration, relativePath: string.Empty, newDirectoryName: string.Empty, jsonFiles4InfoAny: false); foreach (string directory in directories) { if (!Directory.Exists(directory)) continue; Property.Models.Stateless.A_Property.SearchForAbandonedFilesFull(argZero, directory, onlyJson: true); } if (appSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(Property.Models.Stateless.A_Property.SearchForAbandonedFilesFull)); } List directoryCollections = _Rename.GetDirectoryRenameCollections(propertyConfiguration, relativePath: string.Empty, newDirectoryName: string.Empty, jsonFiles4InfoAny: false); foreach (string[] directoryCollection in directoryCollections) { _Log.Information(string.Concat("Cleaning <", directoryCollection[0], ">")); _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(directoryCollection[0]); } string d2FaceLandmarksRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(D2_FaceLandmarks)); _Log.Information(string.Concat("Cleaning <", d2FaceLandmarksRootDirectory, ">")); _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(d2FaceLandmarksRootDirectory); if (appSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(Property.Models.Stateless.IPath.DeleteEmptyDirectories)); } string message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}"; _Log.Information(message); if (_Exceptions.Count != 0) throw new Exception(message); if (configuration.LoadOrCreateThenSaveDirectoryDistanceResults is null) throw new Exception($"{nameof(configuration.LoadOrCreateThenSaveDirectoryDistanceResults)} is null!"); if (configuration.LoadOrCreateThenSaveDirectoryDistanceResults.Value) { long ticks = DateTime.Now.Ticks; foreach (string outputResolution in configuration.OutputResolutions) _Distance.LoadOrCreateThenSaveDirectoryDistanceResults(propertyConfiguration, outputResolution); if (appSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(E_Distance.LoadOrCreateThenSaveDirectoryDistanceResults)); } } private long LogDelta(long ticks, string methodName) { long result; if (_Log is null) throw new Exception($"{nameof(_Log)} is null!"); double delta = new TimeSpan(DateTime.Now.Ticks - ticks).TotalMilliseconds; _Log.Debug($"{methodName} took {Math.Floor(delta)} millisecond(s)"); result = DateTime.Now.Ticks; return result; } #pragma warning disable CA1416 #pragma warning restore CA1416 private (Model Model, PredictorModel PredictorModel) GetTuple(List args, Property.Models.Configuration propertyConfiguration, Models.Configuration configuration) { if (_Log is null) throw new Exception($"{nameof(_Log)} is null!"); (Model Model, PredictorModel PredictorModel) result; Array array; Model? model = null; string[] sourceDirectoryNames; PredictorModel? predictorModel = null; if (!args.Any()) sourceDirectoryNames = Array.Empty(); else { string argZero = Path.GetFullPath(args[0]); sourceDirectoryNames = argZero.Split(Path.DirectorySeparatorChar); if (!argZero.StartsWith(propertyConfiguration.RootDirectory)) throw new Exception($"Source directory must be inside root directory! <{argZero}> <{propertyConfiguration.RootDirectory}>"); if (_ArgZeroIsConfigurationRootDirectory && propertyConfiguration.RootDirectory != argZero) { if (!configuration.MixedYearRelativePaths.Contains(sourceDirectoryNames[0])) { string[] segments = sourceDirectoryNames[0].Split(' '); if (segments.Length < 2 || segments[^1].Length != 4 || (segments[^1][..2] != "19" && segments[^1][..2] != "20")) throw new Exception("root subdirectory must have a year at the end or directory name needs to be added to the exclude list!"); } } } _Log.Information(configuration.ModelDirectory); string[] resizeMatch = (from l in sourceDirectoryNames where configuration.ValidResolutions.Contains(l) select l).ToArray(); if (resizeMatch.Any()) throw new Exception("Input directory should be the source and not a resized directory!"); array = Enum.GetValues(typeof(Model)); foreach (Model check in array) { if (configuration.ModelName.Contains(check.ToString())) { model = check; break; } } if (model is null) throw new Exception("Destination directory must have Model name!"); model = model.Value; array = Enum.GetValues(typeof(PredictorModel)); foreach (PredictorModel check in array) { if (configuration.PredictorModelName.Contains(check.ToString())) { predictorModel = check; break; } } if (predictorModel is null) throw new Exception("Destination directory must have Predictor Model name!"); predictorModel = predictorModel.Value; result = new(model.Value, predictorModel.Value); return result; } private void Verify(Models.Configuration configuration) { if (!configuration.OutputResolutions.Any() || string.IsNullOrEmpty(configuration.OutputResolutions[0]) || !configuration.ValidResolutions.Contains(configuration.OutputResolutions[0])) throw new Exception($"{nameof(configuration.OutputResolutions)} must be a valid outputResolution!"); if ((from l in configuration.OutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.OutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.SaveFaceLandmarkForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.SaveFaceLandmarkForOutputResolutions)} are not in the ValidResolutions list!"); if (configuration.CheckJsonForDistanceResults is null) throw new Exception($"{nameof(configuration.CheckJsonForDistanceResults)} must be set!"); if (configuration.CrossDirectoryMaxItemsInDistanceCollection is null) throw new Exception($"{nameof(configuration.CrossDirectoryMaxItemsInDistanceCollection)} must be set!"); if (configuration.DistanceFactor is null) throw new Exception($"{nameof(configuration.DistanceFactor)} must be set!"); if (configuration.ForceMetadataLastWriteTimeToCreationTime is null) throw new Exception($"{nameof(configuration.ForceMetadataLastWriteTimeToCreationTime)} must be set!"); if (configuration.ForceResizeLastWriteTimeToCreationTime is null) throw new Exception($"{nameof(configuration.ForceResizeLastWriteTimeToCreationTime)} must be set!"); if (configuration.IgnoreExtensions is null) throw new Exception($"{nameof(configuration.IgnoreExtensions)} must be set!"); if (configuration.IgnoreRelativePaths is null) throw new Exception($"{nameof(configuration.IgnoreRelativePaths)} must be set!"); if (configuration.LoadOrCreateThenSaveDirectoryDistanceResults is null) throw new Exception($"{nameof(configuration.LoadOrCreateThenSaveDirectoryDistanceResults)} must be set!"); if (configuration.LoadOrCreateThenSaveDistanceResults is null) throw new Exception($"{nameof(configuration.LoadOrCreateThenSaveDistanceResults)} must be set!"); if (configuration.LoadOrCreateThenSaveImageFacesResults is null) throw new Exception($"{nameof(configuration.LoadOrCreateThenSaveImageFacesResults)} must be set!"); if (configuration.LoadOrCreateThenSaveIndex is null) throw new Exception($"{nameof(configuration.LoadOrCreateThenSaveIndex)} must be set!"); if (configuration.LocationConfidenceFactor is null) throw new Exception($"{nameof(configuration.LocationConfidenceFactor)} must be set!"); if (configuration.MappedMaxIndex is null) throw new Exception($"{nameof(configuration.MappedMaxIndex)} must be set!"); if (configuration.MappedMaxIndex.HasValue && configuration.LoadOrCreateThenSaveIndex.Value && !_IsEnvironment.DebuggerWasAttachedDuringConstructor) throw new Exception($"{nameof(configuration.MappedMaxIndex)} only allowed when debugger is attached!"); if (configuration.MaxItemsInDistanceCollection is null) throw new Exception($"{nameof(configuration.MaxItemsInDistanceCollection)} must be set!"); if (configuration.MixedYearRelativePaths is null) throw new Exception($"{nameof(configuration.MixedYearRelativePaths)} must be set!"); if (configuration.NumJitters is null) throw new Exception($"{nameof(configuration.NumJitters)} must be set!"); if (configuration.OutputQuality is null) throw new Exception($"{nameof(configuration.OutputQuality)} must be set!"); if (configuration.OutputResolutions is null) throw new Exception($"{nameof(configuration.OutputResolutions)} must be set!"); if (configuration.OverrideForFaceImages is null) throw new Exception($"{nameof(configuration.OverrideForFaceImages)} must be set!"); if (configuration.OverrideForFaceLandmarkImages is null) throw new Exception($"{nameof(configuration.OverrideForFaceLandmarkImages)} must be set!"); if (configuration.OverrideForResizeImages is null) throw new Exception($"{nameof(configuration.OverrideForResizeImages)} must be set!"); if (configuration.PaddingLoops is null) throw new Exception($"{nameof(configuration.PaddingLoops)} must be set!"); if (configuration.PropertiesChangedForDistance is null) throw new Exception($"{nameof(configuration.PropertiesChangedForDistance)} must be set!"); if (configuration.PropertiesChangedForFaces is null) throw new Exception($"{nameof(configuration.PropertiesChangedForFaces)} must be set!"); if (configuration.PropertiesChangedForIndex is null) throw new Exception($"{nameof(configuration.PropertiesChangedForIndex)} must be set!"); if (configuration.PropertiesChangedForMetadata is null) throw new Exception($"{nameof(configuration.PropertiesChangedForMetadata)} must be set!"); if (configuration.PropertiesChangedForResize is null) throw new Exception($"{nameof(configuration.PropertiesChangedForResize)} must be set!"); if (configuration.Reverse is null) throw new Exception($"{nameof(configuration.Reverse)} must be set!"); if (configuration.SaveFaceLandmarkForOutputResolutions is null) throw new Exception($"{nameof(configuration.SaveFaceLandmarkForOutputResolutions)} must be set!"); if (configuration.SaveFullYearOfRandomFiles is null) throw new Exception($"{nameof(configuration.SaveFullYearOfRandomFiles)} must be set!"); if (configuration.SaveResizedSubfiles is null) throw new Exception($"{nameof(configuration.SaveResizedSubfiles)} must be set!"); if (configuration.SearchForAbandonedFilesFull is null) throw new Exception($"{nameof(configuration.SearchForAbandonedFilesFull)} must be set!"); if (configuration.SkipSearch is null) throw new Exception($"{nameof(configuration.SkipSearch)} must be set!"); if (configuration.TestDistanceResults is null) throw new Exception($"{nameof(configuration.TestDistanceResults)} must be set!"); if (configuration.ValidResolutions is null) throw new Exception($"{nameof(configuration.ValidResolutions)} must be set!"); if (string.IsNullOrEmpty(configuration.ModelDirectory) || !Directory.Exists(configuration.ModelDirectory)) throw new Exception($"{nameof(configuration.ModelDirectory)} must have a value and exits!"); if (string.IsNullOrEmpty(configuration.ModelName)) throw new Exception($"{nameof(configuration.ModelName)} must have a value!"); if (string.IsNullOrEmpty(configuration.OutputExtension)) throw new Exception($"{nameof(configuration.OutputExtension)} must have a value!"); if (string.IsNullOrEmpty(configuration.PredictorModelName)) throw new Exception($"{nameof(configuration.PredictorModelName)} must have a value!"); if (configuration.DistanceFactor.Value + configuration.LocationConfidenceFactor.Value != 10) throw new Exception($"{nameof(configuration.DistanceFactor)} and {nameof(configuration.LocationConfidenceFactor)} must add up to 10!"); } private int FullParallelWork(object @lock, long ticks, PropertyLogic propertyLogic, string outputResolution, List> sourceDirectoryChanges, List propertyFileInfoCollection, List propertyCollection, List>> metadataCollection, List> resizeKeyValuePairs, List> faceCollections, int g, string sourceDirectory, int r, string[] filteredSourceDirectoryFiles, int count) { int result = 0; if (_Log is null) throw new Exception($"{nameof(_Log)} is null!"); if (_AppSettings.MaxDegreeOfParallelism is null) throw new Exception($"{nameof(AppSettings.MaxDegreeOfParallelism)} is null!"); ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism.Value }; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; if (faceCollections.Count != filteredSourceDirectoryFiles.Length || metadataCollection.Count != filteredSourceDirectoryFiles.Length || resizeKeyValuePairs.Count != filteredSourceDirectoryFiles.Length || propertyCollection.Count != filteredSourceDirectoryFiles.Length) { for (int i = 0; i < filteredSourceDirectoryFiles.Length; i++) { faceCollections.Add(new()); metadataCollection.Add(new()); resizeKeyValuePairs.Add(new()); propertyCollection.Add(new()); propertyFileInfoCollection.Add(null); } } int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); using (ProgressBar progressBar = new(filteredSourceDirectoryFiles.Length, $"{g}) {r + 1:000} / {count:000} - {outputResolution} - {sourceDirectory} - {filteredSourceDirectoryFiles.Length} file(s) - {totalSeconds} total second(s)", options)) { _ = Parallel.For(0, filteredSourceDirectoryFiles.Length, parallelOptions, i => { try { FullParallelForWork(propertyLogic, @lock, outputResolution, sourceDirectoryChanges, propertyFileInfoCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, sourceDirectory, index: i, filteredSourceDirectoryFiles[i]); if (sourceDirectoryChanges.Any()) progressBar.Tick(); } catch (Exception ex) { result += 1; _Log.Error(string.Concat(sourceDirectory, Environment.NewLine, ex.Message, Environment.NewLine, ex.StackTrace), ex); if (result == filteredSourceDirectoryFiles.Length) throw new Exception(string.Concat("All in [", sourceDirectory, "]failed!")); } }); } return result; } private void FullParallelForWork(PropertyLogic propertyLogic, object @lock, string outputResolution, List> sourceDirectoryChanges, List propertyFileInfoCollection, List propertyCollection, List>> metadataCollections, List> resizeKeyValuePairs, List> imageFaceCollections, string sourceDirectory, int index, string filteredSourceDirectoryFile) { if (_AppSettings.MaxDegreeOfParallelism is null) throw new Exception($"{nameof(_AppSettings.MaxDegreeOfParallelism)} is null!"); if (_Configuration.SaveResizedSubfiles is null) throw new Exception($"{nameof(_Configuration.SaveResizedSubfiles)} is null!"); if (_Configuration?.PropertyConfiguration is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration)} must be set!"); if (_Configuration.PropertyConfiguration.WriteBitmapDataBytes is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration.WriteBitmapDataBytes)} is null!"); List faceCollection; string original = "Original"; long ticks = DateTime.Now.Ticks; DateTime dateTime = DateTime.Now; List parseExceptions = new(); Dictionary imageResizeKeyValuePairs; List> subFileTuples = new(); List> metadataCollection; string fileNameWithoutExtension = Path.GetFileNameWithoutExtension(filteredSourceDirectoryFile); string relativePath = Property.Models.Stateless.IPath.GetRelativePath(filteredSourceDirectoryFile, _Configuration.PropertyConfiguration.RootDirectory.Length); FileInfo propertyFileInfo = new(Path.Combine(propertyLogic.AngleBracketCollection[0].Replace("<>", "{}"), string.Concat(fileNameWithoutExtension, ".json"))); A_Property property = propertyLogic.GetProperty(propertyLogic.AngleBracketCollection[0], sourceDirectory, filteredSourceDirectoryFile, subFileTuples, parseExceptions, propertyFileInfo); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(PropertyLogic.GetProperty)); (int metadataGroups, metadataCollection) = _Metadata.GetMetadataCollection(subFileTuples, parseExceptions, filteredSourceDirectoryFile, relativePath, fileNameWithoutExtension); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection)); FileInfo resizedFileInfo = new(Path.Combine(_Resize.AngleBracketCollection[0].Replace("<>", "()"), Path.GetFileName(filteredSourceDirectoryFile))); imageResizeKeyValuePairs = _Resize.GetResizeKeyValuePairs(subFileTuples, parseExceptions, original, metadataCollection, property, filteredSourceDirectoryFile, relativePath, fileNameWithoutExtension); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(C_Resize.GetResizeKeyValuePairs)); if (_Configuration.SaveResizedSubfiles.Value) { _Resize.SaveResizedSubfile(outputResolution, subFileTuples, filteredSourceDirectoryFile, original, property, imageResizeKeyValuePairs, resizedFileInfo); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(C_Resize.SaveResizedSubfile)); resizedFileInfo.Refresh(); } else if (outputResolution == _Configuration.OutputResolutions[0] && false) { byte[] bytes = _Resize.GetResizedBytes(outputResolution, subFileTuples, filteredSourceDirectoryFile, property, imageResizeKeyValuePairs); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(C_Resize.GetResizedBytes)); string path = Path.Combine(resizedFileInfo.DirectoryName, Path.GetFileNameWithoutExtension(resizedFileInfo.Name)); File.WriteAllBytes(path, bytes); } if (_Configuration.LoadOrCreateThenSaveImageFacesResults is null || !_Configuration.LoadOrCreateThenSaveImageFacesResults.Value) faceCollection = new(); else { int[] outputResolutionCollection = imageResizeKeyValuePairs[outputResolution]; int outputResolutionWidth = outputResolutionCollection[0]; int outputResolutionHeight = outputResolutionCollection[1]; int outputResolutionOrientation = outputResolutionCollection[2]; faceCollection = _Faces.GetFaces(_Configuration.PropertyConfiguration, outputResolution, subFileTuples, parseExceptions, relativePath, fileNameWithoutExtension, property, resizedFileInfo, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(D_Face.GetFaces)); _Faces.SaveFaces(_Configuration.PropertyConfiguration, subFileTuples, parseExceptions, relativePath, fileNameWithoutExtension, resizedFileInfo, faceCollection); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(D_Face.SaveFaces)); if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution)) { _FaceLandmarks.SaveFaceLandmarkImages(subFileTuples, parseExceptions, relativePath, fileNameWithoutExtension, resizedFileInfo, faceCollection); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(D2_FaceLandmarks.SaveFaceLandmarkImages)); } } lock (@lock) { propertyCollection[index] = property; imageFaceCollections[index] = faceCollection; metadataCollections[index] = metadataCollection; propertyFileInfoCollection[index] = propertyFileInfo; resizeKeyValuePairs[index] = imageResizeKeyValuePairs; sourceDirectoryChanges.AddRange(from l in subFileTuples where l.Item2 > dateTime select l); } } private static void WriteTab(string checkDirectory, List<(string Id, string Line)> metadataIdLines, string fileName) { string text; FileInfo fileInfo; List duplicates = new(); List metadataIds = new(); fileInfo = new(Path.Combine(checkDirectory, "[()]", Path.ChangeExtension(fileName, "tsv"))); if (fileInfo?.Directory is null) throw new Exception(); if (!fileInfo.Directory.Exists) fileInfo.Directory.Create(); foreach ((string Id, string Line) metadataIdLine in metadataIdLines) { if (metadataIds.Contains(metadataIdLine.Id)) duplicates.Add(metadataIdLine.Id); else metadataIds.Add(metadataIdLine.Id); } for (int i = metadataIdLines.Count - 1; i > -1; i--) { if (duplicates.Contains(metadataIdLines[i].Id)) metadataIdLines.RemoveAt(i); } if (metadataIdLines.Any()) { text = string.Join(Environment.NewLine, from l in metadataIdLines orderby l.Id select l.Line); _ = Property.Models.Stateless.IPath.WriteAllText(fileInfo.FullName, text, compareBeforeWrite: true); } else { if (fileInfo.Exists) File.Delete(fileInfo.FullName); } } private void WriteGroup(PropertyLogic propertyLogic, List propertyCollection, List>> metadataCollection, List> faceCollections, List> resizeKeyValuePairs, string sourceDirectory, string[] filteredSourceDirectoryFiles) { if (_Configuration.PropertiesChangedForMetadata is null) throw new Exception($"{nameof(_Configuration.PropertiesChangedForMetadata)} is null!"); if (_Configuration?.PropertyConfiguration is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration)} must be set!"); if (_Configuration.PropertyConfiguration.PropertiesChangedForProperty is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration.PropertiesChangedForProperty)} is null!"); if (_Configuration.PropertiesChangedForResize is null) throw new Exception($"{nameof(_Configuration.PropertiesChangedForResize)} is null!"); if (_Configuration.PropertiesChangedForFaces is null) throw new Exception($"{nameof(_Configuration.PropertiesChangedForFaces)} is null!"); string key; string json; string checkFile; int sourceDirectoryLength = sourceDirectory.Length; int rootDirectoryLength = _Configuration.PropertyConfiguration.RootDirectory.Length; JsonSerializerOptions writeIndentedJsonSerializerOptions = new() { WriteIndented = false }; if (!(from l in propertyCollection where l?.Width is null select true).Any()) { string checkDirectory; List>> faceCollectionsKeyValuePairs = new(); List> propertyCollectionKeyValuePairs = new(); List>> resizeKeyValuePairsCollections = new(); List>>> metadataCollectionKeyValuePairs = new(); (int level, List directories) = Property.Models.Stateless.IPath.Get(_Configuration.PropertyConfiguration.RootDirectory, sourceDirectory); string fileName = string.Concat(string.Join(_Configuration.PropertyConfiguration.FileNameDirectorySeparator, directories), ".json"); for (int i = 0; i < filteredSourceDirectoryFiles.Length; i++) { key = Property.Models.Stateless.IPath.GetRelativePath(filteredSourceDirectoryFiles[i], sourceDirectoryLength); faceCollectionsKeyValuePairs.Add(new KeyValuePair>(key, faceCollections[i])); propertyCollectionKeyValuePairs.Add(new KeyValuePair(key, propertyCollection[i])); resizeKeyValuePairsCollections.Add(new KeyValuePair>(key, resizeKeyValuePairs[i])); metadataCollectionKeyValuePairs.Add(new KeyValuePair>>(key, metadataCollection[i])); } if (_Metadata.AngleBracketCollection.Any()) { checkDirectory = Property.Models.Stateless.IPath.GetDirectory(_Metadata.AngleBracketCollection[0], level, "[{}]"); checkFile = Path.Combine(checkDirectory, fileName); if (File.Exists(checkFile)) File.Move(checkFile, Path.Combine(checkDirectory, fileName)); checkFile = Path.Combine(checkDirectory, fileName); json = JsonSerializer.Serialize(metadataCollectionKeyValuePairs, writeIndentedJsonSerializerOptions); _ = Property.Models.Stateless.IPath.WriteAllText(checkFile, json, compareBeforeWrite: true); } if (propertyLogic.AngleBracketCollection.Any()) { checkDirectory = Property.Models.Stateless.IPath.GetDirectory(propertyLogic.AngleBracketCollection[0], level, "[{}]"); checkFile = Path.Combine(checkDirectory, fileName); if (File.Exists(checkFile)) File.Move(checkFile, Path.Combine(checkDirectory, fileName)); checkFile = Path.Combine(checkDirectory, fileName); json = JsonSerializer.Serialize(propertyCollectionKeyValuePairs, writeIndentedJsonSerializerOptions); _ = Property.Models.Stateless.IPath.WriteAllText(checkFile, json, compareBeforeWrite: true); } if (_Resize.AngleBracketCollection.Any()) { checkDirectory = Property.Models.Stateless.IPath.GetDirectory(_Resize.AngleBracketCollection[0], level, "[{}]"); checkFile = Path.Combine(checkDirectory, fileName); if (File.Exists(checkFile)) File.Move(checkFile, Path.Combine(checkDirectory, fileName)); checkFile = Path.Combine(checkDirectory, fileName); json = JsonSerializer.Serialize(resizeKeyValuePairsCollections, writeIndentedJsonSerializerOptions); _ = Property.Models.Stateless.IPath.WriteAllText(checkFile, json, compareBeforeWrite: true); } if (_Configuration.LoadOrCreateThenSaveImageFacesResults.HasValue && _Configuration.LoadOrCreateThenSaveImageFacesResults.Value && _Faces.AngleBracketCollection.Any()) { checkDirectory = Property.Models.Stateless.IPath.GetDirectory(_Faces.AngleBracketCollection[0], level, "[{}]"); checkFile = Path.Combine(checkDirectory, fileName); if (File.Exists(checkFile)) File.Move(checkFile, Path.Combine(checkDirectory, fileName)); checkFile = Path.Combine(checkDirectory, fileName); json = JsonSerializer.Serialize(faceCollectionsKeyValuePairs, writeIndentedJsonSerializerOptions); _ = Property.Models.Stateless.IPath.WriteAllText(checkFile, json, compareBeforeWrite: true); } } } private void FullDoWork(List topDirectories, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> groupCollection) { if (_Log is null) throw new Exception($"{nameof(_Log)} is null!"); if (_AppSettings.MaxDegreeOfParallelism is null) throw new Exception($"{nameof(_AppSettings.MaxDegreeOfParallelism)} is null!"); if (_Configuration?.PropertyConfiguration is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration)} must be set!"); if (_Configuration.PropertyConfiguration.ForcePropertyLastWriteTimeToCreationTime is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration.ForcePropertyLastWriteTimeToCreationTime)} is null!"); if (_Configuration.LoadOrCreateThenSaveImageFacesResults is null) throw new Exception($"{nameof(_Configuration.LoadOrCreateThenSaveImageFacesResults)} is null!"); if (_Configuration.LoadOrCreateThenSaveDirectoryDistanceResults is null) throw new Exception($"{nameof(_Configuration.LoadOrCreateThenSaveDirectoryDistanceResults)} is null!"); if (_Configuration.LoadOrCreateThenSaveDistanceResults is null) throw new Exception($"{nameof(_Configuration.LoadOrCreateThenSaveDistanceResults)} is null!"); if (_Configuration.PropertyConfiguration.MaxImagesInDirectoryForTopLevelFirstPass is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration.MaxImagesInDirectoryForTopLevelFirstPass)} is null!"); if (_Configuration.PropertyConfiguration.PopulatePropertyId is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration.PopulatePropertyId)} is null!"); if (_Configuration.PropertyConfiguration.PropertiesChangedForProperty is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration.PropertiesChangedForProperty)} is null!"); if (_Configuration.PropertyConfiguration.WriteBitmapDataBytes is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration.WriteBitmapDataBytes)} is null!"); int exceptionCount; object @lock = new(); long ticks = DateTime.Now.Ticks; string[] filteredSourceDirectoryFiles; List> faceCollections = new(); List propertyCollection = new(); PropertyLogic propertyLogic = GetPropertyLogic(); List propertyFileInfoCollection = new(); List> resizeKeyValuePairs = new(); List> sourceDirectoryChanges = new(); string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property)); List>> metadataCollection = new(); foreach (string outputResolution in _Configuration.OutputResolutions) { _FileKeyValuePairs.Clear(); _FilePropertiesKeyValuePairs.Clear(); foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles, int r) in groupCollection) { if (!topDirectories.Any()) continue; _Faces.AngleBracketCollection.Clear(); _Resize.AngleBracketCollection.Clear(); _Metadata.AngleBracketCollection.Clear(); propertyLogic.AngleBracketCollection.Clear(); _FaceLandmarks.AngleBracketCollection.Clear(); filteredSourceDirectoryFiles = (from l in sourceDirectoryFiles where !_Configuration.IgnoreExtensions.Contains(Path.GetExtension(l)) select l).ToArray(); if (!filteredSourceDirectoryFiles.Any()) continue; propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(_Configuration.PropertyConfiguration, sourceDirectory, nameof(A_Property), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false, contentDescription: string.Empty, singletonDescription: "Properties for each image", collectionDescription: string.Empty)); _Metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(_Configuration.PropertyConfiguration, sourceDirectory, nameof(B_Metadata), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false, contentDescription: string.Empty, singletonDescription: "Metadata as key value pairs", collectionDescription: string.Empty)); _Resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(_Configuration.PropertyConfiguration, sourceDirectory, nameof(C_Resize), outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false, contentDescription: "Resized image", singletonDescription: "Resize deminsions for each resolution", collectionDescription: string.Empty)); if (_Configuration.LoadOrCreateThenSaveImageFacesResults.HasValue && _Configuration.LoadOrCreateThenSaveImageFacesResults.Value) _Faces.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(_Configuration.PropertyConfiguration, sourceDirectory, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true, contentDescription: "n png file(s) for each face found", singletonDescription: string.Empty, collectionDescription: "For each image a json file with all faces found")); if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution)) _FaceLandmarks.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(_Configuration.PropertyConfiguration, sourceDirectory, nameof(D2_FaceLandmarks), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true, contentDescription: "n x 2 png file(s) for each face found", singletonDescription: string.Empty, collectionDescription: string.Empty)); exceptionCount = FullParallelWork(@lock, ticks, propertyLogic, outputResolution, sourceDirectoryChanges, propertyFileInfoCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, g, sourceDirectory, r, filteredSourceDirectoryFiles, groupCollection.Count); if (metadataCollection.Count != filteredSourceDirectoryFiles.Length || propertyCollection.Count != filteredSourceDirectoryFiles.Length || resizeKeyValuePairs.Count != filteredSourceDirectoryFiles.Length || faceCollections.Count != filteredSourceDirectoryFiles.Length) throw new Exception("Counts don't match!"); if (exceptionCount != 0) _Exceptions.Add(sourceDirectory); else { string key; int rootDirectoryLength = _Configuration.PropertyConfiguration.RootDirectory.Length; _FilePropertiesKeyValuePairs.Add(sourceDirectory, new List>()); for (int i = 0; i < filteredSourceDirectoryFiles.Length; i++) { key = Property.Models.Stateless.IPath.GetRelativePath(filteredSourceDirectoryFiles[i], rootDirectoryLength); _FileKeyValuePairs.Add(new KeyValuePair(sourceDirectory, key)); _FilePropertiesKeyValuePairs[sourceDirectory].Add(new Tuple(key, propertyCollection[i])); } } if (exceptionCount == 0 && _ArgZeroIsConfigurationRootDirectory) WriteGroup(propertyLogic, propertyCollection, metadataCollection, faceCollections, resizeKeyValuePairs, sourceDirectory, filteredSourceDirectoryFiles); if (exceptionCount == 0 && _Configuration.LoadOrCreateThenSaveDistanceResults.HasValue && _Configuration.LoadOrCreateThenSaveDistanceResults.Value) _Distance.LoadOrCreateThenSaveDistanceResults(_Configuration.PropertyConfiguration, sourceDirectory, outputResolution, sourceDirectoryChanges, filteredSourceDirectoryFiles, faceCollections); if (_Resize.AngleBracketCollection.Any()) _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(_Resize.AngleBracketCollection[0].Replace("<>", "()")); if (_Faces.AngleBracketCollection.Any()) _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(_Faces.AngleBracketCollection[0].Replace("<>", "()")); if (_FaceLandmarks.AngleBracketCollection.Any()) _ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(_FaceLandmarks.AngleBracketCollection[0].Replace("<>", "()")); if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any()) { for (int y = 0; y < int.MaxValue; y++) { _Log.Information("Press \"Y\" key when ready to continue or close console"); if (Console.ReadKey().Key == ConsoleKey.Y) break; } _Log.Information(". . ."); } } if (_ArgZeroIsConfigurationRootDirectory && outputResolution == _Configuration.OutputResolutions[0]) { int loadLessThan = 7; string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), "{}"); PropertyCompare.Models.PropertyCompareLogic propertyCompareLogic = new(_AppSettings.MaxDegreeOfParallelism.Value, _Configuration.PropertyConfiguration); PropertyCompare.Models.PropertyCompare[] propertyCompares = propertyCompareLogic.Get(aPropertySingletonDirectory, loadLessThan); { string[] lines = (from l in propertyCompares select l.GetSelect()).ToArray(); string aPropertyCollectionDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), "[{}]"); File.WriteAllLines(Path.Join(aPropertyCollectionDirectory, $". . . Ids - {ticks}.txt"), lines); string json = JsonSerializer.Serialize(propertyCompares, new JsonSerializerOptions { WriteIndented = true }); File.WriteAllText(Path.Join(aPropertyCollectionDirectory, $". . . Ids - {ticks}.nosj"), json); } if (!_Configuration.LoadOrCreateThenSaveImageFacesResults.Value && !_Configuration.LoadOrCreateThenSaveDirectoryDistanceResults.Value && !_Configuration.LoadOrCreateThenSaveDistanceResults.Value) break; if (_Exceptions.Count == 0) { if (_FileKeyValuePairs.Any()) _Random.Random(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0], _FileKeyValuePairs); if (_IsEnvironment.Development) continue; G2_Identify identify = new(_Configuration); List identifiedCollection = identify.GetIdentifiedCollection(_Configuration.PropertyConfiguration, _IsEnvironment, _People); _People.WriteAllText(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0], identifiedCollection); identify.WriteAllText(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0], identifiedCollection); if (_Configuration.LoadOrCreateThenSaveIndex.HasValue && _Configuration.LoadOrCreateThenSaveIndex.Value && _FilePropertiesKeyValuePairs.Any()) _Index.SetIndex(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0], _FilePropertiesKeyValuePairs); } } } } private PropertyLogic GetPropertyLogic() { PropertyLogic result; string[] verifyToSeason = Array.Empty(); if (_AppSettings.MaxDegreeOfParallelism is null) throw new Exception($"{nameof(_AppSettings.MaxDegreeOfParallelism)} is null!"); if (_Configuration?.PropertyConfiguration is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration)} must be set!"); result = new(_AppSettings.MaxDegreeOfParallelism.Value, _Configuration.PropertyConfiguration, verifyToSeason); return result; } private void Search(string argZero) { if (_Log is null) throw new Exception($"{nameof(_Log)} is null!"); if (_AppSettings.MaxDegreeOfParallelism is null) throw new Exception($"{nameof(_AppSettings.MaxDegreeOfParallelism)} is null!"); if (_Configuration?.PropertyConfiguration is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration)} must be set!"); if (_Configuration.PropertyConfiguration.ForcePropertyLastWriteTimeToCreationTime is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration.ForcePropertyLastWriteTimeToCreationTime)} is null!"); if (_Configuration.PropertyConfiguration.IgnoreExtensions is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration.IgnoreExtensions)} is null!"); if (_Configuration.PropertyConfiguration.MaxImagesInDirectoryForTopLevelFirstPass is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration.MaxImagesInDirectoryForTopLevelFirstPass)} is null!"); if (_Configuration.PropertyConfiguration.PopulatePropertyId is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration.PopulatePropertyId)} is null!"); if (_Configuration.PropertyConfiguration.PropertiesChangedForProperty is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration.PropertiesChangedForProperty)} is null!"); if (_Configuration.Reverse is null) throw new Exception($"{nameof(_Configuration.Reverse)} is null!"); if (_Configuration.PropertyConfiguration.WriteBitmapDataBytes is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration.WriteBitmapDataBytes)} is null!"); string searchPattern = "*"; long ticks = DateTime.Now.Ticks; List topDirectories = new(); PropertyLogic propertyLogic = GetPropertyLogic(); List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> groupCollection = Property.Models.Stateless.A_Property.GetGroupCollection(argZero, searchPattern, topDirectories, _Configuration.PropertyConfiguration.MaxImagesInDirectoryForTopLevelFirstPass.Value, _Configuration.Reverse.Value); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(Property.Models.Stateless.A_Property.GetGroupCollection)); _Exceptions.AddRange(propertyLogic.DoWork(_Configuration.PropertyConfiguration, topDirectories, groupCollection, firstPass: true)); string message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}"; _Log.Information(message); if (_Exceptions.Count != 0) throw new Exception(message); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(Property.Models.Stateless.A_Property.GetGroupCollection)); groupCollection = Property.Models.Stateless.A_Property.GetGroupCollection(argZero, searchPattern, topDirectories, _Configuration.PropertyConfiguration.MaxImagesInDirectoryForTopLevelFirstPass.Value, _Configuration.Reverse.Value); if (_AppSettings.MaxDegreeOfParallelism.Value < 2) ticks = LogDelta(ticks, nameof(Property.Models.Stateless.A_Property.GetGroupCollection)); FullDoWork(topDirectories, groupCollection); } internal void RenameQueue() { if (_Configuration?.PropertyConfiguration is null) throw new Exception($"{nameof(_Configuration.PropertyConfiguration)} must be set!"); _Rename.RenameQueue(_Configuration.PropertyConfiguration); } }