using Microsoft.Extensions.Configuration; using Phares.Shared; using ShellProgressBar; using System.Diagnostics; using System.Drawing; using System.Drawing.Imaging; using System.Text.Json; using View_by_Distance.Distance.Models; using View_by_Distance.Face.Models; using View_by_Distance.FaceParts.Models; using View_by_Distance.Instance.Models; using View_by_Distance.Map.Models; using View_by_Distance.Metadata.Models; using View_by_Distance.PhotoPrism.Models; using View_by_Distance.Property.Models; using View_by_Distance.Resize.Models; using View_by_Distance.Shared.Models; using View_by_Distance.Shared.Models.Methods; namespace View_by_Distance.Instance; public partial class DlibDotNet { private readonly D_Face _Faces; private readonly G_Index _Index; private readonly C_Resize _Resize; private readonly F_Random _Random; private readonly E3_Rename _Rename; private readonly IConsole _Console; private readonly E_Distance _Distance; private readonly B_Metadata _Metadata; private readonly Serilog.ILogger? _Log; private readonly D2_FaceParts _FaceParts; private readonly AppSettings _AppSettings; private readonly List _Exceptions; private readonly IsEnvironment _IsEnvironment; private readonly bool _PropertyRootExistedBefore; private readonly PersonContainer[] _PersonContainers; private readonly Models.Configuration _Configuration; private readonly bool _ArgZeroIsConfigurationRootDirectory; private readonly Map.Models.Configuration _MapConfiguration; public DlibDotNet( List args, IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, AppSettings appSettings, string workingDirectory, bool isSilent, IConsole console) { string message; _Console = console; _AppSettings = appSettings; _IsEnvironment = isEnvironment; long ticks = DateTime.Now.Ticks; _Exceptions = new List(); _Log = Serilog.Log.ForContext(); Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot); Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration); _Log.Information(propertyConfiguration.RootDirectory); Property.Models.Configuration.Verify(propertyConfiguration, requireExist: false); Verify(configuration); VerifyExtra(args, propertyConfiguration, configuration); _Configuration = configuration; _Index = new(configuration); _Random = new(configuration); _Rename = new(configuration); if (configuration.IgnoreExtensions is null) throw new NullReferenceException(nameof(configuration.IgnoreExtensions)); string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(propertyConfiguration, nameof(A_Property), create: false); _PropertyRootExistedBefore = !Directory.Exists(propertyRoot); string argZero = args.Count > 0 ? Path.GetFullPath(args[0]) : Path.GetFullPath(propertyConfiguration.RootDirectory); _ArgZeroIsConfigurationRootDirectory = propertyConfiguration.RootDirectory == argZero; if (!Directory.Exists(argZero)) _ = Directory.CreateDirectory(argZero); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(argZero); if (!Directory.Exists(argZero)) _ = Directory.CreateDirectory(argZero); _Log.Information(configuration.ModelDirectory); { (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetPngLowQuality(); (ImageCodecInfo hiddenImageCodecInfo, EncoderParameters hiddenEncoderParameters, string hiddenFileNameExtension) = C_Resize.GetGifLowQuality(); _Faces = new D_Face( argZero, configuration.CheckDFaceAndUpWriteDates, configuration.PropertyConfiguration, encoderParameters, configuration.FaceDistanceHiddenImageFactor, filenameExtension, configuration.ForceFaceLastWriteTimeToCreationTime, hiddenEncoderParameters, hiddenFileNameExtension, hiddenImageCodecInfo, imageCodecInfo, configuration.ModelDirectory, configuration.ModelName, configuration.OverrideForFaceImages, configuration.PredictorModelName, configuration.PropertiesChangedForFaces); } { (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetGifLowQuality(); _FaceParts = new D2_FaceParts(imageCodecInfo, encoderParameters, filenameExtension, configuration.CheckDFaceAndUpWriteDates, configuration.OverrideForFaceLandmarkImages); } _Metadata = new(configuration.ForceMetadataLastWriteTimeToCreationTime, configuration.PropertiesChangedForMetadata); _MapConfiguration = Get(configuration, _Faces.FileNameExtension, _Faces.HiddenFileNameExtension, _FaceParts.FileNameExtension); _Distance = new(configuration.DistanceMoveUnableToMatch, configuration.DistanceRenameToMatch, _Configuration.FaceConfidencePercent, configuration.RangeDistanceTolerance, configuration.RangeFaceConfidence); if (_PropertyRootExistedBefore || !_ArgZeroIsConfigurationRootDirectory) _PersonContainers = Array.Empty(); else { int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); message = $") Building People Collection - {totalSeconds} total second(s)"; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; using ProgressBar progressBar = new(1, message, options); progressBar.Tick(); string rootDirectory = propertyConfiguration.RootDirectory; string peopleRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(A2_People)); string? rootResultsDirectory = Path.GetDirectoryName(Path.GetDirectoryName(peopleRootDirectory)); if (rootResultsDirectory is null) throw new Exception(); Storage storage = new(rootDirectory, rootResultsDirectory, peopleRootDirectory); _PersonContainers = Shared.Models.Stateless.Methods.IPersonContainer.GetPersonContainers(storage, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), _Faces.FileNameExtension); } if (!isSilent && configuration.TestDistanceResults) { E2_Navigate e2Navigate = new(console, configuration, argZero); e2Navigate.Navigate(propertyConfiguration, configuration.OutputResolutions[0]); _Log.Information(propertyConfiguration.RootDirectory); } { (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetTuple( configuration.OutputExtension, configuration.OutputQuality); _Resize = new C_Resize( configuration.ForceResizeLastWriteTimeToCreationTime, configuration.OverrideForResizeImages, configuration.PropertiesChangedForResize, configuration.ValidResolutions, imageCodecInfo, encoderParameters, filenameExtension); } if (!configuration.SkipSearch) Search(ticks, argZero, propertyRoot); if (!_PropertyRootExistedBefore && !_IsEnvironment.Development && _Exceptions.Count == 0 && _ArgZeroIsConfigurationRootDirectory) { List directoryCollections = _Rename.GetDirectoryRenameCollections( propertyConfiguration, relativePath: string.Empty, newDirectoryName: string.Empty, jsonFiles4InfoAny: false); foreach (string[] directoryCollection in directoryCollections) { _Log.Information(string.Concat("Cleaning <", directoryCollection[0], ">")); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(directoryCollection[0]); } string d2FacePartsRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(D2_FaceParts)); _Log.Information(string.Concat("Cleaning <", d2FacePartsRootDirectory, ">")); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(d2FacePartsRootDirectory); if (appSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories)); } message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}"; _Log.Information(message); if (_Exceptions.Count != 0) throw new Exception(message); if (_PropertyRootExistedBefore) _Log.Information("First run completed. Run again if wanted"); } private long LogDelta(long ticks, string? methodName) { long result; if (_Log is null) throw new NullReferenceException(nameof(_Log)); double delta = new TimeSpan(DateTime.Now.Ticks - ticks).TotalMilliseconds; _Log.Debug($"{methodName} took {Math.Floor(delta)} millisecond(s)"); result = DateTime.Now.Ticks; return result; } private long LogDeltaInSeconds(long ticks, string methodName) { long result; if (_Log is null) throw new NullReferenceException(nameof(_Log)); double delta = new TimeSpan(DateTime.Now.Ticks - ticks).Seconds; _Log.Debug($"{methodName} took {Math.Floor(delta)} seconds(s)"); result = DateTime.Now.Ticks; return result; } private long LogDeltaInMinutes(long ticks, string methodName) { long result; if (_Log is null) throw new NullReferenceException(nameof(_Log)); double delta = new TimeSpan(DateTime.Now.Ticks - ticks).Minutes; _Log.Debug($"{methodName} took {Math.Floor(delta)} minutes(s)"); result = DateTime.Now.Ticks; return result; } private void Verify(Models.Configuration configuration) { if (!configuration.OutputResolutions.Any() || string.IsNullOrEmpty(configuration.OutputResolutions[0]) || !configuration.ValidResolutions.Contains(configuration.OutputResolutions[0])) throw new NullReferenceException($"{nameof(configuration.OutputResolutions)} must be fileNameToCollection valid outputResolution!"); if ((from l in configuration.OutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.OutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.SaveShortcutsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.SaveShortcutsForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.SaveFaceLandmarkForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.SaveFaceLandmarkForOutputResolutions)} are not in the ValidResolutions list!"); if (string.IsNullOrEmpty(configuration.ModelName)) throw new NullReferenceException(nameof(configuration.ModelName)); if (string.IsNullOrEmpty(configuration.OutputExtension)) throw new NullReferenceException(nameof(configuration.OutputExtension)); if (string.IsNullOrEmpty(configuration.PredictorModelName)) throw new NullReferenceException(nameof(configuration.PredictorModelName)); if (string.IsNullOrEmpty(configuration.ModelDirectory) || !Directory.Exists(configuration.ModelDirectory)) throw new NullReferenceException(nameof(configuration.ModelDirectory)); if (configuration.MappedMaxIndex.HasValue && configuration.LoadOrCreateThenSaveIndex && !_IsEnvironment.DebuggerWasAttachedDuringConstructor) throw new NullReferenceException(nameof(configuration.MappedMaxIndex)); } private void VerifyExtra(List args, Property.Models.Configuration propertyConfiguration, Models.Configuration configuration) { string[] sourceDirectoryNames; if (!args.Any()) sourceDirectoryNames = Array.Empty(); else { string? century; string argZero = Path.GetFullPath(args[0]); sourceDirectoryNames = argZero.Split(Path.DirectorySeparatorChar); if (!argZero.StartsWith(propertyConfiguration.RootDirectory)) throw new Exception($"Source directory must be inside root directory! <{argZero}> <{propertyConfiguration.RootDirectory}>"); if (_ArgZeroIsConfigurationRootDirectory && propertyConfiguration.RootDirectory != argZero) { if (!configuration.MixedYearRelativePaths.Contains(sourceDirectoryNames[0])) { string[] segments = sourceDirectoryNames[0].Split(' '); century = segments[^1].Length == 4 ? segments[^1][..2] : null; if (segments.Length < 2 || century is null || (century != "18" && century != "19" && century != "20")) throw new Exception("root subdirectory must have fileNameToCollection year at the end or directory name needs to be added to the exclude list!"); } } } string[] resizeMatch = (from l in sourceDirectoryNames where configuration.ValidResolutions.Contains(l) select l).ToArray(); if (resizeMatch.Any()) throw new Exception("Input directory should be the source and not fileNameToCollection resized directory!"); if (configuration.LocationDigits != Shared.Models.Stateless.ILocation.Digits) throw new Exception("Configuration has to match interface!"); if (configuration.LocationFactor != Shared.Models.Stateless.ILocation.Factor) throw new Exception("Configuration has to match interface!"); } private static Map.Models.Configuration Get(Models.Configuration configuration, string facesFileNameExtension, string facesHiddenFileNameExtension, string facePartsFileNameExtension) { Map.Models.Configuration result = new( configuration.FaceConfidencePercent, configuration.FaceDistancePermyriad, configuration.MappingDefaultName, configuration.PersonBirthdayFirstYear, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), configuration.PersonCharactersCopyCount, configuration.RangeDaysDeltaTolerance, configuration.RangeDistanceTolerance, configuration.SaveSortingWithoutPerson, configuration.SortingMaximumPerKey, configuration.SortingMinimumToUseSigma, facesFileNameExtension, facesHiddenFileNameExtension, facePartsFileNameExtension); return result; } private void FullParallelForWork( A_Property propertyLogic, Dictionary>> idToMappedFaceFilesWithCollection, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsDateGroupDirectory, string dResultsFullGroupDirectory, string eDistanceContentDirectory, List> sourceDirectoryChanges, List propertyFileHolderCollection, List propertyCollection, List>> metadataCollections, List> resizeKeyValuePairs, Dictionary> fileNameToCollection, List> imageFaceCollections, Container container, int index, Item item, DateTime[] containerDateTimes) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); List faces; Shared.Models.Property property; long ticks = DateTime.Now.Ticks; DateTime dateTime = DateTime.Now; List parseExceptions = new(); List> subFileTuples = new(); List> metadataCollection; if (item.Property is not null && item.Property.Id is not null && !item.Any()) { property = item.Property; if (item.SourceDirectoryFileHolder.LastWriteTime is not null) subFileTuples.Add(new Tuple(nameof(A_Property), item.SourceDirectoryFileHolder.LastWriteTime.Value)); else subFileTuples.Add(new Tuple(nameof(A_Property), new FileInfo(item.SourceDirectoryFileHolder.FullName).LastWriteTime)); bool nameWithoutExtensionIsIdFormat = Shared.Models.Stateless.Methods.IProperty.NameWithoutExtensionIsIdFormat(item.ImageFileHolder); if (nameWithoutExtensionIsIdFormat && item.ImageFileHolder.NameWithoutExtension != item.Property.Id.ToString()) _Log.Information($"Name without extension is Id format but doesn't match id <{item.ImageFileHolder.FullName}>"); } else { int? propertyHashCode = item.Property?.GetHashCode(); if (item.Abandoned.HasValue && item.Abandoned.Value) _Log.Information(string.Concat("Abandoned <", item.ImageFileHolder.FullName, '>')); else if (item.FileSizeChanged.HasValue && item.FileSizeChanged.Value) _Log.Information(string.Concat("FileSizeChanged <", item.ImageFileHolder.FullName, '>')); else if (item.LastWriteTimeChanged.HasValue && item.LastWriteTimeChanged.Value) _Log.Information(string.Concat("LastWriteTimeChanged <", item.ImageFileHolder.FullName, '>')); else if (item.Moved.HasValue && item.Moved.Value) _Log.Information(string.Concat("Moved <", item.ImageFileHolder.FullName, '>')); property = propertyLogic.GetProperty(item, subFileTuples, parseExceptions); item.Update(property); if (propertyHashCode is null) { lock (sourceDirectoryChanges) sourceDirectoryChanges.Add(new Tuple(nameof(A_Property), DateTime.Now)); } else if (propertyHashCode.Value != property.GetHashCode()) { lock (sourceDirectoryChanges) sourceDirectoryChanges.Add(new Tuple(nameof(A_Property), DateTime.Now)); } } if (property is null || item.Property is null) throw new NullReferenceException(nameof(property)); FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(item); item.SetResizedFileHolder(_Resize.FileNameExtension, resizedFileHolder); string facesDirectory = _Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution) ? _Faces.GetFacesDirectory(dResultsFullGroupDirectory, item) : string.Empty; string facePartsDirectory = _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution) ? _FaceParts.GetFacePartsDirectory(_Configuration.PropertyConfiguration, dResultsFullGroupDirectory, item) : string.Empty; MappingFromItem mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item, resizedFileHolder); (int metadataGroups, metadataCollection) = _Metadata.GetMetadataCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection)); Dictionary outputResolutionToResize = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, subFileTuples, parseExceptions, metadataCollection, item.Property, mappingFromItem); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(C_Resize.GetResizeKeyValuePairs)); if (_Configuration.SaveResizedSubfiles) { _Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.Property, mappingFromItem, outputResolutionToResize); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(C_Resize.SaveResizedSubfile)); } if (!mappingFromItem.ResizedFileHolder.Exists && !File.Exists(mappingFromItem.ResizedFileHolder.FullName)) faces = new(); else if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution)) faces = new(); else { List? mappingFromPhotoPrismCollection; List>? collection; if (item.Property?.Id is null) collection = null; else _ = idToMappedFaceFilesWithCollection.TryGetValue(item.Property.Id.Value, out collection); if (!fileNameToCollection.TryGetValue(mappingFromItem.RelativePath[1..], out mappingFromPhotoPrismCollection)) mappingFromPhotoPrismCollection = null; faces = _Faces.GetFaces(outputResolution, dResultsFullGroupDirectory, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionToResize, collection, mappingFromPhotoPrismCollection); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(D_Face.GetFaces)); bool anyFacesSaved = _Faces.SaveFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem, facesDirectory, faces); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(D_Face.SaveFaces)); if ((_Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch) && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution) && !anyFacesSaved && collection is not null) _Distance.LookForMatchFacesAndPossiblyRename(_Faces.FileNameExtension, eDistanceContentDirectory, mappingFromItem, faces, collection); if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution)) { bool saveRotated = !_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution); string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(_Configuration.PropertyConfiguration, container.SourceDirectory); _FaceParts.SaveFaceLandmarkImages(_Configuration.PropertyConfiguration, facePartsDirectory, subFileTuples, parseExceptions, mappingFromItem, faces, saveRotated); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(D2_FaceParts.SaveFaceLandmarkImages)); } } lock (sourceDirectoryChanges) { imageFaceCollections[index] = faces; propertyCollection[index] = property; metadataCollections[index] = metadataCollection; resizeKeyValuePairs[index] = outputResolutionToResize; propertyFileHolderCollection[index] = item.ImageFileHolder; sourceDirectoryChanges.AddRange(from l in subFileTuples where l.Item2 > dateTime select l); } } private int FullParallelWork(int maxDegreeOfParallelism, A_Property propertyLogic, Dictionary>> idToMappedFaceFilesWithCollection, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsDateGroupDirectory, string dResultsFullGroupDirectory, string eDistanceContentDirectory, List> sourceDirectoryChanges, List propertyFileHolderCollection, List propertyCollection, List>> metadataCollection, List> resizeKeyValuePairs, Dictionary> fileNameToCollection, List> imageFaceCollections, Container container, Item[] filteredItems, string message) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); int result = 0; ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; if (imageFaceCollections.Count != filteredItems.Length || metadataCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || propertyCollection.Count != filteredItems.Length) { for (int f = 0; f < filteredItems.Length; f++) { propertyCollection.Add(null); metadataCollection.Add(new()); imageFaceCollections.Add(new()); resizeKeyValuePairs.Add(new()); propertyFileHolderCollection.Add(null); } } DateTime[] containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems); using ProgressBar progressBar = new(filteredItems.Length, message, options); _ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) => { try { FullParallelForWork(propertyLogic, idToMappedFaceFilesWithCollection, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsDateGroupDirectory, dResultsFullGroupDirectory, eDistanceContentDirectory, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, fileNameToCollection, imageFaceCollections, container, index: i, filteredItems[i], containerDateTimes); if (i == 0 || sourceDirectoryChanges.Any()) progressBar.Tick(); } catch (Exception ex) { result += 1; _Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, ex.Message, Environment.NewLine, ex.StackTrace), ex); if (result == filteredItems.Length) throw new Exception(string.Concat("All in [", container.SourceDirectory, "] failed!")); } }); return result; } private static void WriteTab(string checkDirectory, List<(string Id, string Line)> metadataIdLines, string fileName) { string text; FileInfo fileInfo; List duplicates = new(); List metadataIds = new(); fileInfo = new(Path.Combine(checkDirectory, "[()]", Path.ChangeExtension(fileName, "tsv"))); if (fileInfo?.Directory is null) throw new Exception(); if (!fileInfo.Directory.Exists) fileInfo.Directory.Create(); foreach ((string Id, string Line) metadataIdLine in metadataIdLines) { if (metadataIds.Contains(metadataIdLine.Id)) duplicates.Add(metadataIdLine.Id); else metadataIds.Add(metadataIdLine.Id); } for (int i = metadataIdLines.Count - 1; i > -1; i--) { if (duplicates.Contains(metadataIdLines[i].Id)) metadataIdLines.RemoveAt(i); } if (metadataIdLines.Any()) { text = string.Join(Environment.NewLine, from l in metadataIdLines orderby l.Id select l.Line); _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, text, updateDateWhenMatches: true, compareBeforeWrite: true); } else { if (fileInfo.Exists) File.Delete(fileInfo.FullName); } } private void WriteGroup(A_Property propertyLogic, Shared.Models.Property[] propertyCollection, List>> metadataCollection, List> resizeKeyValuePairs, List> imageFaceCollections, string outputResolution, Container container, Item[] filteredItems) { Item item; string key; string json; string checkFile; int sourceDirectoryLength = container.SourceDirectory.Length; JsonSerializerOptions writeIndentedJsonSerializerOptions = new() { WriteIndented = false }; if (!(from l in propertyCollection where l?.Width is null select true).Any()) { string checkDirectory; List> propertyCollectionKeyValuePairs = new(); List>> resizeKeyValuePairsCollections = new(); List>> imageFaceCollectionsKeyValuePairs = new(); List>>> metadataCollectionKeyValuePairs = new(); (int level, List directories) = Shared.Models.Stateless.Methods.IPath.Get( _Configuration.PropertyConfiguration.RootDirectory, container.SourceDirectory); string fileName = string.Concat(string.Join(_Configuration.PropertyConfiguration.FileNameDirectorySeparator, directories), ".json"); for (int i = 0; i < filteredItems.Length; i++) { item = filteredItems[i]; if (item.Property is null) continue; key = Shared.Models.Stateless.Methods.IPath.GetRelativePath(item.ImageFileHolder.FullName, sourceDirectoryLength); propertyCollectionKeyValuePairs.Add(new KeyValuePair(key, propertyCollection[i])); resizeKeyValuePairsCollections.Add(new KeyValuePair>(key, resizeKeyValuePairs[i])); imageFaceCollectionsKeyValuePairs.Add(new KeyValuePair>(key, imageFaceCollections[i])); metadataCollectionKeyValuePairs.Add(new KeyValuePair>>(key, metadataCollection[i])); } if (propertyLogic.AngleBracketCollection.Any()) { checkDirectory = Shared.Models.Stateless.Methods.IPath.GetDirectory(propertyLogic.AngleBracketCollection[0], level, "[{}]"); checkFile = Path.Combine(checkDirectory, fileName); if (File.Exists(checkFile)) File.Move(checkFile, Path.Combine(checkDirectory, fileName)); checkFile = Path.Combine(checkDirectory, fileName); json = JsonSerializer.Serialize(propertyCollectionKeyValuePairs, writeIndentedJsonSerializerOptions); _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true); } if (_Metadata.AngleBracketCollection.Any()) { checkDirectory = Shared.Models.Stateless.Methods.IPath.GetDirectory(_Metadata.AngleBracketCollection[0], level, "[{}]"); checkFile = Path.Combine(checkDirectory, fileName); if (File.Exists(checkFile)) File.Move(checkFile, Path.Combine(checkDirectory, fileName)); checkFile = Path.Combine(checkDirectory, fileName); json = JsonSerializer.Serialize(metadataCollectionKeyValuePairs, writeIndentedJsonSerializerOptions); _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true); } if (_Resize.AngleBracketCollection.Any()) { checkDirectory = Shared.Models.Stateless.Methods.IPath.GetDirectory(_Resize.AngleBracketCollection[0], level, "[{}]"); checkFile = Path.Combine(checkDirectory, fileName); if (File.Exists(checkFile)) File.Move(checkFile, Path.Combine(checkDirectory, fileName)); checkFile = Path.Combine(checkDirectory, fileName); json = JsonSerializer.Serialize(resizeKeyValuePairsCollections, writeIndentedJsonSerializerOptions); _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true); } if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution) && _Faces.AngleBracketCollection.Any()) { checkDirectory = Shared.Models.Stateless.Methods.IPath.GetDirectory(_Faces.AngleBracketCollection[0], level, "[{}]"); checkFile = Path.Combine(checkDirectory, fileName); if (File.Exists(checkFile)) File.Move(checkFile, Path.Combine(checkDirectory, fileName)); checkFile = Path.Combine(checkDirectory, fileName); json = JsonSerializer.Serialize(imageFaceCollectionsKeyValuePairs, writeIndentedJsonSerializerOptions); _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true); } } } private (string, string, string, string, string) GetResultsFullGroupDirectories(string outputResolution) { string aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(A_Property), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false); string bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(B_Metadata), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false); string cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(C_Resize), outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false); string dResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true); string d2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(D2_FaceParts), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true); return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory); } private void SetAngleBracketCollections(A_Property propertyLogic, Container container, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory) { propertyLogic.SetAngleBracketCollection(aResultsFullGroupDirectory, container.SourceDirectory); _Resize.SetAngleBracketCollection(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, container.SourceDirectory); _Metadata.SetAngleBracketCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, container.SourceDirectory); } private void FullDoWork(string argZero, string propertyRoot, long ticks, A_Property propertyLogic, int t, Container[] containers, string eDistanceContentDirectory, Dictionary> fileNameToCollection, Dictionary>> idToMappedFaceFilesWithCollection) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); int total; string message; int totalSeconds; int exceptionCount; Container container; Item[] filteredItems; string aResultsFullGroupDirectory; string bResultsFullGroupDirectory; string cResultsFullGroupDirectory; string dResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; int containersLength = containers.Length; Shared.Models.Property[] propertyCollection; List propertyFileHolderCollection = new(); List> resizeKeyValuePairs = new(); List> imageFaceCollections = new(); List> sourceDirectoryChanges = new(); int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism; List nullablePropertyCollection = new(); List>> metadataCollection = new(); string dResultsDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(D_Face)); foreach (string outputResolution in _Configuration.OutputResolutions) { total = 0; (aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); // _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(dResultsFullGroupDirectory); for (int i = 0; i < containers.Length; i++) { container = containers[i]; if (!container.Items.Any()) continue; if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero)) continue; filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container); if (!filteredItems.Any()) continue; metadataCollection.Clear(); resizeKeyValuePairs.Clear(); imageFaceCollections.Clear(); sourceDirectoryChanges.Clear(); nullablePropertyCollection.Clear(); propertyFileHolderCollection.Clear(); totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); message = $"{i + 1:000}.{container.G} [{filteredItems.Length:000} collection] / {containersLength:000} - {total} / {t} total collection - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}"; if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution)) _Faces.SetAngleBracketCollection(dResultsFullGroupDirectory, container.SourceDirectory); if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution)) _FaceParts.SetAngleBracketCollection(_Configuration.PropertyConfiguration, d2ResultsFullGroupDirectory, container.SourceDirectory); SetAngleBracketCollections(propertyLogic, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory); exceptionCount = FullParallelWork( maxDegreeOfParallelism, propertyLogic, idToMappedFaceFilesWithCollection, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsDateGroupDirectory, dResultsFullGroupDirectory, eDistanceContentDirectory, sourceDirectoryChanges, propertyFileHolderCollection, nullablePropertyCollection, metadataCollection, resizeKeyValuePairs, fileNameToCollection, imageFaceCollections, container, filteredItems, message); if (metadataCollection.Count != filteredItems.Length || nullablePropertyCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || imageFaceCollections.Count != filteredItems.Length) throw new Exception("Counts don't match!"); if (exceptionCount != 0) { _Exceptions.Add(container.SourceDirectory); continue; } for (int f = 0; f < imageFaceCollections.Count; f++) filteredItems[f].Faces.AddRange(from l in imageFaceCollections[f] select l); propertyCollection = (from l in nullablePropertyCollection where l is not null select l).ToArray(); if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0) WriteGroup(propertyLogic, propertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, outputResolution, container, filteredItems); if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any()) { for (int y = 0; y < int.MaxValue; y++) { _Log.Information("Press \"Y\" key when ready to continue or close console"); if (_Console.ReadKey() == ConsoleKey.Y) break; } _Log.Information(". . ."); } // if (isSilent && container.SourceDirectory.EndsWith("Bohdi Ray 2016")) // 7#.1 // break; // if (isSilent && container.SourceDirectory.EndsWith("Halloween 2013")) // 12#.1 // break; // if (isSilent && container.SourceDirectory.EndsWith("zzz =2014.4 Winter Tracy Pictures")) // 30#.2 // break; // if (isSilent && container.SourceDirectory.EndsWith("=2009.0 Winter Logan Michael")) //34#.2 // break; // if (isSilent && container.SourceDirectory.EndsWith("Animal Kingdom")) //35#.3 // break; // if (isSilent && container.SourceDirectory.EndsWith("Texas 2015")) //46#.4 // break; total += container.Items.Count; } } } private void SetMapping(Dictionary> fileNameToCollection, string argZero, Container[] containers) { Mapping mapping; int faceAreaPermille; Item[] filteredItems; int confidencePercent; int normalizedRectangle; DateTime[] containerDateTimes; string deterministicHashCodeKey; MappingFromItem mappingFromItem; MappingFromLocation mappingFromLocation; List? mappingFromPhotoPrismCollection; foreach (Container container in containers) { if (!container.Items.Any()) continue; if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero)) continue; filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container); if (!filteredItems.Any()) continue; containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems); foreach (Item item in filteredItems) { if (item.Property?.Id is null || item.ResizedFileHolder is null) continue; mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item); foreach (Shared.Models.Face face in item.Faces) { if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null) continue; confidencePercent = Shared.Models.Stateless.Methods.ILocation.GetConfidencePercent(_Configuration.FaceConfidencePercent, _Configuration.RangeFaceConfidence, face.Location.Confidence); faceAreaPermille = Shared.Models.Stateless.Methods.IMapping.GetAreaPermille(_Configuration.FaceAreaPermille, face.Location, face.OutputResolution); normalizedRectangle = Shared.Models.Stateless.Methods.ILocation.GetNormalizedRectangle(face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution); deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item.Property.Id.Value, face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution); mappingFromLocation = new(faceAreaPermille, confidencePercent, deterministicHashCodeKey, normalizedRectangle); if (!fileNameToCollection.TryGetValue(mappingFromItem.RelativePath[1..], out mappingFromPhotoPrismCollection)) mappingFromPhotoPrismCollection = null; mapping = new(mappingFromItem, mappingFromLocation, mappingFromPhotoPrismCollection); face.SetMapping(mapping); } } } } private void RenameAbandoned(string argZero, Container[] containers) { List renameCollection = new(); foreach (Container container in containers) { if (!container.Items.Any()) continue; if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero)) continue; foreach (Item item in container.Items) { if (item.Abandoned is not null && item.Abandoned.Value) renameCollection.Add(item.SourceDirectoryFileHolder.FullName); } } if (renameCollection.Any()) { string checkFile; foreach (string file in renameCollection) { if (!File.Exists(file)) continue; checkFile = $"{file}.abd"; if (File.Exists(checkFile)) continue; File.Move(file, checkFile); } throw new Exception($"Renamed {renameCollection.Count}(s) files!"); } } private void RenameMoved(string argZero, Container[] containers) { List renameCollection = new(); foreach (Container container in containers) { if (!container.Items.Any()) continue; if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero)) continue; foreach (Item item in container.Items) { if (item.Moved is not null && item.Moved.Value) renameCollection.Add(item.SourceDirectoryFileHolder.FullName); } } if (renameCollection.Any()) { string checkFile; foreach (string file in renameCollection) { if (!File.Exists(file)) continue; checkFile = $"{file}.mvd"; if (File.Exists(checkFile)) continue; File.Move(file, checkFile); } throw new Exception($"Renamed {renameCollection.Count}(s) files!"); } } private (List, List) GetFilteredDistinct(string argZero, Container[] containers) { List resultIds = new(); List resultFaces = new(); Item[] filteredItems; bool isIgnoreRelativePath; foreach (Container container in containers) { if (!container.Items.Any()) continue; if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero)) continue; filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container); if (!filteredItems.Any()) continue; isIgnoreRelativePath = _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory); foreach (Item item in filteredItems) { if (item.Property?.Id is null || item.ResizedFileHolder is null) continue; if (resultIds.Contains(item.Property.Id.Value)) continue; resultIds.Add(item.Property.Id.Value); if (isIgnoreRelativePath) continue; foreach (Shared.Models.Face face in item.Faces) { if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null) continue; resultFaces.Add(face); } } } return new(resultIds, resultFaces); } private List GetItems(string argZero, Container[] containers) { List items = new(); Item[] filteredItems; bool isIgnoreRelativePath; List collection = new(); foreach (Container container in containers) { if (!container.Items.Any()) continue; if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero)) continue; filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container); if (!filteredItems.Any()) continue; isIgnoreRelativePath = _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory); foreach (Item item in filteredItems) { if (item.Property?.Id is null || item.ResizedFileHolder is null) continue; items.Add(item); } } return items; } private void MapLogic(string argZero, long ticks, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string fPhotoPrismContentDirectory, MapLogicSupport mapLogicSupport, MapLogic mapLogic, string outputResolution, Dictionary> personKeyToIds, List distinctFilteredFaces, Mapping[] mappingCollection, int totalNotMapped) { int? useFiltersCounter = null; SortingContainer[] sortingContainers; string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()"); string d2FacePartsContentDirectory = Path.Combine(d2ResultsFullGroupDirectory, "()"); string dFacesCollectionDirectory = Path.Combine(dResultsFullGroupDirectory, "[]", _Configuration.PropertyConfiguration.ResultAllInOne); Dictionary> idToNormalizedRectangleToMapping = MapLogicSupport.GetIdToNormalizedRectangleToFace(mappingCollection); if (Directory.Exists(fPhotoPrismContentDirectory)) F_PhotoPrism.WriteMatches(fPhotoPrismContentDirectory, _Configuration.PersonBirthdayFormat, ticks, distinctFilteredFaces, mapLogic); if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution)) { List filteredItems = GetItems(argZero, containers); mapLogic.SaveShortcutsForOutputResolutions(personKeyToIds, filteredItems, mappingCollection); } if (_Configuration.PersonCharactersCopyCount > 0 && !string.IsNullOrEmpty(_Configuration.PersonCharacters)) mapLogic.CopyAtLeastOneMappedFiles(dFacesContentDirectory, a2PeopleSingletonDirectory, mappingCollection); mapLogic.CopyManualFiles(dFacesContentDirectory, idToNormalizedRectangleToMapping); if (_Configuration.SaveMappedForOutputResolutions.Contains(outputResolution)) mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, personKeyToIds, mappingCollection, idToNormalizedRectangleToMapping, totalNotMapped); if (_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution)) { MapLogicSupport.SetFaceDistances(_AppSettings.MaxDegreeOfParallelism, ticks, distinctFilteredFaces); List distinctFurtherFilteredFaces = mapLogic.GetFurtherFilterBySkipCollection(distinctFilteredFaces); Dictionary> missingIdThenNormalizedRectangleToPersonContainers = mapLogic.GetMissing(idToNormalizedRectangleToMapping); List missingFaceDistanceContainers = _Distance.GetMissingFaceDistanceContainer(_AppSettings.MaxDegreeOfParallelism, ticks, dFacesCollectionDirectory, missingIdThenNormalizedRectangleToPersonContainers); sortingContainers = mapLogicSupport.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distinctFurtherFilteredFaces, missingFaceDistanceContainers, useFiltersCounter); if (!sortingContainers.Any()) { for (useFiltersCounter = 1; useFiltersCounter < 17; useFiltersCounter++) { sortingContainers = mapLogicSupport.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distinctFurtherFilteredFaces, missingFaceDistanceContainers, useFiltersCounter); if (sortingContainers.Any()) break; } } MapLogicSupport.SaveFaceDistances(_Configuration.PropertyConfiguration, sortingContainers); if (totalNotMapped > 0) { int updated = mapLogic.UpdateFromSortingContainers(sortingContainers); List saveContainers = mapLogic.GetSaveContainers(dFacesContentDirectory, d2FacePartsContentDirectory, mappingCollection, idToNormalizedRectangleToMapping, useFiltersCounter, sortingContainers.Any()); mapLogic.SaveContainers(totalNotMapped, updated, saveContainers); } } } private static Container? AreAllSameEndsWith(string argZero, Container[] containers) { Container? result = null; string[] directoryNames = Shared.Models.Stateless.Methods.IPath.GetDirectoryNames(argZero); if (directoryNames.Length > 2) { string directoryName; string rootDirectoryName = directoryNames[^2]; if (!Directory.Exists(argZero)) throw new Exception(); foreach (Container container in containers) { if (container.SourceDirectory == argZero) result = container; directoryName = Path.GetFileName(container.SourceDirectory); if (!directoryName.EndsWith(rootDirectoryName)) { result = null; break; } } } return result; } private string SaveUrlAndGetNewRootDirectory(Container container) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); string? result = Path.GetDirectoryName(Path.GetDirectoryName(container.SourceDirectory)); if (result is null) throw new Exception(); Uri uri; Item item; string? line; string fileName; Task task; string relativePath; FileHolder fileHolder; string extensionLowered; string sourceDirectoryFile; int length = result.Length; HttpClient httpClient = new(); bool isValidImageFormatExtension; for (int y = 0; y < int.MaxValue; y++) { _Log.Information("Enter fileNameToCollection url for fileNameToCollection image"); line = _Console.ReadLine(); if (string.IsNullOrEmpty(line)) break; uri = new(line); if (uri.HostNameType != UriHostNameType.Dns) continue; task = httpClient.GetByteArrayAsync(uri); fileName = Path.GetFileName(uri.LocalPath); sourceDirectoryFile = Path.Combine(container.SourceDirectory, fileName); File.WriteAllBytes(sourceDirectoryFile, task.Result); extensionLowered = Path.GetExtension(uri.LocalPath); relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFile, length, forceExtensionToLower: true); isValidImageFormatExtension = _Configuration.PropertyConfiguration.ValidImageFormatExtensions.Contains(extensionLowered); fileHolder = new(sourceDirectoryFile); item = new(fileHolder, relativePath, isValidImageFormatExtension); container.Items.Add(item); } _Log.Information(". . ."); return result; } private void ParallelFor(string eDistanceContentDirectory, List> collection, long personKey, string file) { const string lnk = ".lnk"; int? id, normalizedRectangle; IReadOnlyList directories; bool fromDistanceContent = !file.EndsWith(lnk) && file.Contains(eDistanceContentDirectory); if (!file.EndsWith(lnk)) (id, normalizedRectangle) = Shared.Models.Stateless.Methods.IMapping.GetConverted(_MapConfiguration.FacesFileNameExtension, file); else (id, normalizedRectangle) = Shared.Models.Stateless.Methods.IMapping.GetConverted(_MapConfiguration.FacesFileNameExtension, file[..^4]); if (id is null || normalizedRectangle is null) return; if (file.EndsWith(lnk) || (!_Configuration.DistanceMoveUnableToMatch && !_Configuration.DistanceRenameToMatch)) directories = new List(); else directories = MetadataExtractor.ImageMetadataReader.ReadMetadata(file); Rectangle? rectangle = Shared.Models.Stateless.Methods.ILocation.GetNormalizedRectangle(_Configuration.LocationDigits, normalizedRectangle.Value); lock (collection) collection.Add(new(fromDistanceContent, file, personKey, id.Value, normalizedRectangle.Value, directories, rectangle, null)); } private void OpenPossibleDuplicates(List<(long, int, string, double?)> duplicates) { string personKeyFormatted; foreach ((long personKey, int id, string file, double? percent) in duplicates) { if (percent is null) continue; _ = Process.Start("explorer.exe", string.Concat("\"", Path.GetDirectoryName(file), "\"")); personKeyFormatted = Shared.Models.Stateless.Methods.IPersonBirthday.GetFormatted(_Configuration.PersonBirthdayFormat, personKey); } foreach ((long personKey, int id, string file, double? percent) in duplicates) { if (percent is not null && percent.Value == 0) continue; _ = Process.Start("explorer.exe", string.Concat("\"", Path.GetDirectoryName(file), "\"")); personKeyFormatted = Shared.Models.Stateless.Methods.IPersonBirthday.GetFormatted(_Configuration.PersonBirthdayFormat, personKey); } foreach ((long personKey, int id, string file, double? percent) in duplicates) { if (percent is not null && percent.Value > 0) continue; _ = Process.Start("explorer.exe", string.Concat("\"", Path.GetDirectoryName(file), "\"")); personKeyFormatted = Shared.Models.Stateless.Methods.IPersonBirthday.GetFormatted(_Configuration.PersonBirthdayFormat, personKey); } } private void LookForPossibleDuplicates(List> collection) { string key; double? percent; Rectangle? rectangle; List delete = new(); Rectangle intersectRectangle; (string File, int NormalizedRectangle) item; Dictionary distinct = new(); List<(long, int, string, double?)> duplicates = new(); foreach (LocationContainer locationContainer in collection) { key = string.Concat(locationContainer.PersonKey, locationContainer.Id); if (distinct.TryGetValue(key, out item)) { if (item.NormalizedRectangle == locationContainer.NormalizedRectangle) continue; if (locationContainer.Rectangle is null) percent = null; else { rectangle = Shared.Models.Stateless.Methods.ILocation.GetNormalizedRectangle(_Configuration.LocationDigits, item.NormalizedRectangle); if (locationContainer.Rectangle is null || rectangle is null) percent = null; else { intersectRectangle = Rectangle.Intersect(locationContainer.Rectangle.Value, rectangle.Value); percent = intersectRectangle.Width * intersectRectangle.Height; } } delete.Add(item.File); delete.Add(locationContainer.File); duplicates.Add(new(locationContainer.PersonKey, locationContainer.Id, locationContainer.File, percent)); continue; } distinct.Add(key, new(locationContainer.File, locationContainer.NormalizedRectangle)); } if (!_Configuration.DeletePossibleDuplicates) { if (duplicates.Any() && _IsEnvironment.Development) OpenPossibleDuplicates(duplicates); } else { if (delete.Count > 5) throw new Exception("Something maybe wrong!"); foreach (string file in delete) { if (File.Exists(file)) File.Delete(file); } } } private List> GetCollection(long ticks, string? a2PeopleContentDirectory, string eDistanceContentDirectory) { string file; List<(long PersonKey, string File)> collection = new(); List> results = new(); collection.AddRange(Map.Models.Stateless.Methods.IMapLogic.GetDisplayDirectoryAllFiles(_PersonContainers)); collection.AddRange(Map.Models.Stateless.Methods.IMapLogic.DeleteEmptyDirectoriesAndGetMappedFaceFiles(_MapConfiguration, _PersonContainers, ticks, a2PeopleContentDirectory, eDistanceContentDirectory)); for (int i = 0; i < collection.Count; i++) { file = collection[i].File; if (!file.EndsWith(".dup") && !file.EndsWith(".unk")) continue; if (!File.Exists(file)) continue; File.Move(file, file[..^4]); collection[i] = new(collection[i].PersonKey, file[..^4]); } if (collection.Any() && (_Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch)) { int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); string message = $") Building Mapped Face Files Collection - {totalSeconds} total second(s)"; ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism }; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; using ProgressBar progressBar = new(collection.Count, message, options); _ = Parallel.For(0, collection.Count, parallelOptions, (i, state) => { progressBar.Tick(); ParallelFor(eDistanceContentDirectory, results, collection[i].PersonKey, collection[i].File); }); } LookForPossibleDuplicates(results); return results; } private Dictionary>> GetDictionary(long ticks, string? a2PeopleContentDirectory, string eDistanceContentDirectory) { Dictionary>> results = new(); List> collection = GetCollection(ticks, a2PeopleContentDirectory, eDistanceContentDirectory); foreach (LocationContainer locationContainer in collection) { if (!results.ContainsKey(locationContainer.Id)) results.Add(locationContainer.Id, new()); results[locationContainer.Id].Add(locationContainer); } return results; } private static void LookForAbandoned(Dictionary>> idToMappedFaceFilesWithCollection, List distinctFilteredIds) { List renameCollection = new(); foreach (KeyValuePair>> idToCollection in idToMappedFaceFilesWithCollection) { if (distinctFilteredIds.Contains(idToCollection.Key)) continue; foreach (LocationContainer locationContainer in idToCollection.Value) { if (locationContainer.File.Contains('!')) continue; renameCollection.Add(locationContainer.File); } } if (renameCollection.Any()) { string checkFile; foreach (string file in renameCollection) { if (!File.Exists(file)) continue; checkFile = $"{file}.abd"; if (File.Exists(checkFile)) continue; File.Move(file, checkFile); } throw new Exception($"Renamed {renameCollection.Count}(s) files!"); } } private void Search(long ticks, string argZero, string propertyRoot) { int f; int j; int t; Container[] containers; string eDistanceContentDirectory; string? a2PeopleContentDirectory; string aResultsFullGroupDirectory; string bResultsFullGroupDirectory; string cResultsFullGroupDirectory; string dResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; string fPhotoPrismContentDirectory; string fPhotoPrismSingletonDirectory; Dictionary> personKeyToIds; Dictionary> fileNameToCollection; string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "{}"); int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); string message = $") Building Container(s) - {totalSeconds} total second(s)"; A_Property propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse); ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; using (ProgressBar progressBar = new(1, message, options)) { progressBar.Tick(); (j, f, t, containers) = Property.Models.Stateless.Container.GetContainers(_Configuration.PropertyConfiguration, propertyLogic); } RenameMoved(argZero, containers); RenameAbandoned(argZero, containers); Container? container = AreAllSameEndsWith(argZero, containers); if (!_ArgZeroIsConfigurationRootDirectory || container is null) { a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "([])"); eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), "()"); fPhotoPrismContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), "()"); fPhotoPrismSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), "{}"); } else { string resultsGroupDirectory; a2PeopleContentDirectory = null; eDistanceContentDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", "()"); fPhotoPrismContentDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", "()"); fPhotoPrismSingletonDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", "{}"); string? newRootDirectory = SaveUrlAndGetNewRootDirectory(container); for (int i = 1; i < 10; i++) { resultsGroupDirectory = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, string.Empty, create: true); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(resultsGroupDirectory); } argZero = newRootDirectory; _Configuration.PropertyConfiguration.ChangeRootDirectory(newRootDirectory); propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false); propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse); } containers = Shared.Models.Stateless.Methods.IContainer.SortContainers(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, _ArgZeroIsConfigurationRootDirectory, argZero, containers); MapLogicSupport mapLogicSupport = new(_Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeFaceAreaPermilleTolerance, _Configuration.SortingMaximumPerFaceShouldBeHigh); MapLogic? mapLogic = _Configuration.DistanceMoveUnableToMatch ? null : new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, _PersonContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory, mapLogicSupport); personKeyToIds = mapLogic is null ? new() : mapLogic.GetPersonKeyToIds(); fileNameToCollection = !Directory.Exists(fPhotoPrismSingletonDirectory) ? fileNameToCollection = new() : F_PhotoPrism.GetFileNameToCollection(fPhotoPrismSingletonDirectory); Dictionary>> idToMappedFaceFilesWithCollection = GetDictionary(ticks, a2PeopleContentDirectory, eDistanceContentDirectory); FullDoWork(argZero, propertyRoot, ticks, propertyLogic, t, containers, eDistanceContentDirectory, fileNameToCollection, idToMappedFaceFilesWithCollection); _Distance.Clear(); mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, _PersonContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory, mapLogicSupport); SetMapping(fileNameToCollection, argZero, containers); if (!personKeyToIds.Any()) personKeyToIds = mapLogic.GetPersonKeyToIds(); (List distinctFilteredIds, List distinctFilteredFaces) = GetFilteredDistinct(argZero, containers); LookForAbandoned(idToMappedFaceFilesWithCollection, distinctFilteredIds); Mapping[] mappingCollection = MapLogicSupport.GetSelectedMappingCollection(distinctFilteredFaces); int totalNotMapped = mapLogic.UpdateMappingFromPerson(mappingCollection); if (a2PeopleContentDirectory is not null && false) mapLogic.CreateTree(ticks, a2PeopleContentDirectory); foreach (string outputResolution in _Configuration.OutputResolutions) { if (_PropertyRootExistedBefore || container is not null) break; if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution)) mapLogic.SaveShortcutsForOutputResolutions(a2PeopleContentDirectory, personKeyToIds, mappingCollection, totalNotMapped); if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution)) mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, _PersonContainers, a2PeopleContentDirectory, personKeyToIds, mappingCollection, totalNotMapped); (aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); if (_ArgZeroIsConfigurationRootDirectory && _Configuration.SaveResizedSubfiles && outputResolution == _Configuration.OutputResolutions[0] && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution) && _Exceptions.Count == 0) MapLogic(argZero, ticks, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, fPhotoPrismContentDirectory, mapLogicSupport, mapLogic, outputResolution, personKeyToIds, distinctFilteredFaces, mappingCollection, totalNotMapped); if (_Configuration.SaveRandomForOutputResolutions.Contains(outputResolution)) _Random.Random(_Configuration.PropertyConfiguration, outputResolution, personKeyToIds, mappingCollection); if (_IsEnvironment.Development) continue; G2_Identify identify = new(_Configuration); List identifiedCollection = identify.GetIdentifiedCollection(_IsEnvironment, _Configuration.PropertyConfiguration, _Faces.FileNameExtension); identify.WriteAllText(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0], identifiedCollection); if (_Configuration.LoadOrCreateThenSaveIndex) _Index.SetIndex(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0]); if (!_IsEnvironment.Development) { _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(aResultsFullGroupDirectory, "{}")); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(bResultsFullGroupDirectory, "{}")); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(cResultsFullGroupDirectory, "{}")); if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution)) _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(dResultsFullGroupDirectory, "[]")); } } } internal void RenameQueue() => _Rename.RenameQueue(_Configuration.PropertyConfiguration); }