using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Phares.Shared; using ShellProgressBar; using System.Collections.ObjectModel; using System.Drawing.Imaging; using System.Text.Json; using System.Text.RegularExpressions; using View_by_Distance.Distance.Models; using View_by_Distance.Face.Models; using View_by_Distance.FaceParts.Models; using View_by_Distance.Instance.Models; using View_by_Distance.Map.Models; using View_by_Distance.Metadata.Models; using View_by_Distance.PhotoPrism.Models; using View_by_Distance.Property.Models; using View_by_Distance.Resize.Models; using View_by_Distance.Shared.Models; using View_by_Distance.Shared.Models.Methods; using View_by_Distance.Shared.Models.Stateless.Methods; using WindowsShortcutFactory; namespace View_by_Distance.Instance; public partial class DlibDotNet : IDlibDotNet, IDisposable { [GeneratedRegex(@"[\\,\/,\:,\*,\?,\"",\<,\>,\|]")] private static partial Regex CameraRegex(); private readonly D_Face _Faces; private ProgressBar? _ProgressBar; private readonly C_Resize _Resize; private readonly F_Random _Random; private readonly IConsole _Console; private readonly E_Distance _Distance; private readonly D2_FaceParts _FaceParts; private readonly IBlurHasher _BlurHasher; private readonly AppSettings _AppSettings; private readonly List _Exceptions; private readonly ILogger? _Logger; private readonly IsEnvironment _IsEnvironment; private readonly DistanceLimits _DistanceLimits; private readonly bool _PropertyRootExistedBefore; private readonly Models.Configuration _Configuration; private readonly bool _ArgZeroIsConfigurationRootDirectory; private readonly Map.Models.Configuration _MapConfiguration; private readonly List<(string Directory, long PersonKey)> _JLinkResolvedDirectories; public DlibDotNet( List args, ILogger logger, IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, AppSettings appSettings, string workingDirectory, bool isSilent, IConsole console) { string message; _Logger = logger; _Console = console; _AppSettings = appSettings; _IsEnvironment = isEnvironment; long ticks = DateTime.Now.Ticks; _Exceptions = []; _JLinkResolvedDirectories = []; if (ticks.ToString().Last() == '0') ticks += 1; ReadOnlyCollection personContainers; Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot); Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration); _Logger?.LogInformation(propertyConfiguration.RootDirectory); Property.Models.Configuration.Verify(propertyConfiguration, requireExist: false); Verify(configuration); VerifyExtra(args, propertyConfiguration, configuration); _Configuration = configuration; _Random = new(configuration); if (configuration.IgnoreExtensions is null) throw new NullReferenceException(nameof(configuration.IgnoreExtensions)); _BlurHasher = new BlurHash.Models.C2_BlurHasher(configuration.PropertyConfiguration); string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(propertyConfiguration, nameof(A_Property), create: false); _PropertyRootExistedBefore = !Directory.Exists(propertyRoot); string argZero = args.Count > 0 ? Path.GetFullPath(args[0]) : Path.GetFullPath(propertyConfiguration.RootDirectory); _ArgZeroIsConfigurationRootDirectory = propertyConfiguration.RootDirectory == argZero; if (!Directory.Exists(argZero)) _ = Directory.CreateDirectory(argZero); _Logger?.LogInformation(configuration.ModelDirectory); { (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetPngLowQuality(); (ImageCodecInfo hiddenImageCodecInfo, EncoderParameters hiddenEncoderParameters, string hiddenFileNameExtension) = C_Resize.GetJpegLowQuality(); _Faces = new D_Face( argZero, configuration.PropertyConfiguration, configuration.CheckDFaceAndUpWriteDates, encoderParameters, configuration.FaceDistanceHiddenImageFactor, filenameExtension, configuration.ForceFaceLastWriteTimeToCreationTime, hiddenEncoderParameters, hiddenFileNameExtension, hiddenImageCodecInfo, imageCodecInfo, configuration.LoadPhotoPrismLocations, configuration.ModelDirectory, configuration.ModelName, configuration.OverrideForFaceImages, configuration.PredictorModelName, configuration.PropertiesChangedForFaces, configuration.RectangleIntersectMinimums); } { (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetJpegLowQuality(); _FaceParts = new D2_FaceParts(_Configuration.PropertyConfiguration, imageCodecInfo, encoderParameters, filenameExtension, configuration.CheckDFaceAndUpWriteDates, configuration.OverrideForFaceLandmarkImages); } _DistanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaPermyriadTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh); _MapConfiguration = Get(configuration, _DistanceLimits, _Faces.FileNameExtension, _Faces.HiddenFileNameExtension, _FaceParts.FileNameExtension); _Distance = new(configuration.DistanceMoveUnableToMatch, configuration.DistanceRenameToMatch, configuration.FaceConfidencePercent, configuration.RangeDistanceTolerance, configuration.RectangleIntersectMinimums); if (_PropertyRootExistedBefore || !_ArgZeroIsConfigurationRootDirectory) personContainers = new(new List()); else { int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); message = $") Building People Collection - {totalSeconds} total second(s)"; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; using ProgressBar progressBar = new(1, message, options); progressBar.Tick(); string peopleRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(A2_People)); string? rootResultsDirectory = Path.GetDirectoryName(Path.GetDirectoryName(peopleRootDirectory)) ?? throw new Exception(); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(peopleRootDirectory, propertyConfiguration.ResultSingleton)); string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration.PropertyConfiguration, nameof(A2_People), "{}"); string a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration.PropertyConfiguration, nameof(A2_People), "([])"); personContainers = new(IPersonContainer.GetPersonContainers(a2PeopleSingletonDirectory, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), configuration.PropertyConfiguration, _Faces.FileNameExtension)); if (configuration.JLinks.Where(l => !string.IsNullOrEmpty(l)).Any()) { _JLinkResolvedDirectories.AddRange(Map.Models.Stateless.Methods.IMapLogic.GetJLinkDirectories(configuration.GenealogicalDataCommunicationFile, configuration.JLinks, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), a2PeopleSingletonDirectory, a2PeopleContentDirectory)); if (_JLinkResolvedDirectories.Count == 0) throw new Exception(nameof(Map.Models.Stateless.Methods.IMapLogic.GetJLinkDirectories)); } } { (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetTuple( configuration.OutputExtension, configuration.OutputQuality); _Resize = new C_Resize( configuration.PropertyConfiguration, configuration.ForceResizeLastWriteTimeToCreationTime, configuration.OverrideForResizeImages, configuration.PropertiesChangedForResize, configuration.ValidResolutions, imageCodecInfo, encoderParameters, filenameExtension); } if (!configuration.SkipSearch) Search(ticks, personContainers, argZero, propertyRoot); if (!_PropertyRootExistedBefore && !_IsEnvironment.Development && _Exceptions.Count == 0 && _ArgZeroIsConfigurationRootDirectory) { string d2FacePartsRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(D2_FaceParts)); _Logger?.LogInformation(string.Concat("Cleaning <", d2FacePartsRootDirectory, ">")); Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(d2FacePartsRootDirectory, ticks); } message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}"; _Logger?.LogInformation(message); if (_Exceptions.Count != 0) throw new Exception(message); if (_PropertyRootExistedBefore) _Logger?.LogInformation("First run completed. Run again if wanted"); } void IDlibDotNet.Tick() => _ProgressBar?.Tick(); void IDisposable.Dispose() { _ProgressBar?.Dispose(); GC.SuppressFinalize(this); } private static void Verify(Models.Configuration configuration) { if (configuration.RangeDaysDeltaTolerance.Length != 3) throw new NullReferenceException(nameof(configuration.RangeDaysDeltaTolerance)); if (configuration.RangeDistanceTolerance.Length != 3) throw new NullReferenceException(nameof(configuration.RangeDistanceTolerance)); if (configuration.RangeFaceAreaPermyriadTolerance.Length != 3) throw new NullReferenceException(nameof(configuration.RangeFaceAreaPermyriadTolerance)); if (configuration.RangeFaceConfidence.Length != 3) throw new NullReferenceException(nameof(configuration.RangeFaceConfidence)); if (configuration.LocationContainerDistanceTolerance is null && !string.IsNullOrEmpty(configuration.LocationContainerDebugDirectory)) throw new NullReferenceException($"{nameof(configuration.LocationContainerDistanceTolerance)} must have a value when {nameof(configuration.LocationContainerDebugDirectory)} is set!"); _ = DateTime.Now.AddDays(-configuration.RangeDaysDeltaTolerance[1]); if (configuration.OutputResolutions.Length == 0 || string.IsNullOrEmpty(configuration.OutputResolutions[0]) || !configuration.ValidResolutions.Contains(configuration.OutputResolutions[0])) throw new NullReferenceException($"{nameof(configuration.OutputResolutions)} must be fileNameToCollection valid outputResolution!"); if ((from l in configuration.OutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.OutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.SaveShortcutsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.SaveShortcutsForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.SaveFaceLandmarkForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.SaveFaceLandmarkForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.SaveFaceLandmarkForOutputResolutionsV2 where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.SaveFaceLandmarkForOutputResolutionsV2)} are not in the ValidResolutions list!"); if (string.IsNullOrEmpty(configuration.ModelName)) throw new NullReferenceException(nameof(configuration.ModelName)); if (string.IsNullOrEmpty(configuration.OutputExtension)) throw new NullReferenceException(nameof(configuration.OutputExtension)); if (string.IsNullOrEmpty(configuration.PredictorModelName)) throw new NullReferenceException(nameof(configuration.PredictorModelName)); if (string.IsNullOrEmpty(configuration.ModelDirectory) || !Directory.Exists(configuration.ModelDirectory)) throw new NullReferenceException(nameof(configuration.ModelDirectory)); } private void VerifyExtra(List args, Property.Models.Configuration propertyConfiguration, Models.Configuration configuration) { string[] sourceDirectoryNames; if (args.Count == 0) sourceDirectoryNames = []; else { string? century; string argZero = Path.GetFullPath(args[0]); sourceDirectoryNames = argZero.Split(Path.DirectorySeparatorChar); if (!argZero.StartsWith(propertyConfiguration.RootDirectory)) throw new Exception($"Source directory must be inside root directory! <{argZero}> <{propertyConfiguration.RootDirectory}>"); if (_ArgZeroIsConfigurationRootDirectory && propertyConfiguration.RootDirectory != argZero) { if (!configuration.MixedYearRelativePaths.Contains(sourceDirectoryNames[0])) { string[] segments = sourceDirectoryNames[0].Split(' '); century = segments[^1].Length == 4 ? segments[^1][..2] : null; if (segments.Length < 2 || century is null || (century != "18" && century != "19" && century != "20")) throw new Exception("root subdirectory must have fileNameToCollection year at the end or directory name needs to be added to the exclude list!"); } } } string[] resizeMatch = (from l in sourceDirectoryNames where configuration.ValidResolutions.Contains(l) select l).ToArray(); if (resizeMatch.Length > 0) throw new Exception("Input directory should be the source and not fileNameToCollection resized directory!"); if (configuration.LocationDigits != Shared.Models.Stateless.ILocation.Digits) throw new Exception("Configuration has to match interface!"); if (configuration.LocationFactor != Shared.Models.Stateless.ILocation.Factor) throw new Exception("Configuration has to match interface!"); if (configuration.SaveSortingWithoutPerson && configuration.JLinks.Length > 0) throw new Exception("Configuration has SaveSortingWithoutPerson and JLinks!"); if (configuration.SaveSortingWithoutPerson && !string.IsNullOrEmpty(configuration.FocusModel)) throw new Exception("Configuration has SaveSortingWithoutPerson and FocusModel!"); if (configuration.SaveSortingWithoutPerson && !string.IsNullOrEmpty(configuration.FocusDirectory)) throw new Exception("Configuration has SaveSortingWithoutPerson and FocusDirectory!"); } private static void DeleteContinueFiles(ReadOnlyCollection personContainers) { foreach (PersonContainer personContainer in personContainers) { foreach (FilePath filePath in personContainer.DisplayDirectoryAllFilePaths) { if (filePath.ExtensionLowered != ".continue") continue; File.Delete(filePath.FullName); } } } private void MapFaceFileLogic(long ticks, ReadOnlyCollection personContainers, MapLogic mapLogic, string? a2PeopleContentDirectory, string eDistanceContentDirectory, ProgressBarOptions options) { foreach (string outputResolution in _Configuration.OutputResolutions) { if (_Exceptions.Count != 0) break; if (!_Configuration.SaveResizedSubfiles) continue; if (!_ArgZeroIsConfigurationRootDirectory) continue; if (outputResolution != _Configuration.OutputResolutions[0]) continue; if (_PropertyRootExistedBefore || a2PeopleContentDirectory is null) break; if (!_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution)) continue; if (!_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)) continue; List saveContainers = GetSaveContainers(ticks, personContainers, a2PeopleContentDirectory, eDistanceContentDirectory, options, mapLogic, outputResolution); if (saveContainers.Count > 0) { int updated = 0; mapLogic.SaveContainers(updated, saveContainers); throw new NotSupportedException("Done Restart! :)"); } } } private void Search(long ticks, ReadOnlyCollection personContainers, string argZero, string propertyRoot) { string message; MapLogic? mapLogic; A_Property propertyLogic; string eDistanceContentDirectory; string? a2PeopleContentDirectory; string aResultsFullGroupDirectory; string bResultsFullGroupDirectory; string cResultsFullGroupDirectory; string fPhotoPrismContentDirectory; const string fileSearchFilter = "*"; string fPhotoPrismSingletonDirectory; bool filesCollectionCountIsOne = false; const string directorySearchFilter = "*"; string? filesCollectionRootDirectory = null; bool configurationOutputResolutionsHas = false; ReadOnlyDictionary> personKeyToIds; ReadOnlyDictionary? splatNineIdentifiers = null; ReadOnlyCollection>? filePathsCollection = null; bool runToDoCollectionFirst = GetRunToDoCollectionFirst(_Configuration, ticks); (aResultsFullGroupDirectory, bResultsFullGroupDirectory) = GetResultsFullGroupDirectories(); ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(_Configuration.PropertyConfiguration.RootDirectory, ticks); a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "([])"); eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), _Configuration.PropertyConfiguration.ResultContent); string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), _Configuration.PropertyConfiguration.ResultSingleton); _ = Directory.CreateDirectory(Path.Combine(eDistanceContentDirectory, $"{ticks}")); if (runToDoCollectionFirst) mapLogic = null; else mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _Distance, personContainers, ticks, a2PeopleContentDirectory, a2PeopleSingletonDirectory, eDistanceContentDirectory); foreach (string outputResolution in _Configuration.OutputResolutions) { if (outputResolution.Any(char.IsNumber)) continue; configurationOutputResolutionsHas = true; if (!runToDoCollectionFirst) break; (filesCollectionRootDirectory, filePathsCollection, filesCollectionCountIsOne) = GetFilesCollectionThenCopyOrMove(ticks, fileSearchFilter, directorySearchFilter, options, outputResolution); splatNineIdentifiers = GetSplatNineIdentifiersAndHideSplatNine(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, filePathsCollection); break; } fPhotoPrismContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), _Configuration.PropertyConfiguration.ResultContent); fPhotoPrismSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), _Configuration.PropertyConfiguration.ResultSingleton); propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory); if (filesCollectionCountIsOne) { if (filePathsCollection is null) throw new NullReferenceException(nameof(filePathsCollection)); string resultsGroupDirectory; a2PeopleContentDirectory = null; eDistanceContentDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", _Configuration.PropertyConfiguration.ResultContent); fPhotoPrismContentDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", _Configuration.PropertyConfiguration.ResultContent); fPhotoPrismSingletonDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", _Configuration.PropertyConfiguration.ResultSingleton); for (int i = 1; i < 10; i++) { resultsGroupDirectory = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, string.Empty, create: true); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(resultsGroupDirectory); } argZero = SaveUrlAndGetNewRootDirectory(filePathsCollection.First()); _Configuration.PropertyConfiguration.ChangeRootDirectory(argZero); (aResultsFullGroupDirectory, bResultsFullGroupDirectory) = GetResultsFullGroupDirectories(); propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false); propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory); } if (configurationOutputResolutionsHas) { foreach (string outputResolution in _Configuration.OutputResolutions) { if (outputResolution.Any(char.IsNumber)) continue; (cResultsFullGroupDirectory, _, _, _) = GetResultsFullGroupDirectories(outputResolution); filesCollectionRootDirectory = Path.Combine(cResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent); filePathsCollection = IDirectory.GetFilePathCollections(_Configuration.PropertyConfiguration, directorySearchFilter, fileSearchFilter, filesCollectionRootDirectory, useCeilingAverage: true); break; } } if (filesCollectionRootDirectory is null || filePathsCollection is null) throw new NullReferenceException(nameof(filePathsCollection)); string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton); if (!Directory.Exists(aPropertySingletonDirectory)) _ = Directory.CreateDirectory(aPropertySingletonDirectory); int count = filePathsCollection.Select(l => l.Count).Sum(); message = $") Building Container(s) - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)"; _ProgressBar = new(count, message, options); ReadOnlyCollection readOnlyContainers = Shared.Models.Stateless.Methods.IContainer.GetContainers(this, _Configuration.PropertyConfiguration, aPropertySingletonDirectory, filesCollectionRootDirectory, splatNineIdentifiers, filePathsCollection); _ProgressBar.Dispose(); mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _Distance, personContainers, ticks, a2PeopleContentDirectory, a2PeopleSingletonDirectory, eDistanceContentDirectory); DeleteContinueFiles(personContainers); if (!runToDoCollectionFirst) MapFaceFileLogic(ticks, personContainers, mapLogic, a2PeopleContentDirectory, eDistanceContentDirectory, options); FullDoWork(argZero, propertyRoot, ticks, aResultsFullGroupDirectory, bResultsFullGroupDirectory, fPhotoPrismSingletonDirectory, count, readOnlyContainers, propertyLogic, mapLogic); ReadOnlyCollection distinctValidImageItems = Shared.Models.Stateless.Methods.IContainer.GetValidImageItems(_Configuration.PropertyConfiguration, readOnlyContainers, distinctItems: true, filterItems: true); if (_Configuration.LookForAbandoned) { string dResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; foreach (string outputResolution in _Configuration.OutputResolutions) { _ProgressBar = new(5, nameof(mapLogic.LookForAbandoned), options); (cResultsFullGroupDirectory, _, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); mapLogic.LookForAbandoned(this, _Configuration.PropertyConfiguration, bResultsFullGroupDirectory, readOnlyContainers, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory); _ProgressBar.Dispose(); } } _Distance.Clear(); ReadOnlyCollection distinctValidImageFaces = Map.Models.Stateless.Methods.IMapLogic.GetFaces(distinctValidImageItems); ReadOnlyCollection distinctValidImageMappingCollection = GetMappings(_Configuration.PropertyConfiguration, eDistanceContentDirectory, readOnlyContainers, mapLogic, distinctItems: true); if (runToDoCollectionFirst) { if (!Directory.Exists(eDistanceContentDirectory)) _ = Directory.CreateDirectory(eDistanceContentDirectory); string json = JsonSerializer.Serialize(distinctValidImageMappingCollection); File.WriteAllText(Path.Combine(eDistanceContentDirectory, $"{ticks}.json"), json); } foreach (string outputResolution in _Configuration.OutputResolutions) { if (_PropertyRootExistedBefore) break; personKeyToIds = mapLogic.GetPersonKeyToIds(); if (_Configuration.SavePropertyShortcutsForOutputResolutions.Contains(outputResolution)) SavePropertyShortcutsForOutputResolutions(eDistanceContentDirectory, distinctValidImageItems); if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution)) mapLogic.SaveShortcutsForOutputResolutionsPreMapLogic(eDistanceContentDirectory, personKeyToIds, distinctValidImageMappingCollection); if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.JLinks.Where(l => !string.IsNullOrEmpty(l)).Any() && _Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution)) mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, personContainers, a2PeopleContentDirectory, personKeyToIds, distinctValidImageMappingCollection); if (_ArgZeroIsConfigurationRootDirectory && _Configuration.SaveResizedSubfiles && outputResolution == _Configuration.OutputResolutions[0] && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution) && _Exceptions.Count == 0) MapLogic(ticks, readOnlyContainers, fPhotoPrismContentDirectory, mapLogic, outputResolution, new(personKeyToIds), distinctValidImageFaces, distinctValidImageMappingCollection); if (runToDoCollectionFirst && _Configuration.SaveRandomForOutputResolutions.Contains(outputResolution) && personKeyToIds.Count > 0 && splatNineIdentifiers is not null && distinctValidImageMappingCollection.Count > 0) _Random.Random(_Configuration.PropertyConfiguration, _Configuration.ImmichAssetsFile, _Configuration.ImmichOwnerId, _Configuration.ImmichRoot, _Configuration.RadomUseBirthdayMinimum, _Configuration.ValidKeyWordsToIgnoreInRandom, personKeyToIds, splatNineIdentifiers, distinctValidImageMappingCollection); if (_IsEnvironment.Development) continue; if (!_IsEnvironment.Development) { string dResultsFullGroupDirectory; string c2ResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; (cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(aResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton)); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(bResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton)); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(cResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton)); if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution)) _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultCollection)); } } } private bool GetRunToDoCollectionFirst(Models.Configuration configuration, long ticks) { bool result = configuration.SaveSortingWithoutPerson; if (!result) result = !IId.IsOffsetDeterministicHashCode(configuration.PropertyConfiguration); if (!result) { string[] directories; directories = Directory.GetDirectories(_Configuration.PropertyConfiguration.RootDirectory, "*", SearchOption.TopDirectoryOnly); if (directories.Length == 0) result = true; else { string seasonDirectory; DirectoryInfo directoryInfo; DateTime dateTime = new(ticks); string rootDirectory = _Configuration.PropertyConfiguration.RootDirectory; string eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), _Configuration.PropertyConfiguration.ResultContent); (int season, string seasonName) = Shared.Models.Stateless.Methods.IProperty.GetSeason(dateTime.DayOfYear); FileSystemInfo fileSystemInfo = new DirectoryInfo(eDistanceContentDirectory); string[] checkDirectories = [ Path.Combine(rootDirectory, "Ancestry"), Path.Combine(rootDirectory, "Facebook"), Path.Combine(rootDirectory, "LinkedIn") ]; foreach (string checkDirectory in checkDirectories) { if (checkDirectory == rootDirectory) seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName}"); else seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName} {Path.GetFileName(checkDirectory)}"); if (!Directory.Exists(seasonDirectory)) _ = Directory.CreateDirectory(seasonDirectory); if (result) continue; directories = Directory.GetDirectories(checkDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string directory in directories) { directoryInfo = new(directory); if (directoryInfo.LastWriteTime > fileSystemInfo.LastWriteTime) { result = true; break; } } } } } if (result) result = true; if (!result) result = false; return result; } private string SaveUrlAndGetNewRootDirectory(ReadOnlyCollection filePaths) { string result; if (filePaths.Count == 0) throw new NotSupportedException(); string? sourceDirectory = filePaths[0].DirectoryFullPath; if (string.IsNullOrEmpty(sourceDirectory)) throw new NotSupportedException(); Uri uri; string? line; string fileName; FilePath filePath; Task task; string relativePath; FileHolder fileHolder; string extensionLowered; string sourceDirectoryFile; HttpClient httpClient = new(); bool isValidImageFormatExtension; result = Path.GetDirectoryName(Path.GetDirectoryName(sourceDirectory)) ?? throw new NotSupportedException(); int length = result.Length; for (int y = 0; y < int.MaxValue; y++) { _Logger?.LogInformation("Enter fileNameToCollection url for fileNameToCollection image"); line = _Console.ReadLine(); if (string.IsNullOrEmpty(line)) break; uri = new(line); if (uri.HostNameType != UriHostNameType.Dns) continue; task = httpClient.GetByteArrayAsync(uri); fileName = Path.GetFileName(uri.LocalPath); sourceDirectoryFile = Path.Combine(sourceDirectory, fileName); File.WriteAllBytes(sourceDirectoryFile, task.Result); extensionLowered = Path.GetExtension(uri.LocalPath); relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFile, length, forceExtensionToLower: true); isValidImageFormatExtension = _Configuration.PropertyConfiguration.ValidImageFormatExtensions.Contains(extensionLowered); fileHolder = Shared.Models.Stateless.Methods.IFileHolder.Get(sourceDirectoryFile); filePath = FilePath.Get(_Configuration.PropertyConfiguration, fileHolder, index: null); _ = Item.Get(filePath, fileHolder, relativePath, isValidImageFormatExtension); // container.Items.Add(item); } _Logger?.LogInformation(". . ."); return result; } private void FullDoWork(string argZero, string propertyRoot, long ticks, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string fPhotoPrismSingletonDirectory, int count, ReadOnlyCollection readOnlyContainers, A_Property propertyLogic, MapLogic mapLogic) { int total; int notMapped; string message; bool exceptions; int totalSeconds; Container container; int totalNotMapped = 0; bool outputResolutionHasNumber; bool anyNullOrNoIsUniqueFileName; string cResultsFullGroupDirectory; string dResultsFullGroupDirectory; string c2ResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; ReadOnlyCollection filteredItems; int containersLength = readOnlyContainers.Count; List> sourceDirectoryChanges = []; int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism; Dictionary> fileNameToCollection = !Directory.Exists(fPhotoPrismSingletonDirectory) ? fileNameToCollection = [] : F_PhotoPrism.GetFileNameToCollection(fPhotoPrismSingletonDirectory); string dResultsDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(D_Face)); B_Metadata metadata = new(_Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, bResultsFullGroupDirectory); foreach (string outputResolution in _Configuration.OutputResolutions) { total = 0; outputResolutionHasNumber = outputResolution.Any(char.IsNumber); (cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); _Faces.Update(dResultsFullGroupDirectory); _Resize.Update(cResultsFullGroupDirectory); _FaceParts.Update(d2ResultsFullGroupDirectory); _BlurHasher.Update(c2ResultsFullGroupDirectory); for (int i = 0; i < readOnlyContainers.Count; i++) { container = readOnlyContainers[i]; if (container.Items.Count == 0) continue; if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero)) continue; filteredItems = Shared.Models.Stateless.Methods.IContainer.GetValidImageItems(_Configuration.PropertyConfiguration, container); if (filteredItems.Count == 0) continue; sourceDirectoryChanges.Clear(); anyNullOrNoIsUniqueFileName = filteredItems.Any(l => !l.IsUniqueFileName); totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); message = $"{i + 1:000} [{filteredItems.Count:000}] / {containersLength:000} - {total} / {count} total - {totalSeconds} total second(s) - {outputResolution} - <{container.SourceDirectory}> - total not mapped {totalNotMapped:000000}"; propertyLogic.SetAngleBracketCollection(aResultsFullGroupDirectory, container.SourceDirectory, anyNullOrNoIsUniqueFileName); if (outputResolutionHasNumber) _Resize.SetAngleBracketCollection(cResultsFullGroupDirectory, container.SourceDirectory); (notMapped, exceptions) = FullParallelWork(maxDegreeOfParallelism, propertyLogic, metadata, mapLogic, outputResolution, outputResolutionHasNumber, cResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, fileNameToCollection, container, filteredItems, message); totalNotMapped += notMapped; if (exceptions) { _Exceptions.Add(container.SourceDirectory); continue; } if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Length > 0) { for (int y = 0; y < int.MaxValue; y++) { _Logger?.LogInformation("Press \"Y\" key when ready to continue or close console"); if (_Console.ReadKey() == ConsoleKey.Y) break; } _Logger?.LogInformation(". . ."); } total += container.Items.Count; } totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); message = $"### [###] / {containersLength:000} - {total} / {count} total - {totalSeconds} total second(s) - {outputResolution} - <> - total not mapped {totalNotMapped:000000}"; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; using ProgressBar progressBar = new(1, message, options); progressBar.Tick(); } } private static ReadOnlyDictionary> GetKeyValuePairs(ReadOnlyCollection> filePathsCollection) { Dictionary> results = []; List? collection; Dictionary> keyValuePairs = []; foreach (ReadOnlyCollection filePaths in filePathsCollection) { if (filePaths.Count == 0) continue; foreach (FilePath filePath in filePaths) { if (filePath.Id is null) continue; if (!keyValuePairs.TryGetValue(filePath.Id.Value, out collection)) { keyValuePairs.Add(filePath.Id.Value, []); if (!keyValuePairs.TryGetValue(filePath.Id.Value, out collection)) throw new Exception(); } collection.Add(filePath); } } foreach (KeyValuePair> keyValuePair in keyValuePairs) results.Add(keyValuePair.Key, new(keyValuePair.Value)); return new(results); } private static ReadOnlyDictionary GetSplatNineIdentifiersAndHideSplatNine(Property.Models.Configuration propertyConfiguration, string bResultsFullGroupDirectory, ReadOnlyCollection> filePathsCollection) { Dictionary results = []; ReadOnlyDictionary> keyValuePairs = GetKeyValuePairs(filePathsCollection); if (keyValuePairs.Count > 0) { string json; string paddedId; FileInfo fileInfo; FilePath filePath; FileHolder fileHolder; Identifier identifier; string[] directoryNames; List distinct = []; List identifiers = []; string rootDirectory = propertyConfiguration.RootDirectory.Replace('\\', '/'); string bMetadataCollectionDirectory = Path.Combine(bResultsFullGroupDirectory, propertyConfiguration.ResultCollection); if (!Directory.Exists(bMetadataCollectionDirectory)) _ = Directory.CreateDirectory(bMetadataCollectionDirectory); foreach (KeyValuePair> keyValuePair in keyValuePairs) { for (int i = 0; i < keyValuePair.Value.Count; i++) { filePath = keyValuePair.Value[0]; if (filePath.Id is null) continue; directoryNames = keyValuePair.Value.Select(l => l.DirectoryFullPath.Replace('\\', '/')).ToArray(); paddedId = IId.GetPaddedId(propertyConfiguration, filePath.Id.Value, filePath.HasIgnoreKeyword, filePath.HasDateTimeOriginal, index: null); identifier = new(directoryNames, filePath.ExtensionLowered, filePath.HasDateTimeOriginal, filePath.Id.Value, filePath.Length, paddedId, filePath.LastWriteTicks); if (i == 0) identifiers.Add(identifier); if (!filePath.FullName.Contains(" !9")) continue; fileInfo = new(filePath.FullName); fileHolder = Shared.Models.Stateless.Methods.IFileHolder.Get(fileInfo); if (!fileInfo.Attributes.HasFlag(FileAttributes.Hidden)) File.SetAttributes(fileHolder.FullName, FileAttributes.Hidden); if (distinct.Contains(keyValuePair.Key)) continue; distinct.Add(keyValuePair.Key); results.Add(keyValuePair.Key, identifier); } } json = JsonSerializer.Serialize(results.Values.ToArray(), IdentifierCollectionSourceGenerationContext.Default.IdentifierArray); _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(Path.Combine(bMetadataCollectionDirectory, "!9.json"), json, updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null); json = JsonSerializer.Serialize((from l in identifiers orderby l.DirectoryNames.Length descending, l.Id select l).ToArray(), IdentifierCollectionSourceGenerationContext.Default.IdentifierArray); _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(Path.Combine(bMetadataCollectionDirectory, ".json"), json.Replace(rootDirectory, string.Empty), updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null); } return new(results); } private ReadOnlyCollection GetMappings(Property.Models.Configuration propertyConfiguration, string eDistanceContentDirectory, ReadOnlyCollection readOnlyContainers, MapLogic mapLogic, bool distinctItems) { List results = []; int count = 0; int notMapped; Mapping mapping; bool anyValidFaces; List distinct = []; string focusRelativePath; bool? isFocusRelativePath; DateTime[] containerDateTimes; MappingFromItem mappingFromItem; ReadOnlyCollection filteredItems; foreach (Container container in readOnlyContainers) { if (container.Items.Count == 0) continue; filteredItems = Shared.Models.Stateless.Methods.IContainer.GetValidImageItems(propertyConfiguration, container); if (filteredItems.Count == 0) continue; containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems); focusRelativePath = Path.GetFullPath(string.Concat(_Configuration.PropertyConfiguration.RootDirectory, _Configuration.FocusDirectory)); isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath); foreach (Item item in filteredItems) { if (item.Property?.Id is null || item.ResizedFileHolder is null) continue; mappingFromItem = IMappingFromItem.GetMappingFromItem(containerDateTimes, item, item.ResizedFileHolder); if (distinctItems) { if (distinct.Contains(item.Property.Id.Value)) continue; distinct.Add(item.Property.Id.Value); } count++; anyValidFaces = false; foreach (Shared.Models.Face face in item.Faces) { if (face.Mapping is null) throw new NotSupportedException(); if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null) continue; anyValidFaces = true; results.Add(face.Mapping); } if (!anyValidFaces) { (mapping, notMapped) = GetMappingAndUpdateMappingFromPerson(mapLogic, item, isFocusRelativePath, mappingFromItem); results.Add(mapping); } } } if (_Configuration.MoveToDecade && _Configuration.LocationContainerDistanceTolerance is null) _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(eDistanceContentDirectory); return new(results); } private static void SavePropertyShortcutsForOutputResolutions(string eDistanceContentDirectory, ReadOnlyCollection distinctValidImageItems) { #if VerifyItem bool found; List notFound = new(); foreach (Item item in distinctFilteredItems) { found = false; if (item.Property?.Id is null) continue; foreach (Mapping mapping in distinctFilteredMappingCollection) { if (mapping.MappingFromItem.Id != item.Property.Id.Value) continue; found = true; break; } if (!found) notFound.Add(item); } if (notFound.Count > 0) throw new NotSupportedException(); #endif string model; string fileName; string directory; bool? isWrongYear; List dateTimes; List distinct = []; WindowsShortcut windowsShortcut; List<(string, string, string)> collection = []; foreach (Item item in distinctValidImageItems) { if (item.Property?.Id is null) continue; if (item.IsNotUniqueAndNeedsReview is null || !item.IsNotUniqueAndNeedsReview.Value) continue; directory = Path.Combine($"{eDistanceContentDirectory[..^1]}{nameof(item.IsNotUniqueAndNeedsReview)})", item.FilePath.NameWithoutExtension); fileName = Path.Combine(directory, $"{item.FilePath.Length} {item.FilePath.LastWriteTicks}.lnk"); collection.Add((item.FilePath.FullName, directory, fileName)); if (distinct.Contains(directory)) continue; distinct.Add(directory); } foreach (Item item in distinctValidImageItems) { if (item.Property?.Id is null || item.Property.DateTimeOriginal is null) continue; dateTimes = item.Property.GetDateTimes(); (isWrongYear, _) = Shared.Models.Stateless.Methods.IProperty.IsWrongYear(item.FilePath, item.Property.DateTimeOriginal, dateTimes); if (isWrongYear is null || !isWrongYear.Value) continue; // Remove-Item -LiteralPath "\\?\D:\Tmp\a\EX-Z70 " model = string.IsNullOrEmpty(item.Property.Model) ? "Unknown" : CameraRegex().Replace(item.Property.Model.Trim(), "_"); directory = Path.Combine($"{eDistanceContentDirectory[..^1]}{nameof(Item)})", item.Property.DateTimeOriginal.Value.Year.ToString(), model); fileName = item.IsNotUniqueAndNeedsReview is not null && item.IsNotUniqueAndNeedsReview.Value ? Path.Combine(directory, $"{item.FilePath.Name} {item.FilePath.Length}.lnk") : Path.Combine(directory, $"{item.FilePath.Name}.lnk"); collection.Add((item.FilePath.FullName, directory, fileName)); if (distinct.Contains(directory)) continue; distinct.Add(directory); } #if Mapping foreach (Mapping mapping in distinctFilteredMappingCollection) { if (mapping.MappingFromItem.IsWrongYear is null || !mapping.MappingFromItem.IsWrongYear.Value) continue; directory = Path.Combine($"{eDistanceContentDirectory[..^1]}{nameof(Mapping)})", mapping.MappingFromItem.MinimumDateTime.Year.ToString()); fileName = Path.Combine(directory, $"{mapping.MappingFromItem.ResizedFileHolder.Name}.lnk"); collection.Add((mapping.MappingFromItem.ResizedFileHolder.FullName, directory, fileName)); if (distinct.Contains(directory)) continue; distinct.Add(directory); } #endif foreach (string distinctDirectory in distinct) { if (!Directory.Exists(distinctDirectory)) _ = Directory.CreateDirectory(distinctDirectory); } foreach ((string path, string checkDirectory, string checkFile) in collection) { if (File.Exists(checkFile)) continue; windowsShortcut = new() { Path = path }; windowsShortcut.Save(checkFile); windowsShortcut.Dispose(); } } private ReadOnlyCollection GetFilePath(long ticks, string dFacesContentDirectory) { List results = []; FilePath filePath; FileHolder fileHolder; string[] files = Directory.GetFiles(dFacesContentDirectory, $"*{_Faces.FileNameExtension}", SearchOption.AllDirectories); long? skipOlderThan = _Configuration.SkipOlderThanDays is null ? null : new DateTime(ticks).AddDays(-_Configuration.SkipOlderThanDays.Value).Ticks; foreach (string file in files) { fileHolder = Shared.Models.Stateless.Methods.IFileHolder.Get(file); filePath = FilePath.Get(_Configuration.PropertyConfiguration, fileHolder, index: null); if (filePath.Id is null) continue; if (skipOlderThan is not null && (fileHolder.LastWriteTime is null || fileHolder.LastWriteTime.Value.Ticks < skipOlderThan.Value)) continue; results.Add(filePath); } return new(results); } public ReadOnlyDictionary GetOnlyOne(IDistanceLimits distanceLimits, ReadOnlyCollection matrix) { Dictionary results = []; List added = []; LocationContainer? tryGetValue; foreach (LocationContainer locationContainer in matrix) { if (_Configuration.SaveIndividually) break; if (locationContainer.LengthSource is null) continue; if (_Configuration.UseExtraPersonKeyCheck) { if (results.TryGetValue(locationContainer.LengthSource.Name, out tryGetValue)) { if (locationContainer.PersonKey is not null && tryGetValue.PersonKey is not null && locationContainer.PersonKey.Value != tryGetValue.PersonKey) _ = results.Remove(locationContainer.LengthSource.Name); continue; } } if (added.Contains(locationContainer.LengthSource.Name)) continue; added.Add(locationContainer.LengthSource.Name); results.Add(locationContainer.LengthSource.Name, locationContainer); } return new(results); } private List GetSaveContainers(long ticks, ReadOnlyCollection personContainers, string a2PeopleSingletonDirectory, string eDistanceContentDirectory, ProgressBarOptions options, MapLogic mapLogic, string outputResolution) { List results; long[] jLinkResolvedPersonKeys = _JLinkResolvedDirectories.Select(l => l.PersonKey).ToArray(); (string cResultsFullGroupDirectory, string _, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent); ReadOnlyDictionary> mapped = Map.Models.Stateless.Methods.IMapLogic.GetMapped(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, personContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory); if (mapped.Count == 0 && !_Configuration.SaveSortingWithoutPerson) throw new NotSupportedException($"Switch {nameof(_Configuration.SaveSortingWithoutPerson)}!"); ReadOnlyCollection filePaths = GetFilePath(ticks, dFacesContentDirectory); List available = Map.Models.Stateless.Methods.IMapLogic.GetAvailable(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, _Faces, ticks, filePaths); if (!string.IsNullOrEmpty(_Configuration.FocusDirectory) && _Configuration.FocusDirectory.Length != 2) throw new NotSupportedException($"{nameof(_Configuration.FocusDirectory)} currently only works with output directory! Example 00."); ReadOnlyDictionary> mappedWithEncoding = E_Distance.GetMappedWithEncoding(mapped); if (mappedWithEncoding.Count == 0 && !_Configuration.SaveSortingWithoutPerson) throw new NotSupportedException($"Switch {nameof(_Configuration.SaveSortingWithoutPerson)}!"); List preFiltered = E_Distance.GetPreFilterLocationContainer(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, _Configuration.FocusDirectory, _Configuration.FocusModel, _Configuration.SkipPersonWithMoreThen, ticks, mapLogic, jLinkResolvedPersonKeys, mapped, available); if (preFiltered.Count == 0) results = []; else { DistanceLimits distanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaPermyriadTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh); List postFiltered = E_Distance.GetPostFilterLocationContainer(mapLogic, preFiltered, distanceLimits); if (postFiltered.Count == 0) results = []; else { string message = $") Building Matrix - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)"; _ProgressBar = new(postFiltered.Count, message, options); ReadOnlyCollection matrix = E_Distance.GetMatrixLocationContainers(this, _MapConfiguration, ticks, mapLogic, mappedWithEncoding, preFiltered, distanceLimits, postFiltered); _ProgressBar.Dispose(); ReadOnlyDictionary onlyOne = GetOnlyOne(distanceLimits, matrix); if (onlyOne.Count == 0) results = []; else { ReadOnlyDictionary personKeyToPersonContainer = mapLogic.GetPersonKeyToPersonContainer(); results = mapLogic.GetSaveContainers(cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, distanceLimits, onlyOne, personKeyToPersonContainer); } } } return results; } private void MapLogic(long ticks, ReadOnlyCollection containers, string fPhotoPrismContentDirectory, MapLogic mapLogic, string outputResolution, ReadOnlyDictionary> personKeyToIds, ReadOnlyCollection distinctValidImageFaces, ReadOnlyCollection distinctValidImageMappingCollection) { (_, _, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent); string d2FacePartsContentDirectory = Path.Combine(d2ResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent); string d2FacePartsContentCollectionDirectory = Path.Combine(d2ResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContentCollection); if (distinctValidImageMappingCollection.Count > 0) { Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(d2FacePartsContentDirectory, ticks); if (Directory.Exists(d2FacePartsContentCollectionDirectory)) Shared.Models.Stateless.Methods.IPath.MakeHiddenIfAllItemsAreHidden(d2FacePartsContentCollectionDirectory); } if (Directory.Exists(fPhotoPrismContentDirectory)) F_PhotoPrism.WriteMatches(fPhotoPrismContentDirectory, _Configuration.PersonBirthdayFormat, _Configuration.RectangleIntersectMinimums, ticks, distinctValidImageFaces, mapLogic); if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution)) mapLogic.SaveShortcutsForOutputResolutionsDuringMapLogic(containers, personKeyToIds, dFacesContentDirectory, distinctValidImageMappingCollection); ReadOnlyDictionary> idToWholePercentagesToMapping = Map.Models.Stateless.Methods.IMapLogic.GetIdToWholePercentagesToFace(distinctValidImageMappingCollection); if (_Configuration.SaveMappedForOutputResolutions.Contains(outputResolution)) mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, personKeyToIds, distinctValidImageMappingCollection, idToWholePercentagesToMapping); if (_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution)) SaveFaceDistances(ticks, mapLogic, distinctValidImageFaces, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping); } private bool? GetIsFocusModel(Shared.Models.Property? property) { bool? result; if (string.IsNullOrEmpty(_Configuration.FocusModel)) result = null; else if (property is null || string.IsNullOrEmpty(property.Model)) result = null; else result = property.Model.Contains(_Configuration.FocusModel); return result; } private void LogItemPropertyIsNull(Item item) { if (!item.SourceDirectoryFileHolder.Exists) _Logger?.LogInformation(string.Concat("NoJson <", item.FilePath.FullName, '>')); else if (item.FileSizeChanged.HasValue && item.FileSizeChanged.Value) _Logger?.LogInformation(string.Concat("FileSizeChanged <", item.FilePath.FullName, '>')); else if (item.LastWriteTimeChanged.HasValue && item.LastWriteTimeChanged.Value) _Logger?.LogInformation(string.Concat("LastWriteTimeChanged <", item.FilePath.FullName, '>')); else if (item.Moved.HasValue && item.Moved.Value) _Logger?.LogInformation(string.Concat("Moved <", item.FilePath.FullName, '>')); } private int GetNotMappedCountAndUpdateMappingFromPersonThenSetMapping(MapLogic mapLogic, Item item, bool? isFocusRelativePath, MappingFromItem mappingFromItem, List? mappingFromPhotoPrismCollection, List faces) { int result; double? α; int? eyeα; bool? canReMap; bool? eyeReview; Mapping mapping; int notMapped = 0; bool? isFocusPerson; int confidencePercent; int faceAreaPermyriad; bool? inSkipCollection; int wholePercentRectangle; string deterministicHashCodeKey; MappingFromLocation? mappingFromLocation; MappingFromFilterPre mappingFromFilterPre; MappingFromFilterPost mappingFromFilterPost; bool? isFocusModel = GetIsFocusModel(item.Property); ReadOnlyDictionary>? wholePercentagesToPersonContainers = mapLogic.GetWholePercentagesToPersonContainers(item.Property?.Id); long[] jLinkResolvedPersonKeys = _JLinkResolvedDirectories.Select(l => l.PersonKey).ToArray(); foreach (Shared.Models.Face face in faces) { if (item.Property?.Id is null || face.FaceEncoding is null || face.Location is null || face.OutputResolution is null) { canReMap = null; isFocusPerson = null; inSkipCollection = null; mappingFromLocation = null; mappingFromFilterPost = new(canReMap, inSkipCollection, isFocusPerson); mappingFromFilterPre = new(inSkipCollection, isFocusModel, isFocusRelativePath); } else { if (face.FaceParts is null) (eyeα, eyeReview) = (null, null); else { (eyeReview, α) = Shared.Models.Stateless.Methods.IFace.GetEyeα(face.FaceParts); eyeα = α is null ? null : (int)Math.Round(Math.Abs(α.Value)); } confidencePercent = Shared.Models.Stateless.Methods.ILocation.GetConfidencePercent(_Configuration.FaceConfidencePercent, face.Location.Confidence); faceAreaPermyriad = IMapping.GetAreaPermyriad(_Configuration.FaceAreaPermyriad, face.Location, face.OutputResolution); wholePercentRectangle = Shared.Models.Stateless.Methods.ILocation.GetWholePercentages(face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution); deterministicHashCodeKey = IMapping.GetDeterministicHashCodeKey(item.FilePath, face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution); mappingFromLocation = new(faceAreaPermyriad, confidencePercent, deterministicHashCodeKey, eyeα, eyeReview, wholePercentRectangle); inSkipCollection = mapLogic.InSkipCollection(item.Property.Id.Value, mappingFromLocation); mappingFromFilterPre = new(inSkipCollection, isFocusModel, isFocusRelativePath); canReMap = Map.Models.Stateless.Methods.IMapLogic.CanReMap(jLinkResolvedPersonKeys, wholePercentagesToPersonContainers, mappingFromLocation); isFocusPerson = mapLogic.IsFocusPerson(_Configuration.SkipPersonWithMoreThen, jLinkResolvedPersonKeys, wholePercentagesToPersonContainers, mappingFromLocation); mappingFromFilterPost = new(canReMap, inSkipCollection, isFocusPerson); } mapping = Mapping.Get(item.FilePath, mappingFromFilterPost, mappingFromFilterPre, mappingFromItem, mappingFromLocation, mappingFromPhotoPrismCollection); notMapped += mapLogic.UpdateMappingFromPerson(wholePercentagesToPersonContainers, mapping); face.SetMapping(mapping); } result = notMapped; return result; } private static Map.Models.Configuration Get(Models.Configuration configuration, IDistanceLimits distanceLimits, string facesFileNameExtension, string facesHiddenFileNameExtension, string facePartsFileNameExtension) { Map.Models.Configuration result = new(distanceLimits, configuration.PropertyConfiguration, configuration.DeletePossibleDuplicates, configuration.DistanceMoveUnableToMatch, configuration.DistanceRenameToMatch, configuration.FaceConfidencePercent, configuration.FaceDistancePermyriad, configuration.LinkedAlpha, configuration.LocationContainerDebugDirectory, configuration.LocationContainerDirectoryPattern, configuration.LocationContainerDistanceGroupMinimum, configuration.LocationContainerDistanceTake, configuration.LocationContainerDistanceTolerance, configuration.LocationDigits, configuration.MappingDefaultName, configuration.PersonBirthdayFirstYear, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), configuration.RangeDaysDeltaTolerance, configuration.RangeDistanceTolerance, configuration.ReMap, configuration.SaveIndividually, configuration.SaveSortingWithoutPerson, configuration.SkipNotSkipDirectories, configuration.SortingMaximumPerKey, configuration.SortingMinimumToUseSigma, facesFileNameExtension, facesHiddenFileNameExtension, facePartsFileNameExtension); return result; } private (Mapping, int) GetMappingAndUpdateMappingFromPerson(MapLogic mapLogic, Item item, bool? isFocusRelativePath, MappingFromItem mappingFromItem) { Mapping result; int? eyeα = null; bool? eyeReview = null; int confidencePercent = 0; int faceAreaPermyriad = 0; bool? isFocusModel = GetIsFocusModel(item.Property); long[] jLinkResolvedPersonKeys = _JLinkResolvedDirectories.Select(l => l.PersonKey).ToArray(); int wholePercentRectangle = Shared.Models.Stateless.Methods.ILocation.GetWholePercentages(Shared.Models.Stateless.ILocation.Digits); string deterministicHashCodeKey = IMapping.GetDeterministicHashCodeKey(item.FilePath, Shared.Models.Stateless.ILocation.Digits); MappingFromLocation? mappingFromLocation = new(faceAreaPermyriad, confidencePercent, deterministicHashCodeKey, eyeα, eyeReview, wholePercentRectangle); bool? inSkipCollection = mapLogic.InSkipCollection(mappingFromItem.Id, mappingFromLocation); MappingFromFilterPre mappingFromFilterPre = new(inSkipCollection, isFocusModel, isFocusRelativePath); ReadOnlyDictionary>? wholePercentagesToPersonContainers = mapLogic.GetWholePercentagesToPersonContainers(mappingFromItem.Id); bool? canReMap = Map.Models.Stateless.Methods.IMapLogic.CanReMap(jLinkResolvedPersonKeys, wholePercentagesToPersonContainers, mappingFromLocation); bool? isFocusPerson = mapLogic.IsFocusPerson(_Configuration.SkipPersonWithMoreThen, jLinkResolvedPersonKeys, wholePercentagesToPersonContainers, mappingFromLocation); MappingFromFilterPost mappingFromFilterPost = new(canReMap, inSkipCollection, isFocusPerson); result = Mapping.Get(item.FilePath, mappingFromFilterPost, mappingFromFilterPre, mappingFromItem, mappingFromLocation, mappingFromPhotoPrismCollection: null); int notMapped = mapLogic.UpdateMappingFromPerson(wholePercentagesToPersonContainers, result); return (result, notMapped); } private int FullParallelForWork(A_Property propertyLogic, B_Metadata metadata, MapLogic mapLogic, string outputResolution, bool outputResolutionHasNumber, string cResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List> sourceDirectoryChanges, Dictionary> fileNameToCollection, Container container, int index, Item item, DateTime[] containerDateTimes, bool? isFocusRelativePath) { int result = 0; List faces; long ticks = DateTime.Now.Ticks; DateTime dateTime = DateTime.Now; Shared.Models.Property? property; List parseExceptions = []; string[] changesFrom = [nameof(A_Property)]; List> subFileTuples = []; FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(cResultsFullGroupDirectory, item, outputResolutionHasNumber); if (item.Property is null || item.Property.Id is null || !item.SourceDirectoryFileHolder.Exists || item.SourceDirectoryFileHolder.CreationTime is null || item.SourceDirectoryFileHolder.LastWriteTime is null || item.Any()) { LogItemPropertyIsNull(item); int? propertyHashCode = item.Property?.GetHashCode(); property = propertyLogic.GetProperty(metadata, item, subFileTuples, parseExceptions); item.Update(property); if (propertyHashCode is null) { lock (sourceDirectoryChanges) sourceDirectoryChanges.Add(new Tuple(nameof(A_Property), DateTime.Now)); } else if (propertyHashCode.Value != property.GetHashCode()) { lock (sourceDirectoryChanges) sourceDirectoryChanges.Add(new Tuple(nameof(A_Property), DateTime.Now)); } } else { property = item.Property; if (_Configuration.PropertyConfiguration.ForcePropertyLastWriteTimeToCreationTime && item.SourceDirectoryFileHolder.LastWriteTime.Value != item.SourceDirectoryFileHolder.CreationTime.Value) { File.SetLastWriteTime(item.SourceDirectoryFileHolder.FullName, item.SourceDirectoryFileHolder.CreationTime.Value); subFileTuples.Add(new Tuple(nameof(A_Property), item.SourceDirectoryFileHolder.CreationTime.Value)); } else if (item.SourceDirectoryFileHolder.LastWriteTime is not null) subFileTuples.Add(new Tuple(nameof(A_Property), item.SourceDirectoryFileHolder.LastWriteTime.Value)); else subFileTuples.Add(new Tuple(nameof(A_Property), new FileInfo(item.SourceDirectoryFileHolder.FullName).LastWriteTime)); if (resizedFileHolder.Exists && item.Property.Width is not null && item.Property.Width.Value > 4 && _Configuration.SaveBlurHashForOutputResolutions.Contains(outputResolution)) { string? file = _BlurHasher.GetFile(item.FilePath); if (file is not null && !File.Exists(file)) _ = _BlurHasher.EncodeAndSave(item.FilePath, resizedFileHolder); } } bool? shouldIgnore = property is null || property.Keywords is null ? null : _Configuration.PropertyConfiguration.IgnoreRulesKeyWords.Any(l => property.Keywords.Contains(l)); if (shouldIgnore is not null) { if (shouldIgnore.Value) { FileInfo fileInfo = new(resizedFileHolder.FullName); if (!fileInfo.Attributes.HasFlag(FileAttributes.Hidden)) File.SetAttributes(resizedFileHolder.FullName, FileAttributes.Hidden); } if (resizedFileHolder.Exists && item.FilePath.HasIgnoreKeyword is not null && item.FilePath.HasIgnoreKeyword.Value != shouldIgnore.Value) { if (!item.FilePath.DirectoryFullPath.Contains("Results") || !item.FilePath.DirectoryFullPath.Contains("Resize")) throw new NotSupportedException($"Rename File! <{item.FilePath.FileNameFirstSegment}>"); else { File.Delete(resizedFileHolder.FullName); } } } if (property is null || item.Property is null) throw new NullReferenceException(nameof(property)); item.SetResizedFileHolder(_Resize.FileNameExtension, resizedFileHolder); MappingFromItem mappingFromItem = IMappingFromItem.GetMappingFromItem(containerDateTimes, item, resizedFileHolder); ExifDirectory exifDirectory = metadata.GetMetadataCollection(item.FilePath, subFileTuples, parseExceptions, changesFrom, mappingFromItem); if (_AppSettings.Places.Count > 0) { float latitude; float longitude; double? distance; MetadataExtractor.GeoLocation? geoLocation = Metadata.Models.Stateless.Methods.IMetadata.GeoLocation(exifDirectory); foreach (Place place in _AppSettings.Places) { if (geoLocation is null) continue; latitude = Math.Abs(place.Latitude.Degrees) + (place.Latitude.Minutes / 60) + (place.Latitude.Seconds / 3600); if (place.Latitude.Degrees < 0) latitude *= -1; longitude = Math.Abs(place.Longitude.Degrees) + (place.Longitude.Minutes / 60) + (place.Longitude.Seconds / 3600); if (place.Longitude.Degrees < 0) longitude *= -1; distance = geoLocation is null ? null : Metadata.Models.Stateless.Methods.IMetadata.GetDistance(latitude, longitude, geoLocation.Latitude, geoLocation.Longitude); if (distance is null or > 3) continue; distance += 1; } } Dictionary outputResolutionToResize = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, item.Property, mappingFromItem); if (_Configuration.SaveResizedSubfiles) { if (shouldIgnore is not null && item.FilePath.HasIgnoreKeyword is not null && item.FilePath.HasIgnoreKeyword.Value != shouldIgnore.Value) faces = []; else _Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.Property, mappingFromItem, outputResolutionToResize); } if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution)) faces = []; else if (!mappingFromItem.ResizedFileHolder.Exists && !File.Exists(mappingFromItem.ResizedFileHolder.FullName)) faces = []; else { List? mappingFromPhotoPrismCollection; if (!fileNameToCollection.TryGetValue(mappingFromItem.Id, out mappingFromPhotoPrismCollection)) mappingFromPhotoPrismCollection = null; bool move = _Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution); faces = _Faces.GetFaces(outputResolution, cResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionToResize, mappingFromPhotoPrismCollection); result = GetNotMappedCountAndUpdateMappingFromPersonThenSetMapping(mapLogic, item, isFocusRelativePath, mappingFromItem, mappingFromPhotoPrismCollection, faces); List<(Shared.Models.Face, FileHolder?, string, bool Saved)> faceCollection = _Faces.SaveFaces(item.FilePath, subFileTuples, parseExceptions, mappingFromItem, exifDirectory, faces); if (move && faceCollection.All(l => !l.Saved)) { ReadOnlyCollection locationContainers = mapLogic.GetLocationContainers(item); if (locationContainers.Count > 0) { Map.Models.Stateless.Methods.IMapLogic.SetCreationTime(mappingFromItem, locationContainers); if (_Configuration.LocationContainerDistanceTolerance is null && _Configuration.MoveToDecade) Map.Models.Stateless.Methods.IMapLogic.MoveToDecade(_Configuration.PropertyConfiguration, mappingFromItem, locationContainers); _Distance.LookForMatchFacesAndPossiblyRename(_Configuration.OverrideForFaceImages, _DistanceLimits, _Faces, item.FilePath, mappingFromItem, exifDirectory, faces, locationContainers); } } (bool review, int[] eyesCollection) = Shared.Models.Stateless.Methods.IFace.GetEyeCollection(_Configuration.EyeThreshold, faces); if (review || _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution)) { bool saveRotated = false; string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(_Configuration.PropertyConfiguration, container.SourceDirectory); _FaceParts.SaveFaceLandmarkImages(_Configuration.PropertyConfiguration, d2ResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, mappingFromItem, exifDirectory, faces, saveRotated); } if (_Configuration.SaveFaceLandmarkForOutputResolutionsV2.Contains(outputResolution)) _FaceParts.SaveFaceLandmarkImages(d2ResultsFullGroupDirectory, mappingFromItem, exifDirectory, faces); } lock (sourceDirectoryChanges) { item.Faces.AddRange(faces); sourceDirectoryChanges.AddRange(from l in subFileTuples where l.Item2 > dateTime select l); } return result; } private (int, bool) FullParallelWork(int maxDegreeOfParallelism, A_Property propertyLogic, B_Metadata metadata, MapLogic mapLogic, string outputResolution, bool outputResolutionHasNumber, string cResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List> sourceDirectoryChanges, Dictionary> fileNameToCollection, Container container, ReadOnlyCollection filteredItems, string message) { int result = 0; int exceptionsCount = 0; ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; DateTime[] containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems); ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; string focusRelativePath = Path.GetFullPath(string.Concat(_Configuration.PropertyConfiguration.RootDirectory, _Configuration.FocusDirectory)); bool? isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath); bool anyPropertiesChangedForX = _Configuration.PropertyConfiguration.PropertiesChangedForProperty || _Configuration.PropertiesChangedForDistance || _Configuration.PropertiesChangedForFaces || _Configuration.PropertiesChangedForIndex || _Configuration.PropertiesChangedForMetadata || _Configuration.PropertiesChangedForResize; using ProgressBar progressBar = new(filteredItems.Count, message, options); _ = Parallel.For(0, filteredItems.Count, parallelOptions, (i, state) => { try { result += FullParallelForWork(propertyLogic, metadata, mapLogic, outputResolution, outputResolutionHasNumber, cResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, fileNameToCollection, container, index: i, filteredItems[i], containerDateTimes, isFocusRelativePath); if (!anyPropertiesChangedForX && (i == 0 || sourceDirectoryChanges.Count > 0)) progressBar.Tick(); } catch (Exception) { exceptionsCount++; if (exceptionsCount == filteredItems.Count) throw new Exception(string.Concat("All in [", container.SourceDirectory, "] failed!")); } }); return (result, exceptionsCount > 0); } private (string, string) GetResultsFullGroupDirectories() { string aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(A_Property), string.Empty, includeResizeGroup: false, includeModel: false, includePredictorModel: false); string bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(B_Metadata), string.Empty, includeResizeGroup: false, includeModel: false, includePredictorModel: false); return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory); } private (string, string, string, string) GetResultsFullGroupDirectories(string outputResolution) { string cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(C_Resize), outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false); string c2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(BlurHash.Models.C2_BlurHasher), outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false); string dResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true); string d2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(D2_FaceParts), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true); return new(cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory); } private void SaveFaceDistances(long ticks, MapLogic mapLogic, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, ReadOnlyDictionary> idToWholePercentagesToMapping, ReadOnlyCollection faceDistanceEncodings, ReadOnlyCollection faceDistanceContainers) { int? useFiltersCounter = null; DistanceLimits distanceLimits; ReadOnlyCollection sortingContainers; FaceDistanceContainer[] filteredFaceDistanceContainers; long? skipOlderThan = _Configuration.SkipOlderThanDays is null ? null : new DateTime(ticks).AddDays(-_Configuration.SkipOlderThanDays.Value).Ticks; distanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaPermyriadTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh); filteredFaceDistanceContainers = E_Distance.FilteredPostLoadFaceDistanceContainers(mapLogic, faceDistanceContainers, skipOlderThan, distanceLimits); if (filteredFaceDistanceContainers.Length == 0) _Logger?.LogInformation("All images have been filtered!"); else { sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortedSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distanceLimits, faceDistanceEncodings, filteredFaceDistanceContainers); if (sortingContainers.Count == 0) { for (useFiltersCounter = 1; useFiltersCounter < _Configuration.UseFilterTries; useFiltersCounter++) { distanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaPermyriadTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh, useFiltersCounter); filteredFaceDistanceContainers = E_Distance.FilteredPostLoadFaceDistanceContainers(mapLogic, faceDistanceContainers, skipOlderThan, distanceLimits); if (filteredFaceDistanceContainers.Length == 0) _Logger?.LogInformation("All images have been filtered!"); else { sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortedSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distanceLimits, faceDistanceEncodings, filteredFaceDistanceContainers); if (sortingContainers.Count == 0) break; } } } sortingContainers = mapLogic.GetFilterSortingContainers(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, distanceLimits, sortingContainers); if (sortingContainers.Count > 0) E_Distance.SaveFaceDistances(_Configuration.PropertyConfiguration, sortingContainers); if (filteredFaceDistanceContainers.Length > 0) { int updated = sortingContainers.Count == 0 ? 0 : mapLogic.UpdateFromSortingContainers(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, sortingContainers); List saveContainers = mapLogic.GetSaveContainers(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, distanceLimits, useFiltersCounter, sortingContainers); if (saveContainers.Count > 0) mapLogic.SaveContainers(updated, saveContainers); } } } private void SaveFaceDistances(long ticks, MapLogic mapLogic, ReadOnlyCollection distinctValidImageFaces, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, ReadOnlyDictionary> idToWholePercentagesToMapping) { E_Distance.PreFilterSetFaceDistances(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, distinctValidImageFaces); ReadOnlyCollection faceDistanceContainers = E_Distance.GetFaceDistanceContainers(distinctValidImageFaces); if (faceDistanceContainers.Count > 0) { List faceDistanceEncodings = []; foreach (FaceDistanceContainer faceDistanceContainer in faceDistanceContainers) { if (faceDistanceContainer.FaceDistance.Encoding is null) continue; faceDistanceEncodings.Add(faceDistanceContainer.FaceDistance); } SaveFaceDistances(ticks, mapLogic, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, new(faceDistanceEncodings), faceDistanceContainers); } } private static void CheckForAllWindowsLinks(ReadOnlyCollection> filePathsCollection) { string fileFullPath; WindowsShortcut windowsShortcut; foreach (ReadOnlyCollection filePaths in filePathsCollection) { if (filePaths.Count == 0) continue; if (filePaths.All(l => l.ExtensionLowered == ".lnk")) { foreach (FilePath filePath in filePaths) { windowsShortcut = WindowsShortcut.Load(filePath.FullName); if (windowsShortcut.Path is null) { File.Delete(filePath.FullName); continue; } fileFullPath = windowsShortcut.Path; windowsShortcut.Dispose(); File.WriteAllText(Path.ChangeExtension(filePath.FullName, ".url"), fileFullPath); File.Delete(filePath.FullName); } throw new NotSupportedException("All are Windows *.lnk files!"); } } } private static bool IsFilesCollectionCountIsOne(ReadOnlyCollection> filePathsCollection) { bool result = true; int count = 0; foreach (ReadOnlyCollection filePaths in filePathsCollection) { if (filePaths.Count == 0) continue; count += 1; if (count > 1) { result = false; break; } } if (result) CheckForAllWindowsLinks(filePathsCollection); return result; } private (string, ReadOnlyCollection>, bool) GetFilesCollectionThenCopyOrMove(long ticks, string fileSearchFilter, string directorySearchFilter, ProgressBarOptions options, string outputResolution) { ProgressBar progressBar; string filesCollectionRootDirectory = _Configuration.PropertyConfiguration.RootDirectory; (string cResultsFullGroupDirectory, _, _, _) = GetResultsFullGroupDirectories(outputResolution); IReadOnlyDictionary> fileGroups = Shared.Models.Stateless.Methods.IPath.GetKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, [_Configuration.PropertyConfiguration.ResultContent, _Configuration.PropertyConfiguration.ResultContentCollection]); ReadOnlyCollection> filePathsCollection = IDirectory.GetFilePathCollections(_Configuration.PropertyConfiguration, directorySearchFilter, fileSearchFilter, filesCollectionRootDirectory, useCeilingAverage: false); int count = filePathsCollection.Select(l => l.Count).Sum(); bool filesCollectionCountIsOne = IsFilesCollectionCountIsOne(filePathsCollection); string message = $") Selecting for ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)"; progressBar = new(count, message, options); (string[] distinctDirectories, List<(FilePath, string)> toDoCollection) = IDirectory.GetToDoCollection(_Configuration.PropertyConfiguration, filePathsCollection, fileGroups, () => progressBar.Tick()); progressBar.Dispose(); foreach (string distinctDirectory in distinctDirectories) { if (!Directory.Exists(distinctDirectory)) _ = Directory.CreateDirectory(distinctDirectory); } message = $") Copying to ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)"; progressBar = new(count, message, options); _ = IDirectory.CopyOrMove(toDoCollection, move: false, moveBack: false, () => progressBar.Tick()); progressBar.Dispose(); return (filesCollectionRootDirectory, filePathsCollection, filesCollectionCountIsOne); } }