using Microsoft.Extensions.Configuration; using Phares.Shared; using ShellProgressBar; using System.Collections.ObjectModel; using System.Drawing.Imaging; using View_by_Distance.Distance.Models; using View_by_Distance.Face.Models; using View_by_Distance.FaceParts.Models; using View_by_Distance.Instance.Models; using View_by_Distance.Map.Models; using View_by_Distance.Metadata.Models; using View_by_Distance.PhotoPrism.Models; using View_by_Distance.Property.Models; using View_by_Distance.Resize.Models; using View_by_Distance.Shared.Models; using View_by_Distance.Shared.Models.Methods; namespace View_by_Distance.Instance; public partial class DlibDotNet { private readonly D_Face _Faces; private readonly C_Resize _Resize; private readonly F_Random _Random; private readonly IConsole _Console; private readonly E_Distance _Distance; private readonly Serilog.ILogger? _Log; private readonly D2_FaceParts _FaceParts; private readonly AppSettings _AppSettings; private readonly List _Exceptions; private readonly IsEnvironment _IsEnvironment; private readonly bool _PropertyRootExistedBefore; private readonly Models.Configuration _Configuration; private readonly List _PersonContainers; private readonly bool _ArgZeroIsConfigurationRootDirectory; private readonly Map.Models.Configuration _MapConfiguration; private readonly string[]? _GenealogicalDataCommunicationFooterLines; private readonly string[]? _GenealogicalDataCommunicationHeaderLines; public DlibDotNet( List args, IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, AppSettings appSettings, string workingDirectory, bool isSilent, IConsole console) { string message; _Console = console; _AppSettings = appSettings; _IsEnvironment = isEnvironment; long ticks = DateTime.Now.Ticks; _Exceptions = new List(); _Log = Serilog.Log.ForContext(); Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot); Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration); _Log.Information(propertyConfiguration.RootDirectory); Property.Models.Configuration.Verify(propertyConfiguration, requireExist: false); Verify(configuration); VerifyExtra(args, propertyConfiguration, configuration); _Configuration = configuration; _Random = new(configuration); if (configuration.IgnoreExtensions is null) throw new NullReferenceException(nameof(configuration.IgnoreExtensions)); string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(propertyConfiguration, nameof(A_Property), create: false); _PropertyRootExistedBefore = !Directory.Exists(propertyRoot); string argZero = args.Count > 0 ? Path.GetFullPath(args[0]) : Path.GetFullPath(propertyConfiguration.RootDirectory); _ArgZeroIsConfigurationRootDirectory = propertyConfiguration.RootDirectory == argZero; if (!Directory.Exists(argZero)) _ = Directory.CreateDirectory(argZero); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(argZero); if (!Directory.Exists(argZero)) _ = Directory.CreateDirectory(argZero); _Log.Information(configuration.ModelDirectory); { (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetPngLowQuality(); (ImageCodecInfo hiddenImageCodecInfo, EncoderParameters hiddenEncoderParameters, string hiddenFileNameExtension) = C_Resize.GetGifLowQuality(); _Faces = new D_Face( argZero, configuration.CheckDFaceAndUpWriteDates, configuration.PropertyConfiguration, encoderParameters, configuration.FaceDistanceHiddenImageFactor, filenameExtension, configuration.ForceFaceLastWriteTimeToCreationTime, hiddenEncoderParameters, hiddenFileNameExtension, hiddenImageCodecInfo, imageCodecInfo, configuration.ModelDirectory, configuration.ModelName, configuration.OverrideForFaceImages, configuration.PredictorModelName, configuration.PropertiesChangedForFaces); } { (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetGifLowQuality(); _FaceParts = new D2_FaceParts(imageCodecInfo, encoderParameters, filenameExtension, configuration.CheckDFaceAndUpWriteDates, configuration.OverrideForFaceLandmarkImages); } _MapConfiguration = Get(configuration, _Faces.FileNameExtension, _Faces.HiddenFileNameExtension, _FaceParts.FileNameExtension); _Distance = new(configuration.DistanceMoveUnableToMatch, configuration.DistanceRenameToMatch, _Configuration.FaceConfidencePercent, configuration.RangeFaceConfidence); if (_PropertyRootExistedBefore || !_ArgZeroIsConfigurationRootDirectory) { _GenealogicalDataCommunicationFooterLines = null; _GenealogicalDataCommunicationHeaderLines = null; _PersonContainers = new(); } else { int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); message = $") Building People Collection - {totalSeconds} total second(s)"; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; using ProgressBar progressBar = new(1, message, options); progressBar.Tick(); string rootDirectory = propertyConfiguration.RootDirectory; string peopleRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(A2_People)); string? rootResultsDirectory = Path.GetDirectoryName(Path.GetDirectoryName(peopleRootDirectory)); if (rootResultsDirectory is null) throw new Exception(); Storage storage = new(rootDirectory, rootResultsDirectory, peopleRootDirectory); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(peopleRootDirectory, "{}")); (_GenealogicalDataCommunicationHeaderLines, Dictionary> individuals, _GenealogicalDataCommunicationFooterLines) = Shared.Models.Stateless.Methods.IGenealogicalDataCommunication.GetIndividuals(configuration.GenealogicalDataCommunicationFile, requireNickName: true); _PersonContainers = Shared.Models.Stateless.Methods.IPersonContainer.GetPersonContainers(storage, configuration.MappingDefaultName, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), _Faces.FileNameExtension, individuals); } { (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetTuple( configuration.OutputExtension, configuration.OutputQuality); _Resize = new C_Resize( configuration.ForceResizeLastWriteTimeToCreationTime, configuration.OverrideForResizeImages, configuration.PropertiesChangedForResize, configuration.ValidResolutions, imageCodecInfo, encoderParameters, filenameExtension); } if (!configuration.SkipSearch) Search(ticks, argZero, propertyRoot); if (!_PropertyRootExistedBefore && !_IsEnvironment.Development && _Exceptions.Count == 0 && _ArgZeroIsConfigurationRootDirectory) { string d2FacePartsRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(D2_FaceParts)); _Log.Information(string.Concat("Cleaning <", d2FacePartsRootDirectory, ">")); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(d2FacePartsRootDirectory); if (appSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories)); } message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}"; _Log.Information(message); if (_Exceptions.Count != 0) throw new Exception(message); if (_PropertyRootExistedBefore) _Log.Information("First run completed. Run again if wanted"); } private long LogDelta(long ticks, string? methodName) { long result; if (_Log is null) throw new NullReferenceException(nameof(_Log)); double delta = new TimeSpan(DateTime.Now.Ticks - ticks).TotalMilliseconds; _Log.Debug($"{methodName} took {Math.Floor(delta)} millisecond(s)"); result = DateTime.Now.Ticks; return result; } private long LogDeltaInSeconds(long ticks, string methodName) { long result; if (_Log is null) throw new NullReferenceException(nameof(_Log)); double delta = new TimeSpan(DateTime.Now.Ticks - ticks).Seconds; _Log.Debug($"{methodName} took {Math.Floor(delta)} seconds(s)"); result = DateTime.Now.Ticks; return result; } private long LogDeltaInMinutes(long ticks, string methodName) { long result; if (_Log is null) throw new NullReferenceException(nameof(_Log)); double delta = new TimeSpan(DateTime.Now.Ticks - ticks).Minutes; _Log.Debug($"{methodName} took {Math.Floor(delta)} minutes(s)"); result = DateTime.Now.Ticks; return result; } private static void Verify(Models.Configuration configuration) { if (!configuration.OutputResolutions.Any() || string.IsNullOrEmpty(configuration.OutputResolutions[0]) || !configuration.ValidResolutions.Contains(configuration.OutputResolutions[0])) throw new NullReferenceException($"{nameof(configuration.OutputResolutions)} must be fileNameToCollection valid outputResolution!"); if ((from l in configuration.OutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.OutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.SaveShortcutsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.SaveShortcutsForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.SaveFaceLandmarkForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.SaveFaceLandmarkForOutputResolutions)} are not in the ValidResolutions list!"); if (string.IsNullOrEmpty(configuration.ModelName)) throw new NullReferenceException(nameof(configuration.ModelName)); if (string.IsNullOrEmpty(configuration.OutputExtension)) throw new NullReferenceException(nameof(configuration.OutputExtension)); if (string.IsNullOrEmpty(configuration.PredictorModelName)) throw new NullReferenceException(nameof(configuration.PredictorModelName)); if (string.IsNullOrEmpty(configuration.ModelDirectory) || !Directory.Exists(configuration.ModelDirectory)) throw new NullReferenceException(nameof(configuration.ModelDirectory)); } private void VerifyExtra(List args, Property.Models.Configuration propertyConfiguration, Models.Configuration configuration) { string[] sourceDirectoryNames; if (!args.Any()) sourceDirectoryNames = Array.Empty(); else { string? century; string argZero = Path.GetFullPath(args[0]); sourceDirectoryNames = argZero.Split(Path.DirectorySeparatorChar); if (!argZero.StartsWith(propertyConfiguration.RootDirectory)) throw new Exception($"Source directory must be inside root directory! <{argZero}> <{propertyConfiguration.RootDirectory}>"); if (_ArgZeroIsConfigurationRootDirectory && propertyConfiguration.RootDirectory != argZero) { if (!configuration.MixedYearRelativePaths.Contains(sourceDirectoryNames[0])) { string[] segments = sourceDirectoryNames[0].Split(' '); century = segments[^1].Length == 4 ? segments[^1][..2] : null; if (segments.Length < 2 || century is null || (century != "18" && century != "19" && century != "20")) throw new Exception("root subdirectory must have fileNameToCollection year at the end or directory name needs to be added to the exclude list!"); } } } string[] resizeMatch = (from l in sourceDirectoryNames where configuration.ValidResolutions.Contains(l) select l).ToArray(); if (resizeMatch.Any()) throw new Exception("Input directory should be the source and not fileNameToCollection resized directory!"); if (configuration.LocationDigits != Shared.Models.Stateless.ILocation.Digits) throw new Exception("Configuration has to match interface!"); if (configuration.LocationFactor != Shared.Models.Stateless.ILocation.Factor) throw new Exception("Configuration has to match interface!"); } private static Map.Models.Configuration Get(Models.Configuration configuration, string facesFileNameExtension, string facesHiddenFileNameExtension, string facePartsFileNameExtension) { Map.Models.Configuration result = new( configuration.DeletePossibleDuplicates, configuration.DistanceMoveUnableToMatch, configuration.DistanceRenameToMatch, configuration.FaceConfidencePercent, configuration.FaceDistancePermyriad, configuration.LocationDigits, configuration.MappingDefaultName, configuration.PersonBirthdayFirstYear, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), configuration.RangeDaysDeltaTolerance, configuration.RangeDistanceTolerance, configuration.SaveSortingWithoutPerson, configuration.SkipNotSkipDirectories, configuration.SortingMaximumPerKey, configuration.SortingMinimumToUseSigma, facesFileNameExtension, facesHiddenFileNameExtension, facePartsFileNameExtension); return result; } private bool? GetIsFocusModel(ReadOnlyDictionary>> idToLocationContainers, MappingFromItem mappingFromItem) { bool? result; string? model; List>? locationContainers; IReadOnlyList directories; if (string.IsNullOrEmpty(_Configuration.FocusModel)) result = null; else { if (!idToLocationContainers.TryGetValue(mappingFromItem.Id, out locationContainers) || !locationContainers.Any()) result = false; else { directories = locationContainers.First().Directories; model = Metadata.Models.Stateless.Methods.IMetadata.GetModel(directories); if (model is null) directories = MetadataExtractor.ImageMetadataReader.ReadMetadata(mappingFromItem.ResizedFileHolder.FullName); model = Metadata.Models.Stateless.Methods.IMetadata.GetModel(directories); result = model is not null && model.Contains(_Configuration.FocusModel); if (result.Value) result = true; } } return result; } private void SetMapping(ReadOnlyDictionary>> idToLocationContainers, MapLogic mapLogic, Item item, bool? isFocusRelativePath, bool? isIgnoreRelativePath, MappingFromItem mappingFromItem, List? mappingFromPhotoPrismCollection, List faces) { Mapping mapping; int faceAreaPermyriad; int confidencePercent; bool? inSkipCollection; int normalizedRectangle; string deterministicHashCodeKey; MappingFromFilter mappingFromFilter; MappingFromLocation? mappingFromLocation; bool? isFocusModel = GetIsFocusModel(idToLocationContainers, mappingFromItem); foreach (Shared.Models.Face face in faces) { if (item.Property?.Id is null || face.FaceEncoding is null || face.Location is null || face.OutputResolution is null) { inSkipCollection = null; mappingFromLocation = null; mappingFromFilter = new(isFocusModel, isFocusRelativePath, isIgnoreRelativePath, inSkipCollection); } else { confidencePercent = Shared.Models.Stateless.Methods.ILocation.GetConfidencePercent(_Configuration.FaceConfidencePercent, _Configuration.RangeFaceConfidence, face.Location.Confidence); faceAreaPermyriad = Shared.Models.Stateless.Methods.IMapping.GetAreaPermyriad(_Configuration.FaceAreaPermyriad, face.Location, face.OutputResolution); normalizedRectangle = Shared.Models.Stateless.Methods.ILocation.GetNormalizedRectangle(face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution); deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item.Property.Id.Value, face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution); mappingFromLocation = new(faceAreaPermyriad, confidencePercent, deterministicHashCodeKey, normalizedRectangle); inSkipCollection = mapLogic.InSkipCollection(item.Property.Id.Value, mappingFromLocation); mappingFromFilter = new(isFocusModel, isFocusRelativePath, isIgnoreRelativePath, inSkipCollection); } mapping = new(mappingFromItem, mappingFromFilter, mappingFromLocation, mappingFromPhotoPrismCollection); _ = mapLogic.UpdateMappingFromPerson(mapping); face.SetMapping(mapping); } } private Mapping GetMapping(ReadOnlyDictionary>> idToLocationContainers, MapLogic mapLogic, Item item, bool? isFocusRelativePath, bool? isIgnoreRelativePath, MappingFromItem mappingFromItem) { Mapping result; bool? inSkipCollection; int normalizedRectangle; int faceAreaPermyriad = 0; int confidencePercent = 0; string deterministicHashCodeKey; MappingFromFilter mappingFromFilter; MappingFromLocation? mappingFromLocation; bool? isFocusModel = GetIsFocusModel(idToLocationContainers, mappingFromItem); if (item.Property?.Id is null) { inSkipCollection = null; mappingFromLocation = null; mappingFromFilter = new(isFocusModel, isFocusRelativePath, isIgnoreRelativePath, inSkipCollection); } else { normalizedRectangle = Shared.Models.Stateless.Methods.ILocation.GetNormalizedRectangle(Shared.Models.Stateless.ILocation.Digits); deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item.Property.Id.Value, Shared.Models.Stateless.ILocation.Digits); mappingFromLocation = new(faceAreaPermyriad, confidencePercent, deterministicHashCodeKey, normalizedRectangle); inSkipCollection = mapLogic.InSkipCollection(item.Property.Id.Value, mappingFromLocation); mappingFromFilter = new(isFocusModel, isFocusRelativePath, isIgnoreRelativePath, inSkipCollection); } result = new(mappingFromItem, mappingFromFilter, mappingFromLocation, mappingFromPhotoPrismCollection: null); return result; } private void FullParallelForWork(A_Property propertyLogic, B_Metadata metadata, ReadOnlyDictionary>> idToLocationContainers, MapLogic mapLogic, string outputResolution, string cResultsFullGroupDirectory, string dResultsDateGroupDirectory, string dResultsFullGroupDirectory, string eDistanceContentDirectory, List> sourceDirectoryChanges, Dictionary> fileNameToCollection, Container container, int index, Item item, DateTime[] containerDateTimes, bool? isFocusRelativePath, bool? isIgnoreRelativePath, string facePartsCollectionDirectory) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); List faces; Shared.Models.Property property; long ticks = DateTime.Now.Ticks; DateTime dateTime = DateTime.Now; List parseExceptions = new(); List> subFileTuples = new(); List> metadataCollection; if (item.Property is not null && item.Property.Id is not null && !item.Any()) { property = item.Property; if (item.SourceDirectoryFileHolder.LastWriteTime is not null) subFileTuples.Add(new Tuple(nameof(A_Property), item.SourceDirectoryFileHolder.LastWriteTime.Value)); else subFileTuples.Add(new Tuple(nameof(A_Property), new FileInfo(item.SourceDirectoryFileHolder.FullName).LastWriteTime)); bool nameWithoutExtensionIsIdFormat = Shared.Models.Stateless.Methods.IProperty.NameWithoutExtensionIsIdFormat(item.ImageFileHolder); if (nameWithoutExtensionIsIdFormat && item.ImageFileHolder.NameWithoutExtension != item.Property.Id.ToString()) { _Log.Information($"Name without extension is Id format but doesn't match id <{item.ImageFileHolder.FullName}>"); File.Move(item.ImageFileHolder.FullName, $"{item.ImageFileHolder.FullName}.rename"); } } else { int? propertyHashCode = item.Property?.GetHashCode(); if (!item.SourceDirectoryFileHolder.Exists) _Log.Information(string.Concat("NoJson <", item.ImageFileHolder.FullName, '>')); else if (item.FileSizeChanged.HasValue && item.FileSizeChanged.Value) _Log.Information(string.Concat("FileSizeChanged <", item.ImageFileHolder.FullName, '>')); else if (item.LastWriteTimeChanged.HasValue && item.LastWriteTimeChanged.Value) _Log.Information(string.Concat("LastWriteTimeChanged <", item.ImageFileHolder.FullName, '>')); else if (item.Moved.HasValue && item.Moved.Value) _Log.Information(string.Concat("Moved <", item.ImageFileHolder.FullName, '>')); property = propertyLogic.GetProperty(item, subFileTuples, parseExceptions); item.Update(property); if (propertyHashCode is null) { lock (sourceDirectoryChanges) sourceDirectoryChanges.Add(new Tuple(nameof(A_Property), DateTime.Now)); } else if (propertyHashCode.Value != property.GetHashCode()) { lock (sourceDirectoryChanges) sourceDirectoryChanges.Add(new Tuple(nameof(A_Property), DateTime.Now)); } } if (property is null || item.Property is null) throw new NullReferenceException(nameof(property)); FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(item); item.SetResizedFileHolder(_Resize.FileNameExtension, resizedFileHolder); string facesDirectory = _Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution) ? _Faces.GetFacesDirectory(dResultsFullGroupDirectory, item) : string.Empty; string facePartsDirectory = _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution) ? _FaceParts.GetFacePartsDirectory(_Configuration.PropertyConfiguration, dResultsFullGroupDirectory, item, includeNameWithoutExtension: true) : string.Empty; MappingFromItem mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item, resizedFileHolder); (int metadataGroups, metadataCollection) = metadata.GetMetadataCollection(subFileTuples, parseExceptions, mappingFromItem); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection)); Dictionary outputResolutionToResize = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, subFileTuples, parseExceptions, metadataCollection, item.Property, mappingFromItem); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(C_Resize.GetResizeKeyValuePairs)); if (_Configuration.SaveResizedSubfiles) { _Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.Property, mappingFromItem, outputResolutionToResize); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(C_Resize.SaveResizedSubfile)); } if (!mappingFromItem.ResizedFileHolder.Exists && !File.Exists(mappingFromItem.ResizedFileHolder.FullName)) faces = new(); else if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution)) faces = new(); else { List? mappingFromPhotoPrismCollection; List>? collection; if (item.Property?.Id is null) collection = null; else _ = idToLocationContainers.TryGetValue(item.Property.Id.Value, out collection); if (!fileNameToCollection.TryGetValue(mappingFromItem.RelativePath[1..], out mappingFromPhotoPrismCollection)) mappingFromPhotoPrismCollection = null; faces = _Faces.GetFaces(outputResolution, dResultsFullGroupDirectory, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionToResize, collection, mappingFromPhotoPrismCollection); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(D_Face.GetFaces)); List<(Shared.Models.Face, FileInfo?, string, bool Saved)> faceCollection = _Faces.SaveFaces(_FaceParts.FileNameExtension, dResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem, facesDirectory, faces); SetMapping(idToLocationContainers, mapLogic, item, isFocusRelativePath, isIgnoreRelativePath, mappingFromItem, mappingFromPhotoPrismCollection, faces); if (_Configuration.CopyFacesAndSaveFaceLandmarkForOutputResolutions.Contains(outputResolution)) _FaceParts.CopyFacesAndSaveFaceLandmarkImage(facePartsCollectionDirectory, mappingFromItem, faceCollection); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(D_Face.SaveFaces)); if ((_Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch) && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution) && collection is not null && faceCollection.All(l => !l.Saved)) _Distance.LookForMatchFacesAndPossiblyRename(_Faces.FileNameExtension, eDistanceContentDirectory, mappingFromItem, faces, collection); if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution)) { bool saveRotated = false; string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(_Configuration.PropertyConfiguration, container.SourceDirectory); _FaceParts.SaveFaceLandmarkImages(_Configuration.PropertyConfiguration, facePartsDirectory, subFileTuples, parseExceptions, mappingFromItem, faces, saveRotated); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(D2_FaceParts.SaveFaceLandmarkImages)); } } lock (sourceDirectoryChanges) { item.Faces.AddRange(faces); sourceDirectoryChanges.AddRange(from l in subFileTuples where l.Item2 > dateTime select l); } } private int FullParallelWork(int maxDegreeOfParallelism, A_Property propertyLogic, B_Metadata metadata, ReadOnlyDictionary>> idToLocationContainers, MapLogic mapLogic, string outputResolution, string cResultsFullGroupDirectory, string dResultsDateGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string eDistanceContentDirectory, List> sourceDirectoryChanges, Dictionary> fileNameToCollection, Container container, Item[] filteredItems, string message) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); int result = 0; ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; DateTime[] containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems); ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; string focusRelativePath = Path.GetFullPath(string.Concat(_Configuration.PropertyConfiguration.RootDirectory, _Configuration.FocusDirectory)); bool? isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath); bool? isIgnoreRelativePath = !_Configuration.IgnoreRelativePaths.Any() ? null : _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory); string facePartsCollectionDirectory = _Configuration.CopyFacesAndSaveFaceLandmarkForOutputResolutions.Contains(outputResolution) || _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution) ? _FaceParts.GetFacePartsDirectory(_Configuration.PropertyConfiguration, d2ResultsFullGroupDirectory, item: filteredItems.First(), includeNameWithoutExtension: false) : string.Empty; using ProgressBar progressBar = new(filteredItems.Length, message, options); _ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) => { try { FullParallelForWork(propertyLogic, metadata, idToLocationContainers, mapLogic, outputResolution, cResultsFullGroupDirectory, dResultsDateGroupDirectory, dResultsFullGroupDirectory, eDistanceContentDirectory, sourceDirectoryChanges, fileNameToCollection, container, index: i, filteredItems[i], containerDateTimes, isFocusRelativePath, isIgnoreRelativePath, facePartsCollectionDirectory); if (i == 0 || sourceDirectoryChanges.Any()) progressBar.Tick(); } catch (Exception ex) { result += 1; _Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, ex.Message, Environment.NewLine, ex.StackTrace), ex); if (result == filteredItems.Length) throw new Exception(string.Concat("All in [", container.SourceDirectory, "] failed!")); } }); return result; } private static void WriteTab(string checkDirectory, List<(string Id, string Line)> metadataIdLines, string fileName) { string text; FileInfo fileInfo; List duplicates = new(); List metadataIds = new(); fileInfo = new(Path.Combine(checkDirectory, "[()]", Path.ChangeExtension(fileName, "tsv"))); if (fileInfo?.Directory is null) throw new Exception(); if (!fileInfo.Directory.Exists) fileInfo.Directory.Create(); foreach ((string Id, string Line) metadataIdLine in metadataIdLines) { if (metadataIds.Contains(metadataIdLine.Id)) duplicates.Add(metadataIdLine.Id); else metadataIds.Add(metadataIdLine.Id); } for (int i = metadataIdLines.Count - 1; i > -1; i--) { if (duplicates.Contains(metadataIdLines[i].Id)) metadataIdLines.RemoveAt(i); } if (metadataIdLines.Any()) { text = string.Join(Environment.NewLine, from l in metadataIdLines orderby l.Id select l.Line); _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, text, updateDateWhenMatches: true, compareBeforeWrite: true); } else { if (fileInfo.Exists) File.Delete(fileInfo.FullName); } } private (string, string) GetResultsFullGroupDirectories() { string aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(A_Property), string.Empty, includeResizeGroup: false, includeModel: false, includePredictorModel: false); string bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(B_Metadata), string.Empty, includeResizeGroup: false, includeModel: false, includePredictorModel: false); return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory); } private (string, string, string) GetResultsFullGroupDirectories(string outputResolution) { string cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(C_Resize), outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false); string dResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true); string d2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(D2_FaceParts), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true); return new(cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory); } private void FullDoWork(string argZero, string propertyRoot, long ticks, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, int t, Container[] containers, A_Property propertyLogic, B_Metadata metadata, string eDistanceContentDirectory, Dictionary> fileNameToCollection, ReadOnlyDictionary>> idToLocationContainers, MapLogic mapLogic) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); int total; string message; int totalSeconds; int exceptionCount; Container container; Item[] filteredItems; bool anyNullOrNoIsUniqueFileName; string cResultsFullGroupDirectory; string dResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; int containersLength = containers.Length; List> sourceDirectoryChanges = new(); int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism; string dResultsDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(D_Face)); foreach (string outputResolution in _Configuration.OutputResolutions) { total = 0; (cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); for (int i = 0; i < containers.Length; i++) { container = containers[i]; if (!container.Items.Any()) continue; if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero)) continue; filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container); if (!filteredItems.Any()) continue; sourceDirectoryChanges.Clear(); totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); anyNullOrNoIsUniqueFileName = filteredItems.Any(l => l.IsUniqueFileName is null || !l.IsUniqueFileName.Value); message = $"{i + 1:000} [{filteredItems.Length:000} collectionB] / {containersLength:000} - {total} / {t} total collectionB - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}"; if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution)) _Faces.SetAngleBracketCollection(dResultsFullGroupDirectory, container.SourceDirectory); if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution)) _FaceParts.SetAngleBracketCollection(_Configuration.PropertyConfiguration, d2ResultsFullGroupDirectory, container.SourceDirectory); propertyLogic.SetAngleBracketCollection(aResultsFullGroupDirectory, container.SourceDirectory, anyNullOrNoIsUniqueFileName); _Resize.SetAngleBracketCollection(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, container.SourceDirectory); exceptionCount = FullParallelWork(maxDegreeOfParallelism, propertyLogic, metadata, idToLocationContainers, mapLogic, outputResolution, cResultsFullGroupDirectory, dResultsDateGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eDistanceContentDirectory, sourceDirectoryChanges, fileNameToCollection, container, filteredItems, message); if (exceptionCount != 0) { _Exceptions.Add(container.SourceDirectory); continue; } if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any()) { for (int y = 0; y < int.MaxValue; y++) { _Log.Information("Press \"Y\" key when ready to continue or close console"); if (_Console.ReadKey() == ConsoleKey.Y) break; } _Log.Information(". . ."); } total += container.Items.Count; } } } private void SaveFaceDistances(long ticks, MapLogic mapLogic, Mapping[] mappingCollection, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, Dictionary> idToNormalizedRectangleToMapping, List faceDistanceEncodings, FaceDistanceContainer[] faceDistanceContainers) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); DistanceLimits distanceLimits; int? useFiltersCounter = null; SortingContainer[] sortingContainers; FaceDistanceContainer[] filteredFaceDistanceContainers; long? skipOlderThan = _Configuration.SkipOlderThanDays is null ? null : new DateTime(ticks).AddDays(-_Configuration.SkipOlderThanDays.Value).Ticks; distanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh); filteredFaceDistanceContainers = E_Distance.FilteredFaceDistanceContainers(mapLogic, faceDistanceContainers, skipOlderThan, distanceLimits); if (!filteredFaceDistanceContainers.Any()) _Log.Information("All images have been filtered!"); else { sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distanceLimits, faceDistanceEncodings, filteredFaceDistanceContainers); if (!sortingContainers.Any()) { for (useFiltersCounter = 1; useFiltersCounter < _Configuration.UseFilterTries; useFiltersCounter++) { distanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh, useFiltersCounter); filteredFaceDistanceContainers = E_Distance.FilteredFaceDistanceContainers(mapLogic, faceDistanceContainers, skipOlderThan, distanceLimits); if (filteredFaceDistanceContainers.Any()) _Log.Information("All images have been filtered!"); else { sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distanceLimits, faceDistanceEncodings, filteredFaceDistanceContainers); if (sortingContainers.Any()) break; } } } E_Distance.SaveFaceDistances(_Configuration.PropertyConfiguration, sortingContainers); if (filteredFaceDistanceContainers.Length > 0) { int updated = mapLogic.UpdateFromSortingContainers(_Configuration.SaveIndividually, distanceLimits, sortingContainers); List saveContainers = mapLogic.GetSaveContainers(_Configuration.SaveIndividually, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, mappingCollection, idToNormalizedRectangleToMapping, useFiltersCounter, sortingContainers.Any()); mapLogic.SaveContainers(_Configuration.SaveIndividually, filteredFaceDistanceContainers.Length, updated, saveContainers); } } } private void SaveFaceDistances(long ticks, MapLogic mapLogic, List distinctFilteredFaces, Mapping[] mappingCollection, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, string dFacesCollectionDirectory, Dictionary> idToNormalizedRectangleToMapping) { E_Distance.SetFaceDistances(_AppSettings.MaxDegreeOfParallelism, ticks, distinctFilteredFaces); FaceDistanceContainer[] faceDistanceContainers = E_Distance.GetFaceDistanceContainers(distinctFilteredFaces); Dictionary> missingIdThenNormalizedRectangleToPersonContainers = mapLogic.GetMissing(idToNormalizedRectangleToMapping); List missingFaceDistanceContainers = _Distance.GetMissingFaceDistanceContainer(_AppSettings.MaxDegreeOfParallelism, ticks, dFacesCollectionDirectory, missingIdThenNormalizedRectangleToPersonContainers); List faceDistanceEncodings = E_Distance.GetFaceDistanceEncodings(faceDistanceContainers, missingFaceDistanceContainers); if (faceDistanceContainers.Any()) SaveFaceDistances(ticks, mapLogic, mappingCollection, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToNormalizedRectangleToMapping, faceDistanceEncodings, faceDistanceContainers); } private void MapLogic(long ticks, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string fPhotoPrismContentDirectory, MapLogic mapLogic, string outputResolution, Dictionary> personKeyToIds, List distinctFilteredFaces, Mapping[] distinctFilteredMappingCollection, int totalNotMapped) { string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()"); string d2FacePartsContentDirectory = Path.Combine(d2ResultsFullGroupDirectory, "()"); string d2FacePartsContentCollectionDirectory = Path.Combine(d2ResultsFullGroupDirectory, "[()]"); string dFacesCollectionDirectory = Path.Combine(dResultsFullGroupDirectory, "[]", _Configuration.PropertyConfiguration.ResultAllInOne); if (distinctFilteredMappingCollection.Any()) { Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(d2FacePartsContentDirectory, ticks); Shared.Models.Stateless.Methods.IPath.MakeHiddenIfAllItemsAreHidden(d2FacePartsContentCollectionDirectory); } Dictionary> idToNormalizedRectangleToMapping = Map.Models.Stateless.Methods.IMapLogic.GetIdToNormalizedRectangleToFace(distinctFilteredMappingCollection); if (Directory.Exists(fPhotoPrismContentDirectory)) F_PhotoPrism.WriteMatches(fPhotoPrismContentDirectory, _Configuration.PersonBirthdayFormat, ticks, distinctFilteredFaces, mapLogic); if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution)) mapLogic.SaveShortcutsForOutputResolutionsDuringMapLogic(containers, personKeyToIds, dFacesContentDirectory, distinctFilteredMappingCollection); if (!string.IsNullOrEmpty(_Configuration.PersonCharacters)) { if (_Configuration.PersonCharactersCopyCount > 0) mapLogic.CopyAtLeastOneMappedFiles(_Configuration.PersonCharactersCopyCount, dFacesContentDirectory, a2PeopleSingletonDirectory, distinctFilteredMappingCollection); mapLogic.SavePersonKeyToCount(dFacesContentDirectory, a2PeopleSingletonDirectory, distinctFilteredMappingCollection); } mapLogic.CopyManualFiles(dFacesContentDirectory, idToNormalizedRectangleToMapping); if (_Configuration.SaveMappedForOutputResolutions.Contains(outputResolution)) mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, personKeyToIds, distinctFilteredMappingCollection, idToNormalizedRectangleToMapping, totalNotMapped); if (_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution)) SaveFaceDistances(ticks, mapLogic, distinctFilteredFaces, distinctFilteredMappingCollection, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, dFacesCollectionDirectory, idToNormalizedRectangleToMapping); } private string SaveUrlAndGetNewRootDirectory(Container container) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); string? result = Path.GetDirectoryName(Path.GetDirectoryName(container.SourceDirectory)); if (result is null) throw new Exception(); Uri uri; Item item; string? line; string fileName; Task task; string relativePath; FileHolder fileHolder; string extensionLowered; string sourceDirectoryFile; int length = result.Length; HttpClient httpClient = new(); bool isValidImageFormatExtension; for (int y = 0; y < int.MaxValue; y++) { _Log.Information("Enter fileNameToCollection url for fileNameToCollection image"); line = _Console.ReadLine(); if (string.IsNullOrEmpty(line)) break; uri = new(line); if (uri.HostNameType != UriHostNameType.Dns) continue; task = httpClient.GetByteArrayAsync(uri); fileName = Path.GetFileName(uri.LocalPath); sourceDirectoryFile = Path.Combine(container.SourceDirectory, fileName); File.WriteAllBytes(sourceDirectoryFile, task.Result); extensionLowered = Path.GetExtension(uri.LocalPath); relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFile, length, forceExtensionToLower: true); isValidImageFormatExtension = _Configuration.PropertyConfiguration.ValidImageFormatExtensions.Contains(extensionLowered); fileHolder = new(sourceDirectoryFile); item = new(fileHolder, relativePath, isValidImageFormatExtension); container.Items.Add(item); } _Log.Information(". . ."); return result; } private static void LookForAbandoned(List distinctFilteredIds, string directory, string directoryName) { string fileNameWithoutExtension; List renameCollection = new(); string[] distinctFilteredIdsValues = distinctFilteredIds.Select(l => l.ToString()).ToArray(); string[] files = Directory.GetFiles(directory, "*", SearchOption.AllDirectories); foreach (string file in files) { fileNameWithoutExtension = Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(file))); if (distinctFilteredIdsValues.Contains(fileNameWithoutExtension)) continue; if (!Shared.Models.Stateless.Methods.IProperty.NameWithoutExtensionIsIdFormat(fileNameWithoutExtension)) continue; renameCollection.Add(file); } if (renameCollection.Any()) { if (directoryName.Length == 2) Shared.Models.Stateless.Methods.IDirectory.MoveFiles(renameCollection, directoryName, $"{directoryName[0]}abd{directoryName[^1]}"); else if (directoryName.Length == 4) Shared.Models.Stateless.Methods.IDirectory.MoveFiles(renameCollection, directoryName, $"{directoryName[..2]}abd{directoryName[^2..]}"); else throw new NotSupportedException(); } } private static void LookForAbandoned(string bResultsFullGroupDirectory, List distinctFilteredIds) { string[] directories = Directory.GetDirectories(bResultsFullGroupDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string directory in directories) { string? directoryName = Path.GetFileName(directory); if (string.IsNullOrEmpty(directoryName) || (directoryName.Length != 2 && directoryName.Length != 4)) continue; LookForAbandoned(distinctFilteredIds, directory, directoryName); } } private static void LookForAbandoned(ReadOnlyDictionary>> idToLocationContainers, List distinctFilteredIds) { List renameCollection = new(); foreach (KeyValuePair>> idToCollection in idToLocationContainers) { if (distinctFilteredIds.Contains(idToCollection.Key)) continue; foreach (LocationContainer locationContainer in idToCollection.Value) { if (locationContainer.File.Contains('!')) continue; renameCollection.Add(locationContainer.File); } } if (renameCollection.Any()) Shared.Models.Stateless.Methods.IDirectory.MoveFiles(renameCollection, "()", "(abd)"); } private void LookForAbandoned(string bResultsFullGroupDirectory, Container[] containers, ReadOnlyDictionary>> idToLocationContainers) { string[] directories; string? directoryName; string cResultsFullGroupDirectory; string dResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; List distinctFilteredIds = Shared.Models.Stateless.Methods.IContainer.GetFilteredDistinctIds(_Configuration.PropertyConfiguration, containers); LookForAbandoned(idToLocationContainers, distinctFilteredIds); LookForAbandoned(bResultsFullGroupDirectory, distinctFilteredIds); foreach (string outputResolution in _Configuration.OutputResolutions) { (cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); directories = Directory.GetDirectories(cResultsFullGroupDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string directory in directories) { directoryName = Path.GetFileName(directory); if (string.IsNullOrEmpty(directoryName) || (directoryName.Length != 2 && directoryName.Length != 4)) continue; LookForAbandoned(distinctFilteredIds, directory, directoryName); } directories = Directory.GetDirectories(dResultsFullGroupDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string directory in directories) { directoryName = Path.GetFileName(directory); if (string.IsNullOrEmpty(directoryName) || (directoryName.Length != 2 && directoryName.Length != 4)) continue; LookForAbandoned(distinctFilteredIds, directory, directoryName); } directories = Directory.GetDirectories(d2ResultsFullGroupDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string directory in directories) { directoryName = Path.GetFileName(directory); if (string.IsNullOrEmpty(directoryName) || (directoryName.Length != 2 && directoryName.Length != 4)) continue; LookForAbandoned(distinctFilteredIds, directory, directoryName); } } } private Mapping[] GetMappings(Property.Models.Configuration propertyConfiguration, Container[] containers, MapLogic mapLogic, ReadOnlyDictionary>> idToLocationContainers, bool distinctItems) { Mapping[] results; int count = 0; Mapping mapping; bool anyValidFaces; string focusRelativePath; bool? isFocusRelativePath; bool? isIgnoreRelativePath; List distinct = new(); DateTime[] containerDateTimes; IEnumerable filteredItems; MappingFromItem mappingFromItem; List mappingCollection = new(); foreach (Container container in containers) { if (!container.Items.Any()) continue; filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(propertyConfiguration, container); if (!filteredItems.Any()) continue; containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems); focusRelativePath = Path.GetFullPath(string.Concat(_Configuration.PropertyConfiguration.RootDirectory, _Configuration.FocusDirectory)); isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath); isIgnoreRelativePath = !_Configuration.IgnoreRelativePaths.Any() ? null : _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory); foreach (Item item in filteredItems) { if (item.Property?.Id is null || item.ResizedFileHolder is null) continue; mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item, item.ResizedFileHolder); if (distinctItems) { if (distinct.Contains(item.Property.Id.Value)) continue; distinct.Add(item.Property.Id.Value); } count++; anyValidFaces = false; foreach (Shared.Models.Face face in item.Faces) { if (face.Mapping is null) continue; anyValidFaces = true; mappingCollection.Add(face.Mapping); } if (!anyValidFaces) { mapping = GetMapping(idToLocationContainers, mapLogic, item, isFocusRelativePath, isIgnoreRelativePath, mappingFromItem); mappingCollection.Add(mapping); } } } results = (from l in mappingCollection orderby l.MappingFromItem.Id select l).ToArray(); return results; } private void Search(long ticks, string argZero, string propertyRoot) { int t; Container[] containers; A_Property propertyLogic; string eDistanceContentDirectory; string? a2PeopleContentDirectory; string aResultsFullGroupDirectory; string bResultsFullGroupDirectory; string cResultsFullGroupDirectory; string dResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; string fPhotoPrismContentDirectory; string fPhotoPrismSingletonDirectory; Dictionary> personKeyToIds; Dictionary> fileNameToCollection; (aResultsFullGroupDirectory, bResultsFullGroupDirectory) = GetResultsFullGroupDirectories(); string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "{}"); a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "([])"); eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), "()"); fPhotoPrismContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), "()"); fPhotoPrismSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), "{}"); propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory); MapLogic mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, new(_PersonContainers), ticks, a2PeopleSingletonDirectory, eDistanceContentDirectory); _PersonContainers.AddRange(Shared.Models.Stateless.Methods.IPersonContainer.GetNonSpecificPeopleCollection(_Configuration.MappingDefaultName, _Configuration.PersonBirthdayFirstYear, _Configuration.PersonCharacters.ToArray(), _PersonContainers, ticks)); int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); string message = $") Building Container(s) - {totalSeconds} total second(s)"; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; using (ProgressBar progressBar = new(2, message, options)) { progressBar.Tick(); string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, "{}"); if (!Directory.Exists(aPropertySingletonDirectory)) _ = Directory.CreateDirectory(aPropertySingletonDirectory); (t, containers) = Shared.Models.Stateless.Methods.IContainer.GetContainers(_Configuration.PropertyConfiguration, aPropertySingletonDirectory); progressBar.Tick(); } if (containers.Length == 1) { string resultsGroupDirectory; a2PeopleContentDirectory = null; eDistanceContentDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", "()"); fPhotoPrismContentDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", "()"); fPhotoPrismSingletonDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", "{}"); for (int i = 1; i < 10; i++) { resultsGroupDirectory = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, string.Empty, create: true); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(resultsGroupDirectory); } argZero = SaveUrlAndGetNewRootDirectory(containers[0]); _Configuration.PropertyConfiguration.ChangeRootDirectory(argZero); (aResultsFullGroupDirectory, bResultsFullGroupDirectory) = GetResultsFullGroupDirectories(); propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false); propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory); } B_Metadata metadata = new(_Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, bResultsFullGroupDirectory); containers = Shared.Models.Stateless.Methods.IContainer.SortContainers(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, _ArgZeroIsConfigurationRootDirectory, argZero, containers); personKeyToIds = mapLogic.GetPersonKeyToIds(); if (!string.IsNullOrEmpty(_Configuration.GenealogicalDataCommunicationFile) && !string.IsNullOrEmpty(a2PeopleContentDirectory) && _GenealogicalDataCommunicationHeaderLines is not null && _GenealogicalDataCommunicationFooterLines is not null && _GenealogicalDataCommunicationHeaderLines.Any() && _GenealogicalDataCommunicationFooterLines.Any()) Shared.Models.Stateless.Methods.IGenealogicalDataCommunication.CreateTree(_Configuration.MappingDefaultName, _Configuration.PersonBirthdayFormat, _Configuration.PropertyConfiguration.ResultAllInOne, _PersonContainers, _GenealogicalDataCommunicationHeaderLines, _GenealogicalDataCommunicationFooterLines, ticks, a2PeopleContentDirectory, personKeyToIds); ReadOnlyDictionary>> idToLocationContainers = mapLogic.GetIdToLocationContainers(); fileNameToCollection = !Directory.Exists(fPhotoPrismSingletonDirectory) ? fileNameToCollection = new() : F_PhotoPrism.GetFileNameToCollection(fPhotoPrismSingletonDirectory); FullDoWork(argZero, propertyRoot, ticks, aResultsFullGroupDirectory, bResultsFullGroupDirectory, t, containers, propertyLogic, metadata, eDistanceContentDirectory, fileNameToCollection, idToLocationContainers, mapLogic); LookForAbandoned(bResultsFullGroupDirectory, containers, idToLocationContainers); _Distance.Clear(); if (!personKeyToIds.Any()) personKeyToIds = mapLogic.GetPersonKeyToIds(); Mapping[] distinctFilteredMappingCollection = GetMappings(_Configuration.PropertyConfiguration, containers, mapLogic, idToLocationContainers, distinctItems: true); List distinctFilteredItems = Shared.Models.Stateless.Methods.IContainer.GetItems(_Configuration.PropertyConfiguration, containers, distinctItems: true, filterItems: true); List distinctFilteredFaces = Map.Models.Stateless.Methods.IMapLogic.GetFaces(distinctFilteredItems); int totalNotMapped = mapLogic.UpdateMappingFromPerson(distinctFilteredMappingCollection); string json = System.Text.Json.JsonSerializer.Serialize(distinctFilteredMappingCollection); File.WriteAllText(Path.Combine(eDistanceContentDirectory, $"{ticks}.json"), json); for (int i = 1; i < 5; i++) _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(eDistanceContentDirectory); foreach (string outputResolution in _Configuration.OutputResolutions) { if (_PropertyRootExistedBefore) break; if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution)) mapLogic.SaveShortcutsForOutputResolutionsPreMapLogic(eDistanceContentDirectory, personKeyToIds, distinctFilteredMappingCollection); if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution)) mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, _PersonContainers, a2PeopleContentDirectory, personKeyToIds, distinctFilteredMappingCollection, totalNotMapped); (cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); if (_ArgZeroIsConfigurationRootDirectory && _Configuration.SaveResizedSubfiles && outputResolution == _Configuration.OutputResolutions[0] && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution) && _Exceptions.Count == 0) MapLogic(ticks, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, fPhotoPrismContentDirectory, mapLogic, outputResolution, personKeyToIds, distinctFilteredFaces, distinctFilteredMappingCollection, totalNotMapped); if (_Configuration.SaveRandomForOutputResolutions.Contains(outputResolution) && personKeyToIds.Any() && distinctFilteredMappingCollection.Any()) _Random.Random(_Configuration.PropertyConfiguration, outputResolution, personKeyToIds, distinctFilteredMappingCollection); if (_IsEnvironment.Development) continue; if (!_IsEnvironment.Development) { _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(aResultsFullGroupDirectory, "{}")); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(bResultsFullGroupDirectory, "{}")); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(cResultsFullGroupDirectory, "{}")); if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution)) _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(dResultsFullGroupDirectory, "[]")); } } } }