using Microsoft.Extensions.Configuration; using Phares.Shared; using ShellProgressBar; using System.Collections.ObjectModel; using System.Drawing.Imaging; using System.Text.Json; using System.Text.RegularExpressions; using View_by_Distance.Distance.Models; using View_by_Distance.Face.Models; using View_by_Distance.FaceParts.Models; using View_by_Distance.Instance.Models; using View_by_Distance.Map.Models; using View_by_Distance.Metadata.Models; using View_by_Distance.PhotoPrism.Models; using View_by_Distance.Property.Models; using View_by_Distance.Resize.Models; using View_by_Distance.Shared.Models; using View_by_Distance.Shared.Models.Methods; using View_by_Distance.Shared.Models.Stateless.Methods; using WindowsShortcutFactory; namespace View_by_Distance.Instance; public partial class DlibDotNet { private readonly D_Face _Faces; private readonly C_Resize _Resize; private readonly F_Random _Random; private readonly IConsole _Console; private readonly E_Distance _Distance; private readonly Serilog.ILogger? _Log; private readonly IBlurHasher _BlurHasher; private readonly D2_FaceParts _FaceParts; private readonly AppSettings _AppSettings; private readonly List _Exceptions; private readonly IsEnvironment _IsEnvironment; private readonly bool _PropertyRootExistedBefore; private readonly Models.Configuration _Configuration; private readonly bool _ArgZeroIsConfigurationRootDirectory; private readonly Map.Models.Configuration _MapConfiguration; private readonly List<(string, long)> _JLinkResolvedDirectories; public DlibDotNet( List args, IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, AppSettings appSettings, string workingDirectory, bool isSilent, IConsole console) { string message; _Console = console; _AppSettings = appSettings; _IsEnvironment = isEnvironment; long ticks = DateTime.Now.Ticks; _Exceptions = new List(); _JLinkResolvedDirectories = new(); if (ticks.ToString().Last() == '0') ticks += 1; _Log = Serilog.Log.ForContext(); ReadOnlyCollection personContainers; Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot); Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration); _Log.Information(propertyConfiguration.RootDirectory); Property.Models.Configuration.Verify(propertyConfiguration, requireExist: false); Verify(configuration); VerifyExtra(args, propertyConfiguration, configuration); _Configuration = configuration; _Random = new(configuration); if (configuration.IgnoreExtensions is null) throw new NullReferenceException(nameof(configuration.IgnoreExtensions)); _BlurHasher = new BlurHash.Models.C2_BlurHasher(configuration.PropertyConfiguration); string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(propertyConfiguration, nameof(A_Property), create: false); _PropertyRootExistedBefore = !Directory.Exists(propertyRoot); string argZero = args.Count > 0 ? Path.GetFullPath(args[0]) : Path.GetFullPath(propertyConfiguration.RootDirectory); _ArgZeroIsConfigurationRootDirectory = propertyConfiguration.RootDirectory == argZero; if (!Directory.Exists(argZero)) _ = Directory.CreateDirectory(argZero); _Log.Information(configuration.ModelDirectory); { (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetPngLowQuality(); (ImageCodecInfo hiddenImageCodecInfo, EncoderParameters hiddenEncoderParameters, string hiddenFileNameExtension) = C_Resize.GetGifLowQuality(); _Faces = new D_Face( argZero, configuration.PropertyConfiguration, configuration.CheckDFaceAndUpWriteDates, encoderParameters, configuration.FaceDistanceHiddenImageFactor, filenameExtension, configuration.ForceFaceLastWriteTimeToCreationTime, hiddenEncoderParameters, hiddenFileNameExtension, hiddenImageCodecInfo, imageCodecInfo, configuration.LoadPhotoPrismLocations, configuration.ModelDirectory, configuration.ModelName, configuration.OverrideForFaceImages, configuration.PredictorModelName, configuration.PropertiesChangedForFaces, configuration.RectangleIntersectMinimums); } { (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetGifLowQuality(); _FaceParts = new D2_FaceParts(_Configuration.PropertyConfiguration, imageCodecInfo, encoderParameters, filenameExtension, configuration.CheckDFaceAndUpWriteDates, configuration.OverrideForFaceLandmarkImages); } _MapConfiguration = Get(configuration, _Faces.FileNameExtension, _Faces.HiddenFileNameExtension, _FaceParts.FileNameExtension); _Distance = new(configuration.DistanceMoveUnableToMatch, configuration.DistanceRenameToMatch, configuration.FaceConfidencePercent, configuration.RangeDistanceTolerance, configuration.RangeFaceConfidence, configuration.RectangleIntersectMinimums); if (_PropertyRootExistedBefore || !_ArgZeroIsConfigurationRootDirectory) personContainers = new(new List()); else { int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); message = $") Building People Collection - {totalSeconds} total second(s)"; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; using ProgressBar progressBar = new(1, message, options); progressBar.Tick(); string peopleRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(A2_People)); string? rootResultsDirectory = Path.GetDirectoryName(Path.GetDirectoryName(peopleRootDirectory)) ?? throw new Exception(); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(peopleRootDirectory, propertyConfiguration.ResultSingleton)); string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration.PropertyConfiguration, nameof(A2_People), "{}"); string a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration.PropertyConfiguration, nameof(A2_People), "([])"); personContainers = new(IPersonContainer.GetPersonContainers(a2PeopleSingletonDirectory, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), _Faces.FileNameExtension)); if (configuration.JLinks.Where(l => !string.IsNullOrEmpty(l)).Any()) { _JLinkResolvedDirectories.AddRange(Map.Models.Stateless.Methods.IMapLogic.GetJLinkDirectories(configuration.GenealogicalDataCommunicationFile, configuration.JLinks, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), a2PeopleSingletonDirectory, a2PeopleContentDirectory)); if (_JLinkResolvedDirectories.Count == 0) throw new Exception(nameof(Map.Models.Stateless.Methods.IMapLogic.GetJLinkDirectories)); } } { (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetTuple( configuration.OutputExtension, configuration.OutputQuality); _Resize = new C_Resize( configuration.PropertyConfiguration, configuration.ForceResizeLastWriteTimeToCreationTime, configuration.OverrideForResizeImages, configuration.PropertiesChangedForResize, configuration.ValidResolutions, imageCodecInfo, encoderParameters, filenameExtension); } if (!configuration.SkipSearch) Search(ticks, personContainers, argZero, propertyRoot); if (!_PropertyRootExistedBefore && !_IsEnvironment.Development && _Exceptions.Count == 0 && _ArgZeroIsConfigurationRootDirectory) { string d2FacePartsRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(D2_FaceParts)); _Log.Information(string.Concat("Cleaning <", d2FacePartsRootDirectory, ">")); Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(d2FacePartsRootDirectory, ticks); } message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}"; _Log.Information(message); if (_Exceptions.Count != 0) throw new Exception(message); if (_PropertyRootExistedBefore) _Log.Information("First run completed. Run again if wanted"); } private long LogDelta(long ticks, string? methodName) { long result; if (_Log is null) throw new NullReferenceException(nameof(_Log)); double delta = new TimeSpan(DateTime.Now.Ticks - ticks).TotalMilliseconds; _Log.Debug($"{methodName} took {Math.Floor(delta)} millisecond(s)"); result = DateTime.Now.Ticks; return result; } private long LogDeltaInSeconds(long ticks, string methodName) { long result; if (_Log is null) throw new NullReferenceException(nameof(_Log)); double delta = new TimeSpan(DateTime.Now.Ticks - ticks).Seconds; _Log.Debug($"{methodName} took {Math.Floor(delta)} seconds(s)"); result = DateTime.Now.Ticks; return result; } private long LogDeltaInMinutes(long ticks, string methodName) { long result; if (_Log is null) throw new NullReferenceException(nameof(_Log)); double delta = new TimeSpan(DateTime.Now.Ticks - ticks).Minutes; _Log.Debug($"{methodName} took {Math.Floor(delta)} minutes(s)"); result = DateTime.Now.Ticks; return result; } private static void Verify(Models.Configuration configuration) { if (configuration.OutputResolutions.Length == 0 || string.IsNullOrEmpty(configuration.OutputResolutions[0]) || !configuration.ValidResolutions.Contains(configuration.OutputResolutions[0])) throw new NullReferenceException($"{nameof(configuration.OutputResolutions)} must be fileNameToCollection valid outputResolution!"); if ((from l in configuration.OutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.OutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.SaveShortcutsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.SaveShortcutsForOutputResolutions)} are not in the ValidResolutions list!"); if ((from l in configuration.SaveFaceLandmarkForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any()) throw new Exception($"One or more {nameof(configuration.SaveFaceLandmarkForOutputResolutions)} are not in the ValidResolutions list!"); if (string.IsNullOrEmpty(configuration.ModelName)) throw new NullReferenceException(nameof(configuration.ModelName)); if (string.IsNullOrEmpty(configuration.OutputExtension)) throw new NullReferenceException(nameof(configuration.OutputExtension)); if (string.IsNullOrEmpty(configuration.PredictorModelName)) throw new NullReferenceException(nameof(configuration.PredictorModelName)); if (string.IsNullOrEmpty(configuration.ModelDirectory) || !Directory.Exists(configuration.ModelDirectory)) throw new NullReferenceException(nameof(configuration.ModelDirectory)); } private void VerifyExtra(List args, Property.Models.Configuration propertyConfiguration, Models.Configuration configuration) { string[] sourceDirectoryNames; if (args.Count == 0) sourceDirectoryNames = Array.Empty(); else { string? century; string argZero = Path.GetFullPath(args[0]); sourceDirectoryNames = argZero.Split(Path.DirectorySeparatorChar); if (!argZero.StartsWith(propertyConfiguration.RootDirectory)) throw new Exception($"Source directory must be inside root directory! <{argZero}> <{propertyConfiguration.RootDirectory}>"); if (_ArgZeroIsConfigurationRootDirectory && propertyConfiguration.RootDirectory != argZero) { if (!configuration.MixedYearRelativePaths.Contains(sourceDirectoryNames[0])) { string[] segments = sourceDirectoryNames[0].Split(' '); century = segments[^1].Length == 4 ? segments[^1][..2] : null; if (segments.Length < 2 || century is null || (century != "18" && century != "19" && century != "20")) throw new Exception("root subdirectory must have fileNameToCollection year at the end or directory name needs to be added to the exclude list!"); } } } string[] resizeMatch = (from l in sourceDirectoryNames where configuration.ValidResolutions.Contains(l) select l).ToArray(); if (resizeMatch.Length > 0) throw new Exception("Input directory should be the source and not fileNameToCollection resized directory!"); if (configuration.LocationDigits != Shared.Models.Stateless.ILocation.Digits) throw new Exception("Configuration has to match interface!"); if (configuration.LocationFactor != Shared.Models.Stateless.ILocation.Factor) throw new Exception("Configuration has to match interface!"); } private static Map.Models.Configuration Get(Models.Configuration configuration, string facesFileNameExtension, string facesHiddenFileNameExtension, string facePartsFileNameExtension) { Map.Models.Configuration result = new( configuration.DeletePossibleDuplicates, configuration.DistanceMoveUnableToMatch, configuration.DistanceRenameToMatch, configuration.FaceConfidencePercent, configuration.FaceDistancePermyriad, configuration.LocationContainerDistanceTolerance, configuration.LocationDigits, configuration.MappingDefaultName, configuration.PersonBirthdayFirstYear, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), configuration.RangeDaysDeltaTolerance, configuration.RangeDistanceTolerance, configuration.SaveSortingWithoutPerson, configuration.SkipNotSkipDirectories, configuration.SortingMaximumPerKey, configuration.SortingMinimumToUseSigma, facesFileNameExtension, facesHiddenFileNameExtension, facePartsFileNameExtension); return result; } private bool? GetIsFocusModel(Shared.Models.Property? property) { bool? result; if (string.IsNullOrEmpty(_Configuration.FocusModel)) result = null; else if (property is null || string.IsNullOrEmpty(property.Model)) result = null; else result = property.Model.Contains(_Configuration.FocusModel); return result; } private int GetNotMappedCountAndSetMapping(MapLogic mapLogic, Item item, bool? isFocusRelativePath, ReadOnlyCollection> locationContainers, MappingFromItem mappingFromItem, List? mappingFromPhotoPrismCollection, List faces) { int result; double? α; int? eyeα; bool? isUsed; bool? eyeReview; Mapping mapping; int notMapped = 0; bool? isFocusPerson; int confidencePercent; int faceAreaPermyriad; bool? inSkipCollection; int wholePercentRectangle; string deterministicHashCodeKey; MappingFromFilter mappingFromFilter; MappingFromLocation? mappingFromLocation; bool? isFocusModel = GetIsFocusModel(item.Property); bool ignoreXMatches = _JLinkResolvedDirectories.Count > 0; ReadOnlyDictionary>? wholePercentagesToPersonContainers; ReadOnlyCollection locationContainersFiles = new((from l in locationContainers select l.File).ToArray()); foreach (Shared.Models.Face face in faces) { wholePercentagesToPersonContainers = mapLogic.GetWholePercentagesToPersonContainers(item.Property?.Id); if (item.Property?.Id is null || face.FaceEncoding is null || face.Location is null || face.OutputResolution is null) { isUsed = null; isFocusPerson = null; inSkipCollection = null; mappingFromLocation = null; mappingFromFilter = new(isFocusModel, isFocusPerson, isFocusRelativePath, inSkipCollection, isUsed); } else { if (face.FaceParts is null) (eyeα, eyeReview) = (null, null); else { (eyeReview, α) = Shared.Models.Stateless.Methods.IFace.GetEyeα(face.FaceParts); eyeα = α is null ? null : (int)Math.Round(Math.Abs(α.Value)); } confidencePercent = Shared.Models.Stateless.Methods.ILocation.GetConfidencePercent(_Configuration.FaceConfidencePercent, _Configuration.RangeFaceConfidence, face.Location.Confidence); faceAreaPermyriad = IMapping.GetAreaPermyriad(_Configuration.FaceAreaPermyriad, face.Location, face.OutputResolution); wholePercentRectangle = Shared.Models.Stateless.Methods.ILocation.GetWholePercentages(face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution); deterministicHashCodeKey = IMapping.GetDeterministicHashCodeKey(item.Property.Id.Value, face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution); mappingFromLocation = new(faceAreaPermyriad, confidencePercent, deterministicHashCodeKey, eyeα, eyeReview, wholePercentRectangle); isUsed = mapLogic.IsUsed(ignoreXMatches, item.Property.Id.Value, wholePercentagesToPersonContainers, mappingFromLocation); inSkipCollection = mapLogic.InSkipCollection(item.Property.Id.Value, mappingFromLocation); isFocusPerson = mapLogic.IsFocusPerson(_Configuration.SkipPersonWithMoreThen, _JLinkResolvedDirectories, wholePercentagesToPersonContainers, mappingFromLocation); mappingFromFilter = new(isFocusModel, isFocusPerson, isFocusRelativePath, inSkipCollection, isUsed); } mapping = new(mappingFromItem, mappingFromFilter, mappingFromLocation, mappingFromPhotoPrismCollection); notMapped += mapLogic.UpdateMappingFromPerson(locationContainersFiles, wholePercentagesToPersonContainers, mapping); face.SetMapping(mapping); } result = notMapped; return result; } private (Mapping, int) GetMapping(MapLogic mapLogic, ReadOnlyCollection? locationContainersFiles, Item item, bool? isFocusRelativePath, MappingFromItem mappingFromItem) { Mapping result; bool? isUsed; int? eyeα = null; bool? isFocusPerson; bool? eyeReview = null; bool? inSkipCollection; int confidencePercent = 0; int faceAreaPermyriad = 0; int wholePercentRectangle; string deterministicHashCodeKey; MappingFromFilter mappingFromFilter; MappingFromLocation? mappingFromLocation; bool? isFocusModel = GetIsFocusModel(item.Property); bool ignoreXMatches = _JLinkResolvedDirectories.Count > 0; ReadOnlyDictionary>? wholePercentagesToPersonContainers = mapLogic.GetWholePercentagesToPersonContainers(item.Property?.Id); if (item.Property?.Id is null) { isUsed = null; isFocusPerson = null; inSkipCollection = null; mappingFromLocation = null; mappingFromFilter = new(isFocusModel, isFocusPerson, isFocusRelativePath, inSkipCollection, isUsed); } else { wholePercentRectangle = Shared.Models.Stateless.Methods.ILocation.GetWholePercentages(Shared.Models.Stateless.ILocation.Digits); deterministicHashCodeKey = IMapping.GetDeterministicHashCodeKey(item.Property.Id.Value, Shared.Models.Stateless.ILocation.Digits); mappingFromLocation = new(faceAreaPermyriad, confidencePercent, deterministicHashCodeKey, eyeα, eyeReview, wholePercentRectangle); isUsed = mapLogic.IsUsed(ignoreXMatches, item.Property.Id.Value, wholePercentagesToPersonContainers, mappingFromLocation); inSkipCollection = mapLogic.InSkipCollection(item.Property.Id.Value, mappingFromLocation); isFocusPerson = mapLogic.IsFocusPerson(_Configuration.SkipPersonWithMoreThen, _JLinkResolvedDirectories, wholePercentagesToPersonContainers, mappingFromLocation); mappingFromFilter = new(isFocusModel, isFocusPerson, isFocusRelativePath, inSkipCollection, isUsed); } result = new(mappingFromItem, mappingFromFilter, mappingFromLocation, mappingFromPhotoPrismCollection: null); int notMapped = mapLogic.UpdateMappingFromPerson(locationContainersFiles, wholePercentagesToPersonContainers, result); return (result, notMapped); } private void LogItemPropertyIsNull(Item item) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); if (!item.SourceDirectoryFileHolder.Exists) _Log.Information(string.Concat("NoJson <", item.ImageFileHolder.FullName, '>')); else if (item.FileSizeChanged.HasValue && item.FileSizeChanged.Value) _Log.Information(string.Concat("FileSizeChanged <", item.ImageFileHolder.FullName, '>')); else if (item.LastWriteTimeChanged.HasValue && item.LastWriteTimeChanged.Value) _Log.Information(string.Concat("LastWriteTimeChanged <", item.ImageFileHolder.FullName, '>')); else if (item.Moved.HasValue && item.Moved.Value) _Log.Information(string.Concat("Moved <", item.ImageFileHolder.FullName, '>')); } private void LogNameWithoutExtensionIsIdFormatBut(Item item) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); _Log.Information($"Name without extension is Id format but doesn't match id <{item.ImageFileHolder.FullName}>"); File.Move(item.ImageFileHolder.FullName, $"{item.ImageFileHolder.FullName}.rename"); } private int FullParallelForWork(A_Property propertyLogic, B_Metadata metadata, MapLogic mapLogic, string outputResolution, bool outputResolutionHasNumber, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List> sourceDirectoryChanges, Dictionary> fileNameToCollection, Container container, int index, Item item, DateTime[] containerDateTimes, bool? isFocusRelativePath, string facePartsCollectionDirectory) { int result = 0; List faces; long ticks = DateTime.Now.Ticks; DateTime dateTime = DateTime.Now; Shared.Models.Property? property; List parseExceptions = new(); List> subFileTuples = new(); List> metadataCollection; string[] changesFrom = new string[] { nameof(A_Property) }; FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(cResultsFullGroupDirectory, item, outputResolutionHasNumber); ReadOnlyCollection> locationContainers = mapLogic.GetLocationContainers(item); if (item.Property is null || item.Property.Id is null || item.Any()) { LogItemPropertyIsNull(item); int? propertyHashCode = item.Property?.GetHashCode(); property = propertyLogic.GetProperty(metadata, item, subFileTuples, parseExceptions); item.Update(property); if (propertyHashCode is null) { lock (sourceDirectoryChanges) sourceDirectoryChanges.Add(new Tuple(nameof(A_Property), DateTime.Now)); } else if (propertyHashCode.Value != property.GetHashCode()) { lock (sourceDirectoryChanges) sourceDirectoryChanges.Add(new Tuple(nameof(A_Property), DateTime.Now)); } } else { property = item.Property; if (item.SourceDirectoryFileHolder.LastWriteTime is not null) subFileTuples.Add(new Tuple(nameof(A_Property), item.SourceDirectoryFileHolder.LastWriteTime.Value)); else subFileTuples.Add(new Tuple(nameof(A_Property), new FileInfo(item.SourceDirectoryFileHolder.FullName).LastWriteTime)); int sortOrderOnlyLengthIndex = IDirectory.GetSortOrderOnlyLengthIndex(); bool nameWithoutExtensionIsIdFormat = Shared.Models.Stateless.Methods.IProperty.NameWithoutExtensionIsIdFormat(item.ImageFileHolder); bool nameWithoutExtensionIsPaddedIdFormat = IDirectory.NameWithoutExtensionIsPaddedIdFormat(item.ImageFileHolder, sortOrderOnlyLengthIndex); if (nameWithoutExtensionIsIdFormat && item.ImageFileHolder.NameWithoutExtension != item.Property.Id.ToString()) LogNameWithoutExtensionIsIdFormatBut(item); if (nameWithoutExtensionIsPaddedIdFormat && item.ImageFileHolder.NameWithoutExtension.EndsWith(item.Property.Id.Value.ToString()[1..])) LogNameWithoutExtensionIsIdFormatBut(item); if (resizedFileHolder.Exists && item.Property.Width is not null && item.Property.Width.Value > 4 && _Configuration.SaveBlurHashForOutputResolutions.Contains(outputResolution)) { string? file = _BlurHasher.GetFile(resizedFileHolder); if (file is not null && !File.Exists(file)) _ = _BlurHasher.EncodeAndSave(resizedFileHolder); } } if (property is null || item.Property is null) throw new NullReferenceException(nameof(property)); item.SetResizedFileHolder(_Resize.FileNameExtension, resizedFileHolder); MappingFromItem mappingFromItem = IMappingFromItem.GetMappingFromItem(containerDateTimes, item, resizedFileHolder); (int metadataGroups, metadataCollection) = metadata.GetMetadataCollection(subFileTuples, parseExceptions, changesFrom, mappingFromItem); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection)); Dictionary outputResolutionToResize = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, subFileTuples, parseExceptions, metadataCollection, item.Property, mappingFromItem); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(C_Resize.GetResizeKeyValuePairs)); if (_Configuration.SaveResizedSubfiles) { _Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.Property, mappingFromItem, outputResolutionToResize); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(C_Resize.SaveResizedSubfile)); } if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution)) faces = new(); else if (!mappingFromItem.ResizedFileHolder.Exists && !File.Exists(mappingFromItem.ResizedFileHolder.FullName)) faces = new(); else { List? mappingFromPhotoPrismCollection; if (!fileNameToCollection.TryGetValue(mappingFromItem.Id, out mappingFromPhotoPrismCollection)) mappingFromPhotoPrismCollection = null; faces = _Faces.GetFaces(outputResolution, dResultsFullGroupDirectory, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionToResize, locationContainers, mappingFromPhotoPrismCollection); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(D_Face.GetFaces)); result = GetNotMappedCountAndSetMapping(mapLogic, item, isFocusRelativePath, locationContainers, mappingFromItem, mappingFromPhotoPrismCollection, faces); List<(Shared.Models.Face, FileInfo?, string, bool Saved)> faceCollection = _Faces.SaveFaces(_FaceParts.FileNameExtension, dResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem, faces); if (_Configuration.CopyFacesAndSaveFaceLandmarkForOutputResolutions.Contains(outputResolution)) _FaceParts.CopyFacesAndSaveFaceLandmarkImage(facePartsCollectionDirectory, mappingFromItem, faceCollection); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(D_Face.SaveFaces)); if ((_Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch) && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution) && locationContainers is not null && faceCollection.All(l => !l.Saved)) _Distance.LookForMatchFacesAndPossiblyRename(_Faces.FileNameExtension, mappingFromItem, faces, locationContainers); (bool review, int[] eyesCollection) = Shared.Models.Stateless.Methods.IFace.GetEyeCollection(_Configuration.EyeThreshold, faces); if (review || _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution)) { bool saveRotated = false; string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(_Configuration.PropertyConfiguration, container.SourceDirectory); _FaceParts.SaveFaceLandmarkImages(_Configuration.PropertyConfiguration, subFileTuples, parseExceptions, mappingFromItem, faces, saveRotated); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(D2_FaceParts.SaveFaceLandmarkImages)); } } lock (sourceDirectoryChanges) { item.Faces.AddRange(faces); sourceDirectoryChanges.AddRange(from l in subFileTuples where l.Item2 > dateTime select l); } return result; } private (int, bool) FullParallelWork(int maxDegreeOfParallelism, A_Property propertyLogic, B_Metadata metadata, MapLogic mapLogic, string outputResolution, bool outputResolutionHasNumber, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List> sourceDirectoryChanges, Dictionary> fileNameToCollection, Container container, Item[] filteredItems, string message) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); int result = 0; int exceptionsCount = 0; bool ignoreXMatches = _JLinkResolvedDirectories.Count > 0; ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; DateTime[] containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems); ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; string focusRelativePath = Path.GetFullPath(string.Concat(_Configuration.PropertyConfiguration.RootDirectory, _Configuration.FocusDirectory)); bool? isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath); string facePartsCollectionDirectory = _Configuration.CopyFacesAndSaveFaceLandmarkForOutputResolutions.Contains(outputResolution) ? _FaceParts.GetFacePartsDirectory(_Configuration.PropertyConfiguration, d2ResultsFullGroupDirectory, item: filteredItems.First(), includeNameWithoutExtension: false) : string.Empty; bool anyPropertiesChangedForX = _Configuration.PropertyConfiguration.PropertiesChangedForProperty || _Configuration.PropertiesChangedForDistance || _Configuration.PropertiesChangedForFaces || _Configuration.PropertiesChangedForIndex || _Configuration.PropertiesChangedForMetadata || _Configuration.PropertiesChangedForResize; using ProgressBar progressBar = new(filteredItems.Length, message, options); _ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) => { try { result += FullParallelForWork(propertyLogic, metadata, mapLogic, outputResolution, outputResolutionHasNumber, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, fileNameToCollection, container, index: i, filteredItems[i], containerDateTimes, isFocusRelativePath, facePartsCollectionDirectory); if (!anyPropertiesChangedForX && (i == 0 || sourceDirectoryChanges.Count > 0)) progressBar.Tick(); } catch (Exception ex) { exceptionsCount++; _Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, ex.Message, Environment.NewLine, ex.StackTrace), ex); if (exceptionsCount == filteredItems.Length) throw new Exception(string.Concat("All in [", container.SourceDirectory, "] failed!")); } }); return (result, exceptionsCount > 0); } private static void WriteTab(string checkDirectory, List<(string Id, string Line)> metadataIdLines, string fileName) { string text; FileInfo fileInfo; List duplicates = new(); List metadataIds = new(); fileInfo = new(Path.Combine(checkDirectory, "[()]", Path.ChangeExtension(fileName, "tsv"))); if (fileInfo?.Directory is null) throw new Exception(); if (!fileInfo.Directory.Exists) fileInfo.Directory.Create(); foreach ((string Id, string Line) metadataIdLine in metadataIdLines) { if (metadataIds.Contains(metadataIdLine.Id)) duplicates.Add(metadataIdLine.Id); else metadataIds.Add(metadataIdLine.Id); } for (int i = metadataIdLines.Count - 1; i > -1; i--) { if (duplicates.Contains(metadataIdLines[i].Id)) metadataIdLines.RemoveAt(i); } if (metadataIdLines.Count > 0) { text = string.Join(Environment.NewLine, from l in metadataIdLines orderby l.Id select l.Line); _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, text, updateDateWhenMatches: true, compareBeforeWrite: true); } else { if (fileInfo.Exists) File.Delete(fileInfo.FullName); } } private (string, string) GetResultsFullGroupDirectories() { string aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(A_Property), string.Empty, includeResizeGroup: false, includeModel: false, includePredictorModel: false); string bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(B_Metadata), string.Empty, includeResizeGroup: false, includeModel: false, includePredictorModel: false); return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory); } private (string, string, string, string) GetResultsFullGroupDirectories(string outputResolution) { string cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(C_Resize), outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false); string c2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(BlurHash.Models.C2_BlurHasher), outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false); string dResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true); string d2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(D2_FaceParts), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true); return new(cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory); } private void FullDoWork(string argZero, string propertyRoot, long ticks, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, int t, Container[] containers, A_Property propertyLogic, B_Metadata metadata, Dictionary> fileNameToCollection, MapLogic mapLogic) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); int total; int notMapped; string message; bool exceptions; int totalSeconds; Container container; Item[] filteredItems; int totalNotMapped = 0; bool outputResolutionHasNumber; bool anyNullOrNoIsUniqueFileName; string cResultsFullGroupDirectory; string dResultsFullGroupDirectory; string c2ResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; int containersLength = containers.Length; List> sourceDirectoryChanges = new(); int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism; string dResultsDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(D_Face)); foreach (string outputResolution in _Configuration.OutputResolutions) { total = 0; outputResolutionHasNumber = outputResolution.Any(l => char.IsNumber(l)); (cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); _Faces.Update(dResultsFullGroupDirectory); _Resize.Update(cResultsFullGroupDirectory); _FaceParts.Update(d2ResultsFullGroupDirectory); _BlurHasher.Update(c2ResultsFullGroupDirectory); for (int i = 0; i < containers.Length; i++) { container = containers[i]; if (container.Items.Count == 0) continue; if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero)) continue; filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container); if (filteredItems.Length == 0) continue; sourceDirectoryChanges.Clear(); anyNullOrNoIsUniqueFileName = filteredItems.Any(l => !l.IsUniqueFileName); totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); message = $"{i + 1:000} [{filteredItems.Length:000}] / {containersLength:000} - {total} / {t} total - {totalSeconds} total second(s) - {outputResolution} - <{container.SourceDirectory}> - total not mapped {totalNotMapped:000000}"; propertyLogic.SetAngleBracketCollection(aResultsFullGroupDirectory, container.SourceDirectory, anyNullOrNoIsUniqueFileName); if (outputResolutionHasNumber) _Resize.SetAngleBracketCollection(cResultsFullGroupDirectory, container.SourceDirectory); (notMapped, exceptions) = FullParallelWork(maxDegreeOfParallelism, propertyLogic, metadata, mapLogic, outputResolution, outputResolutionHasNumber, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, fileNameToCollection, container, filteredItems, message); totalNotMapped += notMapped; if (exceptions) { _Exceptions.Add(container.SourceDirectory); continue; } if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Length > 0) { for (int y = 0; y < int.MaxValue; y++) { _Log.Information("Press \"Y\" key when ready to continue or close console"); if (_Console.ReadKey() == ConsoleKey.Y) break; } _Log.Information(". . ."); } total += container.Items.Count; } totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); message = $"### [###] / {containersLength:000} - {total} / {t} total - {totalSeconds} total second(s) - {outputResolution} - <> - total not mapped {totalNotMapped:000000}"; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; using ProgressBar progressBar = new(1, message, options); progressBar.Tick(); } } private void SaveFaceDistances(long ticks, MapLogic mapLogic, ReadOnlyCollection mappingCollection, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, ReadOnlyDictionary> idToWholePercentagesToMapping, ReadOnlyCollection faceDistanceEncodings, ReadOnlyCollection faceDistanceContainers) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); DistanceLimits distanceLimits; int? useFiltersCounter = null; SortingContainer[] sortingContainers; FaceDistanceContainer[] filteredFaceDistanceContainers; long? skipOlderThan = _Configuration.SkipOlderThanDays is null ? null : new DateTime(ticks).AddDays(-_Configuration.SkipOlderThanDays.Value).Ticks; distanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh); filteredFaceDistanceContainers = E_Distance.FilteredFaceDistanceContainers(mapLogic, faceDistanceContainers, skipOlderThan, distanceLimits); if (filteredFaceDistanceContainers.Length == 0) _Log.Information("All images have been filtered!"); else { sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distanceLimits, faceDistanceEncodings, filteredFaceDistanceContainers); if (sortingContainers.Length == 0) { for (useFiltersCounter = 1; useFiltersCounter < _Configuration.UseFilterTries; useFiltersCounter++) { distanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh, useFiltersCounter); filteredFaceDistanceContainers = E_Distance.FilteredFaceDistanceContainers(mapLogic, faceDistanceContainers, skipOlderThan, distanceLimits); if (filteredFaceDistanceContainers.Length == 0) _Log.Information("All images have been filtered!"); else { sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distanceLimits, faceDistanceEncodings, filteredFaceDistanceContainers); if (sortingContainers.Length == 0) break; } } } E_Distance.SaveFaceDistances(_Configuration.PropertyConfiguration, sortingContainers); if (filteredFaceDistanceContainers.Length > 0) { int updated = mapLogic.UpdateFromSortingContainers(_Configuration.SaveIndividually, idToWholePercentagesToMapping, distanceLimits, sortingContainers); List saveContainers = mapLogic.GetSaveContainers(_Configuration.SaveIndividually, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, mappingCollection, idToWholePercentagesToMapping, useFiltersCounter, sortingContainers.Length > 0); mapLogic.SaveContainers(_Configuration.SaveIndividually, updated, saveContainers); } } } private void SaveFaceDistances(long ticks, MapLogic mapLogic, ReadOnlyCollection distinctFilteredFaces, ReadOnlyCollection mappingCollection, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, ReadOnlyDictionary> idToWholePercentagesToMapping) { E_Distance.SetFaceDistances(_AppSettings.MaxDegreeOfParallelism, ticks, distinctFilteredFaces); ReadOnlyCollection faceDistanceContainers = E_Distance.GetFaceDistanceContainers(distinctFilteredFaces); if (faceDistanceContainers.Count > 0) { List faceDistanceEncodings = new(); foreach (FaceDistanceContainer faceDistanceContainer in faceDistanceContainers) { if (faceDistanceContainer.FaceDistance.Encoding is null) continue; faceDistanceEncodings.Add(faceDistanceContainer.FaceDistance); } SaveFaceDistances(ticks, mapLogic, mappingCollection, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, new(faceDistanceEncodings), faceDistanceContainers); } } private void MapLogic(long ticks, ReadOnlyCollection containers, string fPhotoPrismContentDirectory, MapLogic mapLogic, string outputResolution, ReadOnlyDictionary> personKeyToIds, ReadOnlyCollection distinctFilteredFaces, ReadOnlyCollection distinctFilteredMappingCollection) { (_, _, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent); string d2FacePartsContentDirectory = Path.Combine(d2ResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent); string d2FacePartsContentCollectionDirectory = Path.Combine(d2ResultsFullGroupDirectory, "[()]"); if (distinctFilteredMappingCollection.Count > 0) { Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(d2FacePartsContentDirectory, ticks); if (Directory.Exists(d2FacePartsContentCollectionDirectory)) Shared.Models.Stateless.Methods.IPath.MakeHiddenIfAllItemsAreHidden(d2FacePartsContentCollectionDirectory); } if (Directory.Exists(fPhotoPrismContentDirectory)) F_PhotoPrism.WriteMatches(fPhotoPrismContentDirectory, _Configuration.PersonBirthdayFormat, _Configuration.RectangleIntersectMinimums, ticks, distinctFilteredFaces, mapLogic); if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution)) mapLogic.SaveShortcutsForOutputResolutionsDuringMapLogic(containers, personKeyToIds, dFacesContentDirectory, distinctFilteredMappingCollection); ReadOnlyDictionary> idToWholePercentagesToMapping = Map.Models.Stateless.Methods.IMapLogic.GetIdToWholePercentagesToFace(distinctFilteredMappingCollection); if (_Configuration.SaveMappedForOutputResolutions.Contains(outputResolution)) mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, personKeyToIds, distinctFilteredMappingCollection, idToWholePercentagesToMapping); if (_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution)) SaveFaceDistances(ticks, mapLogic, distinctFilteredFaces, distinctFilteredMappingCollection, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping); } private string SaveUrlAndGetNewRootDirectory(string[] files) { string result; if (_Log is null) throw new NullReferenceException(nameof(_Log)); if (files.Length == 0) throw new NotSupportedException(); string? sourceDirectory = Path.GetDirectoryName(files.First()); if (string.IsNullOrEmpty(sourceDirectory)) throw new NotSupportedException(); Uri uri; string? line; string fileName; Task task; string relativePath; FileHolder fileHolder; string extensionLowered; string sourceDirectoryFile; HttpClient httpClient = new(); bool isValidImageFormatExtension; result = Path.GetDirectoryName(Path.GetDirectoryName(sourceDirectory)) ?? throw new NotSupportedException(); int length = result.Length; for (int y = 0; y < int.MaxValue; y++) { _Log.Information("Enter fileNameToCollection url for fileNameToCollection image"); line = _Console.ReadLine(); if (string.IsNullOrEmpty(line)) break; uri = new(line); if (uri.HostNameType != UriHostNameType.Dns) continue; task = httpClient.GetByteArrayAsync(uri); fileName = Path.GetFileName(uri.LocalPath); sourceDirectoryFile = Path.Combine(sourceDirectory, fileName); File.WriteAllBytes(sourceDirectoryFile, task.Result); extensionLowered = Path.GetExtension(uri.LocalPath); relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFile, length, forceExtensionToLower: true); isValidImageFormatExtension = _Configuration.PropertyConfiguration.ValidImageFormatExtensions.Contains(extensionLowered); fileHolder = new(sourceDirectoryFile); _ = new Item(fileHolder, relativePath, isValidImageFormatExtension); // container.Items.Add(item); } _Log.Information(". . ."); return result; } private ReadOnlyCollection GetMappings(Property.Models.Configuration propertyConfiguration, Container[] containers, MapLogic mapLogic, bool distinctItems) { ReadOnlyCollection results; int count = 0; int notMapped; Mapping mapping; bool anyValidFaces; string focusRelativePath; bool? isFocusRelativePath; List distinct = new(); DateTime[] containerDateTimes; IEnumerable filteredItems; MappingFromItem mappingFromItem; List mappingCollection = new(); ReadOnlyCollection? locationContainersFiles = null; foreach (Container container in containers) { if (container.Items.Count == 0) continue; filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(propertyConfiguration, container); if (!filteredItems.Any()) continue; containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems); focusRelativePath = Path.GetFullPath(string.Concat(_Configuration.PropertyConfiguration.RootDirectory, _Configuration.FocusDirectory)); isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath); foreach (Item item in filteredItems) { if (item.Property?.Id is null || item.ResizedFileHolder is null) continue; mappingFromItem = IMappingFromItem.GetMappingFromItem(containerDateTimes, item, item.ResizedFileHolder); if (distinctItems) { if (distinct.Contains(item.Property.Id.Value)) continue; distinct.Add(item.Property.Id.Value); } count++; anyValidFaces = false; foreach (Shared.Models.Face face in item.Faces) { if (face.Mapping is null) continue; anyValidFaces = true; mappingCollection.Add(face.Mapping); if (face.Mapping.MappingFromPerson is null || face.Mapping.MappingFromPerson.LocationContainersFiles.Count == 0) continue; if (_Configuration.LocationContainerDistanceTolerance is null) Map.Models.Stateless.Methods.IMapLogic.MoveToDecade(propertyConfiguration, face.Mapping.MappingFromItem, face.Mapping.MappingFromPerson); } if (!anyValidFaces) { (mapping, notMapped) = GetMapping(mapLogic, locationContainersFiles, item, isFocusRelativePath, mappingFromItem); mappingCollection.Add(mapping); } } } results = new((from l in mappingCollection orderby l.MappingFromItem.Id select l).ToArray()); return results; } private static void SavePropertyShortcutsForOutputResolutions(string eDistanceContentDirectory, ReadOnlyCollection distinctFilteredItems) { #if VerifyItem bool found; List notFound = new(); foreach (Item item in distinctFilteredItems) { found = false; if (item.Property?.Id is null) continue; foreach (Mapping mapping in distinctFilteredMappingCollection) { if (mapping.MappingFromItem.Id != item.Property.Id.Value) continue; found = true; break; } if (!found) notFound.Add(item); } if (notFound.Count > 0) throw new NotSupportedException(); #endif string model; string fileName; string directory; bool? isWrongYear; List dateTimes; List distinct = new(); WindowsShortcut windowsShortcut; List<(string, string, string)> collection = new(); foreach (Item item in distinctFilteredItems) { if (item.Property?.Id is null || item.ImageFileHolder.LastWriteTime is null) continue; if (item.IsNotUniqueAndNeedsReview is null || !item.IsNotUniqueAndNeedsReview.Value) continue; directory = Path.Combine($"{eDistanceContentDirectory[..^1]}{nameof(item.IsNotUniqueAndNeedsReview)})", item.ImageFileHolder.NameWithoutExtension); fileName = Path.Combine(directory, $"{item.ImageFileHolder.Length} {item.ImageFileHolder.LastWriteTime.Value.Ticks}.lnk"); collection.Add((item.ImageFileHolder.FullName, directory, fileName)); if (distinct.Contains(directory)) continue; distinct.Add(directory); } foreach (Item item in distinctFilteredItems) { if (item.Property?.Id is null || item.Property.DateTimeOriginal is null) continue; dateTimes = item.Property.GetDateTimes(); (isWrongYear, _) = Shared.Models.Stateless.Methods.IProperty.IsWrongYear(item.ImageFileHolder, item.Property.DateTimeOriginal, dateTimes); if (isWrongYear is null || !isWrongYear.Value) continue; // Remove-Item -LiteralPath "\\?\D:\Tmp\a\EX-Z70 " model = string.IsNullOrEmpty(item.Property.Model) ? "Unknown" : Regex.Replace(item.Property.Model.Trim(), @"[\\,\/,\:,\*,\?,\"",\<,\>,\|]", "_"); directory = Path.Combine($"{eDistanceContentDirectory[..^1]}{nameof(Item)})", item.Property.DateTimeOriginal.Value.Year.ToString(), model); fileName = item.IsNotUniqueAndNeedsReview is not null && item.IsNotUniqueAndNeedsReview.Value ? Path.Combine(directory, $"{item.ImageFileHolder.Name} {item.ImageFileHolder.Length}.lnk") : Path.Combine(directory, $"{item.ImageFileHolder.Name}.lnk"); collection.Add((item.ImageFileHolder.FullName, directory, fileName)); if (distinct.Contains(directory)) continue; distinct.Add(directory); } #if Mapping foreach (Mapping mapping in distinctFilteredMappingCollection) { if (mapping.MappingFromItem.IsWrongYear is null || !mapping.MappingFromItem.IsWrongYear.Value) continue; directory = Path.Combine($"{eDistanceContentDirectory[..^1]}{nameof(Mapping)})", mapping.MappingFromItem.MinimumDateTime.Year.ToString()); fileName = Path.Combine(directory, $"{mapping.MappingFromItem.ResizedFileHolder.Name}.lnk"); collection.Add((mapping.MappingFromItem.ResizedFileHolder.FullName, directory, fileName)); if (distinct.Contains(directory)) continue; distinct.Add(directory); } #endif foreach (string distinctDirectory in distinct) { if (!Directory.Exists(distinctDirectory)) _ = Directory.CreateDirectory(distinctDirectory); } foreach ((string path, string checkDirectory, string checkFile) in collection) { if (File.Exists(checkFile)) continue; windowsShortcut = new() { Path = path }; windowsShortcut.Save(checkFile); windowsShortcut.Dispose(); } } private bool GetRunToDoCollectionFirst(long ticks) { bool result = false; string[] directories; string seasonDirectory; DirectoryInfo directoryInfo; DateTime dateTime = new(ticks); string rootDirectory = _Configuration.PropertyConfiguration.RootDirectory; string eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), _Configuration.PropertyConfiguration.ResultContent); (int season, string seasonName) = Shared.Models.Stateless.Methods.IProperty.GetSeason(dateTime.DayOfYear); FileSystemInfo fileSystemInfo = new DirectoryInfo(eDistanceContentDirectory); string[] checkDirectories = new string[] { Path.Combine(rootDirectory, "Ancestry"), Path.Combine(rootDirectory, "Facebook"), Path.Combine(rootDirectory, "LinkedIn"), rootDirectory, }; foreach (string checkDirectory in checkDirectories) { if (checkDirectory == rootDirectory) seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName}"); else seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName} {Path.GetFileName(checkDirectory)}"); if (!Directory.Exists(seasonDirectory)) _ = Directory.CreateDirectory(seasonDirectory); if (result) continue; directories = Directory.GetDirectories(checkDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string directory in directories) { directoryInfo = new(directory); if (directoryInfo.LastWriteTime > fileSystemInfo.LastWriteTime) { result = true; break; } } } return result; } private void Search(long ticks, ReadOnlyCollection personContainers, string argZero, string propertyRoot) { int t; int count; string message; MapLogic? mapLogic; Container[] containers; A_Property propertyLogic; string eDistanceContentDirectory; string? a2PeopleContentDirectory; string aResultsFullGroupDirectory; string bResultsFullGroupDirectory; string cResultsFullGroupDirectory; string fPhotoPrismContentDirectory; const string fileSearchFilter = "*"; string fPhotoPrismSingletonDirectory; bool filesCollectionCountIsOne = false; List? filesCollection = null; const string directorySearchFilter = "*"; Dictionary> personKeyToIds; bool configurationOutputResolutionsHas = false; bool runToDoCollectionFirst = GetRunToDoCollectionFirst(ticks); Dictionary> fileNameToCollection; (aResultsFullGroupDirectory, bResultsFullGroupDirectory) = GetResultsFullGroupDirectories(); ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(_Configuration.PropertyConfiguration.RootDirectory, ticks); a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "([])"); eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), _Configuration.PropertyConfiguration.ResultContent); string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), _Configuration.PropertyConfiguration.ResultSingleton); _ = Directory.CreateDirectory(Path.Combine(eDistanceContentDirectory, $"({ticks})")); if (runToDoCollectionFirst) mapLogic = null; else mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _Distance, personContainers, ticks, a2PeopleSingletonDirectory, eDistanceContentDirectory); foreach (string outputResolution in _Configuration.OutputResolutions) { if (outputResolution.Any(l => char.IsNumber(l))) continue; configurationOutputResolutionsHas = true; if (!runToDoCollectionFirst) break; ProgressBar progressBar; (cResultsFullGroupDirectory, _, _, _) = GetResultsFullGroupDirectories(outputResolution); IReadOnlyDictionary fileGroups = Shared.Models.Stateless.Methods.IPath.GetKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, new string[] { _Configuration.PropertyConfiguration.ResultContent }); filesCollection = IDirectory.GetFilesCollection(_Configuration.PropertyConfiguration.RootDirectory, directorySearchFilter, fileSearchFilter, useCeilingAverage: false); message = $") Selecting for ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)"; count = filesCollection.Select(l => l.Length).Sum(); filesCollectionCountIsOne = filesCollection.Count == 1; progressBar = new(count, message, options); (string[] distinctDirectories, List<(FileHolder, string)> toDoCollection) = IDirectory.GetToDoCollection(_Configuration.PropertyConfiguration, filesCollection, fileGroups[_Configuration.PropertyConfiguration.ResultContent], () => progressBar.Tick()); progressBar.Dispose(); message = $") Copying to ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)"; progressBar = new(count, message, options); _ = IDirectory.CopyOrMove(toDoCollection, move: false, moveBack: false, () => progressBar.Tick()); progressBar.Dispose(); break; } fPhotoPrismContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), _Configuration.PropertyConfiguration.ResultContent); fPhotoPrismSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), _Configuration.PropertyConfiguration.ResultSingleton); propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory); if (filesCollectionCountIsOne) { if (filesCollection is null) throw new NullReferenceException(nameof(filesCollection)); string resultsGroupDirectory; a2PeopleContentDirectory = null; eDistanceContentDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", _Configuration.PropertyConfiguration.ResultContent); fPhotoPrismContentDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", _Configuration.PropertyConfiguration.ResultContent); fPhotoPrismSingletonDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", _Configuration.PropertyConfiguration.ResultSingleton); for (int i = 1; i < 10; i++) { resultsGroupDirectory = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, string.Empty, create: true); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(resultsGroupDirectory); } argZero = SaveUrlAndGetNewRootDirectory(filesCollection.First()); _Configuration.PropertyConfiguration.ChangeRootDirectory(argZero); (aResultsFullGroupDirectory, bResultsFullGroupDirectory) = GetResultsFullGroupDirectories(); propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false); propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory); } if (configurationOutputResolutionsHas) { foreach (string outputResolution in _Configuration.OutputResolutions) { if (outputResolution.Any(l => char.IsNumber(l))) continue; (cResultsFullGroupDirectory, _, _, _) = GetResultsFullGroupDirectories(outputResolution); filesCollection = IDirectory.GetFilesCollection(Path.Combine(cResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent, _Configuration.PropertyConfiguration.ResultAllInOne), directorySearchFilter, fileSearchFilter, useCeilingAverage: true); count = filesCollection.Select(l => l.Length).Sum(); break; } } if (filesCollection is null) throw new NullReferenceException(nameof(filesCollection)); message = $") Building Container(s) - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)"; using (ProgressBar progressBar = new(2, message, options)) { progressBar.Tick(); string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton); if (!Directory.Exists(aPropertySingletonDirectory)) _ = Directory.CreateDirectory(aPropertySingletonDirectory); (t, containers) = Shared.Models.Stateless.Methods.IContainer.GetContainers(_Configuration.PropertyConfiguration, aPropertySingletonDirectory, filesCollection); progressBar.Tick(); } fileNameToCollection = !Directory.Exists(fPhotoPrismSingletonDirectory) ? fileNameToCollection = new() : F_PhotoPrism.GetFileNameToCollection(fPhotoPrismSingletonDirectory); B_Metadata metadata = new(_Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, bResultsFullGroupDirectory); mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _Distance, personContainers, ticks, a2PeopleSingletonDirectory, eDistanceContentDirectory); FullDoWork(argZero, propertyRoot, ticks, aResultsFullGroupDirectory, bResultsFullGroupDirectory, t, containers, propertyLogic, metadata, fileNameToCollection, mapLogic); ReadOnlyCollection distinctFilteredItems = Shared.Models.Stateless.Methods.IContainer.GetItems(_Configuration.PropertyConfiguration, new(containers), distinctItems: true, filterItems: true); if (_Configuration.LookForAbandoned) { string dResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; foreach (string outputResolution in _Configuration.OutputResolutions) { (cResultsFullGroupDirectory, _, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); mapLogic.LookForAbandoned(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, containers, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory); } } _Distance.Clear(); ReadOnlyCollection distinctFilteredFaces = Map.Models.Stateless.Methods.IMapLogic.GetFaces(distinctFilteredItems); ReadOnlyCollection distinctFilteredMappingCollection = GetMappings(_Configuration.PropertyConfiguration, containers, mapLogic, distinctItems: true); if (runToDoCollectionFirst) { string json = JsonSerializer.Serialize(distinctFilteredMappingCollection); File.WriteAllText(Path.Combine(eDistanceContentDirectory, $"{ticks}.json"), json); } foreach (string outputResolution in _Configuration.OutputResolutions) { if (_PropertyRootExistedBefore) break; personKeyToIds = mapLogic.GetPersonKeyToIds(); if (_Configuration.SavePropertyShortcutsForOutputResolutions.Contains(outputResolution)) SavePropertyShortcutsForOutputResolutions(eDistanceContentDirectory, distinctFilteredItems); if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution)) mapLogic.SaveShortcutsForOutputResolutionsPreMapLogic(eDistanceContentDirectory, personKeyToIds, distinctFilteredMappingCollection); if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.JLinks.Where(l => !string.IsNullOrEmpty(l)).Any() && _Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution)) mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, personContainers, a2PeopleContentDirectory, personKeyToIds, distinctFilteredMappingCollection); if (_ArgZeroIsConfigurationRootDirectory && _Configuration.SaveResizedSubfiles && outputResolution == _Configuration.OutputResolutions[0] && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution) && _Exceptions.Count == 0) MapLogic(ticks, new(containers), fPhotoPrismContentDirectory, mapLogic, outputResolution, new(personKeyToIds), distinctFilteredFaces, distinctFilteredMappingCollection); if (_Configuration.SaveRandomForOutputResolutions.Contains(outputResolution) && personKeyToIds.Count > 0 && distinctFilteredMappingCollection.Count > 0) _Random.Random(_Configuration.PropertyConfiguration, _Configuration.ValidKeyWordsToIgnoreInRandom, outputResolution, personKeyToIds, distinctFilteredMappingCollection); if (_IsEnvironment.Development) continue; if (!_IsEnvironment.Development) { string dResultsFullGroupDirectory; string c2ResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; (cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(aResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton)); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(bResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton)); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(cResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton)); if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution)) _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultCollection)); } } } }