using System.Collections.ObjectModel; using System.Text.Json; using View_by_Distance.Shared.Models; using View_by_Distance.Shared.Models.Properties; using View_by_Distance.Shared.Models.Stateless.Methods; namespace View_by_Distance.Container.Models.Stateless.Methods; internal abstract class Container { private record FilePair(bool IsUnique, List Collection, FilePath FilePath, Item Item); internal static ReadOnlyCollection GetValidImageItems(IPropertyConfiguration propertyConfiguration, Models.Container container) { List results = []; foreach (Item item in container.Items) { if (!item.IsValidImageFormatExtension || propertyConfiguration.IgnoreExtensions.Contains(item.FilePath.ExtensionLowered)) continue; results.Add(item); } return container.Items.Count == results.Count ? container.Items : results.AsReadOnly(); } internal static DateTime[] GetContainerDateTimes(ReadOnlyCollection items) { DateTime[] results; long containerMinimumTicks = (from l in items select l.FilePath.LastWriteTicks).Min(); long containerMaximumTicks = (from l in items select l.FilePath.LastWriteTicks).Max(); results = [new(containerMinimumTicks), new(containerMaximumTicks)]; return results; } internal static List GetFilteredDistinctIds(IPropertyConfiguration propertyConfiguration, ReadOnlyCollection readOnlyContainers) { List results = []; ReadOnlyCollection filteredItems; foreach (Models.Container container in readOnlyContainers) { if (container.Items.Count == 0) continue; filteredItems = GetValidImageItems(propertyConfiguration, container); if (filteredItems.Count == 0) continue; foreach (Item item in filteredItems) { if (item.Property?.Id is null || item.ResizedFileHolder is null) continue; if (results.Contains(item.Property.Id.Value)) continue; results.Add(item.Property.Id.Value); } } return results; } internal static (int, Models.Container[]) GetContainers(IPropertyConfiguration propertyConfiguration, ReadOnlyDictionary? splatNineIdentifiers, string _) { int count; Models.Container[] results; bool useIgnoreExtensions = true; const bool useCeilingAverage = true; const string fileSearchFilter = "*"; IDlibDotNet dlibDotNet = GetDlibDotNet(); const string directorySearchFilter = "*"; ReadOnlyDictionary? exifDirectoriesById = null; ReadOnlyDictionary>? keyValuePairs = null; ReadOnlyCollection filesCollection = IDirectory.GetFilesCollection(propertyConfiguration.RootDirectory, directorySearchFilter, fileSearchFilter, useCeilingAverage); ReadOnlyCollection> filePathsCollection = IDirectory.GetFilePathCollections(propertyConfiguration, filesCollection, useIgnoreExtensions); (count, results) = GetContainers(dlibDotNet, propertyConfiguration, propertyConfiguration.RootDirectory, keyValuePairs, splatNineIdentifiers, filePathsCollection, exifDirectoriesById, directorySearchFilter); return (count, results); } private static IDlibDotNet GetDlibDotNet() => throw new NotImplementedException(); private static (int, Models.Container[]) GetContainers(IDlibDotNet dlibDotNet, IPropertyConfiguration propertyConfiguration, string filesCollectionDirectory, ReadOnlyDictionary>? keyValuePairs, ReadOnlyDictionary? splatNineIdentifiers, ReadOnlyCollection> filePathsCollection, ReadOnlyDictionary? exifDirectoriesById, string directorySearchFilter) { List results = []; string directory; List? items; Models.Container container; List directories = []; Dictionary> directoryToItems = []; foreach (ReadOnlyCollection filePaths in filePathsCollection) { if (filePaths.Count == 0) continue; directory = filePaths[0].DirectoryFullPath; if (directory is null) continue; if (!directories.Contains(directory)) directories.Add(directory); if (!directoryToItems.TryGetValue(directory, out items)) { directoryToItems.Add(directory, []); if (!directoryToItems.TryGetValue(directory, out items)) throw new Exception(); } } (string aResultsFullGroupDirectory, _) = dlibDotNet.GetResultsFullGroupDirectories(); string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, propertyConfiguration.ResultSingleton); if (!Directory.Exists(aPropertySingletonDirectory)) _ = Directory.CreateDirectory(aPropertySingletonDirectory); List filePairs = GetFilePairs(dlibDotNet, propertyConfiguration, aPropertySingletonDirectory, filesCollectionDirectory, keyValuePairs, splatNineIdentifiers, filePathsCollection, exifDirectoriesById, directorySearchFilter); foreach (FilePair filePair in filePairs) { if (!directoryToItems.TryGetValue(filePair.FilePath.DirectoryFullPath, out items)) { directoryToItems.Add(filePair.FilePath.DirectoryFullPath, []); if (!directoryToItems.TryGetValue(filePair.FilePath.DirectoryFullPath, out items)) throw new Exception(); } items.Add(filePair.Item); } foreach (KeyValuePair> keyValuePair in directoryToItems) { if (keyValuePair.Value.Count == 0) continue; container = new(keyValuePair.Key, new(keyValuePair.Value)); results.Add(container); } return (filePairs.Count, results.ToArray()); } private static List GetFilePairs(IDlibDotNet dlibDotNet, IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory, string filesCollectionDirectory, ReadOnlyDictionary>? keyValuePairs, ReadOnlyDictionary? splatNineIdentifiers, ReadOnlyCollection> filePathsCollection, ReadOnlyDictionary? exifDirectoriesById, string directorySearchFilter) { List results = []; const string extension = ".json"; ReadOnlyCollection filePairs; string jsonGroupDirectory = aPropertySingletonDirectory; int maxDegreeOfParallelism = Environment.ProcessorCount; int filesCollectionDirectoryLength = filesCollectionDirectory.Length; ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; filePairs = IFilePair.GetFilePairs(propertyConfiguration, directorySearchFilter, extension, jsonGroupDirectory, filePathsCollection); _ = Parallel.For(0, filePairs.Count, parallelOptions, (i, state) => ParallelFor(dlibDotNet, propertyConfiguration, jsonGroupDirectory, extension, keyValuePairs, splatNineIdentifiers, filesCollectionDirectoryLength, exifDirectoriesById, filePairs[i], results)); return results; } private static void ParallelFor(IDlibDotNet dlibDotNet, IPropertyConfiguration propertyConfiguration, string jsonGroupDirectory, string extension, ReadOnlyDictionary>? keyValuePairs, ReadOnlyDictionary? splatNineIdentifiers, int rootDirectoryLength, ReadOnlyDictionary? __, Shared.Models.FilePair filePair, List results) { dlibDotNet?.Tick(); bool abandoned = false; FileHolder sourceDirectoryFileHolder; Property? property = GetProperty(filePair); bool? fileSizeChanged = property is not null ? property.FileSize != filePair.FilePath.Length : null; bool isValidImageFormatExtension = propertyConfiguration.ValidImageFormatExtensions.Contains(filePair.FilePath.ExtensionLowered); bool? shouldIgnore = property is null || property.Keywords is null ? null : propertyConfiguration.IgnoreRulesKeyWords.Any(l => property.Keywords.Contains(l)); bool? isArchive = filePair.FilePath.Id is null || splatNineIdentifiers is null ? null : splatNineIdentifiers.TryGetValue(filePair.FilePath.Id.Value, out Identifier? identifier); if (property is not null && filePair.FilePath.Id is not null && filePair.FilePath.HasIgnoreKeyword is not null && filePair.FilePath.HasDateTimeOriginal is not null) { char? change; ReadOnlyCollection? filePaths = null; char hasIgnoreKeyword = IId.GetHasIgnoreKeyword(filePair.FilePath).ToString()[0]; char hasDateTimeOriginal = IId.GetHasDateTimeOriginal(propertyConfiguration, filePair.FilePath).ToString()[0]; char missingDateTimeOriginal = IId.GetMissingDateTimeOriginal(propertyConfiguration, filePair.FilePath).ToString()[0]; if (shouldIgnore is not null && shouldIgnore.Value) { if (filePair.FilePath.FileNameFirstSegment[^1] == hasIgnoreKeyword) change = null; else { change = hasIgnoreKeyword; if (keyValuePairs is null || !keyValuePairs.TryGetValue(filePair.FilePath.Id.Value, out filePaths) || filePaths is null) throw new NotSupportedException($"Rename File! <{filePair.FilePath.FileNameFirstSegment}>"); } } else if ((shouldIgnore is null || !shouldIgnore.Value) && property.DateTimeOriginal is null) { if (filePair.FilePath.FileNameFirstSegment[^1] == missingDateTimeOriginal) change = null; else { change = missingDateTimeOriginal; if (keyValuePairs is null || !keyValuePairs.TryGetValue(filePair.FilePath.Id.Value, out filePaths) || filePaths is null) throw new NotSupportedException($"Rename File! <{filePair.FilePath.FileNameFirstSegment}>"); } } else if (filePair.FilePath.FileNameFirstSegment[^1] != hasDateTimeOriginal) { change = hasDateTimeOriginal; if (keyValuePairs is null || !keyValuePairs.TryGetValue(filePair.FilePath.Id.Value, out filePaths) || filePaths is null) throw new NotSupportedException($"Rename File! <{filePair.FilePath.FileNameFirstSegment}>"); } else change = null; if (filePaths is not null && change is not null) RenameFile(filePair, filePair.FilePath, change.Value, filePaths); } string relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(filePair.FilePath.FullName, rootDirectoryLength, forceExtensionToLower: true); bool? lastWriteTimeChanged = property is not null ? propertyConfiguration.PropertiesChangedForProperty || property.LastWriteTime.Ticks != filePair.FilePath.LastWriteTicks : null; if (filePair.Match is not null) sourceDirectoryFileHolder = IFileHolder.Get(filePair.Match); else if (!filePair.IsUnique) sourceDirectoryFileHolder = IFileHolder.Get(Path.GetFullPath(string.Concat(jsonGroupDirectory, relativePath, extension))); else { string fileName = Path.GetFileName(filePair.FilePath.FullName); CombinedEnumAndIndex cei = Shared.Models.Stateless.Methods.IPath.GetCombinedEnumAndIndex(propertyConfiguration, filePair.FilePath); string directory = Path.Combine(jsonGroupDirectory, cei.Combined); string jsonFileName = $"{fileName}{extension}"; string fullFileName = Path.Combine(directory, jsonFileName); MoveIf(jsonFileName, cei, directory, fullFileName); sourceDirectoryFileHolder = IFileHolder.Get(fullFileName); } if (sourceDirectoryFileHolder.CreationTime is not null && sourceDirectoryFileHolder.LastWriteTime is not null && filePair.FilePath.LastWriteTicks != sourceDirectoryFileHolder.CreationTime.Value.Ticks) { File.SetCreationTime(sourceDirectoryFileHolder.FullName, new(filePair.FilePath.LastWriteTicks)); File.SetLastWriteTime(sourceDirectoryFileHolder.FullName, sourceDirectoryFileHolder.LastWriteTime.Value); } Item item = Item.Get(filePair.FilePath, sourceDirectoryFileHolder, relativePath, isArchive, filePair.IsNotUniqueAndNeedsReview, filePair.IsUnique, isValidImageFormatExtension, property, abandoned, fileSizeChanged, lastWriteTimeChanged); lock (results) results.Add(new(filePair.IsUnique, filePair.Collection, filePair.FilePath, item)); } private static Property? GetProperty(Shared.Models.FilePair filePair) { Property? result; if (filePair.Match is null) result = null; else { string json = File.ReadAllText(filePair.Match.FullName); if (string.IsNullOrEmpty(json)) result = null; else result = JsonSerializer.Deserialize(json, PropertyGenerationContext.Default.Property); } return result; } private static void RenameFile(Shared.Models.FilePair filePair, FilePath filePath, char change, ReadOnlyCollection filePaths) { string checkFile; if (filePath.DirectoryFullPath.Contains("Results") && filePath.DirectoryFullPath.Contains("Resize")) File.Delete(filePath.FullName); if (filePair.Match is not null) { string fileNameWithoutExtensionSecond = Path.GetFileNameWithoutExtension(filePair.Match.NameWithoutExtension); string extensionSecond = Path.GetExtension(filePair.Match.Name); checkFile = Path.Combine(filePair.Match.DirectoryFullPath, $"{fileNameWithoutExtensionSecond[..^1]}{change}{extensionSecond}{filePair.Match.ExtensionLowered}"); if (!File.Exists(checkFile)) File.Move(filePair.Match.FullName, checkFile); } foreach (FilePath f in filePaths) { checkFile = Path.Combine(f.DirectoryFullPath, $"{f.NameWithoutExtension[..^1]}{change}{f.ExtensionLowered}"); if (File.Exists(checkFile)) continue; File.Move(f.FullName, checkFile); } } private static void MoveIf(string fileName, CombinedEnumAndIndex cei, string directory, string fullFileName) { string[] segments = directory.Split(cei.Combined); string? checkDirectory = segments.Length == 1 ? Path.Combine(segments[0], $"{cei.Combined[2..]}") : segments.Length == 2 ? $"{segments[0]}{cei.Combined[2..]}{segments[1]}" : null; if (checkDirectory is not null && Directory.Exists(checkDirectory)) { string checkFile = Path.Combine(checkDirectory, fileName); if (File.Exists(checkFile)) File.Move(checkFile, fullFileName); } } internal static List GetFilteredDistinctFileNameFirstSegments(IPropertyConfiguration propertyConfiguration, ReadOnlyCollection readOnlyContainers) { List results = []; ReadOnlyCollection filteredItems; foreach (Models.Container container in readOnlyContainers) { if (container.Items.Count == 0) continue; filteredItems = GetValidImageItems(propertyConfiguration, container); if (filteredItems.Count == 0) continue; foreach (Item item in filteredItems) { if (item.Property?.Id is null || item.ResizedFileHolder is null) continue; if (results.Contains(item.FilePath.FileNameFirstSegment)) continue; results.Add(item.FilePath.FileNameFirstSegment); } } return results; } internal static ReadOnlyCollection GetValidImageItems(IPropertyConfiguration propertyConfiguration, ReadOnlyCollection containers, bool distinctItems, bool filterItems) { List results = []; List distinct = []; ReadOnlyCollection filteredItems; foreach (Models.Container container in containers) { if (container.Items.Count == 0) continue; if (!filterItems) filteredItems = container.Items; else { filteredItems = GetValidImageItems(propertyConfiguration, container); if (filteredItems.Count == 0) continue; } foreach (Item item in filteredItems) { if (item.Property?.Id is null || item.ResizedFileHolder is null) continue; if (distinctItems) { if (distinct.Contains(item.Property.Id.Value)) continue; distinct.Add(item.Property.Id.Value); } results.Add(item); } } return results.AsReadOnly(); } internal static ReadOnlyCollection GetContainers(IDlibDotNet dlibDotNet, IPropertyConfiguration propertyConfiguration, string facesFileNameExtension, string facesHiddenFileNameExtension, string eDistanceContentDirectory, string filesCollectionDirectory, ReadOnlyDictionary>? keyValuePairs, ReadOnlyDictionary? splatNineIdentifiers, ReadOnlyCollection> filePathsCollection, ReadOnlyDictionary exifDirectoriesById) { Models.Container[] results; const string directorySearchFilter = "*"; (_, results) = GetContainers(dlibDotNet, propertyConfiguration, filesCollectionDirectory, keyValuePairs, splatNineIdentifiers, filePathsCollection, exifDirectoriesById, directorySearchFilter); if (keyValuePairs is not null) DoGetFilePairsForRemaining(dlibDotNet, propertyConfiguration, facesFileNameExtension, facesHiddenFileNameExtension, eDistanceContentDirectory, filePathsCollection, directorySearchFilter); return results.AsReadOnly(); } private static void DoGetFilePairsForRemaining(IDlibDotNet dlibDotNet, IPropertyConfiguration propertyConfiguration, string facesFileNameExtension, string facesHiddenFileNameExtension, string eDistanceContentDirectory, ReadOnlyCollection> filePathsCollection, string directorySearchFilter) { const string extension = ".json"; (_, string bResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories(); ReadOnlyDictionary> fileNamesToFiles = FilePath.GetFilesKeyValuePairs(filePathsCollection); string bMetaSingletonDirectory = Path.Combine(bResultsFullGroupDirectory, propertyConfiguration.ResultSingleton); if (!Directory.Exists(bMetaSingletonDirectory)) _ = Directory.CreateDirectory(bMetaSingletonDirectory); _ = IFilePair.GetFilePairs(propertyConfiguration, directorySearchFilter, extension, bMetaSingletonDirectory, filePathsCollection, fileNamesToFiles); (string cResultsFullGroupDirectory, _, string dResultsFullGroupDirectory, _) = dlibDotNet.GetResultsFullGroupDirectories("Original"); string cResizeSingletonDirectory = Path.Combine(cResultsFullGroupDirectory, propertyConfiguration.ResultSingleton); if (!Directory.Exists(cResizeSingletonDirectory)) _ = Directory.CreateDirectory(cResizeSingletonDirectory); _ = IFilePair.GetFilePairs(propertyConfiguration, directorySearchFilter, extension, cResizeSingletonDirectory, filePathsCollection, fileNamesToFiles); string dFaceCollectionDirectory = Path.Combine(dResultsFullGroupDirectory, propertyConfiguration.ResultCollection); if (!Directory.Exists(dFaceCollectionDirectory)) _ = Directory.CreateDirectory(dFaceCollectionDirectory); _ = IFilePair.GetFilePairs(propertyConfiguration, directorySearchFilter, extension, dFaceCollectionDirectory, filePathsCollection, fileNamesToFiles); string dFaceContentDirectory = Path.Combine(dResultsFullGroupDirectory, propertyConfiguration.ResultContent); if (!Directory.Exists(dFaceContentDirectory)) _ = Directory.CreateDirectory(dFaceContentDirectory); AnyMovedFace(propertyConfiguration, facesFileNameExtension, facesHiddenFileNameExtension, fileNamesToFiles, dFaceContentDirectory); AnyMovedDistance(propertyConfiguration, facesFileNameExtension, fileNamesToFiles, eDistanceContentDirectory); } private static void AnyMovedFace(IPropertyConfiguration propertyConfiguration, string extension, string hiddenExtension, IReadOnlyDictionary> fileNamesToFiles, string jsonGroupDirectory) { string directory; string checkFile; string directoryName; List files = []; string[] fileNameSegments; List directories = []; files.AddRange(Directory.GetFiles(jsonGroupDirectory, $"*{extension}", SearchOption.AllDirectories)); files.AddRange(Directory.GetFiles(jsonGroupDirectory, $"*{hiddenExtension}", SearchOption.AllDirectories)); foreach (string file in files) { directory = Path.GetDirectoryName(file) ?? throw new Exception(); if (!directories.Contains(directory)) directories.Add(directory); directoryName = Path.GetFileName(directory); fileNameSegments = Path.GetFileName(file).Split('.'); if (fileNameSegments[0] != directoryName) { fileNameSegments[0] = string.Empty; checkFile = Path.Combine(directory, $"{directoryName}{string.Join('.', fileNameSegments)}"); if (!File.Exists(checkFile)) File.Move(file, checkFile); else { if (new FileInfo(file).LastWriteTime > new FileInfo(checkFile).LastWriteTime) File.Delete(file); else File.Move(file, checkFile, true); } } } if (directories.Count > 0) AnyMovedFace(propertyConfiguration, fileNamesToFiles, directories); } private static void AnyMovedFace(IPropertyConfiguration propertyConfiguration, IReadOnlyDictionary> fileNamesToFiles, List directories) { bool result = false; string checkFile; FilePath filePath; string subDirectory; string directoryName; string checkDirectory; FileHolder fileHolder; string directoryNameWith; List? collection; List directoryNames = []; foreach (string directory in directories) { fileHolder = IFileHolder.Get(Path.GetFileName(directory)); filePath = FilePath.Get(propertyConfiguration, fileHolder, index: null); if (filePath.Id is null) continue; if (!fileNamesToFiles.TryGetValue(filePath.Id.Value, out collection)) throw new Exception(); directoryNames.Clear(); foreach (FilePath f in collection) directoryNames.Add(Path.GetFileName(f.DirectoryFullPath) ?? throw new Exception()); if (directoryNames.Count == 0 || directoryNames.Distinct().Count() != 1) continue; directoryName = Path.GetFileName(Path.GetDirectoryName(directory)) ?? throw new Exception(); if (directoryName != directoryNames[0]) { subDirectory = Path.GetDirectoryName(Path.GetDirectoryName(directory)) ?? throw new Exception(); checkDirectory = Path.Combine(subDirectory, directoryNames[0]); if (!Directory.Exists(checkDirectory)) _ = Directory.CreateDirectory(checkDirectory); directoryNameWith = collection.Count > 1 ? directoryName : $"{collection[0].NameWithoutExtension}"; checkFile = Path.Combine(checkDirectory, directoryNameWith); if (!result) result = true; if (!Directory.Exists(checkFile)) Directory.Move(directory, checkFile); else { if (new DirectoryInfo(directory).LastWriteTime > new DirectoryInfo(checkFile).LastWriteTime) Directory.Delete(directory, recursive: true); else { Directory.Delete(checkFile, recursive: true); Directory.Move(directory, checkFile); } } } } } private static void AnyMovedDistance(IPropertyConfiguration propertyConfiguration, string extension, IReadOnlyDictionary> fileNamesToFiles, string jsonGroupDirectory) { bool result = false; string checkFile; string directory; FilePath filePath; FileHolder fileHolder; string[] fileNameSegments; List? collection; List fileNames = []; string[] files = Directory.GetFiles(jsonGroupDirectory, $"*{extension}", SearchOption.AllDirectories); foreach (string file in files) { fileHolder = IFileHolder.Get(file); if (!fileHolder.Exists) continue; filePath = FilePath.Get(propertyConfiguration, fileHolder, index: null); if (filePath.Id is null) continue; fileNameSegments = filePath.Name.Split('.'); if (!fileNamesToFiles.TryGetValue(filePath.Id.Value, out collection)) continue; fileNames.Clear(); foreach (FilePath f in collection) fileNames.Add(f.NameWithoutExtension ?? throw new Exception()); if (fileNames.Count == 0 || fileNames.Distinct().Count() != 1) continue; if (filePath.FileNameFirstSegment != fileNames[0]) { fileNameSegments[0] = string.Empty; directory = Path.GetDirectoryName(file) ?? throw new Exception(); checkFile = Path.Combine(directory, $"{fileNames[0]}{string.Join('.', fileNameSegments)}"); if (!result) result = true; if (!File.Exists(checkFile)) File.Move(file, checkFile); else { if (new FileInfo(file).LastWriteTime > new FileInfo(checkFile).LastWriteTime) File.Delete(file); else File.Move(file, checkFile, true); } } } } }