using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using Phares.Shared; using ShellProgressBar; using System.Text; using System.Text.Json; using View_by_Distance.Duplicate.Search.Models; using View_by_Distance.Property.Models; using View_by_Distance.Shared.Models; using View_by_Distance.Shared.Models.Methods; namespace View_by_Distance.Duplicate.Search; public class DuplicateSearch { public DuplicateSearch(List args, ILogger logger, IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, AppSettings appSettings, string workingDirectory, bool isSilent, IConsole console) { if (isSilent) { } if (console is null) { } string searchPattern = "*"; long ticks = DateTime.Now.Ticks; Configuration configuration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot); logger?.LogInformation(configuration.RootDirectory); if (appSettings.IndexOnly) WriteIndexData(ticks, configuration, searchPattern); else { Configuration.Verify(configuration, requireExist: false); string argZero = args.Count > 0 ? Path.GetFullPath(args[0]) : Path.GetFullPath(configuration.RootDirectory); bool argZeroIsConfigurationRootDirectory = configuration.RootDirectory == argZero; Container[] containers = GetContainers(appSettings, ticks, argZero, configuration, argZeroIsConfigurationRootDirectory); string destinationRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(configuration, "Z) Moved"); List preloadIds = GetPreloadIds(destinationRoot); Dictionary> idToCollection = GetIdToCollection(argZero, configuration, argZeroIsConfigurationRootDirectory, containers, destinationRoot, preloadIds); int duplicates = (from l in idToCollection where l.Value.Count > 1 select 1).Sum(); if (duplicates == 0) logger?.LogInformation($"Found {duplicates} duplicate file(s)"); else QuestionMove(ticks, logger, destinationRoot, idToCollection, duplicates); } } private static void WriteIndexData(long ticks, Configuration configuration, string searchPattern) { string? alongSideDirectory; string directoryName = Path.GetFileName(configuration.RootDirectory); if (Path.GetPathRoot(configuration.RootDirectory) == configuration.RootDirectory) alongSideDirectory = configuration.RootDirectory; else alongSideDirectory = Path.GetDirectoryName(configuration.RootDirectory); if (alongSideDirectory is null) throw new NullReferenceException(nameof(alongSideDirectory)); IEnumerable<(string, string[])> files = Shared.Models.Stateless.Methods.IFileHolder.GetFiles(configuration.RootDirectory, searchPattern); List fileHolders = Shared.Models.Stateless.Methods.IFileHolder.GetFileHolders(files.ToArray()); File.WriteAllLines(Path.Combine(alongSideDirectory, $"{directoryName}-{ticks}.tsv"), fileHolders.Select(l => l.FullName)); string json = JsonSerializer.Serialize(fileHolders); File.WriteAllText(Path.Combine(alongSideDirectory, $"{directoryName}-{ticks}.json"), json); } private static void Move(ILogger? logger, long ticks, string destinationRoot, List<(FileHolder ImageFileHolder, string Destination)> collection) { StringBuilder stringBuilder = new(); foreach ((FileHolder fileHolder, string destination) in collection) { _ = stringBuilder.AppendLine(fileHolder.FullName); _ = stringBuilder.AppendLine(destination); } _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(Path.Combine(destinationRoot, $"{ticks}file(s).lsv"), stringBuilder.ToString(), updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null); logger?.LogInformation($"Ready to move {collection.Count} file(s)?"); for (int y = 0; y < int.MaxValue; y++) { logger?.LogInformation("Press \"Y\" key to move file(s) or close console to not move files"); if (System.Console.ReadKey().Key == ConsoleKey.Y) break; } logger?.LogInformation(". . ."); foreach ((FileHolder fileHolder, string destination) in collection) { try { File.Move(fileHolder.FullName, destination); } catch (Exception exception) { logger?.LogError(exception, $"Failed to move <{fileHolder.FullName}>"); } } logger?.LogInformation($"{collection.Count} file(s) moved"); for (int y = 0; y < int.MaxValue; y++) { logger?.LogInformation("Press \"Y\" key to move file(s) back or close console to leave them moved"); if (System.Console.ReadKey().Key != ConsoleKey.Y) continue; logger?.LogInformation(". . ."); foreach ((FileHolder fileHolder, string destination) in collection) { if (!File.Exists(destination)) continue; if (File.Exists(fileHolder.FullName)) continue; try { File.Move(destination, fileHolder.FullName); } catch (Exception exception) { logger?.LogError(exception, $"Failed to move <{destination}>"); } } } logger?.LogInformation(". . ."); } private static void QuestionMove(long ticks, ILogger? logger, string destinationRoot, Dictionary> idToCollection, int duplicates) { int[] ids = (from l in idToCollection orderby l.Key where l.Value.Any(m => m is not null) select l.Key).ToArray(); _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(Path.Combine(destinationRoot, $"{ticks}-id(s).lsv"), string.Join(Environment.NewLine, ids), updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null); string json = JsonSerializer.Serialize(idToCollection, new JsonSerializerOptions { WriteIndented = true }); _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(Path.Combine(destinationRoot, $"{ticks}.json"), json, updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null); logger?.LogInformation($"Found {duplicates} duplicate file(s)"); for (int y = 0; y < int.MaxValue; y++) { logger?.LogInformation("Press \"Y\" key to continue or close console to leave them moved"); if (System.Console.ReadKey().Key != ConsoleKey.Y) continue; logger?.LogInformation(". . ."); List<(FileHolder ImageFileHolder, string Destination)> collection = GetCollectionAndCreateDirectories(idToCollection); Move(logger, ticks, destinationRoot, collection); } } private static Container[] GetContainers(AppSettings appSettings, long ticks, string argZero, Configuration configuration, bool argZeroIsConfigurationRootDirectory) { int f; Container[] containers; int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); string message = $") Building Container(s) - {totalSeconds} total second(s)"; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; using (ProgressBar progressBar = new(1, message, options)) { progressBar.Tick(); string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(A_Property), "{}"); (f, containers) = Shared.Models.Stateless.Methods.IContainer.GetContainers(configuration, aPropertySingletonDirectory); } return containers; } private static Dictionary> GetIdToCollection(string argZero, Configuration configuration, bool argZeroIsConfigurationRootDirectory, Container[] containers, string destinationRoot, List preloadIds) { Dictionary> results = new(); string directory; const int zero = 0; Item[] filteredItems; FileHolder resizedFileHolder; DateTime[] containerDateTimes; MappingFromItem? mappingFromItem; List? collection; const string duplicates = "-Duplicate(s)"; if (containers.Any()) { foreach (int id in preloadIds) results.Add(id, new() { null }); } foreach (Container container in containers) { if (!container.Items.Any()) continue; if (!argZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero)) continue; filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(configuration, container); if (!filteredItems.Any()) continue; containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems); foreach (Item item in filteredItems) { if (item.Property?.Id is null) { if (int.TryParse(item.ImageFileHolder.NameWithoutExtension, out int id)) continue; continue; } if (!results.TryGetValue(item.Property.Id.Value, out collection)) results.Add(item.Property.Id.Value, new()); if (collection is null && !results.TryGetValue(item.Property.Id.Value, out collection)) continue; if (collection.Count == 0) directory = $"0{duplicates}"; else directory = $"{collection.Count + 1}{duplicates}"; if (collection.Count == 1) { mappingFromItem = collection[zero]; if (mappingFromItem is not null) { resizedFileHolder = new(mappingFromItem.ResizedFileHolder.FullName.Replace($"0{duplicates}", $"1{duplicates}")); collection[0] = new(mappingFromItem.ContainerDateTimes, item.Property.DateTimeDigitized, item.Property.DateTimeOriginal, mappingFromItem.Id, mappingFromItem.ImageFileHolder, mappingFromItem.IsWrongYear, item.Property.Keywords ?? Array.Empty(), mappingFromItem.MinimumDateTime, item.Property.Model, mappingFromItem.RelativePath, resizedFileHolder); } } resizedFileHolder = new(string.Concat(Path.Combine(destinationRoot, directory), item.RelativePath)); mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item, resizedFileHolder); collection.Add(mappingFromItem); } } return results; } private static List<(FileHolder ImageFileHolder, string Destination)> GetCollectionAndCreateDirectories(Dictionary> idToCollection) { List<(FileHolder ImageFileHolder, string Destination)> results = new(); List collection = new(); foreach (KeyValuePair> keyValuePair in idToCollection) { foreach (MappingFromItem? mappingFromItem in keyValuePair.Value) { if (mappingFromItem?.ResizedFileHolder.DirectoryName is null) continue; if (!mappingFromItem.ImageFileHolder.Exists || mappingFromItem.ResizedFileHolder.Exists) continue; collection.Add(mappingFromItem.ResizedFileHolder.DirectoryName); results.Add(new(mappingFromItem.ImageFileHolder, mappingFromItem.ResizedFileHolder.FullName)); } } foreach (string directory in collection.Distinct()) { if (!Directory.Exists(directory)) _ = Directory.CreateDirectory(directory); } return results; } private static List GetPreloadIds(string destinationRoot) { List results = new(); string[] lines; string preloadDirectory = Path.Combine(destinationRoot, "Preload"); if (!Directory.Exists(preloadDirectory)) _ = Directory.CreateDirectory(preloadDirectory); string[] files = Directory.GetFiles(preloadDirectory, "*.lsv", SearchOption.TopDirectoryOnly); foreach (string file in files) { lines = File.ReadAllLines(file); foreach (string line in lines) { if (string.IsNullOrEmpty(line) || !int.TryParse(line, out int id) || id == 0) continue; results.Add(id); } } return results; } }