using ShellProgressBar; using System.Text.Json; using System.Text.RegularExpressions; using View_by_Distance.Property.Models; namespace View_by_Distance.PropertyCompare.Models; public class PropertyCompareLogic { private readonly Serilog.ILogger? _Log; private readonly string? _DiffRootDirectory; private readonly int _MaxDegreeOfParallelism; private readonly Configuration _Configuration; private readonly List<(string Find, string Replace)>? _SpellingFindReplace; public PropertyCompareLogic(int maxDegreeOfParallelism, Configuration configuration, List<(string Find, string Replace)>? spellingFindReplace = null, string? diffRootDirectory = null) { _Configuration = configuration; _DiffRootDirectory = diffRootDirectory; _SpellingFindReplace = spellingFindReplace; _Log = Serilog.Log.ForContext(); _MaxDegreeOfParallelism = Math.Abs(maxDegreeOfParallelism); } public override string ToString() { string result = JsonSerializer.Serialize(this, new JsonSerializerOptions() { WriteIndented = true }); return result; } private List GetDuplicates(PropertyCompare[] propertyCompares, int i, PropertyCompare[]? diffPropertyCompares) { List results = new(); if (_Log is null) throw new Exception($"{nameof(_Log)} is null!"); int index; string value; long[] distinctNumberValues; List checkValues = new(); List checkNumberValues = new(); if (diffPropertyCompares is null || propertyCompares.Length != diffPropertyCompares.Length) { foreach (PropertyCompare propertyCompare in propertyCompares) { value = $"{propertyCompare.Numbers[i]}\t{propertyCompare.Strings[i]}"; checkNumberValues.Add(propertyCompare.Numbers[i]); checkValues.Add(value); } distinctNumberValues = checkNumberValues.Distinct().ToArray(); if (distinctNumberValues.Length != propertyCompares.Length) _Log.Debug($"A) Distinct {nameof(propertyCompares)} - <{distinctNumberValues.Length} != {propertyCompares.Length}>"); } if (diffPropertyCompares is not null) { foreach (PropertyCompare propertyCompare in diffPropertyCompares) { value = $"{propertyCompare.Numbers[i]}\t{propertyCompare.Strings[i]}"; if (checkNumberValues.Contains(propertyCompare.Numbers[i])) { index = checkValues.IndexOf(value); if (index > -1) { if (index >= propertyCompares.Length - 1) continue; results.Add(propertyCompare); results.Add(propertyCompares[index]); } } checkNumberValues.Add(propertyCompare.Numbers[i]); checkValues.Add(value); } distinctNumberValues = checkNumberValues.Distinct().ToArray(); if (distinctNumberValues.Length != propertyCompares.Length) _Log.Debug($"B) Distinct {nameof(propertyCompares)} - <{distinctNumberValues.Length} != {propertyCompares.Length}>"); } return results; } private (string[] ToDirectories, List FromThenToCollection) Get(string aPropertyCollectionDirectory, PropertyCompare[] propertyCompares, int i, PropertyCompare[]? diffPropertyCompares) { List fromThenToCollection = new(); if (_Log is null) throw new Exception($"{nameof(_Log)} is null!"); int z = 0; string to; string from; bool extensionIsNullOrEmpty; string fileName; string toDirectory; string fromDirectory; List toCollection = new(); List toDirectories = new(); List fromCollection = new(); List bothCollection = new(); foreach (PropertyCompare propertyCompare in propertyCompares) bothCollection.Add(propertyCompare); if (diffPropertyCompares is not null) { foreach (PropertyCompare propertyCompare in diffPropertyCompares) bothCollection.Add(propertyCompare); } foreach (PropertyCompare propertyCompare in bothCollection) { z += 1; if (z % 1000 == 0) { if (!propertyCompare.IsArg) _Log.Debug($"{z}) Loop {_DiffRootDirectory}"); else _Log.Debug($"{z}) Loop {_Configuration.RootDirectory}"); } extensionIsNullOrEmpty = string.IsNullOrEmpty(propertyCompare.Extension); if (propertyCompare.IsArg) fromDirectory = string.Concat(_Configuration.RootDirectory, propertyCompare.RelativeDirectory); else fromDirectory = string.Concat(_DiffRootDirectory, propertyCompare.RelativeDirectory); if (!Directory.Exists(fromDirectory)) _ = Directory.CreateDirectory(fromDirectory); to = string.Empty; if (extensionIsNullOrEmpty) from = Path.GetFullPath(Path.Combine(fromDirectory, $"{propertyCompare.FileNameWithoutExtension}.jpg")); else from = Path.GetFullPath(Path.Combine(fromDirectory, $"{propertyCompare.FileNameWithoutExtension}{propertyCompare.Extension}")); if (fromCollection.Contains(from)) continue; if (!extensionIsNullOrEmpty && !File.Exists(from)) continue; fromCollection.Add(from); for (short c = 65; c < short.MaxValue; c++) { if (c > 95) break; if (extensionIsNullOrEmpty && !propertyCompare.IsArg) c += 1; fileName = Regex.Replace(propertyCompare.Strings[i], @"[\\,\/,\:,\*,\?,\"",\<,\>,\|]", "_"); toDirectory = Path.Combine(aPropertyCollectionDirectory, $". . . {i} - {(char)c}"); if (!extensionIsNullOrEmpty) to = Path.GetFullPath(Path.Combine(toDirectory, $"{propertyCompare.Numbers[i]}{fileName}{propertyCompare.Extension.ToLower()}")); else { to = Path.GetFullPath(Path.Combine(toDirectory, $"{propertyCompare.Numbers[i]}{fileName}.jpg")); break; } if (toCollection.Contains(to)) continue; if (!toDirectories.Contains(toDirectory)) toDirectories.Add(toDirectory); toCollection.Add(to); break; } if (string.IsNullOrEmpty(to)) continue; fromThenToCollection.Add(new string[] { from, to }); } return new(toDirectories.ToArray(), fromThenToCollection); } private void ParallelGet(List? duplicates, List results, List ids, List collection, int loadLessThan, string directory, object @lock, string relativeDirectory, string[] files, List filesWithoutExtension, bool isArg, string jsonFile) { long n; string s; int index; string extension; string corrected; List numbers; string regexResult; List strings; PropertyCompare propertyCompare; string jsonFileNameWithoutExtension = Path.GetFileNameWithoutExtension(jsonFile); string check = Path.Combine(directory, jsonFileNameWithoutExtension); index = filesWithoutExtension.IndexOf(check); if (index == -1) extension = string.Empty; else extension = Path.GetExtension(files[index]); string json = File.ReadAllText(jsonFile); A_Property? property = JsonSerializer.Deserialize(json); if (property?.Id is null) throw new Exception($"{nameof(property)} is null!"); DateTime minimumDateTime = Property.Models.Stateless.A_Property.GetMinimumDateTime(property); corrected = string.Concat(relativeDirectory, jsonFileNameWithoutExtension); if (_SpellingFindReplace is not null && (from l in _SpellingFindReplace where corrected.Contains(l.Find) select true).Any()) { foreach ((string find, string replace) in _SpellingFindReplace) corrected = corrected.Replace(find, replace); } if (string.IsNullOrEmpty(_Configuration.Pattern)) regexResult = corrected; else regexResult = Regex.Replace(corrected, _Configuration.Pattern, string.Empty); numbers = new(); strings = new(); for (int i = 0; i < loadLessThan; i++) { n = i switch { 0 => property.Id.Value, 1 => property.Id.Value, 2 => property.Id.Value, 3 => property.Id.Value, 4 => property.Id.Value, 5 => Property.Models.Stateless.A_Property.GetDateTime(property).Ticks, 6 => property.CreationTime.Ticks, 7 => property.FileSize, 8 => Property.Models.Stateless.A_Property.GetDateTime(property).Ticks, 9 => property.FileSize, _ => throw new Exception() }; s = i switch { 0 => $"{jsonFileNameWithoutExtension.ToLower()}", 1 => $"{property.FileSize}{Property.Models.Stateless.A_Property.GetDateTime(property).Ticks}", 2 => $"{property.FileSize}{property.CreationTime:yyyy-MM-dd_HH-mm-ss}", 3 => $"{property.FileSize}{property.Width}{property.Height}", 4 => string.Empty, 5 => $"{property.FileSize}", 6 => $"{property.FileSize}", 7 => $"{property.Width}{property.Height}", 8 => string.Empty, 9 => string.Empty, _ => throw new Exception() }; numbers.Add(n); strings.Add(s); } propertyCompare = new(extension, jsonFileNameWithoutExtension, isArg, minimumDateTime, numbers, property, regexResult, relativeDirectory, strings); lock (@lock) results.Add(propertyCompare); if (duplicates is not null) { string value = $"{property.Id.Value}\t{property}"; index = ids.IndexOf(property.Id.Value); if (index > -1) { lock (@lock) { duplicates.Add(propertyCompare); duplicates.Add(collection[index]); } } lock (@lock) { ids.Add(property.Id.Value); collection.Add(propertyCompare); } } } public PropertyCompare[] Get(string aPropertySingletonDirectory, int loadLessThan = 7, List? duplicates = null, bool deleteExtension = false) { List results = new(); if (_Log is null) throw new Exception($"{nameof(_Log)} is null!"); string[] files; int totalSeconds; string directory; string searchPattern; object @lock = new(); int exceptionCount = 0; List ids = new(); string relativeDirectory; if (!deleteExtension) searchPattern = "*.json"; else searchPattern = "*.delete"; long ticks = DateTime.Now.Ticks; List filesWithoutExtension; List topDirectories = new(); string extension = searchPattern[1..]; string[] filteredSourceDirectoryFiles; List collection = new(); bool isArg = aPropertySingletonDirectory.Contains(_Configuration.RootDirectory); ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _MaxDegreeOfParallelism }; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> groupCollection = Property.Models.Stateless.A_Property.GetGroupCollection(aPropertySingletonDirectory, searchPattern, topDirectories); int count = groupCollection.Count; foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles, int r) in groupCollection) { if (!topDirectories.Any()) continue; if (!sourceDirectoryFiles.Any()) continue; relativeDirectory = sourceDirectory[aPropertySingletonDirectory.Length..]; if (!isArg) directory = string.Concat(_DiffRootDirectory, relativeDirectory); else directory = string.Concat(_Configuration.RootDirectory, relativeDirectory); if (!Directory.Exists(directory)) _ = Directory.CreateDirectory(directory); files = Directory.GetFiles(directory); filesWithoutExtension = files.Select(l => Path.Combine($"{Path.GetDirectoryName(l)}", Path.GetFileNameWithoutExtension(l))).ToList(); filteredSourceDirectoryFiles = (from l in sourceDirectoryFiles where l.EndsWith(extension) select l).ToArray(); if (!filteredSourceDirectoryFiles.Any()) continue; totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); using ProgressBar progressBar = new(filteredSourceDirectoryFiles.Length, $"{g}) {r + 1:000} / {count:000} - {sourceDirectory} - {filteredSourceDirectoryFiles.Length} file(s) - {totalSeconds} total second(s)", options); _ = Parallel.For(0, filteredSourceDirectoryFiles.Length, parallelOptions, i => { try { ParallelGet(duplicates, results, ids, collection, loadLessThan, directory, @lock, relativeDirectory, files, filesWithoutExtension, isArg, sourceDirectoryFiles[i]); progressBar.Tick(); } catch (Exception ex) { exceptionCount += 1; _Log.Error(string.Concat(sourceDirectory, Environment.NewLine, ex.Message, Environment.NewLine, ex.StackTrace), ex); if (exceptionCount == filteredSourceDirectoryFiles.Length) throw new Exception(string.Concat("All in [", sourceDirectory, "]failed!")); } }); } if (exceptionCount != 0) throw new Exception(); return (from l in results orderby Property.Models.Stateless.A_Property.GetMinimumDateTime(l.Property).Ticks select l).ToArray(); } private void MoveFiles(string[] directories, List fromThenToCollection) { if (_Log is null) throw new Exception($"{nameof(_Log)} is null!"); int z; string to; string from; int moved = 0; _Log.Information("Ready to move?"); ConsoleKey? consoleKey = null; for (int y = 0; y < int.MaxValue; y++) { _Log.Information("Press \"Y\" key when ready to continue"); if (Console.ReadKey().Key == ConsoleKey.Y) break; } _Log.Information(". . ."); foreach (string directory in directories) { if (!Directory.Exists(directory)) _ = Directory.CreateDirectory(directory); } z = 0; foreach (string[] fromThenTo in fromThenToCollection) { z += 1; from = fromThenTo[0]; to = fromThenTo[1]; if (z % 1000 == 0) _Log.Information($"{z})"); if (!File.Exists(from)) continue; if (File.Exists(to)) continue; File.Move(from, to); moved += 1; } _Log.Information($"{moved} file(s) moved"); for (int m = 0; m < int.MaxValue; m++) { moved = 0; _Log.Information($"{m}) Ready to move back?"); for (int y = 0; y < int.MaxValue; y++) { _Log.Information("Press \"Y\" key when ready to continue"); if (Console.ReadKey().Key == ConsoleKey.Y) break; } _Log.Information(". . ."); z = 0; foreach (string[] fromThenTo in fromThenToCollection) { z += 1; from = fromThenTo[1]; to = fromThenTo[0]; if (z % 1000 == 0) _Log.Information($"{z})"); if (!File.Exists(from)) continue; if (File.Exists(to)) continue; File.Move(from, to); moved += 1; } foreach (string directory in directories) { if (Directory.Exists(directory)) { if (!Directory.GetFiles(directory, "*", SearchOption.AllDirectories).Any()) Directory.Delete(directory); } } _Log.Information($"Done moving back {moved} file(s) for loop {m})"); for (int y = 0; y < int.MaxValue; y++) { _Log.Information("Press \"Y\" key when ready to continue"); _Log.Information("Press \"End\" key when ready to break"); consoleKey = Console.ReadKey().Key; if (consoleKey is ConsoleKey.Y or ConsoleKey.End) break; } _Log.Information(". . ."); if (consoleKey.HasValue && consoleKey == ConsoleKey.End) break; } } public void SaveDiffFiles(string aPropertyCollectionDirectory, int loadLessThan, PropertyCompare[] propertyCompares, PropertyCompare[]? diffPropertyCompares) { if (_Log is null) throw new Exception($"{nameof(_Log)} is null!"); string text; string[] lines; string fileName; ConsoleKey? consoleKey = null; List duplicateCollection; for (int i = 0; i < loadLessThan; i++) { if (!propertyCompares.Any()) continue; if (diffPropertyCompares is null || !diffPropertyCompares.Any()) duplicateCollection = GetDuplicates(propertyCompares, i, diffPropertyCompares); else { if (i != 0) { if (diffPropertyCompares is null) continue; } if (propertyCompares.Length == diffPropertyCompares.Length) continue; duplicateCollection = GetDuplicates(propertyCompares, i, diffPropertyCompares); } lines = (from l in duplicateCollection select l.GetSelect()).ToArray(); _Log.Debug($"{i}) loop has {lines.Length} line(s)"); if ((duplicateCollection.Count % 2) != 0) continue; fileName = Path.Join(aPropertyCollectionDirectory, $". . . {i}-Duplicates.txt"); if (!duplicateCollection.Any()) { if (File.Exists(fileName)) File.Delete(fileName); continue; } text = string.Join(Environment.NewLine, lines); File.WriteAllText(fileName, text); for (int y = 0; y < int.MaxValue; y++) { _Log.Information("Press \"Y\" key when ready to continue"); _Log.Information("Press \"End\" key when ready to break"); consoleKey = Console.ReadKey().Key; if (consoleKey is ConsoleKey.Y or ConsoleKey.End) break; } _Log.Information(". . ."); if (consoleKey.HasValue && consoleKey == ConsoleKey.End) break; } } public void SaveLogAndMoveFiles(string aPropertyCollectionDirectory, int loadLessThan, PropertyCompare[] propertyCompares, PropertyCompare[]? diffPropertyCompares, int i) { if (_Log is null) throw new Exception($"{nameof(_Log)} is null!"); List lines; string checkDirectory; string[] toDirectories; ConsoleKey? consoleKey = null; List fromThenToCollection; _Log.Information($"{i}) example - number:{propertyCompares[0].Numbers[i]}; string:{propertyCompares[0].Strings[i]};"); (toDirectories, fromThenToCollection) = Get(aPropertyCollectionDirectory, propertyCompares, i, diffPropertyCompares); if (toDirectories.Length < 2) _Log.Information($"{i}) loop only one directory :)"); else { checkDirectory = toDirectories[1]; _Log.Information($"{i}) loop {toDirectories.Length} directories and . . .{i}) - B will have ~{fromThenToCollection.Where(l => l[1].StartsWith(checkDirectory)).Count()}"); } for (int y = 0; y < int.MaxValue; y++) { _Log.Information($"Press \"Y\" key to {nameof(MoveFiles)} \"N\" to skip"); consoleKey = Console.ReadKey().Key; if (consoleKey is ConsoleKey.Y or ConsoleKey.N) break; } _Log.Information(". . ."); if (consoleKey.HasValue && consoleKey.Value == ConsoleKey.Y) { lines = new(); foreach (string[] fromThenTo in fromThenToCollection) { lines.Add(fromThenTo[0]); lines.Add(fromThenTo[1]); } File.WriteAllLines(Path.Join(aPropertyCollectionDirectory, $". . . {i}-All.txt"), lines); MoveFiles(toDirectories, fromThenToCollection); } } public void WithSubdirectory(string propertyDirectory, bool subDirectoriesAny, string fileName, bool renameCompare, bool deleteArg) { if (_Log is null) throw new Exception($"{nameof(_Log)} is null!"); List fileCollection = new(); if (renameCompare && deleteArg) throw new Exception(); if (!renameCompare && !deleteArg) throw new Exception(); string[] lines; string[] txtFiles = Directory.GetFiles(propertyDirectory, fileName, SearchOption.TopDirectoryOnly); if (txtFiles.Length != 1) lines = Array.Empty(); else lines = File.ReadAllLines(txtFiles[0]); if (lines.Any()) { string argLine; string moveFile; string argFileName; string compareLine; string? argDirectory; string argFullFileName; string compareFileName; string argDirectoryName; string? compareDirectory; string compareFullFileName; string compareDirectoryName; for (int i = 0; i < lines.Length; i++) { argLine = lines[i]; compareLine = lines[i + 1]; i += 1; if (!subDirectoriesAny) { if (!argLine.StartsWith(_Configuration.RootDirectory) || !compareLine.StartsWith(_Configuration.RootDirectory)) throw new Exception(i.ToString()); } else { if (!argLine.StartsWith(_Configuration.RootDirectory) || compareLine.StartsWith(_Configuration.RootDirectory)) throw new Exception(i.ToString()); } } for (int i = 0; i < lines.Length; i++) { argLine = lines[i]; compareLine = lines[i + 1]; i += 1; if (!subDirectoriesAny) { if (!argLine.StartsWith(_Configuration.RootDirectory) || !compareLine.StartsWith(_Configuration.RootDirectory)) throw new Exception(i.ToString()); } else { if (!argLine.StartsWith(_Configuration.RootDirectory) || compareLine.StartsWith(_Configuration.RootDirectory)) throw new Exception(i.ToString()); } argFullFileName = argLine.Split('\t')[0]; compareFullFileName = compareLine.Split('\t')[0]; argFileName = Path.GetFileName(argFullFileName); if (deleteArg) { if (!File.Exists(argFullFileName)) continue; fileCollection.Add(new string[] { argFullFileName }); } else if (renameCompare) { argDirectory = Path.GetDirectoryName(argFullFileName); if (string.IsNullOrEmpty(argDirectory)) continue; compareDirectory = Path.GetDirectoryName(compareFullFileName); if (string.IsNullOrEmpty(compareDirectory)) continue; argDirectoryName = Path.GetFileName(argDirectory); if (argDirectoryName[..3].Equals(argFileName[..3], StringComparison.CurrentCultureIgnoreCase)) continue; compareDirectoryName = Path.GetFileName(argDirectory); compareFileName = Path.GetFileName(compareFullFileName); if (!compareDirectoryName[..3].Equals(compareFileName[..3], StringComparison.CurrentCultureIgnoreCase)) continue; if (!File.Exists(compareFullFileName)) { if (File.Exists(argFullFileName)) fileCollection.Add(new string[] { argFullFileName }); continue; } moveFile = Path.Combine(compareDirectory, argFileName); if (File.Exists(moveFile)) { _Log.Information(argLine); _Log.Information(compareLine); continue; } fileCollection.Add(new string[] { compareFullFileName, moveFile }); } else throw new Exception(); } } foreach (string[] file in fileCollection) { if (deleteArg) File.Delete(Path.GetFullPath(file[0])); else if (renameCompare) File.Move(Path.GetFullPath(file[0]), Path.GetFullPath(file[1])); } } }