using Microsoft.Extensions.Configuration; using Phares.Shared; using System.Text.Json; using System.Text.RegularExpressions; using View_by_Distance.PrepareForOld.Models; using View_by_Distance.Property.Models; using View_by_Distance.Shared.Models.Methods; namespace View_by_Distance.PrepareForOld; public class PrepareForOld { private readonly Serilog.ILogger? _Log; private readonly AppSettings _AppSettings; private readonly List _Exceptions; private readonly IsEnvironment _IsEnvironment; private readonly Models.Configuration _Configuration; private readonly List> _FileKeyValuePairs; private readonly List<(string Find, string Replace)> _SpellingFindReplace; private readonly Dictionary>> _FilePropertiesKeyValuePairs; public PrepareForOld(List args, IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, AppSettings appSettings, string workingDirectory, bool isSilent, IConsole console) { if (isSilent) { } if (console is null) { } string spellingA; string spellingB; _AppSettings = appSettings; if (appSettings.MaxDegreeOfParallelism is null) throw new NullReferenceException(nameof(appSettings.MaxDegreeOfParallelism)); _SpellingFindReplace = new(); _IsEnvironment = isEnvironment; _Exceptions = new List(); _Log = Serilog.Log.ForContext(); _FileKeyValuePairs = new List>(); _FilePropertiesKeyValuePairs = new Dictionary>>(); Property.Models.Configuration propertyConfiguration = Property.Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory); Property.Models.Configuration.Verify(propertyConfiguration); Models.Configuration configuration = Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory, propertyConfiguration); Verify(configuration); if (propertyConfiguration.IgnoreExtensions is null) throw new NullReferenceException(nameof(propertyConfiguration.IgnoreExtensions)); for (int i = 0; i < configuration.Spelling.Length; i++) { spellingA = configuration.Spelling[i]; spellingB = configuration.Spelling[i + 1]; if (spellingB.Contains(spellingA)) throw new Exception("Change configuration!"); _SpellingFindReplace.Add(new(spellingA, spellingB)); i += 1; } string[] dbFiles = Directory.GetFiles(propertyConfiguration.RootDirectory, "*.db", SearchOption.AllDirectories); foreach (string dbFile in dbFiles) File.Delete(dbFile); string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(A_Property), "{}"); if (!Directory.Exists(aPropertySingletonDirectory)) throw new Exception(aPropertySingletonDirectory); ConsoleKey? consoleKey = null; for (int y = 0; y < int.MaxValue; y++) { _Log.Information($"Execute {nameof(VerifyAgainstIndexInfoJsonFiles)} \"Y(es)\" or \"N(o)\"?"); consoleKey = Console.ReadKey().Key; if (consoleKey is ConsoleKey.Y or ConsoleKey.N) break; } _Log.Information(". . ."); if (consoleKey == ConsoleKey.Y) VerifyAgainstIndexInfoJsonFiles(propertyConfiguration, aPropertySingletonDirectory); for (int y = 0; y < int.MaxValue; y++) { _Log.Information($"Execute {nameof(SaveTabSeparatedValues)} \"Y(es)\" or \"N(o)\"?"); consoleKey = Console.ReadKey().Key; if (consoleKey is ConsoleKey.Y or ConsoleKey.N) break; } _Log.Information(". . ."); if (consoleKey == ConsoleKey.Y) SaveTabSeparatedValues(propertyConfiguration, aPropertySingletonDirectory); for (int y = 0; y < int.MaxValue; y++) { _Log.Information($"Execute {nameof(ReSaveJsonFiles)} \"Y(es)\" or \"N(o)\"?"); consoleKey = Console.ReadKey().Key; if (consoleKey is ConsoleKey.Y or ConsoleKey.N) break; } _Log.Information(". . ."); if (consoleKey == ConsoleKey.Y) ReSaveJsonFiles(); for (int y = 0; y < int.MaxValue; y++) { _Log.Information($"Execute {nameof(CopyMissingImagesLogs)} \"Y(es)\" or \"N(o)\"?"); consoleKey = Console.ReadKey().Key; if (consoleKey is ConsoleKey.Y or ConsoleKey.N) break; } _Log.Information(". . ."); if (consoleKey == ConsoleKey.Y) CopyMissingImagesLogs(); string message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}"; _Log.Information(message); if (_Exceptions.Count != 0) throw new Exception(message); _Configuration = configuration; } private static void Verify(Models.Configuration configuration) { if (configuration.Spelling is null || !configuration.Spelling.Any()) throw new NullReferenceException(nameof(configuration.Spelling)); } private static List GetExifCollection(string infoDirectory, string infoDirectoryExtra, bool checkDistinct) { List results = new(); string json; Models.SaveTabSeparatedValues.ImageExifInfo[]? exifCollection; List exifInfoFiles = Directory.GetFiles(infoDirectory, "ImageExifInfo.json", SearchOption.AllDirectories).ToList(); if (!string.IsNullOrEmpty(infoDirectoryExtra)) exifInfoFiles.AddRange(Directory.GetFiles(infoDirectoryExtra, "ImageExifInfo.json", SearchOption.AllDirectories)); foreach (string exifInfoFile in exifInfoFiles) { json = File.ReadAllText(exifInfoFile); exifCollection = JsonSerializer.Deserialize(json); if (exifCollection is null) continue; results.AddRange(exifCollection); } if (checkDistinct) { int[] check = (from l in results select l.Index).Distinct().ToArray(); if (check.Length != results.Count) throw new Exception(); } return results; } private List<(int Index, long Ticks, string RelativeDirectory, string FileNameWithoutExtension, string Extension, string RegexResult)> GetIndexCollection(string infoDirectory, string infoDirectoryExtra, bool checkDistinct) { List<(int Index, long Ticks, string RelativeDirectory, string FileNameWithoutExtension, string Extension, string RegexResult)> results = new(); if (_Configuration?.PropertyConfiguration is null) throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration)); long ticks; string json; string extension; string corrected; string regexResult; string relativeDirectory; List dateTimes; string fileNameWithoutExtension; string firstFileSegmentCollection; Dictionary keyValuePairs = new(); Models.SaveTabSeparatedValues.IndexInfo[]? indexCollection; List exifCollection = GetExifCollection(infoDirectory, infoDirectoryExtra, checkDistinct); List indexInfoFiles = Directory.GetFiles(infoDirectory, "IndexInfo.json", SearchOption.AllDirectories).ToList(); if (!string.IsNullOrEmpty(infoDirectoryExtra)) indexInfoFiles.AddRange(Directory.GetFiles(infoDirectoryExtra, "IndexInfo.json", SearchOption.AllDirectories)); foreach (Models.SaveTabSeparatedValues.ImageExifInfo exifInfo in exifCollection) { dateTimes = Property.Models.Stateless.A_Property.GetDateTimes(exifInfo.CreationTime, exifInfo.LastWriteTime, exifInfo.DateTime, exifInfo.DateTimeDigitized, exifInfo.DateTimeOriginal, exifInfo.GPSDateStamp); if (!checkDistinct && keyValuePairs.ContainsKey(exifInfo.Index)) continue; keyValuePairs.Add(exifInfo.Index, dateTimes.Min().Ticks); } foreach (string indexInfoFile in indexInfoFiles) { json = File.ReadAllText(indexInfoFile); indexCollection = JsonSerializer.Deserialize(json); if (indexCollection is null) continue; foreach (Models.SaveTabSeparatedValues.IndexInfo indexInfo in indexCollection) { if (indexInfo.FileSegmentCollection is null) continue; // if (indexInfo.FileSegmentCollection.Length != 1) // continue; ticks = keyValuePairs[indexInfo.Index]; firstFileSegmentCollection = indexInfo.FileSegmentCollection[0]; extension = Path.GetExtension(firstFileSegmentCollection); fileNameWithoutExtension = Path.GetFileNameWithoutExtension(firstFileSegmentCollection); corrected = firstFileSegmentCollection[..(firstFileSegmentCollection.Length - extension.Length)]; relativeDirectory = firstFileSegmentCollection[..(firstFileSegmentCollection.Length - fileNameWithoutExtension.Length - extension.Length)]; if ((from l in _SpellingFindReplace where corrected.Contains(l.Find) select true).Any()) { foreach ((string find, string replace) in _SpellingFindReplace) corrected = corrected.Replace(find, replace); } if (string.IsNullOrEmpty(_Configuration.PropertyConfiguration.Pattern)) regexResult = corrected; else regexResult = Regex.Replace(corrected, _Configuration.PropertyConfiguration.Pattern, string.Empty); results.Add(new(indexInfo.Index, ticks, relativeDirectory, fileNameWithoutExtension, extension, regexResult)); } } return results; } private static List GetUseTabSeparatedValueIndices(string useDirectory) { List results = new(); string[] lines; string[] segments; string[] tabSeparatedValueFiles = Directory.GetFiles(useDirectory, "*.tsv", SearchOption.TopDirectoryOnly); foreach (string tabSeparatedValueFile in tabSeparatedValueFiles) { lines = File.ReadAllLines(tabSeparatedValueFile); foreach (string line in lines) { if (string.IsNullOrEmpty(line)) continue; segments = line.Split('\t'); if (segments.Length < 1) continue; if (!int.TryParse(segments[0], out int index)) continue; results.Add(index); } } return results.Distinct().OrderBy(l => l).ToList(); } private void SaveTabSeparatedValues(Property.Models.Configuration configuration, string aPropertySingletonDirectory) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); if (_AppSettings.MaxDegreeOfParallelism is null) throw new NullReferenceException(nameof(_AppSettings.MaxDegreeOfParallelism)); if (_Configuration?.PropertyConfiguration is null) throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration)); string? rootDirectoryParent = Path.GetDirectoryName(_Configuration.PropertyConfiguration.RootDirectory); if (string.IsNullOrEmpty(rootDirectoryParent)) throw new NullReferenceException(nameof(rootDirectoryParent)); int z = 0; int mappedIndex; int? propertyId; long? propertyTicks; int loadLessThan = 7; string useDirectory = Path.Combine(rootDirectoryParent, "Use - =="); if (!Directory.Exists(useDirectory)) _ = Directory.CreateDirectory(useDirectory); string tempDirectory = Path.Combine(rootDirectoryParent, "Tmp"); if (!Directory.Exists(tempDirectory)) _ = Directory.CreateDirectory(tempDirectory); string diffRootDirectory = string.Empty; List duplicates = new(); string imageSharedDirectory = Path.Combine(rootDirectoryParent, "Images - Shared"); string namedFaceInfoFile = Path.Combine(imageSharedDirectory, "NamedFaceInfo.json"); string infoDirectory = Path.Combine(imageSharedDirectory, "Images - 4) Info", "2020-06-07"); string infoDirectoryExtra = Path.Combine(imageSharedDirectory, "Images - 4) Info - - - Extra", "2022-04-14"); List mappedIndices = GetUseTabSeparatedValueIndices(useDirectory); List<(int Index, string RelativeDirectory, string FileName, string RegexResult, long Ticks, int? PropertyId, long? PropertyTicks)> collection = new(); PropertyCompare.Models.PropertyCompareLogic propertyCompareLogic = new(_AppSettings.MaxDegreeOfParallelism.Value, _Configuration.PropertyConfiguration, _SpellingFindReplace, diffRootDirectory); PropertyCompare.Models.PropertyCompare[] propertyCompareCollection = propertyCompareLogic.Get(aPropertySingletonDirectory, loadLessThan, duplicates, deleteExtension: false); { long ticks = DateTime.Now.Ticks; string[] lines = (from l in propertyCompareCollection select l.GetSelect()).ToArray(); string aPropertyCollectionDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(A_Property), "[{}]"); File.WriteAllLines(Path.Join(aPropertyCollectionDirectory, $". . . Ids - {ticks}.txt"), lines); string json = JsonSerializer.Serialize(propertyCompareCollection, new JsonSerializerOptions { WriteIndented = true }); File.WriteAllText(Path.Join(aPropertyCollectionDirectory, $". . . Ids - {ticks}.nosj"), json); } List<(int Index, long Ticks, string RelativeDirectory, string FileNameWithoutExtension, string Extension, string RegexResult)> indexCollection = GetIndexCollection(infoDirectory, infoDirectoryExtra, checkDistinct: true); foreach ((int index, long ticks, string relativeDirectory, string fileNameWithoutExtension, string extension, string regexResult) in indexCollection) { z += 1; if (_IsEnvironment.Development && z % 1000 == 0) _Log.Debug($"{z}) Loop {_Configuration.PropertyConfiguration.RootDirectory}"); propertyId = null; propertyTicks = null; foreach (PropertyCompare.Models.PropertyCompare propertyCompare in propertyCompareCollection) { if (propertyCompare?.Property?.Id is null) continue; if (ticks != propertyCompare.MinimumDateTime.Ticks) { // if (ticks > propertyCompare.MinimumDateTime.AddHours(18).Ticks || ticks < propertyCompare.MinimumDateTime.AddHours(-18).Ticks) continue; } if (!regexResult.Equals(propertyCompare.RegexResult, StringComparison.CurrentCultureIgnoreCase)) continue; propertyId = propertyCompare.Property.Id; propertyTicks = propertyCompare.MinimumDateTime.Ticks; mappedIndex = mappedIndices.IndexOf(index); if (mappedIndex > -1) mappedIndices.RemoveAt(mappedIndex); break; } collection.Add(new(index, relativeDirectory, $"{fileNameWithoutExtension}{extension}", regexResult, ticks, propertyId, propertyTicks)); } (int Index, string RelativeDirectory, string FileName, string RegexResult, long Ticks, int? PropertyId, long? PropertyTicks)[] matched = (from l in collection where l.PropertyId.HasValue select l).ToArray(); (int Index, string RelativeDirectory, string FileName, string RegexResult, long Ticks, int? PropertyId, long? PropertyTicks)[] notMatched = (from l in collection where l.PropertyId is null select l).ToArray(); string[] duplicatesLines = ( from l in duplicates select $"{l.Property.Id}\t{l.RegexResult}\t{l.MinimumDateTime:yyyy-MM-dd_HH-mm-ss}\t{l.MinimumDateTime}\t{l.RelativeDirectory}\t{l.FileNameWithoutExtension}{l.Extension}.___" ).ToArray(); string[] matchedLines = ( from l in matched select $"{l.Index}\t{l.PropertyId}\t{l.RegexResult}\t{new DateTime(l.Ticks):yyyy-MM-dd_HH-mm-ss}\t{l.PropertyTicks}\t{l.RelativeDirectory}\t{l.FileName}" ).ToArray(); List notMatchedLines = ( from l in notMatched where !mappedIndices.Contains(l.Index) select $"{l.Index}\t______________\t{l.RegexResult}\t{new DateTime(l.Ticks):yyyy-MM-dd_HH-mm-ss}\t_________________\t{l.RelativeDirectory}\t{l.FileName}" ).ToList(); { long ticks = DateTime.Now.Ticks; notMatchedLines.AddRange(from l in mappedIndices select $"{l}\tFrom other files"); File.WriteAllLines($"{tempDirectory}/{collection.Count}-{ticks}.tsv", matchedLines); File.WriteAllLines($"{tempDirectory}/{collection.Count}-{ticks}.txt", duplicatesLines); File.WriteAllLines($"{tempDirectory}/{collection.Count}-{ticks}-Not.tsv", notMatchedLines); } _Log.Debug($"Done with {nameof(SaveTabSeparatedValues)}"); } private static void GetUseTabSeparatedValueLines(string useDirectory, Dictionary> mappedLines) { string[] lines; string[] segments; string useDirectoryName = Path.GetFileName(useDirectory).Split('-')[1].Trim(); string[] tabSeparatedValueFiles = Directory.GetFiles(useDirectory, "*.tsv", SearchOption.TopDirectoryOnly); foreach (string tabSeparatedValueFile in tabSeparatedValueFiles) { lines = File.ReadAllLines(tabSeparatedValueFile); foreach (string line in lines) { if (string.IsNullOrEmpty(line)) continue; segments = line.Split('\t'); if (segments.Length < 1) continue; if (!int.TryParse(segments[0], out int index)) continue; if (!mappedLines.ContainsKey(index)) mappedLines.Add(index, new()); mappedLines[index].Add($"{line},\t{useDirectoryName}"); } } } private static void Populate(Dictionary> keyValuePairs, Dictionary> mappedLines) { string[] segments; foreach (KeyValuePair> keyValuePair in mappedLines) { segments = keyValuePair.Value[0].Split('\t'); if (segments.Length < 2) continue; if (!int.TryParse(segments[1], out int id)) continue; if (!keyValuePairs.ContainsKey(id)) keyValuePairs.Add(id, new() { id }); keyValuePairs[id].Add(keyValuePair.Key); } } private static Dictionary GetUseTabSeparatedValue(string useEqualsDirectory, string use18HoursDirectory, string useAnyDateDirectory, Dictionary> keyValuePairs, List lines) { Dictionary results = new(); string[] segments; Dictionary> mappedLines = new(); GetUseTabSeparatedValueLines(useEqualsDirectory, mappedLines); GetUseTabSeparatedValueLines(use18HoursDirectory, mappedLines); GetUseTabSeparatedValueLines(useAnyDateDirectory, mappedLines); Populate(keyValuePairs, mappedLines); foreach (KeyValuePair> keyValuePair in mappedLines) { segments = keyValuePair.Value[0].Split('\t'); if (segments.Length < 2) continue; if (!int.TryParse(segments[1], out int id)) continue; foreach (string line in keyValuePair.Value) lines.Add(line); results.Add(keyValuePair.Key, id); } return results; } private static void ReSaveExifFiles(string infoDirectory, string infoDirectoryIgnore, Dictionary findReplace, List missingIndices) { string json; int propertyId; Models.SaveTabSeparatedValues.ImageExifInfo[]? exifCollection; List exifInfoFiles = Directory.GetFiles(infoDirectory, "ImageExifInfo.json", SearchOption.AllDirectories).ToList(); exifInfoFiles.AddRange(Directory.GetFiles(infoDirectoryIgnore, "ImageExifInfo.json", SearchOption.AllDirectories)); foreach (string exifInfoFile in exifInfoFiles) { json = File.ReadAllText(exifInfoFile); exifCollection = JsonSerializer.Deserialize(json); if (exifCollection is null) continue; foreach (Models.SaveTabSeparatedValues.ImageExifInfo exifInfo in exifCollection) { if (missingIndices.Contains(exifInfo.Index)) continue; propertyId = findReplace[exifInfo.Index]; exifInfo.Index = propertyId; } json = JsonSerializer.Serialize(exifCollection); File.WriteAllText(exifInfoFile, json); } } private static void ReSaveNamedFaceInfoFile(string namedFaceInfoFile, Dictionary findReplace, List missingIndices) { Dictionary destination = new(); string json = File.ReadAllText(namedFaceInfoFile); List<(int PropertyId, string[] Birthdays)> collection = new(); JsonSerializerOptions jsonSerializerOptions = new() { WriteIndented = true }; Dictionary? source = JsonSerializer.Deserialize>(json); if (source is null) throw new NullReferenceException(nameof(source)); { int propertyId; foreach (KeyValuePair keyValuePair in source) { if (missingIndices.Contains(keyValuePair.Key)) continue; propertyId = findReplace[keyValuePair.Key]; collection.Add(new(propertyId, keyValuePair.Value)); } } foreach ((int propertyId, string[] birthdays) in collection.OrderBy(l => l.PropertyId)) { if (destination.ContainsKey(propertyId)) { if (birthdays.Length != destination[propertyId].Length) throw new Exception($"{birthdays.Length} != {destination[propertyId].Length}"); if (birthdays[0] != destination[propertyId][0]) throw new Exception($"{birthdays[0]} != {destination[propertyId][0]}"); continue; } destination.Add(propertyId, birthdays); } json = JsonSerializer.Serialize(destination, jsonSerializerOptions); File.WriteAllText(namedFaceInfoFile, json); } private static void ReSaveIndexFiles(string infoDirectory, string infoDirectoryIgnore, Dictionary findReplace, List missingIndices) { string json; int propertyId; Models.SaveTabSeparatedValues.IndexInfo[]? indexCollection; List indexInfoFiles = Directory.GetFiles(infoDirectory, "IndexInfo.json", SearchOption.AllDirectories).ToList(); indexInfoFiles.AddRange(Directory.GetFiles(infoDirectoryIgnore, "IndexInfo.json", SearchOption.AllDirectories)); foreach (string indexInfoFile in indexInfoFiles) { json = File.ReadAllText(indexInfoFile); indexCollection = JsonSerializer.Deserialize(json); if (indexCollection is null) continue; foreach (Models.SaveTabSeparatedValues.IndexInfo indexInfo in indexCollection) { if (missingIndices.Contains(indexInfo.Index)) continue; propertyId = findReplace[indexInfo.Index]; indexInfo.Index = propertyId; } json = JsonSerializer.Serialize(indexCollection); File.WriteAllText(indexInfoFile, json); } } private void ReSaveJsonFiles() { if (_Log is null) throw new NullReferenceException(nameof(_Log)); if (_Configuration?.PropertyConfiguration is null) throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration)); string? rootDirectoryParent = Path.GetDirectoryName(_Configuration.PropertyConfiguration.RootDirectory); if (string.IsNullOrEmpty(rootDirectoryParent)) throw new NullReferenceException(nameof(rootDirectoryParent)); int z = 0; int propertyId; List missingIndices = new(); Dictionary findReplace = new(); List useTabSeparatedValueLines = new(); Dictionary> keyValuePairs = new(); string tempDirectory = Path.Combine(rootDirectoryParent, "Tmp"); string useEqualsDirectory = Path.Combine(rootDirectoryParent, "Use - =="); string use18HoursDirectory = Path.Combine(rootDirectoryParent, "Use - 18 Hours"); string useAnyDateDirectory = Path.Combine(rootDirectoryParent, "Use - Any Date"); string imageSharedDirectory = Path.Combine(rootDirectoryParent, "Images - Shared"); string namedFaceInfoFile = Path.Combine(imageSharedDirectory, "NamedFaceInfo.json"); string infoDirectory = Path.Combine(imageSharedDirectory, "Images - 4) Info", "2020-06-07"); string infoDirectoryIgnore = Path.Combine(imageSharedDirectory, "Images - 4) Info - - - Ignore", "2022-04-14"); Dictionary useTabSeparatedValueCollection = GetUseTabSeparatedValue(useEqualsDirectory, use18HoursDirectory, useAnyDateDirectory, keyValuePairs, useTabSeparatedValueLines); { long ticks = DateTime.Now.Ticks; string json = JsonSerializer.Serialize(keyValuePairs, new JsonSerializerOptions { WriteIndented = true }); File.WriteAllText($"{tempDirectory}/{nameof(keyValuePairs)}-{ticks}.json", json); } List<(int Index, long Ticks, string RelativeDirectory, string FileNameWithoutExtension, string Extension, string RegexResult)> indexCollection = GetIndexCollection(infoDirectory, infoDirectoryIgnore, checkDistinct: true); foreach ((int index, long ticks, string relativeDirectory, string fileNameWithoutExtension, string extension, string regexResult) in indexCollection) { z += 1; if (_IsEnvironment.Development && z % 1000 == 0) _Log.Debug($"{z}) Loop {_Configuration.PropertyConfiguration.RootDirectory}"); if (!useTabSeparatedValueCollection.ContainsKey(index)) { missingIndices.Add(index); continue; } propertyId = useTabSeparatedValueCollection[index]; findReplace.Add(index, propertyId); } ReSaveIndexFiles(infoDirectory, infoDirectoryIgnore, findReplace, missingIndices); ReSaveExifFiles(infoDirectory, infoDirectoryIgnore, findReplace, missingIndices); ReSaveNamedFaceInfoFile(namedFaceInfoFile, findReplace, missingIndices); { long ticks = DateTime.Now.Ticks; File.WriteAllLines($"{tempDirectory}/{nameof(useTabSeparatedValueLines)}-{ticks}.tsv", useTabSeparatedValueLines); File.WriteAllText($"{tempDirectory}/{nameof(missingIndices)}-{ticks}.txt", string.Join(Environment.NewLine, missingIndices)); } _Log.Debug($"Done with {nameof(ReSaveJsonFiles)}"); } private void CopyMissingImagesLogs() { if (_Log is null) throw new NullReferenceException(nameof(_Log)); if (_Configuration?.PropertyConfiguration is null) throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration)); string? rootDirectoryParent = Path.GetDirectoryName(_Configuration.PropertyConfiguration.RootDirectory); if (_Configuration?.PropertyConfiguration is null) throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration)); if (string.IsNullOrEmpty(rootDirectoryParent)) throw new NullReferenceException(nameof(rootDirectoryParent)); int z = 0; int propertyId; Dictionary findReplace = new(); List missingIndices = new(); List useTabSeparatedValueLines = new(); Dictionary> keyValuePairs = new(); string tempDirectory = Path.Combine(rootDirectoryParent, "Tmp"); string currentDirectory = Path.Combine(rootDirectoryParent, "@) Images"); string useEqualsDirectory = Path.Combine(rootDirectoryParent, "Use - =="); string use18HoursDirectory = Path.Combine(rootDirectoryParent, "Use - 18 Hours"); string useAnyDateDirectory = Path.Combine(rootDirectoryParent, "Use - Any Date"); string imageSharedDirectory = Path.Combine(rootDirectoryParent, "Images - Shared"); string namedFaceInfoFile = Path.Combine(imageSharedDirectory, "NamedFaceInfo.json"); string infoDirectory = Path.Combine(imageSharedDirectory, "Images - 4) Info", "2020-06-07"); string infoDirectoryIgnore = Path.Combine(imageSharedDirectory, "Images - 4) Info - - - Ignore", "2022-04-14"); Dictionary useTabSeparatedValueCollection = GetUseTabSeparatedValue(useEqualsDirectory, use18HoursDirectory, useAnyDateDirectory, keyValuePairs, useTabSeparatedValueLines); { long ticks = DateTime.Now.Ticks; string json = JsonSerializer.Serialize(keyValuePairs, new JsonSerializerOptions { WriteIndented = true }); File.WriteAllText($"{tempDirectory}/{nameof(keyValuePairs)}-{ticks}.json", json); } List<(int Index, long Ticks, string RelativeDirectory, string FileNameWithoutExtension, string Extension, string RegexResult)> indexCollection = GetIndexCollection(infoDirectory, infoDirectoryIgnore, checkDistinct: false); foreach ((int index, long ticks, string relativeDirectory, string fileNameWithoutExtension, string extension, string regexResult) in indexCollection) { z += 1; if (_IsEnvironment.Development && z % 1000 == 0) _Log.Debug($"{z}) Loop {_Configuration.PropertyConfiguration.RootDirectory}"); if (!useTabSeparatedValueCollection.ContainsKey(index)) { missingIndices.Add(index); continue; } propertyId = useTabSeparatedValueCollection[index]; findReplace.Add(index, propertyId); } { long ticks = DateTime.Now.Ticks; File.WriteAllLines($"{tempDirectory}/{nameof(useTabSeparatedValueLines)}-{ticks}.tsv", useTabSeparatedValueLines); File.WriteAllText($"{tempDirectory}/{nameof(missingIndices)}-{ticks}.txt", string.Join(Environment.NewLine, missingIndices)); } _Log.Debug($"Done with {nameof(CopyMissingImagesLogs)}"); } private static Dictionary> GetKeyValuePairs(PropertyCompare.Models.PropertyCompare[] propertyCompares) { Dictionary> results = new(); foreach (PropertyCompare.Models.PropertyCompare propertyCompare in propertyCompares) { if (propertyCompare?.Property?.Id is null) continue; if (!results.ContainsKey(propertyCompare.Property.Id.Value)) results.Add(propertyCompare.Property.Id.Value, new()); results[propertyCompare.Property.Id.Value].Add(propertyCompare); } return results; } private void VerifyAgainstIndexInfoJsonFiles(Property.Models.Configuration configuration, string aPropertySingletonDirectory) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); if (_AppSettings.MaxDegreeOfParallelism is null) throw new NullReferenceException(nameof(_AppSettings.MaxDegreeOfParallelism)); if (_Configuration?.PropertyConfiguration is null) throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration)); string? rootDirectoryParent = Path.GetDirectoryName(_Configuration.PropertyConfiguration.RootDirectory); if (string.IsNullOrEmpty(rootDirectoryParent)) throw new NullReferenceException(nameof(rootDirectoryParent)); int z = 0; int? propertyId; long? propertyTicks; int loadLessThan = 7; string diffRootDirectory = string.Empty; PropertyCompare.Models.PropertyCompare propertyCompare; string tempDirectory = Path.Combine(rootDirectoryParent, "Tmp"); if (!Directory.Exists(tempDirectory)) _ = Directory.CreateDirectory(tempDirectory); string imageSharedDirectory = Path.Combine(rootDirectoryParent, "Images - Shared"); string namedFaceInfoFile = Path.Combine(imageSharedDirectory, "NamedFaceInfo.json"); string infoDirectory = Path.Combine(imageSharedDirectory, "Images - 4) Info", "2020-06-07"); List<(int Index, string RelativeDirectory, string FileName, string RegexResult, long Ticks, int? PropertyId, long? PropertyTicks)> collection = new(); PropertyCompare.Models.PropertyCompareLogic propertyCompareLogic = new(_AppSettings.MaxDegreeOfParallelism.Value, _Configuration.PropertyConfiguration, _SpellingFindReplace, diffRootDirectory); PropertyCompare.Models.PropertyCompare[] propertyCompareCollection = propertyCompareLogic.Get(aPropertySingletonDirectory, loadLessThan); { long ticks = DateTime.Now.Ticks; string[] lines = (from l in propertyCompareCollection select l.GetSelect()).ToArray(); string aPropertyCollectionDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(A_Property), "[{}]"); File.WriteAllLines(Path.Join(aPropertyCollectionDirectory, $". . . Ids - {ticks}.txt"), lines); string json = JsonSerializer.Serialize(propertyCompareCollection, new JsonSerializerOptions { WriteIndented = true }); File.WriteAllText(Path.Join(aPropertyCollectionDirectory, $". . . Ids - {ticks}.nosj"), json); } Dictionary> keyValuePairs = GetKeyValuePairs(propertyCompareCollection); List<(int Index, long Ticks, string RelativeDirectory, string FileNameWithoutExtension, string Extension, string RegexResult)> indexCollection = GetIndexCollection(infoDirectory, infoDirectoryExtra: string.Empty, checkDistinct: false); foreach ((int index, long ticks, string relativeDirectory, string fileNameWithoutExtension, string extension, string regexResult) in indexCollection) { z += 1; if (_IsEnvironment.Development && z % 1000 == 0) _Log.Debug($"{z}) Loop {_Configuration.PropertyConfiguration.RootDirectory}"); propertyId = null; propertyTicks = null; if (keyValuePairs.ContainsKey(index)) { propertyCompare = keyValuePairs[index][0]; if (propertyCompare?.Property?.Id is null) continue; propertyId = propertyCompare.Property.Id; propertyTicks = propertyCompare.MinimumDateTime.Ticks; } collection.Add(new(index, relativeDirectory, $"{fileNameWithoutExtension}{extension}", regexResult, ticks, propertyId, propertyTicks)); } (int Index, string RelativeDirectory, string FileName, string RegexResult, long Ticks, int? PropertyId, long? PropertyTicks)[] matched = (from l in collection where l.PropertyId.HasValue select l).ToArray(); (int Index, string RelativeDirectory, string FileName, string RegexResult, long Ticks, int? PropertyId, long? PropertyTicks)[] notMatched = (from l in collection where l.PropertyId is null select l).ToArray(); string[] matchedLines = ( from l in matched select $"{l.Index}\t{l.PropertyId}\t{l.RegexResult}\t{new DateTime(l.Ticks):yyyy-MM-dd_HH-mm-ss}\t{l.PropertyTicks}\t{l.RelativeDirectory}\t{l.FileName}" ).ToArray(); List notMatchedLines = ( from l in notMatched select $"{l.Index}\t______________\t{l.RegexResult}\t{new DateTime(l.Ticks):yyyy-MM-dd_HH-mm-ss}\t_________________\t{l.RelativeDirectory}\t{l.FileName}" ).ToList(); { long ticks = DateTime.Now.Ticks; File.WriteAllLines($"{tempDirectory}/{collection.Count}-{ticks}.tsv", matchedLines); File.WriteAllLines($"{tempDirectory}/{collection.Count}-{ticks}-Not.tsv", notMatchedLines); } _Log.Debug($"Done with {nameof(VerifyAgainstIndexInfoJsonFiles)}"); } }