666 lines
37 KiB
C#
666 lines
37 KiB
C#
using Microsoft.Extensions.Configuration;
|
|
using Phares.Shared;
|
|
using System.Text.Json;
|
|
using System.Text.RegularExpressions;
|
|
using View_by_Distance.PrepareForOld.Models;
|
|
using View_by_Distance.Property.Models;
|
|
using View_by_Distance.Shared.Models.Methods;
|
|
|
|
namespace View_by_Distance.PrepareForOld;
|
|
|
|
public class PrepareForOld
|
|
{
|
|
|
|
private readonly Serilog.ILogger? _Log;
|
|
private readonly AppSettings _AppSettings;
|
|
private readonly List<string> _Exceptions;
|
|
private readonly IsEnvironment _IsEnvironment;
|
|
private readonly Models.Configuration _Configuration;
|
|
private readonly List<KeyValuePair<string, string>> _FileKeyValuePairs;
|
|
private readonly List<(string Find, string Replace)> _SpellingFindReplace;
|
|
private readonly Dictionary<string, List<Tuple<string, A_Property>>> _FilePropertiesKeyValuePairs;
|
|
|
|
public PrepareForOld(List<string> args, IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, AppSettings appSettings, string workingDirectory, bool isSilent, IConsole console)
|
|
{
|
|
if (isSilent)
|
|
{ }
|
|
if (console is null)
|
|
{ }
|
|
string spellingA;
|
|
string spellingB;
|
|
_AppSettings = appSettings;
|
|
if (appSettings.MaxDegreeOfParallelism is null)
|
|
throw new Exception($"{nameof(appSettings.MaxDegreeOfParallelism)} is null!");
|
|
_SpellingFindReplace = new();
|
|
_IsEnvironment = isEnvironment;
|
|
_Exceptions = new List<string>();
|
|
_Log = Serilog.Log.ForContext<PrepareForOld>();
|
|
_FileKeyValuePairs = new List<KeyValuePair<string, string>>();
|
|
_FilePropertiesKeyValuePairs = new Dictionary<string, List<Tuple<string, A_Property>>>();
|
|
Property.Models.Configuration propertyConfiguration = Property.Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory);
|
|
Property.Models.Configuration.Verify(propertyConfiguration);
|
|
Models.Configuration configuration = Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory, propertyConfiguration);
|
|
Verify(configuration);
|
|
if (propertyConfiguration.IgnoreExtensions is null)
|
|
throw new Exception($"{nameof(propertyConfiguration.IgnoreExtensions)} is null!");
|
|
if (propertyConfiguration.MaxImagesInDirectoryForTopLevelFirstPass is null)
|
|
throw new Exception($"{nameof(propertyConfiguration.MaxImagesInDirectoryForTopLevelFirstPass)} is null!");
|
|
for (int i = 0; i < configuration.Spelling.Length; i++)
|
|
{
|
|
spellingA = configuration.Spelling[i];
|
|
spellingB = configuration.Spelling[i + 1];
|
|
if (spellingB.Contains(spellingA))
|
|
throw new Exception("Change configuration!");
|
|
_SpellingFindReplace.Add(new(spellingA, spellingB));
|
|
i += 1;
|
|
}
|
|
string[] dbFiles = Directory.GetFiles(propertyConfiguration.RootDirectory, "*.db", SearchOption.AllDirectories);
|
|
foreach (string dbFile in dbFiles)
|
|
File.Delete(dbFile);
|
|
string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(A_Property), "{}");
|
|
if (!Directory.Exists(aPropertySingletonDirectory))
|
|
throw new Exception(aPropertySingletonDirectory);
|
|
ConsoleKey? consoleKey = null;
|
|
for (int y = 0; y < int.MaxValue; y++)
|
|
{
|
|
_Log.Information($"Execute {nameof(VerifyAgainstIndexInfoJsonFiles)} \"Y(es)\" or \"N(o)\"?");
|
|
consoleKey = Console.ReadKey().Key;
|
|
if (consoleKey is ConsoleKey.Y or ConsoleKey.N)
|
|
break;
|
|
}
|
|
_Log.Information(". . .");
|
|
if (consoleKey == ConsoleKey.Y)
|
|
VerifyAgainstIndexInfoJsonFiles(propertyConfiguration, aPropertySingletonDirectory);
|
|
for (int y = 0; y < int.MaxValue; y++)
|
|
{
|
|
_Log.Information($"Execute {nameof(SaveTabSeparatedValues)} \"Y(es)\" or \"N(o)\"?");
|
|
consoleKey = Console.ReadKey().Key;
|
|
if (consoleKey is ConsoleKey.Y or ConsoleKey.N)
|
|
break;
|
|
}
|
|
_Log.Information(". . .");
|
|
if (consoleKey == ConsoleKey.Y)
|
|
SaveTabSeparatedValues(propertyConfiguration, aPropertySingletonDirectory);
|
|
for (int y = 0; y < int.MaxValue; y++)
|
|
{
|
|
_Log.Information($"Execute {nameof(ReSaveJsonFiles)} \"Y(es)\" or \"N(o)\"?");
|
|
consoleKey = Console.ReadKey().Key;
|
|
if (consoleKey is ConsoleKey.Y or ConsoleKey.N)
|
|
break;
|
|
}
|
|
_Log.Information(". . .");
|
|
if (consoleKey == ConsoleKey.Y)
|
|
ReSaveJsonFiles();
|
|
for (int y = 0; y < int.MaxValue; y++)
|
|
{
|
|
_Log.Information($"Execute {nameof(CopyMissingImagesLogs)} \"Y(es)\" or \"N(o)\"?");
|
|
consoleKey = Console.ReadKey().Key;
|
|
if (consoleKey is ConsoleKey.Y or ConsoleKey.N)
|
|
break;
|
|
}
|
|
_Log.Information(". . .");
|
|
if (consoleKey == ConsoleKey.Y)
|
|
CopyMissingImagesLogs();
|
|
string message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}";
|
|
_Log.Information(message);
|
|
if (_Exceptions.Count != 0)
|
|
throw new Exception(message);
|
|
_Configuration = configuration;
|
|
}
|
|
|
|
private static void Verify(Models.Configuration configuration)
|
|
{
|
|
if (configuration.Spelling is null || !configuration.Spelling.Any())
|
|
throw new Exception($"{nameof(configuration.Spelling)} should have at least one!");
|
|
}
|
|
|
|
private static List<Models.SaveTabSeparatedValues.ImageExifInfo> GetExifCollection(string infoDirectory, string infoDirectoryExtra, bool checkDistinct)
|
|
{
|
|
List<Models.SaveTabSeparatedValues.ImageExifInfo> results = new();
|
|
string json;
|
|
Models.SaveTabSeparatedValues.ImageExifInfo[]? exifCollection;
|
|
List<string> exifInfoFiles = Directory.GetFiles(infoDirectory, "ImageExifInfo.json", SearchOption.AllDirectories).ToList();
|
|
if (!string.IsNullOrEmpty(infoDirectoryExtra))
|
|
exifInfoFiles.AddRange(Directory.GetFiles(infoDirectoryExtra, "ImageExifInfo.json", SearchOption.AllDirectories));
|
|
foreach (string exifInfoFile in exifInfoFiles)
|
|
{
|
|
json = File.ReadAllText(exifInfoFile);
|
|
exifCollection = JsonSerializer.Deserialize<Models.SaveTabSeparatedValues.ImageExifInfo[]>(json);
|
|
if (exifCollection is null)
|
|
continue;
|
|
results.AddRange(exifCollection);
|
|
}
|
|
if (checkDistinct)
|
|
{
|
|
int[] check = (from l in results select l.Index).Distinct().ToArray();
|
|
if (check.Length != results.Count)
|
|
throw new Exception();
|
|
}
|
|
return results;
|
|
}
|
|
|
|
private List<(int Index, long Ticks, string RelativeDirectory, string FileNameWithoutExtension, string Extension, string RegexResult)> GetIndexCollection(string infoDirectory, string infoDirectoryExtra, bool checkDistinct)
|
|
{
|
|
List<(int Index, long Ticks, string RelativeDirectory, string FileNameWithoutExtension, string Extension, string RegexResult)> results = new();
|
|
if (_Configuration?.PropertyConfiguration is null)
|
|
throw new Exception($"{nameof(_Configuration.PropertyConfiguration)} must be set!");
|
|
long ticks;
|
|
string json;
|
|
string extension;
|
|
string corrected;
|
|
string regexResult;
|
|
string relativeDirectory;
|
|
List<DateTime> dateTimes;
|
|
string fileNameWithoutExtension;
|
|
string firstFileSegmentCollection;
|
|
Dictionary<int, long> keyValuePairs = new();
|
|
Models.SaveTabSeparatedValues.IndexInfo[]? indexCollection;
|
|
List<Models.SaveTabSeparatedValues.ImageExifInfo> exifCollection = GetExifCollection(infoDirectory, infoDirectoryExtra, checkDistinct);
|
|
List<string> indexInfoFiles = Directory.GetFiles(infoDirectory, "IndexInfo.json", SearchOption.AllDirectories).ToList();
|
|
if (!string.IsNullOrEmpty(infoDirectoryExtra))
|
|
indexInfoFiles.AddRange(Directory.GetFiles(infoDirectoryExtra, "IndexInfo.json", SearchOption.AllDirectories));
|
|
foreach (Models.SaveTabSeparatedValues.ImageExifInfo exifInfo in exifCollection)
|
|
{
|
|
dateTimes = Property.Models.Stateless.A_Property.GetDateTimes(exifInfo.CreationTime, exifInfo.LastWriteTime, exifInfo.DateTime, exifInfo.DateTimeDigitized, exifInfo.DateTimeOriginal, exifInfo.GPSDateStamp);
|
|
if (!checkDistinct && keyValuePairs.ContainsKey(exifInfo.Index))
|
|
continue;
|
|
keyValuePairs.Add(exifInfo.Index, dateTimes.Min().Ticks);
|
|
}
|
|
foreach (string indexInfoFile in indexInfoFiles)
|
|
{
|
|
json = File.ReadAllText(indexInfoFile);
|
|
indexCollection = JsonSerializer.Deserialize<Models.SaveTabSeparatedValues.IndexInfo[]>(json);
|
|
if (indexCollection is null)
|
|
continue;
|
|
foreach (Models.SaveTabSeparatedValues.IndexInfo indexInfo in indexCollection)
|
|
{
|
|
if (indexInfo.FileSegmentCollection is null)
|
|
continue;
|
|
// if (indexInfo.FileSegmentCollection.Length != 1)
|
|
// continue;
|
|
ticks = keyValuePairs[indexInfo.Index];
|
|
firstFileSegmentCollection = indexInfo.FileSegmentCollection[0];
|
|
extension = Path.GetExtension(firstFileSegmentCollection);
|
|
fileNameWithoutExtension = Path.GetFileNameWithoutExtension(firstFileSegmentCollection);
|
|
corrected = firstFileSegmentCollection[..(firstFileSegmentCollection.Length - extension.Length)];
|
|
relativeDirectory = firstFileSegmentCollection[..(firstFileSegmentCollection.Length - fileNameWithoutExtension.Length - extension.Length)];
|
|
if ((from l in _SpellingFindReplace where corrected.Contains(l.Find) select true).Any())
|
|
{
|
|
foreach ((string find, string replace) in _SpellingFindReplace)
|
|
corrected = corrected.Replace(find, replace);
|
|
}
|
|
if (string.IsNullOrEmpty(_Configuration.PropertyConfiguration.Pattern))
|
|
regexResult = corrected;
|
|
else
|
|
regexResult = Regex.Replace(corrected, _Configuration.PropertyConfiguration.Pattern, string.Empty);
|
|
results.Add(new(indexInfo.Index, ticks, relativeDirectory, fileNameWithoutExtension, extension, regexResult));
|
|
}
|
|
}
|
|
return results;
|
|
}
|
|
|
|
private static List<int> GetUseTabSeparatedValueIndices(string useDirectory)
|
|
{
|
|
List<int> results = new();
|
|
string[] lines;
|
|
string[] segments;
|
|
string[] tabSeparatedValueFiles = Directory.GetFiles(useDirectory, "*.tsv", SearchOption.TopDirectoryOnly);
|
|
foreach (string tabSeparatedValueFile in tabSeparatedValueFiles)
|
|
{
|
|
lines = File.ReadAllLines(tabSeparatedValueFile);
|
|
foreach (string line in lines)
|
|
{
|
|
if (string.IsNullOrEmpty(line))
|
|
continue;
|
|
segments = line.Split('\t');
|
|
if (segments.Length < 1)
|
|
continue;
|
|
if (!int.TryParse(segments[0], out int index))
|
|
continue;
|
|
results.Add(index);
|
|
}
|
|
}
|
|
return results.Distinct().OrderBy(l => l).ToList();
|
|
}
|
|
|
|
private void SaveTabSeparatedValues(Property.Models.Configuration configuration, string aPropertySingletonDirectory)
|
|
{
|
|
if (_Log is null)
|
|
throw new Exception($"{nameof(_Log)} is null!");
|
|
if (_AppSettings.MaxDegreeOfParallelism is null)
|
|
throw new Exception($"{nameof(_AppSettings.MaxDegreeOfParallelism)} is null!");
|
|
if (_Configuration?.PropertyConfiguration is null)
|
|
throw new Exception($"{nameof(_Configuration.PropertyConfiguration)} must be set!");
|
|
string? rootDirectoryParent = Path.GetDirectoryName(_Configuration.PropertyConfiguration.RootDirectory);
|
|
if (string.IsNullOrEmpty(rootDirectoryParent))
|
|
throw new Exception($"{nameof(rootDirectoryParent)} is null!");
|
|
int z = 0;
|
|
int mappedIndex;
|
|
int? propertyId;
|
|
long? propertyTicks;
|
|
int loadLessThan = 7;
|
|
string useDirectory = Path.Combine(rootDirectoryParent, "Use - ==");
|
|
if (!Directory.Exists(useDirectory))
|
|
_ = Directory.CreateDirectory(useDirectory);
|
|
string tempDirectory = Path.Combine(rootDirectoryParent, "Tmp");
|
|
if (!Directory.Exists(tempDirectory))
|
|
_ = Directory.CreateDirectory(tempDirectory);
|
|
string diffRootDirectory = string.Empty;
|
|
List<PropertyCompare.Models.PropertyCompare> duplicates = new();
|
|
string imageSharedDirectory = Path.Combine(rootDirectoryParent, "Images - Shared");
|
|
string namedFaceInfoFile = Path.Combine(imageSharedDirectory, "NamedFaceInfo.json");
|
|
string infoDirectory = Path.Combine(imageSharedDirectory, "Images - 4) Info", "2020-06-07");
|
|
string infoDirectoryExtra = Path.Combine(imageSharedDirectory, "Images - 4) Info - - - Extra", "2022-04-14");
|
|
List<int> mappedIndices = GetUseTabSeparatedValueIndices(useDirectory);
|
|
List<(int Index, string RelativeDirectory, string FileName, string RegexResult, long Ticks, int? PropertyId, long? PropertyTicks)> collection = new();
|
|
PropertyCompare.Models.PropertyCompareLogic propertyCompareLogic = new(_AppSettings.MaxDegreeOfParallelism.Value, _Configuration.PropertyConfiguration, _SpellingFindReplace, diffRootDirectory);
|
|
PropertyCompare.Models.PropertyCompare[] propertyCompareCollection = propertyCompareLogic.Get(aPropertySingletonDirectory, loadLessThan, duplicates, deleteExtension: false);
|
|
{
|
|
long ticks = DateTime.Now.Ticks;
|
|
string[] lines = (from l in propertyCompareCollection select l.GetSelect()).ToArray();
|
|
string aPropertyCollectionDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(A_Property), "[{}]");
|
|
File.WriteAllLines(Path.Join(aPropertyCollectionDirectory, $". . . Ids - {ticks}.txt"), lines);
|
|
string json = JsonSerializer.Serialize(propertyCompareCollection, new JsonSerializerOptions { WriteIndented = true });
|
|
File.WriteAllText(Path.Join(aPropertyCollectionDirectory, $". . . Ids - {ticks}.nosj"), json);
|
|
}
|
|
List<(int Index, long Ticks, string RelativeDirectory, string FileNameWithoutExtension, string Extension, string RegexResult)> indexCollection = GetIndexCollection(infoDirectory, infoDirectoryExtra, checkDistinct: true);
|
|
foreach ((int index, long ticks, string relativeDirectory, string fileNameWithoutExtension, string extension, string regexResult) in indexCollection)
|
|
{
|
|
z += 1;
|
|
if (_IsEnvironment.Development && z % 1000 == 0)
|
|
_Log.Debug($"{z}) Loop {_Configuration.PropertyConfiguration.RootDirectory}");
|
|
propertyId = null;
|
|
propertyTicks = null;
|
|
foreach (PropertyCompare.Models.PropertyCompare propertyCompare in propertyCompareCollection)
|
|
{
|
|
if (propertyCompare?.Property?.Id is null)
|
|
continue;
|
|
if (ticks != propertyCompare.MinimumDateTime.Ticks)
|
|
{
|
|
// if (ticks > propertyCompare.MinimumDateTime.AddHours(18).Ticks || ticks < propertyCompare.MinimumDateTime.AddHours(-18).Ticks)
|
|
continue;
|
|
}
|
|
if (!regexResult.Equals(propertyCompare.RegexResult, StringComparison.CurrentCultureIgnoreCase))
|
|
continue;
|
|
propertyId = propertyCompare.Property.Id;
|
|
propertyTicks = propertyCompare.MinimumDateTime.Ticks;
|
|
mappedIndex = mappedIndices.IndexOf(index);
|
|
if (mappedIndex > -1)
|
|
mappedIndices.RemoveAt(mappedIndex);
|
|
break;
|
|
}
|
|
collection.Add(new(index, relativeDirectory, $"{fileNameWithoutExtension}{extension}", regexResult, ticks, propertyId, propertyTicks));
|
|
}
|
|
(int Index, string RelativeDirectory, string FileName, string RegexResult, long Ticks, int? PropertyId, long? PropertyTicks)[] matched = (from l in collection where l.PropertyId.HasValue select l).ToArray();
|
|
(int Index, string RelativeDirectory, string FileName, string RegexResult, long Ticks, int? PropertyId, long? PropertyTicks)[] notMatched = (from l in collection where l.PropertyId is null select l).ToArray();
|
|
string[] duplicatesLines = (
|
|
from l
|
|
in duplicates
|
|
select $"{l.Property.Id}\t{l.RegexResult}\t{l.MinimumDateTime:yyyy-MM-dd_HH-mm-ss}\t{l.MinimumDateTime}\t{l.RelativeDirectory}\t{l.FileNameWithoutExtension}{l.Extension}.___"
|
|
).ToArray();
|
|
string[] matchedLines = (
|
|
from l
|
|
in matched
|
|
select $"{l.Index}\t{l.PropertyId}\t{l.RegexResult}\t{new DateTime(l.Ticks):yyyy-MM-dd_HH-mm-ss}\t{l.PropertyTicks}\t{l.RelativeDirectory}\t{l.FileName}"
|
|
).ToArray();
|
|
List<string> notMatchedLines = (
|
|
from l
|
|
in notMatched
|
|
where !mappedIndices.Contains(l.Index)
|
|
select $"{l.Index}\t______________\t{l.RegexResult}\t{new DateTime(l.Ticks):yyyy-MM-dd_HH-mm-ss}\t_________________\t{l.RelativeDirectory}\t{l.FileName}"
|
|
).ToList();
|
|
{
|
|
long ticks = DateTime.Now.Ticks;
|
|
notMatchedLines.AddRange(from l in mappedIndices select $"{l}\tFrom other files");
|
|
File.WriteAllLines($"{tempDirectory}/{collection.Count}-{ticks}.tsv", matchedLines);
|
|
File.WriteAllLines($"{tempDirectory}/{collection.Count}-{ticks}.txt", duplicatesLines);
|
|
File.WriteAllLines($"{tempDirectory}/{collection.Count}-{ticks}-Not.tsv", notMatchedLines);
|
|
}
|
|
_Log.Debug($"Done with {nameof(SaveTabSeparatedValues)}");
|
|
}
|
|
|
|
private static void GetUseTabSeparatedValueLines(string useDirectory, Dictionary<int, List<string>> mappedLines)
|
|
{
|
|
string[] lines;
|
|
string[] segments;
|
|
string useDirectoryName = Path.GetFileName(useDirectory).Split('-')[1].Trim();
|
|
string[] tabSeparatedValueFiles = Directory.GetFiles(useDirectory, "*.tsv", SearchOption.TopDirectoryOnly);
|
|
foreach (string tabSeparatedValueFile in tabSeparatedValueFiles)
|
|
{
|
|
lines = File.ReadAllLines(tabSeparatedValueFile);
|
|
foreach (string line in lines)
|
|
{
|
|
if (string.IsNullOrEmpty(line))
|
|
continue;
|
|
segments = line.Split('\t');
|
|
if (segments.Length < 1)
|
|
continue;
|
|
if (!int.TryParse(segments[0], out int index))
|
|
continue;
|
|
if (!mappedLines.ContainsKey(index))
|
|
mappedLines.Add(index, new());
|
|
mappedLines[index].Add($"{line},\t{useDirectoryName}");
|
|
}
|
|
}
|
|
}
|
|
|
|
private static void Populate(Dictionary<int, List<int>> keyValuePairs, Dictionary<int, List<string>> mappedLines)
|
|
{
|
|
string[] segments;
|
|
foreach (KeyValuePair<int, List<string>> keyValuePair in mappedLines)
|
|
{
|
|
segments = keyValuePair.Value[0].Split('\t');
|
|
if (segments.Length < 2)
|
|
continue;
|
|
if (!int.TryParse(segments[1], out int id))
|
|
continue;
|
|
if (!keyValuePairs.ContainsKey(id))
|
|
keyValuePairs.Add(id, new() { id });
|
|
keyValuePairs[id].Add(keyValuePair.Key);
|
|
}
|
|
}
|
|
|
|
private static Dictionary<int, int> GetUseTabSeparatedValue(string useEqualsDirectory, string use18HoursDirectory, string useAnyDateDirectory, Dictionary<int, List<int>> keyValuePairs, List<string> lines)
|
|
{
|
|
Dictionary<int, int> results = new();
|
|
string[] segments;
|
|
Dictionary<int, List<string>> mappedLines = new();
|
|
GetUseTabSeparatedValueLines(useEqualsDirectory, mappedLines);
|
|
GetUseTabSeparatedValueLines(use18HoursDirectory, mappedLines);
|
|
GetUseTabSeparatedValueLines(useAnyDateDirectory, mappedLines);
|
|
Populate(keyValuePairs, mappedLines);
|
|
foreach (KeyValuePair<int, List<string>> keyValuePair in mappedLines)
|
|
{
|
|
segments = keyValuePair.Value[0].Split('\t');
|
|
if (segments.Length < 2)
|
|
continue;
|
|
if (!int.TryParse(segments[1], out int id))
|
|
continue;
|
|
foreach (string line in keyValuePair.Value)
|
|
lines.Add(line);
|
|
results.Add(keyValuePair.Key, id);
|
|
}
|
|
return results;
|
|
}
|
|
|
|
private static void ReSaveExifFiles(string infoDirectory, string infoDirectoryIgnore, Dictionary<int, int> findReplace, List<int> missingIndices)
|
|
{
|
|
string json;
|
|
int propertyId;
|
|
Models.SaveTabSeparatedValues.ImageExifInfo[]? exifCollection;
|
|
List<string> exifInfoFiles = Directory.GetFiles(infoDirectory, "ImageExifInfo.json", SearchOption.AllDirectories).ToList();
|
|
exifInfoFiles.AddRange(Directory.GetFiles(infoDirectoryIgnore, "ImageExifInfo.json", SearchOption.AllDirectories));
|
|
foreach (string exifInfoFile in exifInfoFiles)
|
|
{
|
|
json = File.ReadAllText(exifInfoFile);
|
|
exifCollection = JsonSerializer.Deserialize<Models.SaveTabSeparatedValues.ImageExifInfo[]>(json);
|
|
if (exifCollection is null)
|
|
continue;
|
|
foreach (Models.SaveTabSeparatedValues.ImageExifInfo exifInfo in exifCollection)
|
|
{
|
|
if (missingIndices.Contains(exifInfo.Index))
|
|
continue;
|
|
propertyId = findReplace[exifInfo.Index];
|
|
exifInfo.Index = propertyId;
|
|
}
|
|
json = JsonSerializer.Serialize(exifCollection);
|
|
File.WriteAllText(exifInfoFile, json);
|
|
}
|
|
}
|
|
|
|
private static void ReSaveNamedFaceInfoFile(string namedFaceInfoFile, Dictionary<int, int> findReplace, List<int> missingIndices)
|
|
{
|
|
Dictionary<int, string[]> destination = new();
|
|
string json = File.ReadAllText(namedFaceInfoFile);
|
|
List<(int PropertyId, string[] Birthdays)> collection = new();
|
|
JsonSerializerOptions jsonSerializerOptions = new() { WriteIndented = true };
|
|
Dictionary<int, string[]>? source = JsonSerializer.Deserialize<Dictionary<int, string[]>>(json);
|
|
if (source is null)
|
|
throw new Exception($"{nameof(source)} is null!");
|
|
{
|
|
int propertyId;
|
|
foreach (KeyValuePair<int, string[]> keyValuePair in source)
|
|
{
|
|
if (missingIndices.Contains(keyValuePair.Key))
|
|
continue;
|
|
propertyId = findReplace[keyValuePair.Key];
|
|
collection.Add(new(propertyId, keyValuePair.Value));
|
|
}
|
|
}
|
|
foreach ((int propertyId, string[] birthdays) in collection.OrderBy(l => l.PropertyId))
|
|
{
|
|
if (destination.ContainsKey(propertyId))
|
|
{
|
|
if (birthdays.Length != destination[propertyId].Length)
|
|
throw new Exception($"{birthdays.Length} != {destination[propertyId].Length}");
|
|
if (birthdays[0] != destination[propertyId][0])
|
|
throw new Exception($"{birthdays[0]} != {destination[propertyId][0]}");
|
|
continue;
|
|
}
|
|
destination.Add(propertyId, birthdays);
|
|
}
|
|
json = JsonSerializer.Serialize(destination, jsonSerializerOptions);
|
|
File.WriteAllText(namedFaceInfoFile, json);
|
|
}
|
|
|
|
private static void ReSaveIndexFiles(string infoDirectory, string infoDirectoryIgnore, Dictionary<int, int> findReplace, List<int> missingIndices)
|
|
{
|
|
string json;
|
|
int propertyId;
|
|
Models.SaveTabSeparatedValues.IndexInfo[]? indexCollection;
|
|
List<string> indexInfoFiles = Directory.GetFiles(infoDirectory, "IndexInfo.json", SearchOption.AllDirectories).ToList();
|
|
indexInfoFiles.AddRange(Directory.GetFiles(infoDirectoryIgnore, "IndexInfo.json", SearchOption.AllDirectories));
|
|
foreach (string indexInfoFile in indexInfoFiles)
|
|
{
|
|
json = File.ReadAllText(indexInfoFile);
|
|
indexCollection = JsonSerializer.Deserialize<Models.SaveTabSeparatedValues.IndexInfo[]>(json);
|
|
if (indexCollection is null)
|
|
continue;
|
|
foreach (Models.SaveTabSeparatedValues.IndexInfo indexInfo in indexCollection)
|
|
{
|
|
if (missingIndices.Contains(indexInfo.Index))
|
|
continue;
|
|
propertyId = findReplace[indexInfo.Index];
|
|
indexInfo.Index = propertyId;
|
|
}
|
|
json = JsonSerializer.Serialize(indexCollection);
|
|
File.WriteAllText(indexInfoFile, json);
|
|
}
|
|
}
|
|
|
|
private void ReSaveJsonFiles()
|
|
{
|
|
if (_Log is null)
|
|
throw new Exception($"{nameof(_Log)} is null!");
|
|
if (_Configuration?.PropertyConfiguration is null)
|
|
throw new Exception($"{nameof(_Configuration.PropertyConfiguration)} must be set!");
|
|
string? rootDirectoryParent = Path.GetDirectoryName(_Configuration.PropertyConfiguration.RootDirectory);
|
|
if (string.IsNullOrEmpty(rootDirectoryParent))
|
|
throw new Exception($"{nameof(rootDirectoryParent)} is null!");
|
|
int z = 0;
|
|
int propertyId;
|
|
List<int> missingIndices = new();
|
|
Dictionary<int, int> findReplace = new();
|
|
List<string> useTabSeparatedValueLines = new();
|
|
Dictionary<int, List<int>> keyValuePairs = new();
|
|
string tempDirectory = Path.Combine(rootDirectoryParent, "Tmp");
|
|
string useEqualsDirectory = Path.Combine(rootDirectoryParent, "Use - ==");
|
|
string use18HoursDirectory = Path.Combine(rootDirectoryParent, "Use - 18 Hours");
|
|
string useAnyDateDirectory = Path.Combine(rootDirectoryParent, "Use - Any Date");
|
|
string imageSharedDirectory = Path.Combine(rootDirectoryParent, "Images - Shared");
|
|
string namedFaceInfoFile = Path.Combine(imageSharedDirectory, "NamedFaceInfo.json");
|
|
string infoDirectory = Path.Combine(imageSharedDirectory, "Images - 4) Info", "2020-06-07");
|
|
string infoDirectoryIgnore = Path.Combine(imageSharedDirectory, "Images - 4) Info - - - Ignore", "2022-04-14");
|
|
Dictionary<int, int> useTabSeparatedValueCollection = GetUseTabSeparatedValue(useEqualsDirectory, use18HoursDirectory, useAnyDateDirectory, keyValuePairs, useTabSeparatedValueLines);
|
|
{
|
|
long ticks = DateTime.Now.Ticks;
|
|
string json = JsonSerializer.Serialize(keyValuePairs, new JsonSerializerOptions { WriteIndented = true });
|
|
File.WriteAllText($"{tempDirectory}/{nameof(keyValuePairs)}-{ticks}.json", json);
|
|
}
|
|
List<(int Index, long Ticks, string RelativeDirectory, string FileNameWithoutExtension, string Extension, string RegexResult)> indexCollection = GetIndexCollection(infoDirectory, infoDirectoryIgnore, checkDistinct: true);
|
|
foreach ((int index, long ticks, string relativeDirectory, string fileNameWithoutExtension, string extension, string regexResult) in indexCollection)
|
|
{
|
|
z += 1;
|
|
if (_IsEnvironment.Development && z % 1000 == 0)
|
|
_Log.Debug($"{z}) Loop {_Configuration.PropertyConfiguration.RootDirectory}");
|
|
if (!useTabSeparatedValueCollection.ContainsKey(index))
|
|
{
|
|
missingIndices.Add(index);
|
|
continue;
|
|
}
|
|
propertyId = useTabSeparatedValueCollection[index];
|
|
findReplace.Add(index, propertyId);
|
|
}
|
|
ReSaveIndexFiles(infoDirectory, infoDirectoryIgnore, findReplace, missingIndices);
|
|
ReSaveExifFiles(infoDirectory, infoDirectoryIgnore, findReplace, missingIndices);
|
|
ReSaveNamedFaceInfoFile(namedFaceInfoFile, findReplace, missingIndices);
|
|
{
|
|
long ticks = DateTime.Now.Ticks;
|
|
File.WriteAllLines($"{tempDirectory}/{nameof(useTabSeparatedValueLines)}-{ticks}.tsv", useTabSeparatedValueLines);
|
|
File.WriteAllText($"{tempDirectory}/{nameof(missingIndices)}-{ticks}.txt", string.Join(Environment.NewLine, missingIndices));
|
|
}
|
|
_Log.Debug($"Done with {nameof(ReSaveJsonFiles)}");
|
|
}
|
|
|
|
private void CopyMissingImagesLogs()
|
|
{
|
|
if (_Log is null)
|
|
throw new Exception($"{nameof(_Log)} is null!");
|
|
if (_Configuration?.PropertyConfiguration is null)
|
|
throw new Exception($"{nameof(_Configuration.PropertyConfiguration)} must be set!");
|
|
string? rootDirectoryParent = Path.GetDirectoryName(_Configuration.PropertyConfiguration.RootDirectory);
|
|
if (_Configuration?.PropertyConfiguration is null)
|
|
throw new Exception($"{nameof(_Configuration.PropertyConfiguration)} must be set!");
|
|
if (string.IsNullOrEmpty(rootDirectoryParent))
|
|
throw new Exception($"{nameof(rootDirectoryParent)} is null!");
|
|
int z = 0;
|
|
int propertyId;
|
|
Dictionary<int, int> findReplace = new();
|
|
List<int> missingIndices = new();
|
|
List<string> useTabSeparatedValueLines = new();
|
|
Dictionary<int, List<int>> keyValuePairs = new();
|
|
string tempDirectory = Path.Combine(rootDirectoryParent, "Tmp");
|
|
string currentDirectory = Path.Combine(rootDirectoryParent, "@) Images");
|
|
string useEqualsDirectory = Path.Combine(rootDirectoryParent, "Use - ==");
|
|
string use18HoursDirectory = Path.Combine(rootDirectoryParent, "Use - 18 Hours");
|
|
string useAnyDateDirectory = Path.Combine(rootDirectoryParent, "Use - Any Date");
|
|
string imageSharedDirectory = Path.Combine(rootDirectoryParent, "Images - Shared");
|
|
string namedFaceInfoFile = Path.Combine(imageSharedDirectory, "NamedFaceInfo.json");
|
|
string infoDirectory = Path.Combine(imageSharedDirectory, "Images - 4) Info", "2020-06-07");
|
|
string infoDirectoryIgnore = Path.Combine(imageSharedDirectory, "Images - 4) Info - - - Ignore", "2022-04-14");
|
|
Dictionary<int, int> useTabSeparatedValueCollection = GetUseTabSeparatedValue(useEqualsDirectory, use18HoursDirectory, useAnyDateDirectory, keyValuePairs, useTabSeparatedValueLines);
|
|
{
|
|
long ticks = DateTime.Now.Ticks;
|
|
string json = JsonSerializer.Serialize(keyValuePairs, new JsonSerializerOptions { WriteIndented = true });
|
|
File.WriteAllText($"{tempDirectory}/{nameof(keyValuePairs)}-{ticks}.json", json);
|
|
}
|
|
List<(int Index, long Ticks, string RelativeDirectory, string FileNameWithoutExtension, string Extension, string RegexResult)> indexCollection = GetIndexCollection(infoDirectory, infoDirectoryIgnore, checkDistinct: false);
|
|
foreach ((int index, long ticks, string relativeDirectory, string fileNameWithoutExtension, string extension, string regexResult) in indexCollection)
|
|
{
|
|
z += 1;
|
|
if (_IsEnvironment.Development && z % 1000 == 0)
|
|
_Log.Debug($"{z}) Loop {_Configuration.PropertyConfiguration.RootDirectory}");
|
|
if (!useTabSeparatedValueCollection.ContainsKey(index))
|
|
{
|
|
missingIndices.Add(index);
|
|
continue;
|
|
}
|
|
propertyId = useTabSeparatedValueCollection[index];
|
|
findReplace.Add(index, propertyId);
|
|
}
|
|
{
|
|
long ticks = DateTime.Now.Ticks;
|
|
File.WriteAllLines($"{tempDirectory}/{nameof(useTabSeparatedValueLines)}-{ticks}.tsv", useTabSeparatedValueLines);
|
|
File.WriteAllText($"{tempDirectory}/{nameof(missingIndices)}-{ticks}.txt", string.Join(Environment.NewLine, missingIndices));
|
|
}
|
|
_Log.Debug($"Done with {nameof(CopyMissingImagesLogs)}");
|
|
}
|
|
|
|
private static Dictionary<int, List<PropertyCompare.Models.PropertyCompare>> GetKeyValuePairs(PropertyCompare.Models.PropertyCompare[] propertyCompares)
|
|
{
|
|
Dictionary<int, List<PropertyCompare.Models.PropertyCompare>> results = new();
|
|
foreach (PropertyCompare.Models.PropertyCompare propertyCompare in propertyCompares)
|
|
{
|
|
if (propertyCompare?.Property?.Id is null)
|
|
continue;
|
|
if (!results.ContainsKey(propertyCompare.Property.Id.Value))
|
|
results.Add(propertyCompare.Property.Id.Value, new());
|
|
results[propertyCompare.Property.Id.Value].Add(propertyCompare);
|
|
}
|
|
return results;
|
|
}
|
|
|
|
private void VerifyAgainstIndexInfoJsonFiles(Property.Models.Configuration configuration, string aPropertySingletonDirectory)
|
|
{
|
|
if (_Log is null)
|
|
throw new Exception($"{nameof(_Log)} is null!");
|
|
if (_AppSettings.MaxDegreeOfParallelism is null)
|
|
throw new Exception($"{nameof(_AppSettings.MaxDegreeOfParallelism)} is null!");
|
|
if (_Configuration?.PropertyConfiguration is null)
|
|
throw new Exception($"{nameof(_Configuration.PropertyConfiguration)} must be set!");
|
|
string? rootDirectoryParent = Path.GetDirectoryName(_Configuration.PropertyConfiguration.RootDirectory);
|
|
if (string.IsNullOrEmpty(rootDirectoryParent))
|
|
throw new Exception($"{nameof(rootDirectoryParent)} is null!");
|
|
int z = 0;
|
|
int? propertyId;
|
|
long? propertyTicks;
|
|
int loadLessThan = 7;
|
|
string diffRootDirectory = string.Empty;
|
|
PropertyCompare.Models.PropertyCompare propertyCompare;
|
|
string tempDirectory = Path.Combine(rootDirectoryParent, "Tmp");
|
|
if (!Directory.Exists(tempDirectory))
|
|
_ = Directory.CreateDirectory(tempDirectory);
|
|
string imageSharedDirectory = Path.Combine(rootDirectoryParent, "Images - Shared");
|
|
string namedFaceInfoFile = Path.Combine(imageSharedDirectory, "NamedFaceInfo.json");
|
|
string infoDirectory = Path.Combine(imageSharedDirectory, "Images - 4) Info", "2020-06-07");
|
|
List<(int Index, string RelativeDirectory, string FileName, string RegexResult, long Ticks, int? PropertyId, long? PropertyTicks)> collection = new();
|
|
PropertyCompare.Models.PropertyCompareLogic propertyCompareLogic = new(_AppSettings.MaxDegreeOfParallelism.Value, _Configuration.PropertyConfiguration, _SpellingFindReplace, diffRootDirectory);
|
|
PropertyCompare.Models.PropertyCompare[] propertyCompareCollection = propertyCompareLogic.Get(aPropertySingletonDirectory, loadLessThan);
|
|
{
|
|
long ticks = DateTime.Now.Ticks;
|
|
string[] lines = (from l in propertyCompareCollection select l.GetSelect()).ToArray();
|
|
string aPropertyCollectionDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(A_Property), "[{}]");
|
|
File.WriteAllLines(Path.Join(aPropertyCollectionDirectory, $". . . Ids - {ticks}.txt"), lines);
|
|
string json = JsonSerializer.Serialize(propertyCompareCollection, new JsonSerializerOptions { WriteIndented = true });
|
|
File.WriteAllText(Path.Join(aPropertyCollectionDirectory, $". . . Ids - {ticks}.nosj"), json);
|
|
}
|
|
Dictionary<int, List<PropertyCompare.Models.PropertyCompare>> keyValuePairs = GetKeyValuePairs(propertyCompareCollection);
|
|
List<(int Index, long Ticks, string RelativeDirectory, string FileNameWithoutExtension, string Extension, string RegexResult)> indexCollection = GetIndexCollection(infoDirectory, infoDirectoryExtra: string.Empty, checkDistinct: false);
|
|
foreach ((int index, long ticks, string relativeDirectory, string fileNameWithoutExtension, string extension, string regexResult) in indexCollection)
|
|
{
|
|
z += 1;
|
|
if (_IsEnvironment.Development && z % 1000 == 0)
|
|
_Log.Debug($"{z}) Loop {_Configuration.PropertyConfiguration.RootDirectory}");
|
|
propertyId = null;
|
|
propertyTicks = null;
|
|
if (keyValuePairs.ContainsKey(index))
|
|
{
|
|
propertyCompare = keyValuePairs[index][0];
|
|
if (propertyCompare?.Property?.Id is null)
|
|
continue;
|
|
propertyId = propertyCompare.Property.Id;
|
|
propertyTicks = propertyCompare.MinimumDateTime.Ticks;
|
|
}
|
|
collection.Add(new(index, relativeDirectory, $"{fileNameWithoutExtension}{extension}", regexResult, ticks, propertyId, propertyTicks));
|
|
}
|
|
(int Index, string RelativeDirectory, string FileName, string RegexResult, long Ticks, int? PropertyId, long? PropertyTicks)[] matched = (from l in collection where l.PropertyId.HasValue select l).ToArray();
|
|
(int Index, string RelativeDirectory, string FileName, string RegexResult, long Ticks, int? PropertyId, long? PropertyTicks)[] notMatched = (from l in collection where l.PropertyId is null select l).ToArray();
|
|
string[] matchedLines = (
|
|
from l
|
|
in matched
|
|
select $"{l.Index}\t{l.PropertyId}\t{l.RegexResult}\t{new DateTime(l.Ticks):yyyy-MM-dd_HH-mm-ss}\t{l.PropertyTicks}\t{l.RelativeDirectory}\t{l.FileName}"
|
|
).ToArray();
|
|
List<string> notMatchedLines = (
|
|
from l
|
|
in notMatched
|
|
select $"{l.Index}\t______________\t{l.RegexResult}\t{new DateTime(l.Ticks):yyyy-MM-dd_HH-mm-ss}\t_________________\t{l.RelativeDirectory}\t{l.FileName}"
|
|
).ToList();
|
|
{
|
|
long ticks = DateTime.Now.Ticks;
|
|
File.WriteAllLines($"{tempDirectory}/{collection.Count}-{ticks}.tsv", matchedLines);
|
|
File.WriteAllLines($"{tempDirectory}/{collection.Count}-{ticks}-Not.tsv", notMatchedLines);
|
|
}
|
|
_Log.Debug($"Done with {nameof(VerifyAgainstIndexInfoJsonFiles)}");
|
|
}
|
|
|
|
} |