using System.Text.Json; using View_by_Distance.Shared.Models.Stateless; namespace View_by_Distance.Property.Models.Stateless; public static class A_Property { public static string DateTimeFormat() => "yyyy:MM:dd HH:mm:ss"; public static (int Season, string seasonName) GetSeason(int dayOfYear) { (int Season, string seasonName) result = dayOfYear switch { < 78 => new(0, "Winter"), < 171 => new(1, "Spring"), < 264 => new(2, "Summer"), < 354 => new(3, "Fall"), _ => new(4, "Winter") }; return result; } public static List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> GetGroupCollection(string rootDirectory, string searchPattern, List topDirectories, int maxImagesInDirectoryForTopLevelFirstPass = 50, bool reverse = false) { List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> results = new(); string? parentDirectory; string[] subDirectories; string[] sourceDirectoryFiles; List fileCollections = new(); if (!topDirectories.Any()) topDirectories.AddRange(from l in Directory.GetDirectories(rootDirectory, "*", SearchOption.TopDirectoryOnly) select Path.GetFullPath(l)); for (int g = 1; g < 5; g++) { if (g == 4) { for (int i = fileCollections.Count - 1; i > -1; i--) { parentDirectory = Path.GetDirectoryName(fileCollections[i][0]); if (string.IsNullOrEmpty(parentDirectory)) continue; results.Add(new(g, parentDirectory, fileCollections[i], results.Count)); fileCollections.RemoveAt(i); } } else if (g == 2) { fileCollections = (from l in fileCollections orderby l.Length descending select l).ToList(); for (int i = fileCollections.Count - 1; i > -1; i--) { if (fileCollections[i].Length > maxImagesInDirectoryForTopLevelFirstPass * g) break; parentDirectory = Path.GetDirectoryName(fileCollections[i][0]); if (string.IsNullOrEmpty(parentDirectory)) continue; results.Add(new(g, parentDirectory, fileCollections[i], results.Count)); fileCollections.RemoveAt(i); } } else if (g == 3) { subDirectories = Directory.GetDirectories(rootDirectory, "*", SearchOption.AllDirectories); if (reverse) subDirectories = subDirectories.Reverse().ToArray(); foreach (string subDirectory in subDirectories) { sourceDirectoryFiles = Directory.GetFiles(subDirectory, "*", SearchOption.TopDirectoryOnly); if (!topDirectories.Contains(subDirectory)) results.Add(new(g, subDirectory, sourceDirectoryFiles, results.Count)); } } else if (g == 1) { sourceDirectoryFiles = Directory.GetFiles(rootDirectory, searchPattern, SearchOption.TopDirectoryOnly); if (sourceDirectoryFiles.Length > maxImagesInDirectoryForTopLevelFirstPass) fileCollections.Add(sourceDirectoryFiles); else results.Add(new(g, rootDirectory, sourceDirectoryFiles, results.Count)); if (reverse) topDirectories.Reverse(); subDirectories = topDirectories.ToArray(); foreach (string subDirectory in subDirectories) { sourceDirectoryFiles = Directory.GetFiles(subDirectory, searchPattern, SearchOption.TopDirectoryOnly); if (sourceDirectoryFiles.Length > maxImagesInDirectoryForTopLevelFirstPass) fileCollections.Add(sourceDirectoryFiles); else { if (sourceDirectoryFiles.Any() || Directory.GetDirectories(subDirectory, "*", SearchOption.TopDirectoryOnly).Any()) results.Add(new(g, subDirectory, sourceDirectoryFiles, results.Count)); else if (searchPattern == "*") { sourceDirectoryFiles = Directory.GetFiles(subDirectory, searchPattern, SearchOption.TopDirectoryOnly); foreach (string subFile in sourceDirectoryFiles) File.Delete(subFile); Directory.Delete(subDirectory); } } } fileCollections.Reverse(); } else throw new Exception(); } return results; } public static List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> GetJsonGroupCollection(string rootDirectory) { List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> results; bool reverse = false; string searchPattern = "*.json"; List topDirectories = new(); int maxImagesInDirectoryForTopLevelFirstPass = 50; results = GetGroupCollection(rootDirectory, searchPattern, topDirectories, maxImagesInDirectoryForTopLevelFirstPass, reverse); return results; } public static List<(int g, string sourceDirectory, FileInfo[] sourceDirectoryFiles, int r)> GetFileInfoGroupCollection(Models.Configuration configuration, bool reverse, string searchPattern, List topDirectories) { if (configuration.MaxImagesInDirectoryForTopLevelFirstPass is null) throw new Exception($"{nameof(configuration.MaxImagesInDirectoryForTopLevelFirstPass)} is null!"); List<(int g, string sourceDirectory, FileInfo[] sourceDirectoryFiles, int r)> results = new(); List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)>? collection = GetGroupCollection(configuration.RootDirectory, searchPattern, topDirectories, configuration.MaxImagesInDirectoryForTopLevelFirstPass.Value, reverse); foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles, int r) in collection) results.Add(new(g, sourceDirectory, (from l in sourceDirectoryFiles select new FileInfo(l)).ToArray(), r)); return results; } private static List<(int g, string sourceDirectory, List<(string sourceDirectoryFile, Models.A_Property? property)> collection, int r)> GetCollection(string rootDirectory, List<(int g, string sourceDirectory, string[] SourceDirectoryFiles, int r)> jsonCollection) { List<(int, string, List<(string, Models.A_Property?)>, int)> results = new(); int length = rootDirectory.Length; ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = Environment.ProcessorCount }; _ = Parallel.For(0, jsonCollection.Count, parallelOptions, i => ParallelFor(jsonCollection, i, length, results)); return results; } private static List Populate(Models.Configuration configuration, string aPropertySingletonDirectory, List<(int, string, FileInfo[], int)> fileInfoGroupCollection, List<(int, string, List<(string, Models.A_Property?)>, int)> collectionFromJson) { List results = new(); if (configuration.PropertiesChangedForProperty is null) throw new Exception($"{configuration.PropertiesChangedForProperty} is null"); int length; string inferred; string relativePath; FileInfo keyFileInfo; string keySourceDirectory; List propertyHolderCollection; Dictionary fileInfoKeyValuePairs = new(); length = configuration.RootDirectory.Length; foreach ((int g, string sourceDirectory, FileInfo[] sourceDirectoryFileInfoCollection, int r) in fileInfoGroupCollection) { foreach (FileInfo sourceDirectoryFileInfo in sourceDirectoryFileInfoCollection) { relativePath = $"{XPath.GetRelativePath(sourceDirectoryFileInfo.FullName, length)}.json"; fileInfoKeyValuePairs.Add(relativePath, new(sourceDirectory, sourceDirectoryFileInfo)); } } length = aPropertySingletonDirectory.Length; foreach ((int g, string _, List<(string, Models.A_Property?)> collection, int r) in collectionFromJson) { if (!collection.Any()) continue; propertyHolderCollection = new(); foreach ((string sourceDirectoryFile, Models.A_Property? property) in collection) { relativePath = XPath.GetRelativePath(sourceDirectoryFile, length); if (!fileInfoKeyValuePairs.ContainsKey(relativePath)) { inferred = string.Concat(configuration.RootDirectory, relativePath); keyFileInfo = new(inferred[..^5]); if (keyFileInfo.Extension is ".json") continue; keySourceDirectory = string.Concat(keyFileInfo.DirectoryName); propertyHolderCollection.Add(new(g, keySourceDirectory, sourceDirectoryFile, relativePath, r, keyFileInfo, property, true, null, null, null)); } else { keyFileInfo = fileInfoKeyValuePairs[relativePath].FileInfo; keySourceDirectory = fileInfoKeyValuePairs[relativePath].SourceDirectory; if (!fileInfoKeyValuePairs.Remove(relativePath)) throw new Exception(); if (keyFileInfo.Extension is ".json") continue; if (property?.Id is null || property?.Width is null || property?.Height is null) propertyHolderCollection.Add(new(g, keySourceDirectory, sourceDirectoryFile, relativePath, r, keyFileInfo, property, false, null, null, null)); else if (configuration.PropertiesChangedForProperty.Value || property.LastWriteTime != keyFileInfo.LastWriteTime || property.FileSize != keyFileInfo.Length) propertyHolderCollection.Add(new(g, keySourceDirectory, sourceDirectoryFile, relativePath, r, keyFileInfo, property, false, true, null, null)); else propertyHolderCollection.Add(new(g, keySourceDirectory, sourceDirectoryFile, relativePath, r, keyFileInfo, property, false, false, null, null)); } } if (propertyHolderCollection.Any()) results.Add(propertyHolderCollection.ToArray()); } length = configuration.RootDirectory.Length; foreach ((int g, string sourceDirectory, FileInfo[] sourceDirectoryFileInfoCollection, int r) in fileInfoGroupCollection) { propertyHolderCollection = new(); foreach (FileInfo sourceDirectoryFileInfo in sourceDirectoryFileInfoCollection) { relativePath = $"{XPath.GetRelativePath(sourceDirectoryFileInfo.FullName, length)}.json"; if (!fileInfoKeyValuePairs.ContainsKey(relativePath)) continue; if (!fileInfoKeyValuePairs.Remove(relativePath)) throw new Exception(); if (sourceDirectoryFileInfo.Extension is ".json") continue; propertyHolderCollection.Add(new(g, sourceDirectory, relativePath, sourceDirectoryFileInfo.FullName, r, sourceDirectoryFileInfo, null, null, null, null, null)); } if (propertyHolderCollection.Any()) results.Add(propertyHolderCollection.ToArray()); } if (fileInfoKeyValuePairs.Any()) throw new Exception(); results = (from l in results orderby l[0].G, l[0].R select l).ToList(); return results; } private static void ParallelFor(List<(int, string, string[], int)> jsonCollection, int i, int length, List<(int, string, List<(string, Models.A_Property?)>, int)> results) { string key; string json; Models.A_Property? property; (int g, string sourceDirectory, string[] sourceDirectoryFiles, int r) = jsonCollection[i]; List<(string, Models.A_Property?)> collection = new(); foreach (string sourceDirectoryFile in sourceDirectoryFiles) { json = File.ReadAllText(sourceDirectoryFile); key = XPath.GetRelativePath(sourceDirectoryFile, length); property = JsonSerializer.Deserialize(json); collection.Add(new(sourceDirectoryFile, property)); } lock (results) results.Add(new(g, sourceDirectory, collection, r)); } public static List Get(Models.Configuration configuration, bool reverse, Model? model, PredictorModel? predictorModel, PropertyLogic propertyLogic) { List results; string searchPattern = "*"; long ticks = DateTime.Now.Ticks; List topDirectories = new(); List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> jsonCollection; List<(int g, string sourceDirectory, FileInfo[] sourceDirectoryFiles, int r)> fileInfoGroupCollection; string aPropertySingletonDirectory = IResult.GetResultsDateGroupDirectory(configuration, nameof(A_Property), "{}"); List<(int g, string sourceDirectory, List<(string sourceDirectoryFile, Models.A_Property? property)> collection, int r)> collectionFromJson; jsonCollection = GetJsonGroupCollection(aPropertySingletonDirectory); fileInfoGroupCollection = GetFileInfoGroupCollection(configuration, reverse, searchPattern, topDirectories); collectionFromJson = GetCollection(aPropertySingletonDirectory, jsonCollection); results = Populate(configuration, aPropertySingletonDirectory, fileInfoGroupCollection, collectionFromJson); propertyLogic.ParallelWork(configuration, model, predictorModel, ticks, results, firstPass: false); if (propertyLogic.ExceptionsDirectories.Any()) throw new Exception(); return results; } public static int GetDeterministicHashCode(byte[] value) { int result; unchecked { int hash1 = (5381 << 16) + 5381; int hash2 = hash1; for (int i = 0; i < value.Length; i += 2) { hash1 = ((hash1 << 5) + hash1) ^ value[i]; if (i == value.Length - 1) break; hash2 = ((hash2 << 5) + hash2) ^ value[i + 1]; } result = hash1 + (hash2 * 1566083941); } return result; } public static int GetDeterministicHashCode(string value) { int result; unchecked { int hash1 = (5381 << 16) + 5381; int hash2 = hash1; for (int i = 0; i < value.Length; i += 2) { hash1 = ((hash1 << 5) + hash1) ^ value[i]; if (i == value.Length - 1) break; hash2 = ((hash2 << 5) + hash2) ^ value[i + 1]; } result = hash1 + (hash2 * 1566083941); } return result; } public static (bool?, string[]) IsWrongYear(string[] segments, string year) { bool? result; string[] results = ( from l in segments where l?.Length > 2 && ( l[..2] is "19" or "20" || (l.Length == 5 && l.Substring(1, 2) is "19" or "20" && (l[0] is '~' or '=' or '-' or '^' or '#')) || (l.Length == 6 && l[..2] is "19" or "20" && l[4] == '.') || (l.Length == 7 && l.Substring(1, 2) is "19" or "20" && l[5] == '.') ) select l ).ToArray(); string[] matches = ( from l in results where l == year || (l.Length == 5 && l.Substring(1, 4) == year && (l[0] is '~' or '=' or '-' or '^' or '#')) || (l.Length == 6 && l[..4] == year && l[4] == '.') || (l.Length == 7 && l.Substring(1, 4) == year && l[5] == '.') select l ).ToArray(); if (!results.Any()) result = null; else result = !matches.Any(); return new(result, results); } public static List GetDateTimes(DateTime creationTime, DateTime lastWriteTime, DateTime? dateTime, DateTime? dateTimeDigitized, DateTime? dateTimeOriginal, DateTime? gpsDateStamp) { List results = new() { creationTime, lastWriteTime }; if (dateTime.HasValue) results.Add(dateTime.Value); if (dateTimeDigitized.HasValue) results.Add(dateTimeDigitized.Value); if (dateTimeOriginal.HasValue) results.Add(dateTimeOriginal.Value); if (gpsDateStamp.HasValue) results.Add(gpsDateStamp.Value); return results; } public static DateTime GetDateTime(Models.A_Property? property) { DateTime result; if (property is null) result = DateTime.MinValue; else { List dateTimes = new() { property.CreationTime, property.LastWriteTime }; if (property.DateTime.HasValue) dateTimes.Add(property.DateTime.Value); if (property.DateTimeDigitized.HasValue) dateTimes.Add(property.DateTimeDigitized.Value); if (property.DateTimeOriginal.HasValue) dateTimes.Add(property.DateTimeOriginal.Value); if (property.GPSDateStamp.HasValue) dateTimes.Add(property.GPSDateStamp.Value); result = dateTimes.Min(); } return result; } public static List GetDateTimes(Models.A_Property property) => GetDateTimes(property.CreationTime, property.LastWriteTime, property.DateTime, property.DateTimeDigitized, property.DateTimeOriginal, property.GPSDateStamp); public static DateTime GetMinimumDateTime(Models.A_Property? property) { DateTime result; List dateTimes; if (property is null) result = DateTime.MinValue; else { dateTimes = GetDateTimes(property); result = dateTimes.Min(); } return result; } public static string GetDiffRootDirectory(string diffPropertyDirectory) { string result = string.Empty; string results = " - Results"; string? checkDirectory = diffPropertyDirectory; for (int i = 0; i < int.MaxValue; i++) { checkDirectory = Path.GetDirectoryName(checkDirectory); if (string.IsNullOrEmpty(checkDirectory)) break; if (checkDirectory.EndsWith(results)) { result = checkDirectory[..^results.Length]; break; } } return result; } public static double GetStandardDeviation(IEnumerable values, double average) { double result = 0; if (!values.Any()) throw new Exception("Collection must have at least one value!"); double sum = values.Sum(l => (l - average) * (l - average)); result = Math.Sqrt(sum / values.Count()); return result; } public static TimeSpan GetThreeStandardDeviationHigh(int minimum, PropertyHolder[] propertyHolderCollection) { TimeSpan result; List ticksCollection = new(); foreach (PropertyHolder propertyHolder in propertyHolderCollection) { if (propertyHolder.Property is null || propertyHolder.MinimumDateTime is null) continue; ticksCollection.Add(propertyHolder.MinimumDateTime.Value.Ticks); } long threeStandardDeviationHigh; long min; if (!ticksCollection.Any()) min = 0; else min = ticksCollection.Min(); if (ticksCollection.Count < minimum) threeStandardDeviationHigh = long.MaxValue; else { ticksCollection = (from l in ticksCollection select l - min).ToList(); double sum = ticksCollection.Sum(); double average = sum / ticksCollection.Count; double standardDeviation = GetStandardDeviation(ticksCollection, average); threeStandardDeviationHigh = (long)Math.Ceiling(average + min + (standardDeviation * 3)); } result = new TimeSpan(threeStandardDeviationHigh - min); return result; } public static (int, List, List) Get(PropertyHolder[] propertyHolderCollection, TimeSpan threeStandardDeviationHigh, int i) { List results = new(); int j = i; long? ticks; TimeSpan timeSpan; PropertyHolder propertyHolder; List dateTimes = new(); PropertyHolder nextPropertyHolder; for (; j < propertyHolderCollection.Length; j++) { ticks = null; propertyHolder = propertyHolderCollection[j]; if (propertyHolder.Property is null || propertyHolder.MinimumDateTime is null) continue; for (int k = j + 1; k < propertyHolderCollection.Length; k++) { nextPropertyHolder = propertyHolderCollection[k]; if (nextPropertyHolder.Property is not null && nextPropertyHolder.MinimumDateTime is not null) { ticks = nextPropertyHolder.MinimumDateTime.Value.Ticks; break; } } results.Add(propertyHolder); dateTimes.Add(propertyHolder.MinimumDateTime.Value); if (ticks.HasValue) { timeSpan = new(ticks.Value - propertyHolder.MinimumDateTime.Value.Ticks); if (timeSpan > threeStandardDeviationHigh) break; } } return new(j, dateTimes, results); } public static bool Any(List propertyHolderCollections) { bool result = false; foreach (PropertyHolder[] propertyHolderCollection in propertyHolderCollections) { if (!propertyHolderCollection.Any()) continue; if ((from l in propertyHolderCollection where l.Any() select true).Any()) { result = true; break; } } return result; } }