using ShellProgressBar; using System.Diagnostics; using System.Drawing; using System.Drawing.Imaging; using System.Globalization; using System.Runtime.InteropServices; using System.Text; using System.Text.Json; using View_by_Distance.Property.Models.Stateless; using View_by_Distance.Shared.Models.Properties; using View_by_Distance.Shared.Models.Stateless; namespace View_by_Distance.Property.Models; public class PropertyLogic { protected readonly List<(int, string[])> _AllCollection; protected readonly List _ExceptionsDirectories; protected readonly Dictionary _KeyValuePairs; protected readonly Dictionary _IndicesFromNew; protected readonly string _DeterministicHashCodeRootDirectory; protected readonly Dictionary _SixCharacterNamedFaceInfo; protected readonly Dictionary _NamedFaceInfoDeterministicHashCodeKeyValuePairs; protected readonly Dictionary _NamedDeterministicHashCodeKeyValuePairs; protected readonly Dictionary _IncorrectDeterministicHashCodeKeyValuePairs; public bool Reverse { get; } public List AngleBracketCollection { get; } public Dictionary KeyValuePairs => _KeyValuePairs; public Dictionary IndicesFromNew => _IndicesFromNew; public List ExceptionsDirectories => _ExceptionsDirectories; public string DeterministicHashCodeRootDirectory => _DeterministicHashCodeRootDirectory; public Dictionary NamedDeterministicHashCodeKeyValuePairs => _NamedDeterministicHashCodeKeyValuePairs; public Dictionary IncorrectDeterministicHashCodeKeyValuePairs => _IncorrectDeterministicHashCodeKeyValuePairs; public Dictionary NamedFaceInfoDeterministicHashCodeKeyValuePairs => _NamedFaceInfoDeterministicHashCodeKeyValuePairs; private readonly Model? _Model; private readonly Serilog.ILogger? _Log; private readonly string[] _VerifyToSeason; private readonly int _MaxDegreeOfParallelism; private readonly ASCIIEncoding _ASCIIEncoding; private readonly Configuration _Configuration; private readonly PredictorModel? _PredictorModel; private readonly JsonSerializerOptions _WriteIndentedJsonSerializerOptions; public PropertyLogic(int maxDegreeOfParallelism, Configuration configuration, bool reverse, Model? model, PredictorModel? predictorModel) { _Model = model; Reverse = reverse; _AllCollection = new(); _Configuration = configuration; _ExceptionsDirectories = new(); _PredictorModel = predictorModel; _ASCIIEncoding = new ASCIIEncoding(); AngleBracketCollection = new List(); _Log = Serilog.Log.ForContext(); _MaxDegreeOfParallelism = maxDegreeOfParallelism; Dictionary? namedFaceInfoDeterministicHashCode; _WriteIndentedJsonSerializerOptions = new JsonSerializerOptions { WriteIndented = true }; if (configuration.VerifyToSeason is null || !configuration.VerifyToSeason.Any()) throw new Exception(); _VerifyToSeason = configuration.VerifyToSeason.Select(l => Path.Combine(configuration.RootDirectory, l)).ToArray(); string json; string[] files; string fullPath; Dictionary? keyValuePairs; string deterministicHashCodeRootDirectory; List>? collection; Dictionary indicesFromNew = new(); Dictionary? sixCharacterNamedFaceInfo; Dictionary namedDeterministicHashCode = new(); Dictionary incorrectDeterministicHashCode = new(); string? rootDirectoryParent = Path.GetDirectoryName(configuration.RootDirectory); if (string.IsNullOrEmpty(rootDirectoryParent)) throw new NullReferenceException(nameof(rootDirectoryParent)); files = Directory.GetFiles(rootDirectoryParent, "*DeterministicHashCode*.json", SearchOption.TopDirectoryOnly); if (files.Length != 1) namedFaceInfoDeterministicHashCode = new(); else { json = File.ReadAllText(files[0]); namedFaceInfoDeterministicHashCode = JsonSerializer.Deserialize>(json); if (namedFaceInfoDeterministicHashCode is null) throw new NullReferenceException(nameof(namedFaceInfoDeterministicHashCode)); } if (namedFaceInfoDeterministicHashCode.Any()) { deterministicHashCodeRootDirectory = string.Empty; files = Directory.GetFiles(rootDirectoryParent, "*SixCharacter*.json", SearchOption.TopDirectoryOnly); if (files.Length != 1) sixCharacterNamedFaceInfo = new(); else { json = File.ReadAllText(files[0]); sixCharacterNamedFaceInfo = JsonSerializer.Deserialize>(json); if (sixCharacterNamedFaceInfo is null) throw new NullReferenceException(nameof(sixCharacterNamedFaceInfo)); } } else { sixCharacterNamedFaceInfo = new(); string[] directories = Directory.GetDirectories(rootDirectoryParent, "*DeterministicHashCode*", SearchOption.TopDirectoryOnly); if (!directories.Any()) deterministicHashCodeRootDirectory = string.Empty; else { Dictionary> faces = new(); deterministicHashCodeRootDirectory = directories[0]; SetKeyValuePairs(deterministicHashCodeRootDirectory, namedDeterministicHashCode, incorrectDeterministicHashCode, faces); } } files = Directory.GetFiles(rootDirectoryParent, "*keyValuePairs*.json", SearchOption.TopDirectoryOnly); if (files.Length != 1) keyValuePairs = new(); else { json = File.ReadAllText(files[0]); keyValuePairs = JsonSerializer.Deserialize>(json); if (keyValuePairs is null) throw new NullReferenceException(nameof(keyValuePairs)); } foreach (string propertyContentCollectionFile in configuration.PropertyContentCollectionFiles) { fullPath = Path.GetFullPath(string.Concat(rootDirectoryParent, propertyContentCollectionFile)); if (fullPath.Contains(configuration.RootDirectory)) continue; if (!File.Exists(fullPath)) continue; json = File.ReadAllText(fullPath); collection = JsonSerializer.Deserialize>>(json); if (collection is null) throw new NullReferenceException(nameof(collection)); foreach (KeyValuePair keyValuePair in collection) { if (indicesFromNew.ContainsKey(keyValuePair.Key)) continue; indicesFromNew.Add(keyValuePair.Key, keyValuePair.Value); } } _KeyValuePairs = keyValuePairs; _IndicesFromNew = indicesFromNew; _SixCharacterNamedFaceInfo = sixCharacterNamedFaceInfo; _NamedDeterministicHashCodeKeyValuePairs = namedDeterministicHashCode; _DeterministicHashCodeRootDirectory = deterministicHashCodeRootDirectory; _IncorrectDeterministicHashCodeKeyValuePairs = incorrectDeterministicHashCode; _NamedFaceInfoDeterministicHashCodeKeyValuePairs = namedFaceInfoDeterministicHashCode; } private static void SetKeyValuePairs(string deterministicHashCodeRootDirectory, List<(string, float)> named, List<(string, float)> incorrect, Dictionary> keyValuePairs) { string[] files; string fileName; string personKey; string[] yearDirectories; string[] personKeyDirectories; string[] personNameDirectories; float? idAndNormalizedPixelPercentage; string[] ticksDirectories = Directory.GetDirectories(deterministicHashCodeRootDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string ticksDirectory in ticksDirectories) { if (!ticksDirectory.EndsWith(')')) continue; personKeyDirectories = Directory.GetDirectories(ticksDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string personKeyDirectory in personKeyDirectories) { personKey = Path.GetFileName(personKeyDirectory); yearDirectories = Directory.GetDirectories(personKeyDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string yearDirectory in yearDirectories) { files = Directory.GetFiles(yearDirectory, "*", SearchOption.TopDirectoryOnly); personNameDirectories = Directory.GetDirectories(yearDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string file in files) { if (file.EndsWith(".lnk")) continue; fileName = Path.GetFileName(file); idAndNormalizedPixelPercentage = Named.GetReversedDeterministicHashCode(keyValuePairs, fileName); if (idAndNormalizedPixelPercentage is null) break; incorrect.Add(new(personKey, idAndNormalizedPixelPercentage.Value)); } foreach (string personNameDirectory in personNameDirectories) { files = Directory.GetFiles(personNameDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string file in files) { if (file.EndsWith(".lnk")) continue; fileName = Path.GetFileNameWithoutExtension(file); idAndNormalizedPixelPercentage = Named.GetReversedDeterministicHashCode(keyValuePairs, fileName); if (idAndNormalizedPixelPercentage is null) break; named.Add(new(personKey, idAndNormalizedPixelPercentage.Value)); } } } } } } private static void SetKeyValuePairs(string deterministicHashCodeRootDirectory, Dictionary namedDeterministicHashCode, Dictionary incorrectDeterministicHashCode, Dictionary> keyValuePairs) { Dictionary> namedKeyValuePairs = new(); Dictionary> incorrectKeyValuePairs = new(); List<(string PersonKey, float IdAndNormalizedPixelPercentage)> named = new(); List<(string PersonKey, float IdAndNormalizedPixelPercentage)> incorrect = new(); SetKeyValuePairs(deterministicHashCodeRootDirectory, named, incorrect, keyValuePairs); named = (from l in named orderby l.IdAndNormalizedPixelPercentage select l).ToList(); incorrect = (from l in incorrect orderby l.IdAndNormalizedPixelPercentage select l).ToList(); foreach ((string personKey, float idAndNormalizedPixelPercentage) in named) { if (!namedKeyValuePairs.ContainsKey(idAndNormalizedPixelPercentage)) namedKeyValuePairs.Add(idAndNormalizedPixelPercentage, new()); namedKeyValuePairs[idAndNormalizedPixelPercentage].Add(personKey); } foreach ((string personKey, float idAndNormalizedPixelPercentage) in incorrect) { if (!incorrectKeyValuePairs.ContainsKey(idAndNormalizedPixelPercentage)) incorrectKeyValuePairs.Add(idAndNormalizedPixelPercentage, new()); incorrectKeyValuePairs[idAndNormalizedPixelPercentage].Add(personKey); } foreach (KeyValuePair> keyValuePair in namedKeyValuePairs) namedDeterministicHashCode.Add(keyValuePair.Key, keyValuePair.Value.Distinct().ToArray()); foreach (KeyValuePair> keyValuePair in incorrectKeyValuePairs) incorrectDeterministicHashCode.Add(keyValuePair.Key, keyValuePair.Value.Distinct().ToArray()); } public void UpdateKeyValuePairs(List containers) { Dictionary> keyValuePairs = new(); Dictionary namedDeterministicHashCode = new(); Dictionary incorrectDeterministicHashCode = new(); foreach (Container container in containers) { foreach (Item item in container.Items) { if (item.ImageFileHolder is null || item.Property?.Id is null || !item.Faces.Any()) continue; keyValuePairs.Add(item.Property.Id.Value, item.Faces); } } SetKeyValuePairs(_DeterministicHashCodeRootDirectory, namedDeterministicHashCode, incorrectDeterministicHashCode, keyValuePairs); foreach (KeyValuePair keyValuePair in namedDeterministicHashCode) _NamedDeterministicHashCodeKeyValuePairs.Add(keyValuePair.Key, keyValuePair.Value); foreach (KeyValuePair keyValuePair in incorrectDeterministicHashCode) _IncorrectDeterministicHashCodeKeyValuePairs.Add(keyValuePair.Key, keyValuePair.Value); } public override string ToString() { string result = JsonSerializer.Serialize(this, new JsonSerializerOptions() { WriteIndented = true }); return result; } private long LogDelta(long ticks, string methodName) { long result; if (_Log is null) throw new NullReferenceException(nameof(_Log)); double delta = new TimeSpan(DateTime.Now.Ticks - ticks).TotalMilliseconds; _Log.Debug($"{methodName} took {Math.Floor(delta)} millisecond(s)"); result = DateTime.Now.Ticks; return result; } public static List GetMetadataDateTimesByPattern(string dateTimeFormat, string filteredSourceDirectoryFile) { List results = new(); try { DateTime checkDateTime; DateTime kristy = new(1976, 3, 8); IReadOnlyList directories = MetadataExtractor.ImageMetadataReader.ReadMetadata(filteredSourceDirectoryFile); foreach (MetadataExtractor.Directory directory in directories) { foreach (MetadataExtractor.Tag tag in directory.Tags) { if (string.IsNullOrEmpty(tag.Description) || tag.Description.Length != dateTimeFormat.Length) continue; if (!DateTime.TryParseExact(tag.Description, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) continue; if (checkDateTime < kristy) continue; results.Add(checkDateTime); } } } catch (Exception) { } return results; } public static List GetMetadataDateTimesByPattern(string dateTimeFormat, FileHolder filteredSourceDirectoryFileHolder) { List results = GetMetadataDateTimesByPattern(dateTimeFormat, filteredSourceDirectoryFileHolder.FullName); return results; } #pragma warning disable CA1416 private A_Property GetImageProperty(FileHolder filteredSourceDirectoryFileHolder, bool populateId, bool isIgnoreExtension, bool isValidImageFormatExtension, bool isValidMetadataExtensions, int? id, List indices) { A_Property result; if (_Log is null) throw new NullReferenceException(nameof(_Log)); if (_Configuration.WriteBitmapDataBytes is null) throw new NullReferenceException(nameof(_Configuration.WriteBitmapDataBytes)); long ticks; byte[] bytes; string value; long fileLength; int encodingHash; int? width = null; int? height = null; string dateTimeFormat; DateTime checkDateTime; DateTime? dateTime = null; PropertyItem? propertyItem; string make = string.Empty; string model = string.Empty; DateTime? gpsDateStamp = null; DateTime? dateTimeOriginal = null; string orientation = string.Empty; DateTime? dateTimeDigitized = null; if (!isValidImageFormatExtension && isValidMetadataExtensions) { dateTimeFormat = "ddd MMM dd HH:mm:ss yyyy"; List dateTimes = GetMetadataDateTimesByPattern(dateTimeFormat, filteredSourceDirectoryFileHolder); if (dateTimes.Any()) dateTimeOriginal = dateTimes.Min(); } else if (!isIgnoreExtension && isValidImageFormatExtension) { if (populateId && (id is null || !indices.Any()) && !_IndicesFromNew.Any() && !_KeyValuePairs.Any()) throw new Exception("In order to keep six character indices at least one need to have an item!"); try { using Image image = Image.FromFile(filteredSourceDirectoryFileHolder.FullName); if (populateId && (id is null || !indices.Any())) { using Bitmap bitmap = new(image); string angleBracket = AngleBracketCollection[0]; Rectangle rectangle = new(0, 0, image.Width, image.Height); BitmapData bitmapData = bitmap.LockBits(rectangle, ImageLockMode.ReadOnly, bitmap.PixelFormat); IntPtr intPtr = bitmapData.Scan0; int length = bitmapData.Stride * bitmap.Height; bytes = new byte[length]; Marshal.Copy(intPtr, bytes, 0, length); bitmap.UnlockBits(bitmapData); if (id is null) { ticks = DateTime.Now.Ticks; id = Stateless.A_Property.GetDeterministicHashCode(bytes); if (_MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Stateless.A_Property.GetDeterministicHashCode)); } if (_Configuration.WriteBitmapDataBytes.Value) { FileInfo contentFileInfo = new(Path.Combine(angleBracket.Replace("<>", "()"), filteredSourceDirectoryFileHolder.Name)); File.WriteAllBytes(Path.ChangeExtension(contentFileInfo.FullName, string.Empty), bytes); } if (_IndicesFromNew.ContainsKey(id.Value) && _IndicesFromNew[id.Value].Any()) indices.AddRange(_IndicesFromNew[id.Value]); else { ticks = DateTime.Now.Ticks; string encoding = Encoding.Default.GetString(bytes); if (_MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Encoding.Default.GetString)); encodingHash = Stateless.A_Property.GetDeterministicHashCode(encoding); if (_MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Stateless.A_Property.GetDeterministicHashCode)); if (!_KeyValuePairs.ContainsKey(encodingHash)) indices.Add(encodingHash); else indices.AddRange(_KeyValuePairs[encodingHash]); } } width = image.Width; height = image.Height; dateTimeFormat = Stateless.A_Property.DateTimeFormat(); if (image.PropertyIdList.Contains((int)IExif.Tags.DateTime)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.DateTime); if (propertyItem?.Value is not null) { value = _ASCIIEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); if (value.Length > dateTimeFormat.Length) value = value[..dateTimeFormat.Length]; if (value.Length == dateTimeFormat.Length && DateTime.TryParseExact(value, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) dateTime = checkDateTime; } } if (image.PropertyIdList.Contains((int)IExif.Tags.DateTimeDigitized)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.DateTimeDigitized); if (propertyItem?.Value is not null) { value = _ASCIIEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); if (value.Length > dateTimeFormat.Length) value = value[..dateTimeFormat.Length]; if (value.Length == dateTimeFormat.Length && DateTime.TryParseExact(value, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) dateTimeDigitized = checkDateTime; } } if (image.PropertyIdList.Contains((int)IExif.Tags.DateTimeOriginal)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.DateTimeOriginal); if (propertyItem?.Value is not null) { value = _ASCIIEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); if (value.Length > dateTimeFormat.Length) value = value[..dateTimeFormat.Length]; if (value.Length == dateTimeFormat.Length && DateTime.TryParseExact(value, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) dateTimeOriginal = checkDateTime; } } if (image.PropertyIdList.Contains((int)IExif.Tags.GPSDateStamp)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.GPSDateStamp); if (propertyItem?.Value is not null) { value = _ASCIIEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); if (value.Length > dateTimeFormat.Length) value = value[..dateTimeFormat.Length]; if (value.Length == dateTimeFormat.Length && DateTime.TryParseExact(value, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) gpsDateStamp = checkDateTime; } } if (image.PropertyIdList.Contains((int)IExif.Tags.Make)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.Make); if (propertyItem?.Value is not null) { value = _ASCIIEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); make = value; } } if (image.PropertyIdList.Contains((int)IExif.Tags.Model)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.Model); if (propertyItem?.Value is not null) { value = _ASCIIEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); model = value; } } if (image.PropertyIdList.Contains((int)IExif.Tags.Orientation)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.Orientation); if (propertyItem?.Value is not null) { value = BitConverter.ToInt16(propertyItem.Value, 0).ToString(); orientation = value; } } } catch (Exception) { _Log.Info(string.Concat(new StackFrame().GetMethod()?.Name, " <", filteredSourceDirectoryFileHolder.Name, ">")); } } else dateTimeOriginal = null; if (filteredSourceDirectoryFileHolder.Length is null) fileLength = 0; else fileLength = filteredSourceDirectoryFileHolder.Length.Value; result = new(filteredSourceDirectoryFileHolder.CreationTime, dateTime, dateTimeDigitized, dateTimeOriginal, fileLength, gpsDateStamp, height, id, indices.ToArray(), filteredSourceDirectoryFileHolder.LastWriteTime, make, model, orientation, width); return result; } #pragma warning restore CA1416 private A_Property GetPropertyOfPrivate(Item item, bool firstPass, List> filteredSourceDirectoryFileTuples, List parseExceptions, bool isIgnoreExtension, bool isValidMetadataExtensions) { A_Property? result; if (_Configuration.ForcePropertyLastWriteTimeToCreationTime is null) throw new NullReferenceException(nameof(_Configuration.ForcePropertyLastWriteTimeToCreationTime)); if (_Configuration.PopulatePropertyId is null) throw new NullReferenceException(nameof(_Configuration.PopulatePropertyId)); if (_Configuration.PropertiesChangedForProperty is null) throw new NullReferenceException(nameof(_Configuration.PropertiesChangedForProperty)); if (item.ImageFileHolder is null) throw new NullReferenceException(nameof(item.ImageFileHolder)); string json; int? id = null; List indices = new(); bool hasWrongYearProperty = false; string[] changesFrom = Array.Empty(); string angleBracket = AngleBracketCollection[0]; bool populateId = !firstPass && _Configuration.PopulatePropertyId.Value; string without = Path.Combine(angleBracket.Replace("<>", "{}"), $"{item.ImageFileHolder.NameWithoutExtension}.json"); FileInfo fileInfo = new(Path.Combine(angleBracket.Replace("<>", "{}"), $"{item.ImageFileHolder.NameWithoutExtension}{item.ImageFileHolder.ExtensionLowered}.json")); if (item.ValidImageFormatExtension && File.Exists(without)) { File.Move(without, fileInfo.FullName); fileInfo.Refresh(); } List dateTimes = (from l in filteredSourceDirectoryFileTuples where changesFrom.Contains(l.Item1) select l.Item2).ToList(); if (!fileInfo.Exists) { if (fileInfo.Directory?.Parent is null) throw new Exception(); string parentCheck = Path.Combine(fileInfo.Directory.Parent.FullName, fileInfo.Name); if (File.Exists(parentCheck)) { File.Move(parentCheck, fileInfo.FullName); fileInfo.Refresh(); } } if (_Configuration.ForcePropertyLastWriteTimeToCreationTime.Value && !fileInfo.Exists && File.Exists(Path.ChangeExtension(fileInfo.FullName, ".delete"))) { File.Move(Path.ChangeExtension(fileInfo.FullName, ".delete"), fileInfo.FullName); fileInfo.Refresh(); } if (_Configuration.ForcePropertyLastWriteTimeToCreationTime.Value && fileInfo.Exists && fileInfo.LastWriteTime != fileInfo.CreationTime) { File.SetLastWriteTime(fileInfo.FullName, fileInfo.CreationTime); fileInfo.Refresh(); } if (_Configuration.PropertiesChangedForProperty.Value) result = null; else if (!fileInfo.Exists) result = null; else if (!fileInfo.FullName.EndsWith(".json") && !fileInfo.FullName.EndsWith(".old")) throw new ArgumentException("must be a *.json file"); else if (dateTimes.Any() && dateTimes.Max() > fileInfo.LastWriteTime) result = null; else { json = File.ReadAllText(fileInfo.FullName); try { if (item.ImageFileHolder is null) throw new NullReferenceException(nameof(item.ImageFileHolder)); bool check = true; A_Property? property = JsonSerializer.Deserialize(json); if (!isIgnoreExtension && item.ValidImageFormatExtension && ((populateId && property?.Id is null) || property?.Width is null || property?.Height is null)) { check = false; id = property?.Id; if (property is not null && property.Indices.Any()) indices = property.Indices.ToList(); property = GetImageProperty(item.ImageFileHolder, populateId, isIgnoreExtension, item.ValidImageFormatExtension, isValidMetadataExtensions, id, indices); } if (!isIgnoreExtension && item.ValidImageFormatExtension && populateId && property is not null && !property.Indices.Any()) { check = false; id = property?.Id; if (property is not null && property.Indices.Any()) indices = property.Indices.ToList(); property = GetImageProperty(item.ImageFileHolder, populateId, isIgnoreExtension, item.ValidImageFormatExtension, isValidMetadataExtensions, id, indices); } if (!isIgnoreExtension && item.ValidImageFormatExtension && populateId && property is not null && property.LastWriteTime != item.ImageFileHolder.LastWriteTime) { check = false; id = null; indices.Clear(); property = GetImageProperty(item.ImageFileHolder, populateId, isIgnoreExtension, item.ValidImageFormatExtension, isValidMetadataExtensions, id, indices); } if (!isIgnoreExtension && item.ValidImageFormatExtension && property?.Width is not null && property?.Height is not null && property.Width.Value == property.Height.Value && item.ImageFileHolder.Exists) { check = false; id = property?.Id; if (property is not null && property.Indices.Any()) indices = property.Indices.ToList(); property = GetImageProperty(item.ImageFileHolder, populateId, isIgnoreExtension, item.ValidImageFormatExtension, isValidMetadataExtensions, id, indices); if (property?.Width is not null && property?.Height is not null && property.Width.Value != property.Height.Value) throw new Exception("Was square!"); } // if (filteredSourceDirectoryFileFileInfo.CreationTime != property?.CreationTime || filteredSourceDirectoryFileFileInfo.LastWriteTime != property?.LastWriteTime) // { // check = false; // id = null; // indices.Clear(); // property = GetImagePropertyB(filteredSourceDirectoryFile, populateId, isIgnoreExtension, isValidImageFormatExtension, isValidMetadataExtensions, id, indices); // } if (json.Contains("WrongYear")) { id = property?.Id; hasWrongYearProperty = true; } if (property is null) throw new Exception(); if (!check) result = null; else { result = property; filteredSourceDirectoryFileTuples.Add(new Tuple(nameof(A_Property), fileInfo.LastWriteTime)); } } catch (Exception) { result = null; parseExceptions.Add(nameof(A_Property)); } } if (result is null) { if (item.ImageFileHolder is null) throw new NullReferenceException(nameof(item.ImageFileHolder)); result = GetImageProperty(item.ImageFileHolder, populateId, isIgnoreExtension, item.ValidImageFormatExtension, isValidMetadataExtensions, id, indices); json = JsonSerializer.Serialize(result, _WriteIndentedJsonSerializerOptions); if (populateId && IPath.WriteAllText(fileInfo.FullName, json, updateDateWhenMatches: true, compareBeforeWrite: true)) { if (!_Configuration.ForcePropertyLastWriteTimeToCreationTime.Value && (!fileInfo.Exists || fileInfo.LastWriteTime == fileInfo.CreationTime)) filteredSourceDirectoryFileTuples.Add(new Tuple(nameof(A_Property), DateTime.Now)); else { File.SetLastWriteTime(fileInfo.FullName, fileInfo.CreationTime); fileInfo.Refresh(); filteredSourceDirectoryFileTuples.Add(new Tuple(nameof(A_Property), fileInfo.CreationTime)); } } } else if (hasWrongYearProperty) { json = JsonSerializer.Serialize(result, _WriteIndentedJsonSerializerOptions); if (IPath.WriteAllText(fileInfo.FullName, json, updateDateWhenMatches: true, compareBeforeWrite: true)) { File.SetLastWriteTime(fileInfo.FullName, fileInfo.CreationTime); fileInfo.Refresh(); filteredSourceDirectoryFileTuples.Add(new Tuple(nameof(A_Property), fileInfo.CreationTime)); } } return result; } private bool AnyFilesMoved(string sourceDirectory, Item[] filteredItems) { bool result = false; if (_Log is null) throw new NullReferenceException(nameof(_Log)); int season; string[] matches; string deleteFile; bool? isWrongYear; string seasonName; DateTime dateTime; string destinationFile; DateTime minimumDateTime; string destinationDirectory; string[] sourceDirectorySegments; DateTime directoryMaximumOfMinimumDateTime = DateTime.MinValue; foreach (Item filteredItem in filteredItems) { if (!filteredItem.ValidImageFormatExtension || filteredItem.Property is null || filteredItem.ImageFileHolder is null) continue; minimumDateTime = Stateless.A_Property.GetMinimumDateTime(filteredItem.Property); if (minimumDateTime > directoryMaximumOfMinimumDateTime) directoryMaximumOfMinimumDateTime = minimumDateTime; if (minimumDateTime != filteredItem.ImageFileHolder.CreationTime) { (isWrongYear, matches) = filteredItem.Property.IsWrongYear(filteredItem.ImageFileHolder.FullName, minimumDateTime); if (isWrongYear is null || !isWrongYear.Value) dateTime = minimumDateTime; else { if (!matches.Any()) continue; if (!DateTime.TryParseExact(matches[0], "yyyy", CultureInfo.InvariantCulture, DateTimeStyles.None, out dateTime)) continue; } try { File.SetCreationTime(filteredItem.ImageFileHolder.FullName, dateTime); } catch (Exception) { } } if (!_VerifyToSeason.Contains(sourceDirectory)) continue; if (!filteredItem.ImageFileHolder.FullName.Contains("zzz ") && !filteredItem.ImageFileHolder.FullName.Contains("Camera ") && filteredItem.Property.DateTimeOriginal.HasValue) { TimeSpan timeSpan = new(filteredItem.Property.DateTimeOriginal.Value.Ticks - filteredItem.Property.LastWriteTime.Ticks); if (timeSpan.TotalHours > 6) { _Log.Warning($"*** propertyHolder.FileInfo.FullName <{filteredItem.ImageFileHolder.FullName}>"); _Log.Warning($"*** DateTimeOriginal <{filteredItem.Property.DateTimeOriginal.Value}>"); _Log.Warning($"*** LastWriteTime <{filteredItem.Property.LastWriteTime}>"); _Log.Warning($"*** TotalHours <{timeSpan.TotalHours}>"); } } sourceDirectorySegments = Path.GetFileName(sourceDirectory).Split(' '); (season, seasonName) = Stateless.A_Property.GetSeason(minimumDateTime.DayOfYear); if (sourceDirectorySegments[0] == "zzz") destinationDirectory = Path.Combine(_Configuration.RootDirectory, $"zzz ={minimumDateTime:yyyy}.{season} {seasonName} {string.Join(' ', sourceDirectorySegments.Skip(3))}"); else if (sourceDirectorySegments.Length > 2) destinationDirectory = Path.Combine(_Configuration.RootDirectory, $"={minimumDateTime:yyyy}.{season} {seasonName} {string.Join(' ', sourceDirectorySegments.Skip(2))}"); else destinationDirectory = Path.Combine(_Configuration.RootDirectory, $"={minimumDateTime:yyyy}.{season} {seasonName}"); if (destinationDirectory == sourceDirectory) continue; lock (filteredItem) filteredItem.SetMoved(true); if (!result) result = true; if (!Directory.Exists(destinationDirectory)) _ = Directory.CreateDirectory(destinationDirectory); destinationFile = Path.Combine(destinationDirectory, filteredItem.ImageFileHolder.Name); if (File.Exists(destinationFile)) { if (destinationFile.EndsWith(".jpg", ignoreCase: true, CultureInfo.CurrentCulture)) destinationFile = Path.Combine(destinationDirectory, Path.ChangeExtension(filteredItem.ImageFileHolder.Name, ".jpeg")); else if (destinationFile.EndsWith(".jpeg", ignoreCase: true, CultureInfo.CurrentCulture)) destinationFile = Path.Combine(destinationDirectory, Path.ChangeExtension(filteredItem.ImageFileHolder.Name, ".jpg")); } if (File.Exists(destinationFile)) { _Log.Information($"*** source <{filteredItem.ImageFileHolder.FullName}>"); _Log.Information($"*** destination <{destinationFile}>"); if (filteredItem.ImageFileHolder.Exists) { deleteFile = Path.ChangeExtension(filteredItem.ImageFileHolder.FullName, ".delete"); if (File.Exists(deleteFile)) File.Delete(deleteFile); File.Move(filteredItem.ImageFileHolder.FullName, deleteFile); } } else { File.Move(filteredItem.ImageFileHolder.FullName, destinationFile); if (filteredItem.ImageFileHolder.Exists) { deleteFile = Path.ChangeExtension(filteredItem.ImageFileHolder.FullName, ".delete"); if (File.Exists(deleteFile)) File.Delete(deleteFile); File.Move(filteredItem.ImageFileHolder.FullName, deleteFile); } } } if (directoryMaximumOfMinimumDateTime != DateTime.MinValue) { System.IO.DirectoryInfo directoryInfo = new(sourceDirectory); if (directoryInfo.LastWriteTime != directoryMaximumOfMinimumDateTime) Directory.SetLastWriteTime(sourceDirectory, directoryMaximumOfMinimumDateTime); } return result; } private void ParallelForWork(bool firstPass, string sourceDirectory, List> filteredSourceDirectoryFileTuples, List> sourceDirectoryChanges, Item item) { if (item.ImageFileHolder is null) throw new NullReferenceException(nameof(item.ImageFileHolder)); A_Property property; List parseExceptions = new(); bool isValidMetadataExtensions = _Configuration.ValidMetadataExtensions.Contains(item.ImageFileHolder.ExtensionLowered); bool isIgnoreExtension = item.ValidImageFormatExtension && _Configuration.IgnoreExtensions.Contains(item.ImageFileHolder.ExtensionLowered); string filteredSourceDirectoryFileExtensionLowered = Path.Combine(sourceDirectory, $"{item.ImageFileHolder.NameWithoutExtension}{item.ImageFileHolder.ExtensionLowered}"); if (item.ValidImageFormatExtension && item.ImageFileHolder.FullName.Length == filteredSourceDirectoryFileExtensionLowered.Length && item.ImageFileHolder.FullName != filteredSourceDirectoryFileExtensionLowered) File.Move(item.ImageFileHolder.FullName, filteredSourceDirectoryFileExtensionLowered); if (item.Changed is null || item.Changed.Value || item.Property is null) { property = GetPropertyOfPrivate(item, firstPass, filteredSourceDirectoryFileTuples, parseExceptions, isIgnoreExtension, isValidMetadataExtensions); lock (sourceDirectoryChanges) sourceDirectoryChanges.Add(new Tuple(nameof(A_Property), DateTime.Now)); lock (item) item.Update(property); } } private void ParallelWork(bool firstPass, List exceptions, List> sourceDirectoryChanges, int containersCount, Container container, Item[] filteredItems, int totalSeconds) { List> filteredSourceDirectoryFileTuples = new(); ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _MaxDegreeOfParallelism }; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; string message = $"{container.R:000}.{container.G} / {containersCount:000}) {filteredItems.Length:000} file(s) - {totalSeconds} total second(s) - {container.SourceDirectory}"; using ProgressBar progressBar = new(filteredItems.Length, message, options); _ = Parallel.For(0, filteredItems.Length, parallelOptions, i => { try { long ticks = DateTime.Now.Ticks; DateTime dateTime = DateTime.Now; List> collection; ParallelForWork(firstPass, container.SourceDirectory, sourceDirectoryChanges, filteredSourceDirectoryFileTuples, filteredItems[i]); if (i == 0 || sourceDirectoryChanges.Any()) progressBar.Tick(); lock (filteredSourceDirectoryFileTuples) collection = (from l in filteredSourceDirectoryFileTuples where l.Item2 > dateTime select l).ToList(); lock (sourceDirectoryChanges) sourceDirectoryChanges.AddRange(collection); } catch (Exception ex) { lock (exceptions) exceptions.Add(ex); } }); } private void SetAngleBracketCollection(string sourceDirectory) { AngleBracketCollection.Clear(); AngleBracketCollection.AddRange(IResult.GetDirectoryInfoCollection(_Configuration, _Model, _PredictorModel, sourceDirectory, nameof(A_Property), string.Empty, includeResizeGroup: false, includeModel: false, includePredictorModel: false, contentDescription: string.Empty, singletonDescription: "Properties for each image", collectionDescription: string.Empty)); } public void ParallelWork(long ticks, List containers, bool firstPass) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); if (_Configuration.PopulatePropertyId is null) throw new NullReferenceException(nameof(_Configuration.PopulatePropertyId)); int totalSeconds; bool? anyFilesMoved; Item[] filteredItems; List exceptions = new(); int containersCount = containers.Count; List> sourceDirectoryChanges = new(); string propertyRoot = IResult.GetResultsGroupDirectory(_Configuration, nameof(A_Property)); foreach (Container container in containers) { if (!container.Items.Any()) continue; sourceDirectoryChanges.Clear(); if (firstPass) filteredItems = (from l in container.Items where l.NoJson is null || !l.NoJson.Value && (l.Changed is null || l.Changed.Value) select l).ToArray(); else filteredItems = (from l in container.Items where l.ImageFileHolder is not null && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l).ToArray(); if (!filteredItems.Any()) continue; totalSeconds = (int)Math.Truncate(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); SetAngleBracketCollection(container.SourceDirectory); ParallelWork(firstPass, exceptions, sourceDirectoryChanges, containersCount, container, filteredItems, totalSeconds); foreach (Exception exception in exceptions) _Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, exception.Message, Environment.NewLine, exception.StackTrace), exception); if (exceptions.Count == filteredItems.Length) throw new Exception(string.Concat("All in [", container.SourceDirectory, "]failed!")); if (exceptions.Count != 0) _ExceptionsDirectories.Add(container.SourceDirectory); if (!firstPass || exceptions.Count != 0) anyFilesMoved = null; else anyFilesMoved = AnyFilesMoved(container.SourceDirectory, filteredItems); if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any()) { for (int y = 0; y < int.MaxValue; y++) { _Log.Information("Press \"Y\" key when ready to continue or close console"); if (Console.ReadKey().Key == ConsoleKey.Y) break; } _Log.Information(". . ."); } } } public A_Property GetProperty(Item item, List> filteredSourceDirectoryFileTuples, List parseExceptions) { A_Property result; if (item.ImageFileHolder is null) throw new NullReferenceException(nameof(item.ImageFileHolder)); bool firstPass = false; bool isValidMetadataExtensions = _Configuration.ValidMetadataExtensions.Contains(item.ImageFileHolder.ExtensionLowered); bool isIgnoreExtension = item.ValidImageFormatExtension && _Configuration.IgnoreExtensions.Contains(item.ImageFileHolder.ExtensionLowered); result = GetPropertyOfPrivate(item, firstPass, filteredSourceDirectoryFileTuples, parseExceptions, isIgnoreExtension, isValidMetadataExtensions); return result; } public (long Ticks, string FilteredSourceDirectoryFile, string PropertyDirectory, int PropertyId)[] GetPropertyIds(List groupCollection, bool saveToCollection) { List<(long Ticks, string FilteredSourceDirectoryFile, string PropertyDirectory, int PropertyId)> results = new(); int level; A_Property? property; string checkDirectory; List directories; string propertyDirectory; string angleBracket = AngleBracketCollection[0]; foreach (DirectoryInfo group in groupCollection) { SetAngleBracketCollection(group.SourceDirectory); if (string.IsNullOrEmpty(group.SourceDirectory)) throw new Exception(); if (!saveToCollection) propertyDirectory = angleBracket.Replace("<>", "()"); else { (level, directories) = IPath.Get(_Configuration.RootDirectory, group.SourceDirectory); checkDirectory = IPath.GetDirectory(angleBracket, level, "[()]"); propertyDirectory = Path.Combine(checkDirectory, string.Join(_Configuration.FileNameDirectorySeparator, directories)); } if (!Directory.Exists(propertyDirectory)) _ = Directory.CreateDirectory(propertyDirectory); for (int i = 0; i < group.SourceDirectoryFileHolderCollection.Length; i++) { property = group.PropertyCollection[i]; if (property?.Id is null) continue; results.Add(new(property.GetDateTimes().Min().Ticks, group.FilteredSourceDirectoryFiles[i], propertyDirectory, property.Id.Value)); } } return results.OrderBy(l => l.Ticks).ToArray(); } public void AddToPropertyLogicAllCollection(Item[] filteredItems) { if (_SixCharacterNamedFaceInfo.Any()) { string[] keys; Item item; for (int i = 0; i < filteredItems.Length; i++) { item = filteredItems[i]; if (item.Property?.Id is null) continue; foreach (int sixCharacterIndex in item.Property.Indices) { if (!_SixCharacterNamedFaceInfo.ContainsKey(sixCharacterIndex)) continue; keys = _SixCharacterNamedFaceInfo[sixCharacterIndex]; _AllCollection.Add(new(item.Property.Id.Value, keys)); } } } } public void SaveAllCollection() { if (_AllCollection.Any()) { string[] keys; string? rootDirectoryParent = Path.GetDirectoryName(_Configuration.RootDirectory); if (string.IsNullOrEmpty(rootDirectoryParent)) throw new NullReferenceException(nameof(rootDirectoryParent)); Dictionary namedFaceInfoDeterministicHashCodeIndices = new(); List<(int, string[])> allCollection = _AllCollection.OrderBy(l => l.Item1).ToList(); foreach ((int deterministicHashCode, string[] values) in allCollection) { if (namedFaceInfoDeterministicHashCodeIndices.ContainsKey(deterministicHashCode)) { keys = namedFaceInfoDeterministicHashCodeIndices[deterministicHashCode]; if (JsonSerializer.Serialize(values) == JsonSerializer.Serialize(keys)) continue; throw new Exception(); } namedFaceInfoDeterministicHashCodeIndices.Add(deterministicHashCode, values); } string json = JsonSerializer.Serialize(namedFaceInfoDeterministicHashCodeIndices, new JsonSerializerOptions { WriteIndented = true }); string checkFile = Path.Combine(rootDirectoryParent, "NamedFaceInfoDeterministicHashCodeIndices.json"); _ = IPath.WriteAllText(checkFile, json, updateDateWhenMatches: true, compareBeforeWrite: true); } } }