using ShellProgressBar; using System.Diagnostics; using System.Drawing; using System.Drawing.Imaging; using System.Globalization; using System.Runtime.InteropServices; using System.Text; using System.Text.Json; using View_by_Distance.Property.Models.Stateless; using View_by_Distance.Shared.Models.Stateless; namespace View_by_Distance.Property.Models; public class PropertyLogic { protected readonly List<(int, string[])> _AllCollection; protected readonly List _ExceptionsDirectories; protected readonly Dictionary _KeyValuePairs; protected readonly Dictionary _IndicesFromNew; protected readonly Dictionary _SixCharacterNamedFaceInfo; protected readonly Dictionary _NamedFaceInfoDeterministicHashCodeIndices; public List AngleBracketCollection { get; } public Dictionary KeyValuePairs => _KeyValuePairs; public Dictionary IndicesFromNew => _IndicesFromNew; public List ExceptionsDirectories => _ExceptionsDirectories; public Dictionary NamedFaceInfoDeterministicHashCodeIndices => _NamedFaceInfoDeterministicHashCodeIndices; private readonly Serilog.ILogger? _Log; private readonly string[] _VerifyToSeason; private readonly int _MaxDegreeOfParallelism; private readonly ASCIIEncoding _ASCIIEncoding; private readonly Configuration _Configuration; private readonly JsonSerializerOptions _WriteIndentedJsonSerializerOptions; public PropertyLogic(int maxDegreeOfParallelism, Configuration configuration) { _AllCollection = new(); _Configuration = configuration; _ExceptionsDirectories = new(); _ASCIIEncoding = new ASCIIEncoding(); AngleBracketCollection = new List(); _Log = Serilog.Log.ForContext(); _MaxDegreeOfParallelism = maxDegreeOfParallelism; Dictionary? namedFaceInfoDeterministicHashCodeIndices; _WriteIndentedJsonSerializerOptions = new JsonSerializerOptions { WriteIndented = true }; if (configuration.VerifyToSeason is null || !configuration.VerifyToSeason.Any()) throw new Exception(); _VerifyToSeason = configuration.VerifyToSeason.Select(l => Path.Combine(configuration.RootDirectory, l)).ToArray(); string json; string[] files; string fullPath; Dictionary? keyValuePairs; List>? collection; Dictionary indicesFromNew = new(); Dictionary? sixCharacterNamedFaceInfo; string? rootDirectoryParent = Path.GetDirectoryName(configuration.RootDirectory); if (string.IsNullOrEmpty(rootDirectoryParent)) throw new Exception($"{nameof(rootDirectoryParent)} is null!"); files = Directory.GetFiles(rootDirectoryParent, "*DeterministicHashCode*.json", SearchOption.TopDirectoryOnly); if (files.Length != 1) namedFaceInfoDeterministicHashCodeIndices = new(); else { json = File.ReadAllText(files[0]); namedFaceInfoDeterministicHashCodeIndices = JsonSerializer.Deserialize>(json); if (namedFaceInfoDeterministicHashCodeIndices is null) throw new Exception($"{nameof(namedFaceInfoDeterministicHashCodeIndices)} is null!"); } if (namedFaceInfoDeterministicHashCodeIndices.Any()) sixCharacterNamedFaceInfo = new(); else { files = Directory.GetFiles(rootDirectoryParent, "*SixCharacter*.json", SearchOption.TopDirectoryOnly); if (files.Length != 1) sixCharacterNamedFaceInfo = new(); else { json = File.ReadAllText(files[0]); sixCharacterNamedFaceInfo = JsonSerializer.Deserialize>(json); if (sixCharacterNamedFaceInfo is null) throw new Exception($"{nameof(sixCharacterNamedFaceInfo)} is null!"); } } files = Directory.GetFiles(rootDirectoryParent, "*keyValuePairs*.json", SearchOption.TopDirectoryOnly); if (files.Length != 1) keyValuePairs = new(); else { json = File.ReadAllText(files[0]); keyValuePairs = JsonSerializer.Deserialize>(json); if (keyValuePairs is null) throw new Exception($"{nameof(keyValuePairs)} is null!"); } foreach (string propertyContentCollectionFile in configuration.PropertyContentCollectionFiles) { fullPath = Path.GetFullPath(string.Concat(rootDirectoryParent, propertyContentCollectionFile)); if (fullPath.Contains(configuration.RootDirectory)) continue; if (!File.Exists(fullPath)) continue; json = File.ReadAllText(fullPath); collection = JsonSerializer.Deserialize>>(json); if (collection is null) throw new Exception($"{nameof(collection)} is null!"); foreach (KeyValuePair keyValuePair in collection) { if (indicesFromNew.ContainsKey(keyValuePair.Key)) continue; indicesFromNew.Add(keyValuePair.Key, keyValuePair.Value); } } _KeyValuePairs = keyValuePairs; _IndicesFromNew = indicesFromNew; _SixCharacterNamedFaceInfo = sixCharacterNamedFaceInfo; _NamedFaceInfoDeterministicHashCodeIndices = namedFaceInfoDeterministicHashCodeIndices; } public override string ToString() { string result = JsonSerializer.Serialize(this, new JsonSerializerOptions() { WriteIndented = true }); return result; } private long LogDelta(long ticks, string methodName) { long result; if (_Log is null) throw new Exception($"{nameof(_Log)} is null!"); double delta = new TimeSpan(DateTime.Now.Ticks - ticks).TotalMilliseconds; _Log.Debug($"{methodName} took {Math.Floor(delta)} millisecond(s)"); result = DateTime.Now.Ticks; return result; } public static List GetMetadataDateTimesByPattern(string dateTimeFormat, string filteredSourceDirectoryFile) { List results = new(); try { DateTime checkDateTime; DateTime kristy = new(1976, 3, 8); IReadOnlyList directories = MetadataExtractor.ImageMetadataReader.ReadMetadata(filteredSourceDirectoryFile); foreach (MetadataExtractor.Directory directory in directories) { foreach (MetadataExtractor.Tag tag in directory.Tags) { if (string.IsNullOrEmpty(tag.Description) || tag.Description.Length != dateTimeFormat.Length) continue; if (!DateTime.TryParseExact(tag.Description, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) continue; if (checkDateTime < kristy) continue; results.Add(checkDateTime); } } } catch (Exception) { } return results; } public static List GetMetadataDateTimesByPattern(string dateTimeFormat, FileInfo filteredSourceDirectoryFileInfo) { List results = GetMetadataDateTimesByPattern(dateTimeFormat, filteredSourceDirectoryFileInfo.FullName); return results; } #pragma warning disable CA1416 private A_Property GetImageProperty(string angleBracket, FileInfo filteredSourceDirectoryFileInfo, bool populateId, bool isIgnoreExtension, bool isValidImageFormatExtension, bool isValidMetadataExtensions, int? id, List indices) { A_Property result; if (_Log is null) throw new Exception($"{nameof(_Log)} is null!"); if (_Configuration.WriteBitmapDataBytes is null) throw new Exception($"{nameof(_Configuration.WriteBitmapDataBytes)} is null!"); long ticks; byte[] bytes; string value; int encodingHash; int? width = null; int? height = null; string dateTimeFormat; DateTime checkDateTime; DateTime? dateTime = null; PropertyItem? propertyItem; string make = string.Empty; string model = string.Empty; DateTime? gpsDateStamp = null; DateTime? dateTimeOriginal = null; string orientation = string.Empty; DateTime? dateTimeDigitized = null; if (!isValidImageFormatExtension && isValidMetadataExtensions) { dateTimeFormat = "ddd MMM dd HH:mm:ss yyyy"; List dateTimes = GetMetadataDateTimesByPattern(dateTimeFormat, filteredSourceDirectoryFileInfo); if (dateTimes.Any()) dateTimeOriginal = dateTimes.Min(); } else if (!isIgnoreExtension && isValidImageFormatExtension) { if (!_IndicesFromNew.Any() && !_KeyValuePairs.Any()) throw new Exception("In order to keep six character indices at least one need to have an item!"); try { using Image image = Image.FromFile(filteredSourceDirectoryFileInfo.FullName); if (populateId && (id is null || !indices.Any())) { using Bitmap bitmap = new(image); Rectangle rectangle = new(0, 0, image.Width, image.Height); BitmapData bitmapData = bitmap.LockBits(rectangle, ImageLockMode.ReadOnly, bitmap.PixelFormat); IntPtr intPtr = bitmapData.Scan0; int length = bitmapData.Stride * bitmap.Height; bytes = new byte[length]; Marshal.Copy(intPtr, bytes, 0, length); bitmap.UnlockBits(bitmapData); if (id is null) { ticks = DateTime.Now.Ticks; id = Stateless.A_Property.GetDeterministicHashCode(bytes); if (_MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Stateless.A_Property.GetDeterministicHashCode)); } if (_Configuration.WriteBitmapDataBytes.Value) { FileInfo contentFileInfo = new(Path.Combine(angleBracket.Replace("<>", "()"), filteredSourceDirectoryFileInfo.Name)); File.WriteAllBytes(Path.ChangeExtension(contentFileInfo.FullName, string.Empty), bytes); } if (_IndicesFromNew.ContainsKey(id.Value) && _IndicesFromNew[id.Value].Any()) indices.AddRange(_IndicesFromNew[id.Value]); else { ticks = DateTime.Now.Ticks; string encoding = Encoding.Default.GetString(bytes); if (_MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Encoding.Default.GetString)); encodingHash = Stateless.A_Property.GetDeterministicHashCode(encoding); if (_MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Stateless.A_Property.GetDeterministicHashCode)); if (!_KeyValuePairs.ContainsKey(encodingHash)) indices.Add(encodingHash); else indices.AddRange(_KeyValuePairs[encodingHash]); } } width = image.Width; height = image.Height; dateTimeFormat = Stateless.A_Property.DateTimeFormat(); if (image.PropertyIdList.Contains((int)IExif.Tags.DateTime)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.DateTime); if (propertyItem?.Value is not null) { value = _ASCIIEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); if (value.Length > dateTimeFormat.Length) value = value[..dateTimeFormat.Length]; if (value.Length == dateTimeFormat.Length && DateTime.TryParseExact(value, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) dateTime = checkDateTime; } } if (image.PropertyIdList.Contains((int)IExif.Tags.DateTimeDigitized)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.DateTimeDigitized); if (propertyItem?.Value is not null) { value = _ASCIIEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); if (value.Length > dateTimeFormat.Length) value = value[..dateTimeFormat.Length]; if (value.Length == dateTimeFormat.Length && DateTime.TryParseExact(value, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) dateTimeDigitized = checkDateTime; } } if (image.PropertyIdList.Contains((int)IExif.Tags.DateTimeOriginal)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.DateTimeOriginal); if (propertyItem?.Value is not null) { value = _ASCIIEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); if (value.Length > dateTimeFormat.Length) value = value[..dateTimeFormat.Length]; if (value.Length == dateTimeFormat.Length && DateTime.TryParseExact(value, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) dateTimeOriginal = checkDateTime; } } if (image.PropertyIdList.Contains((int)IExif.Tags.GPSDateStamp)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.GPSDateStamp); if (propertyItem?.Value is not null) { value = _ASCIIEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); if (value.Length > dateTimeFormat.Length) value = value[..dateTimeFormat.Length]; if (value.Length == dateTimeFormat.Length && DateTime.TryParseExact(value, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) gpsDateStamp = checkDateTime; } } if (image.PropertyIdList.Contains((int)IExif.Tags.Make)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.Make); if (propertyItem?.Value is not null) { value = _ASCIIEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); make = value; } } if (image.PropertyIdList.Contains((int)IExif.Tags.Model)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.Model); if (propertyItem?.Value is not null) { value = _ASCIIEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); model = value; } } if (image.PropertyIdList.Contains((int)IExif.Tags.Orientation)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.Orientation); if (propertyItem?.Value is not null) { value = BitConverter.ToInt16(propertyItem.Value, 0).ToString(); orientation = value; } } } catch (Exception) { _Log.Info(string.Concat(new StackFrame().GetMethod()?.Name, " <", filteredSourceDirectoryFileInfo.Name, ">")); } } else dateTimeOriginal = null; result = new(filteredSourceDirectoryFileInfo.CreationTime, dateTime, dateTimeDigitized, dateTimeOriginal, filteredSourceDirectoryFileInfo.Length, gpsDateStamp, height, id, indices.ToArray(), filteredSourceDirectoryFileInfo.LastWriteTime, make, model, orientation, width); return result; } #pragma warning restore CA1416 private A_Property GetPropertyOfPrivate(string angleBracket, PropertyHolder propertyHolder, bool firstPass, List> filteredSourceDirectoryFileTuples, List parseExceptions, bool isIgnoreExtension, bool isValidImageFormatExtension, bool isValidMetadataExtensions, string extensionLowered) { A_Property? result; if (_Configuration.ForcePropertyLastWriteTimeToCreationTime is null) throw new Exception($"{nameof(_Configuration.ForcePropertyLastWriteTimeToCreationTime)} is null!"); if (_Configuration.PopulatePropertyId is null) throw new Exception($"{nameof(_Configuration.PopulatePropertyId)} is null!"); if (_Configuration.PropertiesChangedForProperty is null) throw new Exception($"{nameof(_Configuration.PropertiesChangedForProperty)} is null!"); string json; int? id = null; List indices = new(); bool hasWrongYearProperty = false; string[] changesFrom = Array.Empty(); bool populateId = !firstPass && _Configuration.PopulatePropertyId.Value; string without = Path.Combine(angleBracket.Replace("<>", "{}"), $"{propertyHolder.ImageFileNameWithoutExtension}.json"); FileInfo fileInfo = new(Path.Combine(angleBracket.Replace("<>", "{}"), $"{propertyHolder.ImageFileNameWithoutExtension}{extensionLowered}.json")); if (isValidImageFormatExtension && File.Exists(without)) { File.Move(without, fileInfo.FullName); fileInfo.Refresh(); } List dateTimes = (from l in filteredSourceDirectoryFileTuples where changesFrom.Contains(l.Item1) select l.Item2).ToList(); if (!fileInfo.Exists) { if (fileInfo.Directory?.Parent is null) throw new Exception(); string parentCheck = Path.Combine(fileInfo.Directory.Parent.FullName, fileInfo.Name); if (File.Exists(parentCheck)) { File.Move(parentCheck, fileInfo.FullName); fileInfo.Refresh(); } } if (_Configuration.ForcePropertyLastWriteTimeToCreationTime.Value && !fileInfo.Exists && File.Exists(Path.ChangeExtension(fileInfo.FullName, ".delete"))) { File.Move(Path.ChangeExtension(fileInfo.FullName, ".delete"), fileInfo.FullName); fileInfo.Refresh(); } if (_Configuration.ForcePropertyLastWriteTimeToCreationTime.Value && fileInfo.Exists && fileInfo.LastWriteTime != fileInfo.CreationTime) { File.SetLastWriteTime(fileInfo.FullName, fileInfo.CreationTime); fileInfo.Refresh(); } if (_Configuration.PropertiesChangedForProperty.Value) result = null; else if (!fileInfo.Exists) result = null; else if (!fileInfo.FullName.EndsWith(".json") && !fileInfo.FullName.EndsWith(".old")) throw new ArgumentException("must be a *.json file"); else if (dateTimes.Any() && dateTimes.Max() > fileInfo.LastWriteTime) result = null; else { json = File.ReadAllText(fileInfo.FullName); try { if (propertyHolder.ImageFileInfo is null) throw new ArgumentException($"{propertyHolder.ImageFileInfo} is null!"); bool check = true; A_Property? property = JsonSerializer.Deserialize(json); if (!isIgnoreExtension && isValidImageFormatExtension && ((populateId && property?.Id is null) || property?.Width is null || property?.Height is null)) { check = false; id = property?.Id; if (property is not null && property.Indices.Any()) indices = property.Indices.ToList(); property = GetImageProperty(angleBracket, propertyHolder.ImageFileInfo, populateId, isIgnoreExtension, isValidImageFormatExtension, isValidMetadataExtensions, id, indices); } if (!isIgnoreExtension && isValidImageFormatExtension && populateId && property is not null && !property.Indices.Any()) { check = false; id = property?.Id; if (property is not null && property.Indices.Any()) indices = property.Indices.ToList(); property = GetImageProperty(angleBracket, propertyHolder.ImageFileInfo, populateId, isIgnoreExtension, isValidImageFormatExtension, isValidMetadataExtensions, id, indices); } if (!isIgnoreExtension && isValidImageFormatExtension && populateId && property is not null && property.LastWriteTime != propertyHolder.ImageFileInfo.LastWriteTime) { check = false; id = null; indices.Clear(); property = GetImageProperty(angleBracket, propertyHolder.ImageFileInfo, populateId, isIgnoreExtension, isValidImageFormatExtension, isValidMetadataExtensions, id, indices); } if (!isIgnoreExtension && isValidImageFormatExtension && property?.Width is not null && property?.Height is not null && property.Width.Value == property.Height.Value && propertyHolder.ImageFileInfo.Exists) { check = false; id = property?.Id; if (property is not null && property.Indices.Any()) indices = property.Indices.ToList(); property = GetImageProperty(angleBracket, propertyHolder.ImageFileInfo, populateId, isIgnoreExtension, isValidImageFormatExtension, isValidMetadataExtensions, id, indices); if (property?.Width is not null && property?.Height is not null && property.Width.Value != property.Height.Value) throw new Exception("Was square!"); } // if (filteredSourceDirectoryFileFileInfo.CreationTime != property?.CreationTime || filteredSourceDirectoryFileFileInfo.LastWriteTime != property?.LastWriteTime) // { // check = false; // id = null; // indices.Clear(); // property = GetImagePropertyB(angleBracket, filteredSourceDirectoryFile, populateId, isIgnoreExtension, isValidImageFormatExtension, isValidMetadataExtensions, id, indices); // } if (json.Contains("WrongYear")) { id = property?.Id; hasWrongYearProperty = true; } if (property is null) throw new Exception(); if (!check) result = null; else { result = property; filteredSourceDirectoryFileTuples.Add(new Tuple(nameof(A_Property), fileInfo.LastWriteTime)); } } catch (Exception) { result = null; parseExceptions.Add(nameof(A_Property)); } } if (result is null) { if (propertyHolder.ImageFileInfo is null) throw new ArgumentException($"{propertyHolder.ImageFileInfo} is null!"); result = GetImageProperty(angleBracket, propertyHolder.ImageFileInfo, populateId, isIgnoreExtension, isValidImageFormatExtension, isValidMetadataExtensions, id, indices); json = JsonSerializer.Serialize(result, _WriteIndentedJsonSerializerOptions); if (populateId && IPath.WriteAllText(fileInfo.FullName, json, compareBeforeWrite: true)) { if (!_Configuration.ForcePropertyLastWriteTimeToCreationTime.Value && (!fileInfo.Exists || fileInfo.LastWriteTime == fileInfo.CreationTime)) filteredSourceDirectoryFileTuples.Add(new Tuple(nameof(A_Property), DateTime.Now)); else { File.SetLastWriteTime(fileInfo.FullName, fileInfo.CreationTime); fileInfo.Refresh(); filteredSourceDirectoryFileTuples.Add(new Tuple(nameof(A_Property), fileInfo.CreationTime)); } } } else if (hasWrongYearProperty) { json = JsonSerializer.Serialize(result, _WriteIndentedJsonSerializerOptions); if (IPath.WriteAllText(fileInfo.FullName, json, compareBeforeWrite: true)) { File.SetLastWriteTime(fileInfo.FullName, fileInfo.CreationTime); fileInfo.Refresh(); filteredSourceDirectoryFileTuples.Add(new Tuple(nameof(A_Property), fileInfo.CreationTime)); } } return result; } private bool AnyFilesMoved(string sourceDirectory, PropertyHolder[] filteredPropertyHolderCollection) { bool result = false; if (_Log is null) throw new Exception($"{nameof(_Log)} is null!"); int season; string[] matches; string deleteFile; bool? isWrongYear; string seasonName; DateTime dateTime; string destinationFile; DateTime minimumDateTime; string destinationDirectory; string[] sourceDirectorySegments; DateTime directoryMaximumOfMinimumDateTime = DateTime.MinValue; foreach (PropertyHolder propertyHolder in filteredPropertyHolderCollection) { if (propertyHolder.ValidImageFormatExtension is null || !propertyHolder.ValidImageFormatExtension.Value) continue; if (propertyHolder.Property is null) continue; if (propertyHolder.ImageFileInfo is null) continue; minimumDateTime = Stateless.A_Property.GetMinimumDateTime(propertyHolder.Property); if (minimumDateTime > directoryMaximumOfMinimumDateTime) directoryMaximumOfMinimumDateTime = minimumDateTime; if (minimumDateTime != propertyHolder.ImageFileInfo.CreationTime) { (isWrongYear, matches) = propertyHolder.Property.IsWrongYear(propertyHolder.ImageFileInfo.FullName, minimumDateTime); if (isWrongYear is null || !isWrongYear.Value) dateTime = minimumDateTime; else { if (isWrongYear.HasValue && isWrongYear.Value) { lock (propertyHolder) propertyHolder.SetWrongYear(true); } if (!matches.Any()) continue; if (!DateTime.TryParseExact(matches[0], "yyyy", CultureInfo.InvariantCulture, DateTimeStyles.None, out dateTime)) continue; } try { File.SetCreationTime(propertyHolder.ImageFileInfo.FullName, dateTime); } catch (Exception) { } } if (!_VerifyToSeason.Contains(sourceDirectory)) continue; if (!propertyHolder.ImageFileInfo.FullName.Contains("zzz ") && !propertyHolder.ImageFileInfo.FullName.Contains("Camera ") && propertyHolder.Property.DateTimeOriginal.HasValue) { TimeSpan timeSpan = new(propertyHolder.Property.DateTimeOriginal.Value.Ticks - propertyHolder.Property.LastWriteTime.Ticks); if (timeSpan.TotalHours > 6) { _Log.Warning($"*** propertyHolder.FileInfo.FullName <{propertyHolder.ImageFileInfo.FullName}>"); _Log.Warning($"*** DateTimeOriginal <{propertyHolder.Property.DateTimeOriginal.Value}>"); _Log.Warning($"*** LastWriteTime <{propertyHolder.Property.LastWriteTime}>"); _Log.Warning($"*** TotalHours <{timeSpan.TotalHours}>"); } } sourceDirectorySegments = Path.GetFileName(sourceDirectory).Split(' '); (season, seasonName) = Stateless.A_Property.GetSeason(minimumDateTime.DayOfYear); if (sourceDirectorySegments[0] == "zzz") destinationDirectory = Path.Combine(_Configuration.RootDirectory, $"zzz ={minimumDateTime:yyyy}.{season} {seasonName} {string.Join(' ', sourceDirectorySegments.Skip(3))}"); else if (sourceDirectorySegments.Length > 2) destinationDirectory = Path.Combine(_Configuration.RootDirectory, $"={minimumDateTime:yyyy}.{season} {seasonName} {string.Join(' ', sourceDirectorySegments.Skip(2))}"); else destinationDirectory = Path.Combine(_Configuration.RootDirectory, $"={minimumDateTime:yyyy}.{season} {seasonName}"); if (destinationDirectory == sourceDirectory) continue; lock (propertyHolder) propertyHolder.SetMoved(true); if (!result) result = true; if (!Directory.Exists(destinationDirectory)) _ = Directory.CreateDirectory(destinationDirectory); destinationFile = Path.Combine(destinationDirectory, propertyHolder.ImageFileInfo.Name); if (File.Exists(destinationFile)) { if (destinationFile.EndsWith(".jpg", ignoreCase: true, CultureInfo.CurrentCulture)) destinationFile = Path.Combine(destinationDirectory, Path.ChangeExtension(propertyHolder.ImageFileInfo.Name, ".jpeg")); else if (destinationFile.EndsWith(".jpeg", ignoreCase: true, CultureInfo.CurrentCulture)) destinationFile = Path.Combine(destinationDirectory, Path.ChangeExtension(propertyHolder.ImageFileInfo.Name, ".jpg")); } if (File.Exists(destinationFile)) { _Log.Information($"*** source <{propertyHolder.ImageFileInfo.FullName}>"); _Log.Information($"*** destination <{destinationFile}>"); if (propertyHolder.ImageFileInfo.Exists) { deleteFile = Path.ChangeExtension(propertyHolder.ImageFileInfo.FullName, ".delete"); if (File.Exists(deleteFile)) File.Delete(deleteFile); File.Move(propertyHolder.ImageFileInfo.FullName, deleteFile); } } else { File.Move(propertyHolder.ImageFileInfo.FullName, destinationFile); if (propertyHolder.ImageFileInfo.Exists) { deleteFile = Path.ChangeExtension(propertyHolder.ImageFileInfo.FullName, ".delete"); if (File.Exists(deleteFile)) File.Delete(deleteFile); File.Move(propertyHolder.ImageFileInfo.FullName, deleteFile); } } } if (directoryMaximumOfMinimumDateTime != DateTime.MinValue) { System.IO.DirectoryInfo directoryInfo = new(sourceDirectory); if (directoryInfo.LastWriteTime != directoryMaximumOfMinimumDateTime) Directory.SetLastWriteTime(sourceDirectory, directoryMaximumOfMinimumDateTime); } return result; } private void WriteGroup(int sourceDirectoryLength, PropertyHolder[] filteredPropertyHolderCollection, string angleBracket) { if (!(from l in filteredPropertyHolderCollection where l?.Property?.Width is null select true).Any()) { string key; string json; string checkFile; string checkDirectory; List> propertyCollectionKeyValuePairs = new(); JsonSerializerOptions writeIndentedJsonSerializerOptions = new() { WriteIndented = false }; (int level, List directories) = IPath.Get(_Configuration.RootDirectory, filteredPropertyHolderCollection[0].SourceDirectory); string fileName = string.Concat(string.Join(_Configuration.FileNameDirectorySeparator, directories), ".json"); foreach (PropertyHolder propertyHolder in filteredPropertyHolderCollection) { if (propertyHolder.Property is null) continue; if (propertyHolder.ImageFileInfo is null) continue; key = IPath.GetRelativePath(propertyHolder.ImageFileInfo.FullName, sourceDirectoryLength); propertyCollectionKeyValuePairs.Add(new KeyValuePair(key, propertyHolder.Property)); } checkDirectory = IPath.GetDirectory(angleBracket, level, "[{}]"); checkFile = Path.Combine(checkDirectory, fileName); if (File.Exists(checkFile)) File.Move(checkFile, Path.Combine(checkDirectory, fileName)); checkFile = Path.Combine(checkDirectory, fileName); json = JsonSerializer.Serialize(propertyCollectionKeyValuePairs, writeIndentedJsonSerializerOptions); _ = IPath.WriteAllText(checkFile, json, compareBeforeWrite: true); } } private void ParallelForWork(bool firstPass, string angleBracket, string sourceDirectory, List> filteredSourceDirectoryFileTuples, PropertyHolder propertyHolder) { if (propertyHolder.ImageFileInfo is null) throw new Exception($"{nameof(propertyHolder.ImageFileInfo)} is null!"); A_Property property; List parseExceptions = new(); string extensionLowered = propertyHolder.ImageFileInfo.Extension.ToLower(); bool isValidMetadataExtensions = _Configuration.ValidMetadataExtensions.Contains(extensionLowered); bool isValidImageFormatExtension = _Configuration.ValidImageFormatExtensions.Contains(extensionLowered); lock (propertyHolder) propertyHolder.SetValidImageFormatExtension(isValidImageFormatExtension); bool isIgnoreExtension = isValidImageFormatExtension && _Configuration.IgnoreExtensions.Contains(extensionLowered); string filteredSourceDirectoryFileExtensionLowered = Path.Combine(sourceDirectory, $"{propertyHolder.ImageFileNameWithoutExtension}{extensionLowered}"); if (isValidImageFormatExtension && propertyHolder.ImageFileInfo.FullName.Length == filteredSourceDirectoryFileExtensionLowered.Length && propertyHolder.ImageFileInfo.FullName != filteredSourceDirectoryFileExtensionLowered) File.Move(propertyHolder.ImageFileInfo.FullName, filteredSourceDirectoryFileExtensionLowered); if (propertyHolder.Changed is null || propertyHolder.Changed.Value || propertyHolder.Property is null) { property = GetPropertyOfPrivate(angleBracket, propertyHolder, firstPass, filteredSourceDirectoryFileTuples, parseExceptions, isIgnoreExtension, isValidImageFormatExtension, isValidMetadataExtensions, extensionLowered); lock (propertyHolder) propertyHolder.Update(property); } } private void ParallelWork(bool firstPass, List exceptions, List> sourceDirectoryChanges, int propertyHolderCollectionsCount, int g, string sourceDirectory, int r, PropertyHolder[] filteredPropertyHolderCollection, int totalSeconds, string angleBracket) { List> filteredSourceDirectoryFileTuples = new(); ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _MaxDegreeOfParallelism }; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; using ProgressBar progressBar = new(filteredPropertyHolderCollection.Length, $"{r + 1:000}.{g} / {propertyHolderCollectionsCount:000}) {filteredPropertyHolderCollection.Length:000} file(s) - {totalSeconds} total second(s) - {sourceDirectory}", options); _ = Parallel.For(0, filteredPropertyHolderCollection.Length, parallelOptions, i => { try { long ticks = DateTime.Now.Ticks; DateTime dateTime = DateTime.Now; List> collection; ParallelForWork(firstPass, angleBracket, sourceDirectory, filteredSourceDirectoryFileTuples, filteredPropertyHolderCollection[i]); progressBar.Tick(); lock (filteredSourceDirectoryFileTuples) collection = (from l in filteredSourceDirectoryFileTuples where l.Item2 > dateTime select l).ToList(); lock (sourceDirectoryChanges) sourceDirectoryChanges.AddRange(collection); } catch (Exception ex) { lock (exceptions) exceptions.Add(ex); } }); } private string SetAngleBracketCollectionAndGetZero(Configuration configuration, Model? model, PredictorModel? predictorModel, string sourceDirectory) { string result; AngleBracketCollection.Clear(); AngleBracketCollection.AddRange(IResult.GetDirectoryInfoCollection(configuration, model, predictorModel, sourceDirectory, nameof(A_Property), string.Empty, includeResizeGroup: false, includeModel: false, includePredictorModel: false, contentDescription: string.Empty, singletonDescription: "Properties for each image", collectionDescription: string.Empty)); result = AngleBracketCollection[0]; return result; } public void ParallelWork(Configuration configuration, Model? model, PredictorModel? predictorModel, long ticks, List propertyHolderCollections, bool firstPass) { if (_Log is null) throw new Exception($"{nameof(_Log)} is null!"); if (_Configuration.PopulatePropertyId is null) throw new Exception($"{nameof(_Configuration.PopulatePropertyId)} is null!"); int g; int r; int totalSeconds; string angleBracket; string sourceDirectory; List exceptions = new(); PropertyHolder[] filteredPropertyHolderCollection; List> sourceDirectoryChanges = new(); int sourceDirectoryLength = configuration.RootDirectory.Length; int propertyHolderCollectionsCount = propertyHolderCollections.Count; string propertyRoot = IResult.GetResultsGroupDirectory(configuration, nameof(A_Property)); foreach (PropertyHolder[] propertyHolderCollection in propertyHolderCollections) { if (!propertyHolderCollection.Any()) continue; sourceDirectoryChanges.Clear(); if (firstPass) filteredPropertyHolderCollection = (from l in propertyHolderCollection where l.NoJson is null || !l.NoJson.Value && (l.Changed is null || l.Changed.Value) select l).ToArray(); else filteredPropertyHolderCollection = (from l in propertyHolderCollection where l.ImageFileInfo is not null && !_Configuration.IgnoreExtensions.Contains(l.ImageFileInfo.Extension) select l).ToArray(); if (!filteredPropertyHolderCollection.Any()) continue; g = filteredPropertyHolderCollection[0].G; r = filteredPropertyHolderCollection[0].R; sourceDirectory = filteredPropertyHolderCollection[0].SourceDirectory; totalSeconds = (int)Math.Truncate(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); angleBracket = SetAngleBracketCollectionAndGetZero(configuration, model, predictorModel, sourceDirectory); ParallelWork(firstPass, exceptions, sourceDirectoryChanges, propertyHolderCollectionsCount, g, sourceDirectory, r, filteredPropertyHolderCollection, totalSeconds, angleBracket); foreach (Exception exception in exceptions) _Log.Error(string.Concat(sourceDirectory, Environment.NewLine, exception.Message, Environment.NewLine, exception.StackTrace), exception); if (exceptions.Count == filteredPropertyHolderCollection.Length) throw new Exception(string.Concat("All in [", sourceDirectory, "]failed!")); if (exceptions.Count != 0) _ExceptionsDirectories.Add(sourceDirectory); bool? anyFilesMoved; if (!firstPass || exceptions.Count != 0) anyFilesMoved = null; else anyFilesMoved = AnyFilesMoved(sourceDirectory, filteredPropertyHolderCollection); if (exceptions.Count == 0 && !firstPass && _Configuration.PopulatePropertyId.Value && (anyFilesMoved is null || !anyFilesMoved.Value)) WriteGroup(sourceDirectoryLength, filteredPropertyHolderCollection, angleBracket); if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any()) { for (int y = 0; y < int.MaxValue; y++) { _Log.Information("Press \"Y\" key when ready to continue or close console"); if (Console.ReadKey().Key == ConsoleKey.Y) break; } _Log.Information(". . ."); } } } public A_Property GetProperty(string angleBracket, PropertyHolder propertyHolder, List> filteredSourceDirectoryFileTuples, List parseExceptions) { A_Property result; if (propertyHolder.ImageFileInfo is null) throw new ArgumentException($"{propertyHolder.ImageFileInfo} is null!"); bool firstPass = false; string extensionLowered = propertyHolder.ImageFileInfo.Extension.ToLower(); bool isValidMetadataExtensions = _Configuration.ValidMetadataExtensions.Contains(extensionLowered); bool isValidImageFormatExtension = _Configuration.ValidImageFormatExtensions.Contains(extensionLowered); bool isIgnoreExtension = isValidImageFormatExtension && _Configuration.IgnoreExtensions.Contains(extensionLowered); result = GetPropertyOfPrivate(angleBracket, propertyHolder, firstPass, filteredSourceDirectoryFileTuples, parseExceptions, isIgnoreExtension, isValidImageFormatExtension, isValidMetadataExtensions, extensionLowered); return result; } public (long Ticks, string FilteredSourceDirectoryFile, string PropertyDirectory, int PropertyId)[] GetPropertyIds(Configuration configuration, Model? model, PredictorModel? predictorModel, List groupCollection, bool saveToCollection) { List<(long Ticks, string FilteredSourceDirectoryFile, string PropertyDirectory, int PropertyId)> results = new(); int level; string angleBracket; A_Property? property; string checkDirectory; List directories; string propertyDirectory; foreach (DirectoryInfo group in groupCollection) { angleBracket = SetAngleBracketCollectionAndGetZero(configuration, model, predictorModel, group.SourceDirectory); if (string.IsNullOrEmpty(group.SourceDirectory)) throw new Exception(); if (!saveToCollection) propertyDirectory = angleBracket.Replace("<>", "()"); else { (level, directories) = IPath.Get(_Configuration.RootDirectory, group.SourceDirectory); checkDirectory = IPath.GetDirectory(angleBracket, level, "[()]"); propertyDirectory = Path.Combine(checkDirectory, string.Join(_Configuration.FileNameDirectorySeparator, directories)); } if (!Directory.Exists(propertyDirectory)) _ = Directory.CreateDirectory(propertyDirectory); for (int i = 0; i < group.SourceDirectoryFileInfoCollection.Length; i++) { property = group.PropertyCollection[i]; if (property?.Id is null) continue; results.Add(new(property.GetDateTimes().Min().Ticks, group.FilteredSourceDirectoryFiles[i], propertyDirectory, property.Id.Value)); } } return results.OrderBy(l => l.Ticks).ToArray(); } public void AddToPropertyLogicAllCollection(PropertyHolder[] filteredPropertyHolderCollection) { if (_SixCharacterNamedFaceInfo.Any()) { string[] keys; PropertyHolder propertyHolder; for (int i = 0; i < filteredPropertyHolderCollection.Length; i++) { propertyHolder = filteredPropertyHolderCollection[i]; if (propertyHolder.Property?.Id is null) continue; foreach (int sixCharacterIndex in propertyHolder.Property.Indices) { if (!_SixCharacterNamedFaceInfo.ContainsKey(sixCharacterIndex)) continue; keys = _SixCharacterNamedFaceInfo[sixCharacterIndex]; _AllCollection.Add(new(propertyHolder.Property.Id.Value, keys)); } } } } public void SaveAllCollection() { if (_AllCollection.Any()) { string[] keys; string? rootDirectoryParent = Path.GetDirectoryName(_Configuration.RootDirectory); if (string.IsNullOrEmpty(rootDirectoryParent)) throw new Exception($"{nameof(rootDirectoryParent)} is null!"); Dictionary namedFaceInfoDeterministicHashCodeIndices = new(); List<(int, string[])> allCollection = _AllCollection.OrderBy(l => l.Item1).ToList(); foreach ((int deterministicHashCode, string[] values) in allCollection) { if (namedFaceInfoDeterministicHashCodeIndices.ContainsKey(deterministicHashCode)) { keys = namedFaceInfoDeterministicHashCodeIndices[deterministicHashCode]; if (JsonSerializer.Serialize(values) == JsonSerializer.Serialize(keys)) continue; throw new Exception(); } namedFaceInfoDeterministicHashCodeIndices.Add(deterministicHashCode, values); } string json = JsonSerializer.Serialize(namedFaceInfoDeterministicHashCodeIndices, new JsonSerializerOptions { WriteIndented = true }); string checkFile = Path.Combine(rootDirectoryParent, "NamedFaceInfoDeterministicHashCodeIndices.json"); _ = IPath.WriteAllText(checkFile, json, compareBeforeWrite: true); } } }