using ShellProgressBar; using System.Drawing; using System.Drawing.Imaging; using System.Globalization; using System.Runtime.InteropServices; using System.Text; using System.Text.Json; using View_by_Distance.Property.Models.Stateless; using View_by_Distance.Shared.Models; using View_by_Distance.Shared.Models.Stateless; namespace View_by_Distance.Property.Models; public class A_Property { protected readonly List _ExceptionsDirectories; public bool Reverse { get; } public List ExceptionsDirectories => _ExceptionsDirectories; private readonly Serilog.ILogger? _Log; private readonly string _OutputExtension; private readonly int _MaxDegreeOfParallelism; private readonly ASCIIEncoding _ASCIIEncoding; private readonly Configuration _Configuration; private readonly List _AngleBracketCollection; private readonly IReadOnlyDictionary _JsonGroups; private readonly JsonSerializerOptions _WriteIndentedJsonSerializerOptions; public A_Property(int maxDegreeOfParallelism, Configuration configuration, string outputExtension, bool reverse, string aResultsFullGroupDirectory) { Reverse = reverse; _Configuration = configuration; _ExceptionsDirectories = new(); _OutputExtension = outputExtension; _ASCIIEncoding = new ASCIIEncoding(); _Log = Serilog.Log.ForContext(); _AngleBracketCollection = new List(); _MaxDegreeOfParallelism = maxDegreeOfParallelism; _WriteIndentedJsonSerializerOptions = new JsonSerializerOptions { WriteIndented = true }; string checkDirectory; List collection = new(); for (int i = 0; i < 12; i++) { if (i == 10) checkDirectory = Path.Combine(aResultsFullGroupDirectory, "{}", configuration.ResultAllInOne, "-"); else if (i == 11) checkDirectory = Path.Combine(aResultsFullGroupDirectory, "{}", configuration.ResultAllInOne, "_"); else checkDirectory = Path.Combine(aResultsFullGroupDirectory, "{}", configuration.ResultAllInOne, i.ToString()); if (!Directory.Exists(checkDirectory)) _ = Directory.CreateDirectory(checkDirectory); collection.Add(checkDirectory); } Dictionary jsonGroups = new() { { "{}", collection.ToArray() } }; _JsonGroups = jsonGroups; } public override string ToString() { string result = JsonSerializer.Serialize(this, new JsonSerializerOptions() { WriteIndented = true }); return result; } private long LogDelta(long ticks, string? methodName) { long result; if (_Log is null) throw new NullReferenceException(nameof(_Log)); double delta = new TimeSpan(DateTime.Now.Ticks - ticks).TotalMilliseconds; _Log.Debug($"{methodName} took {Math.Floor(delta)} millisecond(s)"); result = DateTime.Now.Ticks; return result; } #pragma warning disable CA1416 private static List GetMetadataDateTimesByPattern(string dateTimeFormat, FileHolder fileHolder) { List results = new(); try { DateTime checkDateTime; DateTime kristy = new(1976, 3, 8); IReadOnlyList directories = MetadataExtractor.ImageMetadataReader.ReadMetadata(fileHolder.FullName); foreach (MetadataExtractor.Directory directory in directories) { foreach (MetadataExtractor.Tag tag in directory.Tags) { if (string.IsNullOrEmpty(tag.Description) || tag.Description.Length != dateTimeFormat.Length) continue; if (!DateTime.TryParseExact(tag.Description, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) continue; if (checkDateTime < kristy) continue; results.Add(checkDateTime); } } } catch (Exception) { } return results; } private static Shared.Models.Property GetImageProperty(FileHolder fileHolder, Shared.Models.Property? property, bool populateId, bool isIgnoreExtension, bool isValidImageFormatExtension, bool isValidMetadataExtensions, int? id, ASCIIEncoding asciiEncoding, bool writeBitmapDataBytes, string? angleBracket) { Shared.Models.Property result; byte[] bytes; string value; long fileLength; int? width = null; int? height = null; string? make = null; string? model = null; string dateTimeFormat; DateTime checkDateTime; DateTime? dateTime = null; PropertyItem? propertyItem; string? orientation = null; DateTime? gpsDateStamp = null; DateTime? dateTimeOriginal = null; DateTime? dateTimeDigitized = null; DateTime? dateTimeFromName = Shared.Models.Stateless.Methods.IProperty.GetDateTimeFromName(fileHolder); if (!isValidImageFormatExtension && isValidMetadataExtensions && fileHolder.Exists) { dateTimeFormat = "ddd MMM dd HH:mm:ss yyyy"; List dateTimes = GetMetadataDateTimesByPattern(dateTimeFormat, fileHolder); if (dateTimes.Any()) dateTimeOriginal = dateTimes.Min(); } else if (!isIgnoreExtension && isValidImageFormatExtension && fileHolder.Exists) { try { using Image image = Image.FromFile(fileHolder.FullName); width = image.Width; height = image.Height; if (populateId && id is null) { using Bitmap bitmap = new(image); Rectangle rectangle = new(0, 0, image.Width, image.Height); BitmapData bitmapData = bitmap.LockBits(rectangle, ImageLockMode.ReadOnly, bitmap.PixelFormat); IntPtr intPtr = bitmapData.Scan0; int length = bitmapData.Stride * bitmap.Height; bytes = new byte[length]; Marshal.Copy(intPtr, bytes, 0, length); bitmap.UnlockBits(bitmapData); id ??= Shared.Models.Stateless.Methods.IProperty.GetDeterministicHashCode(bytes); if (writeBitmapDataBytes && !string.IsNullOrEmpty(angleBracket)) { FileInfo contentFileInfo = new(Path.Combine(angleBracket.Replace("<>", "()"), fileHolder.Name)); File.WriteAllBytes(Path.ChangeExtension(contentFileInfo.FullName, string.Empty), bytes); } } dateTimeFormat = Shared.Models.Stateless.Methods.IProperty.DateTimeFormat(); if (image.PropertyIdList.Contains((int)IExif.Tags.DateTime)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.DateTime); if (propertyItem?.Value is not null) { value = asciiEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); if (value.Length > dateTimeFormat.Length) value = value[..dateTimeFormat.Length]; if (value.Length == dateTimeFormat.Length && DateTime.TryParseExact(value, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) dateTime = checkDateTime; } } if (image.PropertyIdList.Contains((int)IExif.Tags.DateTimeDigitized)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.DateTimeDigitized); if (propertyItem?.Value is not null) { value = asciiEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); if (value.Length > dateTimeFormat.Length) value = value[..dateTimeFormat.Length]; if (value.Length == dateTimeFormat.Length && DateTime.TryParseExact(value, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) dateTimeDigitized = checkDateTime; } } if (image.PropertyIdList.Contains((int)IExif.Tags.DateTimeOriginal)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.DateTimeOriginal); if (propertyItem?.Value is not null) { value = asciiEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); if (value.Length > dateTimeFormat.Length) value = value[..dateTimeFormat.Length]; if (value.Length == dateTimeFormat.Length && DateTime.TryParseExact(value, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) dateTimeOriginal = checkDateTime; } } if (image.PropertyIdList.Contains((int)IExif.Tags.GPSDateStamp)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.GPSDateStamp); if (propertyItem?.Value is not null) { value = asciiEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); if (value.Length > dateTimeFormat.Length) value = value[..dateTimeFormat.Length]; if (value.Length == dateTimeFormat.Length && DateTime.TryParseExact(value, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) gpsDateStamp = checkDateTime; } } if (image.PropertyIdList.Contains((int)IExif.Tags.Make)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.Make); if (propertyItem?.Value is not null) { value = asciiEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); make = value; } } if (image.PropertyIdList.Contains((int)IExif.Tags.Model)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.Model); if (propertyItem?.Value is not null) { value = asciiEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1); model = value; } } if (image.PropertyIdList.Contains((int)IExif.Tags.Orientation)) { propertyItem = image.GetPropertyItem((int)IExif.Tags.Orientation); if (propertyItem?.Value is not null) { value = BitConverter.ToInt16(propertyItem.Value, 0).ToString(); orientation = value; } } } catch (Exception) { } } else dateTimeOriginal = null; if (fileHolder.Length is null) fileLength = 0; else fileLength = fileHolder.Length.Value; if (fileHolder.CreationTime is null && property?.CreationTime is null) throw new NullReferenceException(nameof(fileHolder.CreationTime)); if (fileHolder.LastWriteTime is null && property?.LastWriteTime is null) throw new NullReferenceException(nameof(fileHolder.LastWriteTime)); if (fileHolder.CreationTime is not null && fileHolder.LastWriteTime is not null) result = new(fileHolder.CreationTime.Value, dateTime, dateTimeDigitized, dateTimeFromName, dateTimeOriginal, fileLength, gpsDateStamp, height, id, fileHolder.LastWriteTime.Value, make, model, orientation, width); else if (property is not null) result = new(property.CreationTime, dateTime, dateTimeDigitized, dateTimeFromName, dateTimeOriginal, fileLength, gpsDateStamp, height, id, property.LastWriteTime, make, model, orientation, width); else throw new NullReferenceException(nameof(property)); return result; } public static Shared.Models.Property GetImageProperty(string fileName) { int? id = null; bool populateId = true; string? angleBracket = null; bool isIgnoreExtension = false; bool writeBitmapDataBytes = false; ASCIIEncoding asciiEncoding = new(); bool isValidMetadataExtensions = true; FileHolder fileHolder = new(fileName); bool isValidImageFormatExtension = true; Shared.Models.Property? property = null; Shared.Models.Property result = GetImageProperty(fileHolder, property, populateId, isIgnoreExtension, isValidImageFormatExtension, isValidMetadataExtensions, id, asciiEncoding, writeBitmapDataBytes, angleBracket); return result; } #pragma warning restore CA1416 private Shared.Models.Property GetPropertyOfPrivate(Item item, List> sourceDirectoryFileTuples, List parseExceptions, bool isIgnoreExtension, bool isValidMetadataExtensions) { Shared.Models.Property? result; int? id = null; FileInfo fileInfo; string? json = null; bool hasWrongYearProperty = false; string[] changesFrom = Array.Empty(); string angleBracket = _AngleBracketCollection[0]; bool populateId = _Configuration.PopulatePropertyId; char directory = Shared.Models.Stateless.Methods.IDirectory.GetDirectory(item.ImageFileHolder.Name); int directoryIndex = Shared.Models.Stateless.Methods.IDirectory.GetDirectory(directory); if (!item.IsUniqueFileName) fileInfo = new(Path.Combine(angleBracket.Replace("<>", "{}"), $"{item.ImageFileHolder.NameWithoutExtension}{item.ImageFileHolder.ExtensionLowered}.json")); else fileInfo = new(Path.Combine(_JsonGroups["{}"][directoryIndex], $"{item.ImageFileHolder.NameWithoutExtension}{item.ImageFileHolder.ExtensionLowered}.json")); List dateTimes = (from l in sourceDirectoryFileTuples where l is not null && changesFrom.Contains(l.Item1) select l.Item2).ToList(); if (_Configuration.ForcePropertyLastWriteTimeToCreationTime && !fileInfo.Exists && File.Exists(Path.ChangeExtension(fileInfo.FullName, ".delete"))) { File.Move(Path.ChangeExtension(fileInfo.FullName, ".delete"), fileInfo.FullName); fileInfo.Refresh(); } if (_Configuration.ForcePropertyLastWriteTimeToCreationTime && fileInfo.Exists && fileInfo.LastWriteTime != fileInfo.CreationTime) { File.SetLastWriteTime(fileInfo.FullName, fileInfo.CreationTime); fileInfo.Refresh(); } if (_Configuration.PropertiesChangedForProperty) result = null; else if (!fileInfo.Exists) result = null; else if (!fileInfo.FullName.EndsWith(".json") && !fileInfo.FullName.EndsWith(".old")) throw new ArgumentException("must be a *.json file"); else if (dateTimes.Any() && dateTimes.Max() > fileInfo.LastWriteTime) result = null; else { json = File.ReadAllText(fileInfo.FullName); try { if (item.Property is not null) result = item.Property; else result = JsonSerializer.Deserialize(json); if (result is not null && json.Contains("WrongYear")) { id = result.Id; hasWrongYearProperty = true; result = null; } if (!isIgnoreExtension && item.IsValidImageFormatExtension && ((populateId && result?.Id is null) || result?.Width is null || result.Height is null)) { id = result?.Id; result = null; } if (!isIgnoreExtension && item.IsValidImageFormatExtension && populateId && result is not null && result.LastWriteTime != item.ImageFileHolder.LastWriteTime) { id = null; result = null; } if (!isIgnoreExtension && item.IsValidImageFormatExtension && result?.Width is not null && result.Height is not null && result.Width.Value == result.Height.Value && item.ImageFileHolder.Exists) { id = result.Id; result = null; if (result?.Width is not null && result.Height is not null && result.Width.Value != result.Height.Value) throw new Exception("Was square!"); } if (!isIgnoreExtension && item.IsValidImageFormatExtension && result is not null && result.FileSize != item.ImageFileHolder.Length) { id = result.Id; result = null; } if (result is not null) { sourceDirectoryFileTuples.Add(new Tuple(nameof(A_Property), fileInfo.LastWriteTime)); if (fileInfo.CreationTime != result.LastWriteTime) { File.SetCreationTime(fileInfo.FullName, result.LastWriteTime); File.SetLastWriteTime(fileInfo.FullName, fileInfo.LastWriteTime); } } } catch (Exception) { result = null; parseExceptions.Add(nameof(A_Property)); } } if (result is null) { id ??= item.ImageFileHolder.Id; result = GetImageProperty(item.ImageFileHolder, result, populateId, isIgnoreExtension, item.IsValidImageFormatExtension, isValidMetadataExtensions, id, _ASCIIEncoding, _Configuration.WriteBitmapDataBytes, angleBracket); json = JsonSerializer.Serialize(result, _WriteIndentedJsonSerializerOptions); if (populateId && Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, json, updateDateWhenMatches: true, compareBeforeWrite: true)) { File.SetCreationTime(fileInfo.FullName, result.LastWriteTime); if (!_Configuration.ForcePropertyLastWriteTimeToCreationTime) sourceDirectoryFileTuples.Add(new Tuple(nameof(A_Property), DateTime.Now)); else { File.SetLastWriteTime(fileInfo.FullName, fileInfo.CreationTime); fileInfo.Refresh(); sourceDirectoryFileTuples.Add(new Tuple(nameof(A_Property), fileInfo.CreationTime)); } } } else if (hasWrongYearProperty) { json = JsonSerializer.Serialize(result, _WriteIndentedJsonSerializerOptions); if (Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, json, updateDateWhenMatches: true, compareBeforeWrite: true)) { File.SetCreationTime(fileInfo.FullName, result.LastWriteTime); File.SetLastWriteTime(fileInfo.FullName, fileInfo.CreationTime); fileInfo.Refresh(); sourceDirectoryFileTuples.Add(new Tuple(nameof(A_Property), fileInfo.CreationTime)); } } return result; } private void SavePropertyParallelForWork(string sourceDirectory, List> sourceDirectoryFileTuples, List> sourceDirectoryChanges, Item item) { Shared.Models.Property property; List parseExceptions = new(); bool isValidMetadataExtensions = _Configuration.ValidMetadataExtensions.Contains(item.ImageFileHolder.ExtensionLowered); bool isIgnoreExtension = item.IsValidImageFormatExtension && _Configuration.IgnoreExtensions.Contains(item.ImageFileHolder.ExtensionLowered); string filteredSourceDirectoryFileExtensionLowered = Path.Combine(sourceDirectory, $"{item.ImageFileHolder.NameWithoutExtension}{item.ImageFileHolder.ExtensionLowered}"); if (item.IsValidImageFormatExtension && item.ImageFileHolder.FullName.Length == filteredSourceDirectoryFileExtensionLowered.Length && item.ImageFileHolder.FullName != filteredSourceDirectoryFileExtensionLowered) File.Move(item.ImageFileHolder.FullName, filteredSourceDirectoryFileExtensionLowered); if (item.FileSizeChanged is null || item.FileSizeChanged.Value || item.LastWriteTimeChanged is null || item.LastWriteTimeChanged.Value || item.Property is null) { property = GetPropertyOfPrivate(item, sourceDirectoryFileTuples, parseExceptions, isIgnoreExtension, isValidMetadataExtensions); lock (sourceDirectoryChanges) sourceDirectoryChanges.Add(new Tuple(nameof(A_Property), DateTime.Now)); lock (item) item.Update(property); } } private void SavePropertyParallelWork(int maxDegreeOfParallelism, List exceptions, List> sourceDirectoryChanges, Container container, List items, string message) { List> sourceDirectoryFileTuples = new(); ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; using ProgressBar progressBar = new(items.Count, message, options); _ = Parallel.For(0, items.Count, parallelOptions, (i, state) => { try { long ticks = DateTime.Now.Ticks; DateTime dateTime = DateTime.Now; List> collection; SavePropertyParallelForWork(container.SourceDirectory, sourceDirectoryChanges, sourceDirectoryFileTuples, items[i]); if (i == 0 || sourceDirectoryChanges.Any()) progressBar.Tick(); lock (sourceDirectoryFileTuples) collection = (from l in sourceDirectoryFileTuples where l.Item2 > dateTime select l).ToList(); lock (sourceDirectoryChanges) sourceDirectoryChanges.AddRange(collection); } catch (Exception ex) { lock (exceptions) exceptions.Add(ex); } }); } public void SetAngleBracketCollection(string aResultsFullGroupDirectory, string sourceDirectory, bool anyNullOrNoIsUniqueFileName = true) { _AngleBracketCollection.Clear(); if (!anyNullOrNoIsUniqueFileName) _AngleBracketCollection.AddRange(new[] { Path.Combine(aResultsFullGroupDirectory, "<>") }); else _AngleBracketCollection.AddRange(IResult.GetDirectoryInfoCollection(_Configuration, sourceDirectory, aResultsFullGroupDirectory, contentDescription: string.Empty, singletonDescription: "Properties for each image", collectionDescription: string.Empty, converted: false)); } private void SetAngleBracketCollection(string sourceDirectory, bool anyNullOrNoIsUniqueFileName) { _AngleBracketCollection.Clear(); string aResultsFullGroupDirectory = IResult.GetResultsFullGroupDirectory(_Configuration, nameof(A_Property), string.Empty, includeResizeGroup: false, includeModel: false, includePredictorModel: false); SetAngleBracketCollection(aResultsFullGroupDirectory, sourceDirectory, anyNullOrNoIsUniqueFileName); } public void SavePropertyParallelWork(long ticks, int t, Container[] containers, Shared.Models.Methods.IThumbHasher? thumbHasher) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); int total = 0; string message; int totalSeconds; Container container; bool anyNullOrNoIsUniqueFileName; List exceptions = new(); int containersLength = containers.Length; const string outputResolution = "Original"; List> sourceDirectoryChanges = new(); string propertyRoot = IResult.GetResultsGroupDirectory(_Configuration, nameof(A_Property)); for (int i = 0; i < containers.Length; i++) { container = containers[i]; if (!container.Items.Any()) continue; sourceDirectoryChanges.Clear(); if (!container.Items.Any()) continue; anyNullOrNoIsUniqueFileName = container.Items.Any(l => !l.IsUniqueFileName); SetAngleBracketCollection(container.SourceDirectory, anyNullOrNoIsUniqueFileName); totalSeconds = (int)Math.Truncate(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); message = $"{i + 1:000} [{container.Items.Count:000}] / {containersLength:000} - {total} / {t} total - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}"; SavePropertyParallelWork(_MaxDegreeOfParallelism, exceptions, sourceDirectoryChanges, container, container.Items, message); foreach (Exception exception in exceptions) _Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, exception.Message, Environment.NewLine, exception.StackTrace), exception); if (exceptions.Count == container.Items.Count) throw new Exception(string.Concat("All in [", container.SourceDirectory, "]failed!")); if (exceptions.Count != 0) _ExceptionsDirectories.Add(container.SourceDirectory); if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any()) { for (int y = 0; y < int.MaxValue; y++) { _Log.Information("Press \"Y\" key when ready to continue or close console"); if (System.Console.ReadKey().Key == ConsoleKey.Y) break; } _Log.Information(". . ."); } total += container.Items.Count; } } public Shared.Models.Property GetProperty(Item item, List> sourceDirectoryFileTuples, List parseExceptions) { Shared.Models.Property result; bool angleBracketCollectionAny = _AngleBracketCollection.Any(); if (!angleBracketCollectionAny) { if (item.ImageFileHolder.DirectoryName is null) throw new NullReferenceException(nameof(item.ImageFileHolder.DirectoryName)); SetAngleBracketCollection(item.ImageFileHolder.DirectoryName, !item.IsUniqueFileName); } bool isValidMetadataExtensions = _Configuration.ValidMetadataExtensions.Contains(item.ImageFileHolder.ExtensionLowered); bool isIgnoreExtension = item.IsValidImageFormatExtension && _Configuration.IgnoreExtensions.Contains(item.ImageFileHolder.ExtensionLowered); result = GetPropertyOfPrivate(item, sourceDirectoryFileTuples, parseExceptions, isIgnoreExtension, isValidMetadataExtensions); if (!angleBracketCollectionAny) _AngleBracketCollection.Clear(); return result; } }