517 lines
28 KiB
C#

using ShellProgressBar;
using System.Drawing;
using System.Drawing.Imaging;
using System.Globalization;
using System.Runtime.InteropServices;
using System.Text;
using System.Text.Json;
using View_by_Distance.Property.Models.Stateless;
using View_by_Distance.Shared.Models;
using View_by_Distance.Shared.Models.Stateless;
namespace View_by_Distance.Property.Models;
public class A_Property
{
protected readonly List<string> _ExceptionsDirectories;
public bool Reverse { get; }
public List<string> ExceptionsDirectories => _ExceptionsDirectories;
private readonly Serilog.ILogger? _Log;
private readonly string _OutputExtension;
private readonly int _MaxDegreeOfParallelism;
private readonly ASCIIEncoding _ASCIIEncoding;
private readonly Configuration _Configuration;
private readonly List<string> _AngleBracketCollection;
private readonly IReadOnlyDictionary<string, string[]> _JsonGroups;
private readonly JsonSerializerOptions _WriteIndentedJsonSerializerOptions;
public A_Property(int maxDegreeOfParallelism, Configuration configuration, string outputExtension, bool reverse, string aResultsFullGroupDirectory)
{
Reverse = reverse;
_Configuration = configuration;
_ExceptionsDirectories = new();
_OutputExtension = outputExtension;
_ASCIIEncoding = new ASCIIEncoding();
_Log = Serilog.Log.ForContext<A_Property>();
_AngleBracketCollection = new List<string>();
_MaxDegreeOfParallelism = maxDegreeOfParallelism;
_WriteIndentedJsonSerializerOptions = new JsonSerializerOptions { WriteIndented = true };
_JsonGroups = Shared.Models.Stateless.Methods.IPath.GetKeyValuePairs(configuration.ResultAllInOne, aResultsFullGroupDirectory);
}
public override string ToString()
{
string result = JsonSerializer.Serialize(this, new JsonSerializerOptions() { WriteIndented = true });
return result;
}
private long LogDelta(long ticks, string? methodName)
{
long result;
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
double delta = new TimeSpan(DateTime.Now.Ticks - ticks).TotalMilliseconds;
_Log.Debug($"{methodName} took {Math.Floor(delta)} millisecond(s)");
result = DateTime.Now.Ticks;
return result;
}
#pragma warning disable CA1416
private static List<DateTime> GetMetadataDateTimesByPattern(string dateTimeFormat, FileHolder fileHolder)
{
List<DateTime> results = new();
try
{
DateTime checkDateTime;
DateTime kristy = new(1976, 3, 8);
IReadOnlyList<MetadataExtractor.Directory> directories = MetadataExtractor.ImageMetadataReader.ReadMetadata(fileHolder.FullName);
foreach (MetadataExtractor.Directory directory in directories)
{
foreach (MetadataExtractor.Tag tag in directory.Tags)
{
if (string.IsNullOrEmpty(tag.Description) || tag.Description.Length != dateTimeFormat.Length)
continue;
if (!DateTime.TryParseExact(tag.Description, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime))
continue;
if (checkDateTime < kristy)
continue;
results.Add(checkDateTime);
}
}
}
catch (Exception) { }
return results;
}
private static Shared.Models.Property GetImageProperty(FileHolder fileHolder, Shared.Models.Property? property, bool populateId, bool isIgnoreExtension, bool isValidImageFormatExtension, bool isValidMetadataExtensions, int? id, ASCIIEncoding asciiEncoding, bool writeBitmapDataBytes, string? angleBracket)
{
Shared.Models.Property result;
byte[] bytes;
string value;
long fileLength;
int? width = null;
int? height = null;
string? make = null;
string? model = null;
string dateTimeFormat;
DateTime checkDateTime;
DateTime? dateTime = null;
PropertyItem? propertyItem;
string? orientation = null;
DateTime? gpsDateStamp = null;
DateTime? dateTimeOriginal = null;
DateTime? dateTimeDigitized = null;
DateTime? dateTimeFromName = Shared.Models.Stateless.Methods.IProperty.GetDateTimeFromName(fileHolder);
if (!isValidImageFormatExtension && isValidMetadataExtensions && fileHolder.Exists)
{
dateTimeFormat = "ddd MMM dd HH:mm:ss yyyy";
List<DateTime> dateTimes = GetMetadataDateTimesByPattern(dateTimeFormat, fileHolder);
if (dateTimes.Any())
dateTimeOriginal = dateTimes.Min();
}
else if (!isIgnoreExtension && isValidImageFormatExtension && fileHolder.Exists)
{
try
{
using Image image = Image.FromFile(fileHolder.FullName);
width = image.Width;
height = image.Height;
if (populateId && id is null)
{
using Bitmap bitmap = new(image);
Rectangle rectangle = new(0, 0, image.Width, image.Height);
BitmapData bitmapData = bitmap.LockBits(rectangle, ImageLockMode.ReadOnly, bitmap.PixelFormat);
IntPtr intPtr = bitmapData.Scan0;
int length = bitmapData.Stride * bitmap.Height;
bytes = new byte[length];
Marshal.Copy(intPtr, bytes, 0, length);
bitmap.UnlockBits(bitmapData);
id ??= Shared.Models.Stateless.Methods.IProperty.GetDeterministicHashCode(bytes);
if (writeBitmapDataBytes && !string.IsNullOrEmpty(angleBracket))
{
FileInfo contentFileInfo = new(Path.Combine(angleBracket.Replace("<>", "()"), fileHolder.Name));
File.WriteAllBytes(Path.ChangeExtension(contentFileInfo.FullName, string.Empty), bytes);
}
}
dateTimeFormat = Shared.Models.Stateless.Methods.IProperty.DateTimeFormat();
if (image.PropertyIdList.Contains((int)IExif.Tags.DateTime))
{
propertyItem = image.GetPropertyItem((int)IExif.Tags.DateTime);
if (propertyItem?.Value is not null)
{
value = asciiEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1);
if (value.Length > dateTimeFormat.Length)
value = value[..dateTimeFormat.Length];
if (value.Length == dateTimeFormat.Length && DateTime.TryParseExact(value, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime))
dateTime = checkDateTime;
}
}
if (image.PropertyIdList.Contains((int)IExif.Tags.DateTimeDigitized))
{
propertyItem = image.GetPropertyItem((int)IExif.Tags.DateTimeDigitized);
if (propertyItem?.Value is not null)
{
value = asciiEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1);
if (value.Length > dateTimeFormat.Length)
value = value[..dateTimeFormat.Length];
if (value.Length == dateTimeFormat.Length && DateTime.TryParseExact(value, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime))
dateTimeDigitized = checkDateTime;
}
}
if (image.PropertyIdList.Contains((int)IExif.Tags.DateTimeOriginal))
{
propertyItem = image.GetPropertyItem((int)IExif.Tags.DateTimeOriginal);
if (propertyItem?.Value is not null)
{
value = asciiEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1);
if (value.Length > dateTimeFormat.Length)
value = value[..dateTimeFormat.Length];
if (value.Length == dateTimeFormat.Length && DateTime.TryParseExact(value, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime))
dateTimeOriginal = checkDateTime;
}
}
if (image.PropertyIdList.Contains((int)IExif.Tags.GPSDateStamp))
{
propertyItem = image.GetPropertyItem((int)IExif.Tags.GPSDateStamp);
if (propertyItem?.Value is not null)
{
value = asciiEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1);
if (value.Length > dateTimeFormat.Length)
value = value[..dateTimeFormat.Length];
if (value.Length == dateTimeFormat.Length && DateTime.TryParseExact(value, dateTimeFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime))
gpsDateStamp = checkDateTime;
}
}
if (image.PropertyIdList.Contains((int)IExif.Tags.Make))
{
propertyItem = image.GetPropertyItem((int)IExif.Tags.Make);
if (propertyItem?.Value is not null)
{
value = asciiEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1);
make = value;
}
}
if (image.PropertyIdList.Contains((int)IExif.Tags.Model))
{
propertyItem = image.GetPropertyItem((int)IExif.Tags.Model);
if (propertyItem?.Value is not null)
{
value = asciiEncoding.GetString(propertyItem.Value, 0, propertyItem.Len - 1);
model = value;
}
}
if (image.PropertyIdList.Contains((int)IExif.Tags.Orientation))
{
propertyItem = image.GetPropertyItem((int)IExif.Tags.Orientation);
if (propertyItem?.Value is not null)
{
value = BitConverter.ToInt16(propertyItem.Value, 0).ToString();
orientation = value;
}
}
}
catch (Exception) { }
}
else
dateTimeOriginal = null;
if (fileHolder.Length is null)
fileLength = 0;
else
fileLength = fileHolder.Length.Value;
if (fileHolder.CreationTime is null && property?.CreationTime is null)
throw new NullReferenceException(nameof(fileHolder.CreationTime));
if (fileHolder.LastWriteTime is null && property?.LastWriteTime is null)
throw new NullReferenceException(nameof(fileHolder.LastWriteTime));
if (fileHolder.CreationTime is not null && fileHolder.LastWriteTime is not null)
result = new(fileHolder.CreationTime.Value, dateTime, dateTimeDigitized, dateTimeFromName, dateTimeOriginal, fileLength, gpsDateStamp, height, id, fileHolder.LastWriteTime.Value, make, model, orientation, width);
else if (property is not null)
result = new(property.CreationTime, dateTime, dateTimeDigitized, dateTimeFromName, dateTimeOriginal, fileLength, gpsDateStamp, height, id, property.LastWriteTime, make, model, orientation, width);
else
throw new NullReferenceException(nameof(property));
return result;
}
public static Shared.Models.Property GetImageProperty(string fileName)
{
int? id = null;
bool populateId = true;
string? angleBracket = null;
bool isIgnoreExtension = false;
bool writeBitmapDataBytes = false;
ASCIIEncoding asciiEncoding = new();
bool isValidMetadataExtensions = true;
FileHolder fileHolder = new(fileName);
bool isValidImageFormatExtension = true;
Shared.Models.Property? property = null;
Shared.Models.Property result = GetImageProperty(fileHolder, property, populateId, isIgnoreExtension, isValidImageFormatExtension, isValidMetadataExtensions, id, asciiEncoding, writeBitmapDataBytes, angleBracket);
return result;
}
#pragma warning restore CA1416
private Shared.Models.Property GetPropertyOfPrivate(Item item, List<Tuple<string, DateTime>> sourceDirectoryFileTuples, List<string> parseExceptions, bool isIgnoreExtension, bool isValidMetadataExtensions)
{
Shared.Models.Property? result;
int? id = null;
FileInfo fileInfo;
string? json = null;
bool hasWrongYearProperty = false;
string[] changesFrom = Array.Empty<string>();
string angleBracket = _AngleBracketCollection[0];
bool populateId = _Configuration.PopulatePropertyId;
char directory = Shared.Models.Stateless.Methods.IDirectory.GetDirectory(item.ImageFileHolder.Name);
int directoryIndex = Shared.Models.Stateless.Methods.IDirectory.GetDirectory(directory);
if (!item.IsUniqueFileName)
fileInfo = new(Path.Combine(angleBracket.Replace("<>", "{}"), $"{item.ImageFileHolder.NameWithoutExtension}{item.ImageFileHolder.ExtensionLowered}.json"));
else
fileInfo = new(Path.Combine(_JsonGroups["{}"][directoryIndex], $"{item.ImageFileHolder.NameWithoutExtension}{item.ImageFileHolder.ExtensionLowered}.json"));
List<DateTime> dateTimes = (from l in sourceDirectoryFileTuples where l is not null && changesFrom.Contains(l.Item1) select l.Item2).ToList();
if (_Configuration.ForcePropertyLastWriteTimeToCreationTime && !fileInfo.Exists && File.Exists(Path.ChangeExtension(fileInfo.FullName, ".delete")))
{
File.Move(Path.ChangeExtension(fileInfo.FullName, ".delete"), fileInfo.FullName);
fileInfo.Refresh();
}
if (_Configuration.ForcePropertyLastWriteTimeToCreationTime && fileInfo.Exists && fileInfo.LastWriteTime != fileInfo.CreationTime)
{
File.SetLastWriteTime(fileInfo.FullName, fileInfo.CreationTime);
fileInfo.Refresh();
}
if (_Configuration.PropertiesChangedForProperty)
result = null;
else if (!fileInfo.Exists)
result = null;
else if (!fileInfo.FullName.EndsWith(".json") && !fileInfo.FullName.EndsWith(".old"))
throw new ArgumentException("must be a *.json file");
else if (dateTimes.Any() && dateTimes.Max() > fileInfo.LastWriteTime)
result = null;
else
{
json = File.ReadAllText(fileInfo.FullName);
try
{
if (item.Property is not null)
result = item.Property;
else
result = JsonSerializer.Deserialize<Shared.Models.Property>(json);
if (result is not null && json.Contains("WrongYear"))
{
id = result.Id;
hasWrongYearProperty = true;
result = null;
}
if (!isIgnoreExtension && item.IsValidImageFormatExtension && ((populateId && result?.Id is null) || result?.Width is null || result.Height is null))
{
id = result?.Id;
result = null;
}
if (!isIgnoreExtension && item.IsValidImageFormatExtension && populateId && result is not null && result.LastWriteTime != item.ImageFileHolder.LastWriteTime)
{
id = null;
result = null;
}
if (!isIgnoreExtension && item.IsValidImageFormatExtension && result?.Width is not null && result.Height is not null && result.Width.Value == result.Height.Value && item.ImageFileHolder.Exists)
{
id = result.Id;
result = null;
if (result?.Width is not null && result.Height is not null && result.Width.Value != result.Height.Value)
throw new Exception("Was square!");
}
if (!isIgnoreExtension && item.IsValidImageFormatExtension && result is not null && result.FileSize != item.ImageFileHolder.Length)
{
id = result.Id;
result = null;
}
if (result is not null)
{
sourceDirectoryFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), fileInfo.LastWriteTime));
if (fileInfo.CreationTime != result.LastWriteTime)
{
File.SetCreationTime(fileInfo.FullName, result.LastWriteTime);
File.SetLastWriteTime(fileInfo.FullName, fileInfo.LastWriteTime);
}
}
}
catch (Exception)
{
result = null;
parseExceptions.Add(nameof(A_Property));
}
}
if (result is null)
{
id ??= item.ImageFileHolder.Id;
result = GetImageProperty(item.ImageFileHolder, result, populateId, isIgnoreExtension, item.IsValidImageFormatExtension, isValidMetadataExtensions, id, _ASCIIEncoding, _Configuration.WriteBitmapDataBytes, angleBracket);
json = JsonSerializer.Serialize(result, _WriteIndentedJsonSerializerOptions);
if (populateId && Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, json, updateDateWhenMatches: true, compareBeforeWrite: true))
{
File.SetCreationTime(fileInfo.FullName, result.LastWriteTime);
if (!_Configuration.ForcePropertyLastWriteTimeToCreationTime)
sourceDirectoryFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), DateTime.Now));
else
{
File.SetLastWriteTime(fileInfo.FullName, fileInfo.CreationTime);
fileInfo.Refresh();
sourceDirectoryFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), fileInfo.CreationTime));
}
}
}
else if (hasWrongYearProperty)
{
json = JsonSerializer.Serialize(result, _WriteIndentedJsonSerializerOptions);
if (Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, json, updateDateWhenMatches: true, compareBeforeWrite: true))
{
File.SetCreationTime(fileInfo.FullName, result.LastWriteTime);
File.SetLastWriteTime(fileInfo.FullName, fileInfo.CreationTime);
fileInfo.Refresh();
sourceDirectoryFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), fileInfo.CreationTime));
}
}
return result;
}
private void SavePropertyParallelForWork(string sourceDirectory, List<Tuple<string, DateTime>> sourceDirectoryFileTuples, List<Tuple<string, DateTime>> sourceDirectoryChanges, Item item)
{
Shared.Models.Property property;
List<string> parseExceptions = new();
bool isValidMetadataExtensions = _Configuration.ValidMetadataExtensions.Contains(item.ImageFileHolder.ExtensionLowered);
bool isIgnoreExtension = item.IsValidImageFormatExtension && _Configuration.IgnoreExtensions.Contains(item.ImageFileHolder.ExtensionLowered);
string filteredSourceDirectoryFileExtensionLowered = Path.Combine(sourceDirectory, $"{item.ImageFileHolder.NameWithoutExtension}{item.ImageFileHolder.ExtensionLowered}");
if (item.IsValidImageFormatExtension && item.ImageFileHolder.FullName.Length == filteredSourceDirectoryFileExtensionLowered.Length && item.ImageFileHolder.FullName != filteredSourceDirectoryFileExtensionLowered)
File.Move(item.ImageFileHolder.FullName, filteredSourceDirectoryFileExtensionLowered);
if (item.FileSizeChanged is null || item.FileSizeChanged.Value || item.LastWriteTimeChanged is null || item.LastWriteTimeChanged.Value || item.Property is null)
{
property = GetPropertyOfPrivate(item, sourceDirectoryFileTuples, parseExceptions, isIgnoreExtension, isValidMetadataExtensions);
lock (sourceDirectoryChanges)
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), DateTime.Now));
lock (item)
item.Update(property);
}
}
private void SavePropertyParallelWork(int maxDegreeOfParallelism, List<Exception> exceptions, List<Tuple<string, DateTime>> sourceDirectoryChanges, Container container, List<Item> items, string message)
{
List<Tuple<string, DateTime>> sourceDirectoryFileTuples = new();
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
using ProgressBar progressBar = new(items.Count, message, options);
_ = Parallel.For(0, items.Count, parallelOptions, (i, state) =>
{
try
{
long ticks = DateTime.Now.Ticks;
DateTime dateTime = DateTime.Now;
List<Tuple<string, DateTime>> collection;
SavePropertyParallelForWork(container.SourceDirectory, sourceDirectoryChanges, sourceDirectoryFileTuples, items[i]);
if (i == 0 || sourceDirectoryChanges.Any())
progressBar.Tick();
lock (sourceDirectoryFileTuples)
collection = (from l in sourceDirectoryFileTuples where l.Item2 > dateTime select l).ToList();
lock (sourceDirectoryChanges)
sourceDirectoryChanges.AddRange(collection);
}
catch (Exception ex)
{
lock (exceptions)
exceptions.Add(ex);
}
});
}
public void SetAngleBracketCollection(string aResultsFullGroupDirectory, string sourceDirectory, bool anyNullOrNoIsUniqueFileName = true)
{
_AngleBracketCollection.Clear();
if (!anyNullOrNoIsUniqueFileName)
_AngleBracketCollection.AddRange(new[] { Path.Combine(aResultsFullGroupDirectory, "<>") });
else
_AngleBracketCollection.AddRange(IResult.GetDirectoryInfoCollection(_Configuration,
sourceDirectory,
aResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Properties for each image",
collectionDescription: string.Empty,
converted: false));
}
private void SetAngleBracketCollection(string sourceDirectory, bool anyNullOrNoIsUniqueFileName)
{
_AngleBracketCollection.Clear();
string aResultsFullGroupDirectory = IResult.GetResultsFullGroupDirectory(_Configuration,
nameof(A_Property),
string.Empty,
includeResizeGroup: false,
includeModel: false,
includePredictorModel: false);
SetAngleBracketCollection(aResultsFullGroupDirectory, sourceDirectory, anyNullOrNoIsUniqueFileName);
}
public void SavePropertyParallelWork(long ticks, int t, Container[] containers)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
int total = 0;
string message;
int totalSeconds;
Container container;
bool anyNullOrNoIsUniqueFileName;
List<Exception> exceptions = new();
int containersLength = containers.Length;
const string outputResolution = "Original";
List<Tuple<string, DateTime>> sourceDirectoryChanges = new();
string propertyRoot = IResult.GetResultsGroupDirectory(_Configuration, nameof(A_Property));
for (int i = 0; i < containers.Length; i++)
{
container = containers[i];
if (!container.Items.Any())
continue;
sourceDirectoryChanges.Clear();
if (!container.Items.Any())
continue;
anyNullOrNoIsUniqueFileName = container.Items.Any(l => !l.IsUniqueFileName);
SetAngleBracketCollection(container.SourceDirectory, anyNullOrNoIsUniqueFileName);
totalSeconds = (int)Math.Truncate(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
message = $"{i + 1:000} [{container.Items.Count:000}] / {containersLength:000} - {total} / {t} total - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}";
SavePropertyParallelWork(_MaxDegreeOfParallelism, exceptions, sourceDirectoryChanges, container, container.Items, message);
foreach (Exception exception in exceptions)
_Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, exception.Message, Environment.NewLine, exception.StackTrace), exception);
if (exceptions.Count == container.Items.Count)
throw new Exception(string.Concat("All in [", container.SourceDirectory, "]failed!"));
if (exceptions.Count != 0)
_ExceptionsDirectories.Add(container.SourceDirectory);
if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any())
{
for (int y = 0; y < int.MaxValue; y++)
{
_Log.Information("Press \"Y\" key when ready to continue or close console");
if (System.Console.ReadKey().Key == ConsoleKey.Y)
break;
}
_Log.Information(". . .");
}
total += container.Items.Count;
}
}
public Shared.Models.Property GetProperty(Item item, List<Tuple<string, DateTime>> sourceDirectoryFileTuples, List<string> parseExceptions)
{
Shared.Models.Property result;
bool angleBracketCollectionAny = _AngleBracketCollection.Any();
if (!angleBracketCollectionAny)
{
if (item.ImageFileHolder.DirectoryName is null)
throw new NullReferenceException(nameof(item.ImageFileHolder.DirectoryName));
SetAngleBracketCollection(item.ImageFileHolder.DirectoryName, !item.IsUniqueFileName);
}
bool isValidMetadataExtensions = _Configuration.ValidMetadataExtensions.Contains(item.ImageFileHolder.ExtensionLowered);
bool isIgnoreExtension = item.IsValidImageFormatExtension && _Configuration.IgnoreExtensions.Contains(item.ImageFileHolder.ExtensionLowered);
result = GetPropertyOfPrivate(item, sourceDirectoryFileTuples, parseExceptions, isIgnoreExtension, isValidMetadataExtensions);
if (!angleBracketCollectionAny)
_AngleBracketCollection.Clear();
return result;
}
}