From 82143850ac6d49ba890c9bee9f7131d37fa0e8b4 Mon Sep 17 00:00:00 2001 From: Mike Phares Date: Mon, 22 May 2023 20:36:02 -0700 Subject: [PATCH] Remove ThumbHasher from Property --- Drag-Drop-Move/DragDropMove.cs | 11 ++--- Instance/DlibDotNet.cs | 48 ++----------------- Property/Models/A_Property.cs | 46 ++++++------------ Shared/Models/Properties/IProperty.cs | 1 - Shared/Models/Property.cs | 4 +- Tests/UnitTestResize.cs | 3 +- .../UnitTestFace.cs | 3 +- 7 files changed, 26 insertions(+), 90 deletions(-) diff --git a/Drag-Drop-Move/DragDropMove.cs b/Drag-Drop-Move/DragDropMove.cs index e469871..79e1f0f 100644 --- a/Drag-Drop-Move/DragDropMove.cs +++ b/Drag-Drop-Move/DragDropMove.cs @@ -202,11 +202,10 @@ public partial class DragDropMove : Form List<(string, int, DateTime)> results = new(); DateTime dateTime; Shared.Models.Property property; - Shared.Models.Methods.IThumbHasher? thumbHasher = null; string[] files = Directory.GetFiles(checkDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string file in files) { - property = Property.Models.A_Property.GetImageProperty(thumbHasher, file); + property = Property.Models.A_Property.GetImageProperty(file); if (property.Id is null || property.DateTimeOriginal is null) continue; dateTime = property.DateTimeOriginal.Value.AddTicks(ticks); @@ -240,7 +239,6 @@ public partial class DragDropMove : Form ticks++; } - Shared.Models.Methods.IThumbHasher? thumbHasher = null; List<(string, int, DateTime)> collection = GetCollection(checkDirectory, minimumDateTime, maximumDateTime, ticks); ConstructorInfo? constructorInfo = typeof(PropertyItem).GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.Public, null, Array.Empty(), null) ?? throw new Exception(); foreach ((string file, int id, DateTime dateTime) in collection) @@ -255,7 +253,7 @@ public partial class DragDropMove : Form bitmap.SetPropertyItem(propertyItem); bitmap.Save(checkFile); bitmap.Dispose(); - property = Property.Models.A_Property.GetImageProperty(thumbHasher, checkFile); + property = Property.Models.A_Property.GetImageProperty(checkFile); if (property.Id is null || property.Id.Value != id) throw new Exception(); } @@ -290,11 +288,10 @@ public partial class DragDropMove : Form _Logger.Error("bad file(s) or target file(s) or maximum directory doesn't equal 1!"); else { - Shared.Models.Methods.IThumbHasher? thumbHasher = null; DateTime minimumDateTime = DateTime.ParseExact(Path.GetFileName(minimumDirectory.First()), format, null, System.Globalization.DateTimeStyles.None); DateTime maximumDateTime = DateTime.ParseExact(Path.GetFileName(maximumDirectory.First()), format, null, System.Globalization.DateTimeStyles.None).AddHours(23); - Shared.Models.Property badProperty = Property.Models.A_Property.GetImageProperty(thumbHasher, badFiles.First()); - Shared.Models.Property targetProperty = Property.Models.A_Property.GetImageProperty(thumbHasher, targetFiles.First()); + Shared.Models.Property badProperty = Property.Models.A_Property.GetImageProperty(badFiles.First()); + Shared.Models.Property targetProperty = Property.Models.A_Property.GetImageProperty(targetFiles.First()); if (badProperty.DateTimeOriginal is null || targetProperty.DateTimeOriginal is null) _Logger.Error("Date is null!"); else diff --git a/Instance/DlibDotNet.cs b/Instance/DlibDotNet.cs index d2ad46a..ecd6eca 100644 --- a/Instance/DlibDotNet.cs +++ b/Instance/DlibDotNet.cs @@ -376,45 +376,7 @@ public partial class DlibDotNet List> subFileTuples = new(); List> metadataCollection; FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(item); - if (item.Property is not null && item.Property.Id is not null && resizedFileHolder.Exists && item.Property.ThumbHashBytes is null) - { - (string aResultsFullGroupDirectory, _) = GetResultsFullGroupDirectories(); - string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, "{}"); - string[] files = Directory.GetFiles(aPropertySingletonDirectory, $"{item.Property.Id.Value}*", SearchOption.AllDirectories); - if (files.Length == 1) - { - string matchFile = files.First(); - string json = File.ReadAllText(matchFile); - string find = "\"CreationTime\":"; - if (!json.Contains(find)) - throw new NotImplementedException(); - byte[]? thumbHashBytes = _IThumbHasher.Encode(resizedFileHolder.FullName); - string thumbHashJson = JsonSerializer.Serialize(thumbHashBytes); - json = json.Replace(find, $"\"{nameof(property.ThumbHashBytes)}\": {thumbHashJson}, {find}"); - property = JsonSerializer.Deserialize(json); - if (property is null || property.ThumbHashBytes is null) - throw new NullReferenceException(nameof(property)); - json = JsonSerializer.Serialize(property, new JsonSerializerOptions { WriteIndented = true }); - if (thumbHashBytes is null || thumbHashBytes.Length != property.ThumbHashBytes.Length) - throw new Exception(nameof(property.ThumbHashBytes)); - for (int i = 0; i < thumbHashBytes.Length; i++) - { - if (thumbHashBytes[i] != property.ThumbHashBytes[i]) - throw new Exception(nameof(property.ThumbHashBytes)); - } - thumbHashBytes = JsonSerializer.Deserialize(thumbHashJson); - if (thumbHashBytes is null || thumbHashBytes.Length != property.ThumbHashBytes.Length) - throw new Exception(nameof(property.ThumbHashBytes)); - for (int i = 0; i < thumbHashBytes.Length; i++) - { - if (thumbHashBytes[i] != property.ThumbHashBytes[i]) - throw new Exception(nameof(property.ThumbHashBytes)); - } - File.WriteAllText(matchFile, json); - File.SetLastWriteTime(matchFile, item.Property.LastWriteTime); - } - } - if (item.Property is not null && item.Property.Id is not null && resizedFileHolder.Exists && item.Property.Width is not null && item.Property.Height is not null && item.Property.ThumbHashBytes is not null) + if (item.Property is not null && item.Property.Id is not null && resizedFileHolder.Exists && item.Property.Width is not null && item.Property.Height is not null) { string fileName; string c2ThumbHasherContentDirectory = Path.Combine(c2ResultsFullGroupDirectory, "()"); @@ -423,13 +385,13 @@ public partial class DlibDotNet _ = Directory.CreateDirectory(c2ThumbHasherContentDirectory); if (!Directory.Exists(c2ThumbHasherSingletonDirectory)) _ = Directory.CreateDirectory(c2ThumbHasherSingletonDirectory); - MemoryStream memoryStream = _IThumbHasher.GetMemoryStream(item.Property.ThumbHashBytes, item.Property.Width.Value, item.Property.Height.Value); - string thumbHashJson = JsonSerializer.Serialize(item.Property.ThumbHashBytes)[1..^1]; + MemoryStream memoryStream = _IThumbHasher.GetMemoryStream(Array.Empty(), item.Property.Width.Value, item.Property.Height.Value); + string thumbHashJson = JsonSerializer.Serialize(Array.Empty())[1..^1]; if (!Regex.Matches(thumbHashJson, @"[\\,\/,\:,\*,\?,\"",\<,\>,\|]").Any()) fileName = Path.Combine(c2ThumbHasherSingletonDirectory, $"{thumbHashJson}.png"); else { - // string thumbHash = BitConverter.ToString(item.Property.ThumbHashBytes).Replace("-", string.Empty); + // string thumbHash = BitConverter.ToString(Array.Empty()).Replace("-", string.Empty); // fileName = Path.Combine(c2ThumbHasherContentDirectory, $"{thumbHash}.png"); fileName = Path.Combine(c2ThumbHasherContentDirectory, $"{resizedFileHolder.NameWithoutExtension}.png"); } @@ -467,7 +429,7 @@ public partial class DlibDotNet _Log.Information(string.Concat("LastWriteTimeChanged <", item.ImageFileHolder.FullName, '>')); else if (item.Moved.HasValue && item.Moved.Value) _Log.Information(string.Concat("Moved <", item.ImageFileHolder.FullName, '>')); - property = propertyLogic.GetProperty(_IThumbHasher, item, subFileTuples, parseExceptions); + property = propertyLogic.GetProperty(item, subFileTuples, parseExceptions); item.Update(property); if (propertyHashCode is null) { diff --git a/Property/Models/A_Property.cs b/Property/Models/A_Property.cs index 70ad49d..df01bf8 100644 --- a/Property/Models/A_Property.cs +++ b/Property/Models/A_Property.cs @@ -102,7 +102,7 @@ public class A_Property return results; } - private static Shared.Models.Property GetImageProperty(Shared.Models.Methods.IThumbHasher? thumbHasher, FileHolder fileHolder, Shared.Models.Property? property, bool populateId, bool isIgnoreExtension, bool isValidImageFormatExtension, bool isValidMetadataExtensions, int? id, ASCIIEncoding asciiEncoding, bool writeBitmapDataBytes, string? angleBracket) + private static Shared.Models.Property GetImageProperty(FileHolder fileHolder, Shared.Models.Property? property, bool populateId, bool isIgnoreExtension, bool isValidImageFormatExtension, bool isValidMetadataExtensions, int? id, ASCIIEncoding asciiEncoding, bool writeBitmapDataBytes, string? angleBracket) { Shared.Models.Property result; byte[] bytes; @@ -117,7 +117,6 @@ public class A_Property DateTime? dateTime = null; PropertyItem? propertyItem; string? orientation = null; - byte[]? thumbHashBytes = null; DateTime? gpsDateStamp = null; DateTime? dateTimeOriginal = null; DateTime? dateTimeDigitized = null; @@ -242,18 +241,16 @@ public class A_Property throw new NullReferenceException(nameof(fileHolder.CreationTime)); if (fileHolder.LastWriteTime is null && property?.LastWriteTime is null) throw new NullReferenceException(nameof(fileHolder.LastWriteTime)); - if (thumbHasher is not null && property?.ThumbHashBytes is null) - thumbHashBytes = thumbHasher.Encode(fileHolder.FullName); if (fileHolder.CreationTime is not null && fileHolder.LastWriteTime is not null) - result = new(fileHolder.CreationTime.Value, dateTime, dateTimeDigitized, dateTimeFromName, dateTimeOriginal, fileLength, gpsDateStamp, height, id, fileHolder.LastWriteTime.Value, make, model, orientation, thumbHashBytes, width); + result = new(fileHolder.CreationTime.Value, dateTime, dateTimeDigitized, dateTimeFromName, dateTimeOriginal, fileLength, gpsDateStamp, height, id, fileHolder.LastWriteTime.Value, make, model, orientation, width); else if (property is not null) - result = new(property.CreationTime, dateTime, dateTimeDigitized, dateTimeFromName, dateTimeOriginal, fileLength, gpsDateStamp, height, id, property.LastWriteTime, make, model, orientation, thumbHashBytes, width); + result = new(property.CreationTime, dateTime, dateTimeDigitized, dateTimeFromName, dateTimeOriginal, fileLength, gpsDateStamp, height, id, property.LastWriteTime, make, model, orientation, width); else throw new NullReferenceException(nameof(property)); return result; } - public static Shared.Models.Property GetImageProperty(Shared.Models.Methods.IThumbHasher? thumbHasher, string fileName) + public static Shared.Models.Property GetImageProperty(string fileName) { int? id = null; bool populateId = true; @@ -265,13 +262,13 @@ public class A_Property FileHolder fileHolder = new(fileName); bool isValidImageFormatExtension = true; Shared.Models.Property? property = null; - Shared.Models.Property result = GetImageProperty(thumbHasher, fileHolder, property, populateId, isIgnoreExtension, isValidImageFormatExtension, isValidMetadataExtensions, id, asciiEncoding, writeBitmapDataBytes, angleBracket); + Shared.Models.Property result = GetImageProperty(fileHolder, property, populateId, isIgnoreExtension, isValidImageFormatExtension, isValidMetadataExtensions, id, asciiEncoding, writeBitmapDataBytes, angleBracket); return result; } #pragma warning restore CA1416 - private Shared.Models.Property GetPropertyOfPrivate(Shared.Models.Methods.IThumbHasher? thumbHasher, Item item, List> sourceDirectoryFileTuples, List parseExceptions, bool isIgnoreExtension, bool isValidMetadataExtensions) + private Shared.Models.Property GetPropertyOfPrivate(Item item, List> sourceDirectoryFileTuples, List parseExceptions, bool isIgnoreExtension, bool isValidMetadataExtensions) { Shared.Models.Property? result; int? id = null; @@ -359,25 +356,10 @@ public class A_Property parseExceptions.Add(nameof(A_Property)); } } - if (!string.IsNullOrEmpty(json) && result is not null && thumbHasher is not null && result.ThumbHashBytes is null) - { - string find = "\"CreationTime\":"; - if (!json.Contains(find)) - throw new NotImplementedException(); - byte[] thumbHashBytes = thumbHasher.Encode(item.ImageFileHolder.FullName); - string thumbHashJson = JsonSerializer.Serialize(thumbHashBytes); - json = json.Replace(find, $"\"{nameof(result.ThumbHashBytes)}\": {thumbHashJson}, {find}"); - result = JsonSerializer.Deserialize(json); - if (result is null || result.ThumbHashBytes is null) - throw new NullReferenceException(nameof(result)); - json = JsonSerializer.Serialize(result, _WriteIndentedJsonSerializerOptions); - File.WriteAllText(fileInfo.FullName, json); - File.SetLastWriteTime(fileInfo.FullName, fileInfo.LastWriteTime); - } if (result is null) { id ??= item.ImageFileHolder.Id; - result = GetImageProperty(thumbHasher, item.ImageFileHolder, result, populateId, isIgnoreExtension, item.IsValidImageFormatExtension, isValidMetadataExtensions, id, _ASCIIEncoding, _Configuration.WriteBitmapDataBytes, angleBracket); + result = GetImageProperty(item.ImageFileHolder, result, populateId, isIgnoreExtension, item.IsValidImageFormatExtension, isValidMetadataExtensions, id, _ASCIIEncoding, _Configuration.WriteBitmapDataBytes, angleBracket); json = JsonSerializer.Serialize(result, _WriteIndentedJsonSerializerOptions); if (populateId && Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, json, updateDateWhenMatches: true, compareBeforeWrite: true)) { @@ -406,7 +388,7 @@ public class A_Property return result; } - private void SavePropertyParallelForWork(Shared.Models.Methods.IThumbHasher? thumbHasher, string sourceDirectory, List> sourceDirectoryFileTuples, List> sourceDirectoryChanges, Item item) + private void SavePropertyParallelForWork(string sourceDirectory, List> sourceDirectoryFileTuples, List> sourceDirectoryChanges, Item item) { Shared.Models.Property property; List parseExceptions = new(); @@ -417,7 +399,7 @@ public class A_Property File.Move(item.ImageFileHolder.FullName, filteredSourceDirectoryFileExtensionLowered); if (item.FileSizeChanged is null || item.FileSizeChanged.Value || item.LastWriteTimeChanged is null || item.LastWriteTimeChanged.Value || item.Property is null) { - property = GetPropertyOfPrivate(thumbHasher, item, sourceDirectoryFileTuples, parseExceptions, isIgnoreExtension, isValidMetadataExtensions); + property = GetPropertyOfPrivate(item, sourceDirectoryFileTuples, parseExceptions, isIgnoreExtension, isValidMetadataExtensions); lock (sourceDirectoryChanges) sourceDirectoryChanges.Add(new Tuple(nameof(A_Property), DateTime.Now)); lock (item) @@ -425,7 +407,7 @@ public class A_Property } } - private void SavePropertyParallelWork(int maxDegreeOfParallelism, Shared.Models.Methods.IThumbHasher? thumbHasher, List exceptions, List> sourceDirectoryChanges, Container container, List items, string message) + private void SavePropertyParallelWork(int maxDegreeOfParallelism, List exceptions, List> sourceDirectoryChanges, Container container, List items, string message) { List> sourceDirectoryFileTuples = new(); ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; @@ -438,7 +420,7 @@ public class A_Property long ticks = DateTime.Now.Ticks; DateTime dateTime = DateTime.Now; List> collection; - SavePropertyParallelForWork(thumbHasher, container.SourceDirectory, sourceDirectoryChanges, sourceDirectoryFileTuples, items[i]); + SavePropertyParallelForWork(container.SourceDirectory, sourceDirectoryChanges, sourceDirectoryFileTuples, items[i]); if (i == 0 || sourceDirectoryChanges.Any()) progressBar.Tick(); lock (sourceDirectoryFileTuples) @@ -507,7 +489,7 @@ public class A_Property SetAngleBracketCollection(container.SourceDirectory, anyNullOrNoIsUniqueFileName); totalSeconds = (int)Math.Truncate(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); message = $"{i + 1:000} [{container.Items.Count:000}] / {containersLength:000} - {total} / {t} total - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}"; - SavePropertyParallelWork(_MaxDegreeOfParallelism, thumbHasher, exceptions, sourceDirectoryChanges, container, container.Items, message); + SavePropertyParallelWork(_MaxDegreeOfParallelism, exceptions, sourceDirectoryChanges, container, container.Items, message); foreach (Exception exception in exceptions) _Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, exception.Message, Environment.NewLine, exception.StackTrace), exception); if (exceptions.Count == container.Items.Count) @@ -528,7 +510,7 @@ public class A_Property } } - public Shared.Models.Property GetProperty(Shared.Models.Methods.IThumbHasher? thumbHasher, Item item, List> sourceDirectoryFileTuples, List parseExceptions) + public Shared.Models.Property GetProperty(Item item, List> sourceDirectoryFileTuples, List parseExceptions) { Shared.Models.Property result; bool angleBracketCollectionAny = _AngleBracketCollection.Any(); @@ -540,7 +522,7 @@ public class A_Property } bool isValidMetadataExtensions = _Configuration.ValidMetadataExtensions.Contains(item.ImageFileHolder.ExtensionLowered); bool isIgnoreExtension = item.IsValidImageFormatExtension && _Configuration.IgnoreExtensions.Contains(item.ImageFileHolder.ExtensionLowered); - result = GetPropertyOfPrivate(thumbHasher, item, sourceDirectoryFileTuples, parseExceptions, isIgnoreExtension, isValidMetadataExtensions); + result = GetPropertyOfPrivate(item, sourceDirectoryFileTuples, parseExceptions, isIgnoreExtension, isValidMetadataExtensions); if (!angleBracketCollectionAny) _AngleBracketCollection.Clear(); return result; diff --git a/Shared/Models/Properties/IProperty.cs b/Shared/Models/Properties/IProperty.cs index 838bcf8..f1454da 100644 --- a/Shared/Models/Properties/IProperty.cs +++ b/Shared/Models/Properties/IProperty.cs @@ -16,7 +16,6 @@ public interface IProperty public string? Make { init; get; } public string? Model { init; get; } public string? Orientation { init; get; } - public byte[]? ThumbHashBytes { init; get; } public int? Width { init; get; } } \ No newline at end of file diff --git a/Shared/Models/Property.cs b/Shared/Models/Property.cs index 3b6c49f..c8bec8a 100644 --- a/Shared/Models/Property.cs +++ b/Shared/Models/Property.cs @@ -19,11 +19,10 @@ public class Property : Properties.IProperty public string? Make { init; get; } public string? Model { init; get; } public string? Orientation { init; get; } - public byte[]? ThumbHashBytes { init; get; } public int? Width { init; get; } [JsonConstructor] - public Property(DateTime creationTime, DateTime? dateTime, DateTime? dateTimeDigitized, DateTime? dateTimeFromName, DateTime? dateTimeOriginal, long fileSize, DateTime? gpsDateStamp, int? height, int? id, DateTime lastWriteTime, string? make, string? model, string? orientation, byte[]? thumbHashBytes, int? width) + public Property(DateTime creationTime, DateTime? dateTime, DateTime? dateTimeDigitized, DateTime? dateTimeFromName, DateTime? dateTimeOriginal, long fileSize, DateTime? gpsDateStamp, int? height, int? id, DateTime lastWriteTime, string? make, string? model, string? orientation, int? width) { DateTimeFromName = dateTimeFromName; CreationTime = creationTime; @@ -38,7 +37,6 @@ public class Property : Properties.IProperty Make = make; Model = model; Orientation = orientation; - ThumbHashBytes = thumbHashBytes; Width = width; } diff --git a/Tests/UnitTestResize.cs b/Tests/UnitTestResize.cs index 722539a..a95925f 100644 --- a/Tests/UnitTestResize.cs +++ b/Tests/UnitTestResize.cs @@ -168,8 +168,7 @@ public class UnitTestResize Assert.IsNotNull(item.ImageFileHolder); if (item.Property is null) { - Shared.Models.Methods.IThumbHasher? thumbHasher = new ThumbHash.Models.C2_ThumbHasher(); - property = propertyLogic.GetProperty(thumbHasher, item, subFileTuples, parseExceptions); + property = propertyLogic.GetProperty(item, subFileTuples, parseExceptions); item.Update(property); } if (property is null || item.Property is null) diff --git a/TestsWithFaceRecognitionDotNet/UnitTestFace.cs b/TestsWithFaceRecognitionDotNet/UnitTestFace.cs index 9881281..65bbc4a 100644 --- a/TestsWithFaceRecognitionDotNet/UnitTestFace.cs +++ b/TestsWithFaceRecognitionDotNet/UnitTestFace.cs @@ -242,8 +242,7 @@ public class UnitTestFace Assert.IsNotNull(item.ImageFileHolder); if (item.Property is null) { - Shared.Models.Methods.IThumbHasher? thumbHasher = new ThumbHash.Models.C2_ThumbHasher(); - property = propertyLogic.GetProperty(thumbHasher, item, subFileTuples, parseExceptions); + property = propertyLogic.GetProperty(item, subFileTuples, parseExceptions); item.Update(property); } if (property is null || item.Property is null)