phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Switched to ThumbHasher over BlurHasher

Browse Source
This commit is contained in:
Mike Phares
2023-05-21 23:56:10 -07:00
parent 514637b9c6
commit a0c880c7ba
26 changed files with 803 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

102
Instance/DlibDotNet.cs
View File

@ -2,7 +2,6 @@
using Phares.Shared;
using ShellProgressBar;
using System.Collections.ObjectModel;
using System.Drawing;
using System.Drawing.Imaging;
using System.Text.Json;
using System.Text.RegularExpressions;
@ -32,8 +31,8 @@ public partial class DlibDotNet
private readonly Serilog.ILogger? _Log;
private readonly D2_FaceParts _FaceParts;
private readonly AppSettings _AppSettings;
private readonly IBlurHasher _IBlurHasher;
private readonly List<string> _Exceptions;
private readonly IThumbHasher _IThumbHasher;
private readonly IsEnvironment _IsEnvironment;
private readonly bool _PropertyRootExistedBefore;
private readonly Models.Configuration _Configuration;
@ -59,7 +58,7 @@ public partial class DlibDotNet
long ticks = DateTime.Now.Ticks;
_Exceptions = new List<string>();
_Log = Serilog.Log.ForContext<DlibDotNet>();
_IBlurHasher = new BlurHash.Models.BlurHasher();
_IThumbHasher = new ThumbHash.Models.C2_ThumbHasher();
Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot);
Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration);
_Log.Information(propertyConfiguration.RootDirectory);
@ -355,9 +354,8 @@ public partial class DlibDotNet
MapLogic mapLogic,
string outputResolution,
string cResultsFullGroupDirectory,
string dResultsDateGroupDirectory,
string c2ResultsFullGroupDirectory,
string dResultsFullGroupDirectory,
string eDistanceContentDirectory,
List<Tuple<string, DateTime>> sourceDirectoryChanges,
Dictionary<string, List<MappingFromPhotoPrism>> fileNameToCollection,
Container container,
@ -377,7 +375,8 @@ public partial class DlibDotNet
List<string> parseExceptions = new();
List<Tuple<string, DateTime>> subFileTuples = new();
List<KeyValuePair<string, string>> metadataCollection;
if (item.Property is not null && item.Property.Id is not null && !item.Any() && item.Property.BlurHash is null)
FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(item);
if (item.Property is not null && item.Property.Id is not null && resizedFileHolder.Exists && item.Property.ThumbHashBytes is null)
{
(string aResultsFullGroupDirectory, _) = GetResultsFullGroupDirectories();
string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, "{}");
@ -389,19 +388,60 @@ public partial class DlibDotNet
string find = "\"CreationTime\":";
if (!json.Contains(find))
throw new NotImplementedException();
#pragma warning disable CA1416
using Image image = Image.FromFile(item.ImageFileHolder.FullName);
#pragma warning restore CA1416
string blurHash = _IBlurHasher.Encode(image);
json = json.Replace(find, $"\"{nameof(item.Property.BlurHash)}\": \"{blurHash}\", {find}");
byte[]? thumbHashBytes = _IThumbHasher.Encode(resizedFileHolder.FullName);
string thumbHashJson = JsonSerializer.Serialize(thumbHashBytes);
json = json.Replace(find, $"\"{nameof(property.ThumbHashBytes)}\": {thumbHashJson}, {find}");
property = JsonSerializer.Deserialize<Shared.Models.Property>(json);
if (property is null || property.BlurHash is null)
if (property is null || property.ThumbHashBytes is null)
throw new NullReferenceException(nameof(property));
json = JsonSerializer.Serialize(property, new JsonSerializerOptions { WriteIndented = true });
if (thumbHashBytes is null || thumbHashBytes.Length != property.ThumbHashBytes.Length)
throw new Exception(nameof(property.ThumbHashBytes));
for (int i = 0; i < thumbHashBytes.Length; i++)
{
if (thumbHashBytes[i] != property.ThumbHashBytes[i])
throw new Exception(nameof(property.ThumbHashBytes));
}
thumbHashBytes = JsonSerializer.Deserialize<byte[]>(thumbHashJson);
if (thumbHashBytes is null || thumbHashBytes.Length != property.ThumbHashBytes.Length)
throw new Exception(nameof(property.ThumbHashBytes));
for (int i = 0; i < thumbHashBytes.Length; i++)
{
if (thumbHashBytes[i] != property.ThumbHashBytes[i])
throw new Exception(nameof(property.ThumbHashBytes));
}
File.WriteAllText(matchFile, json);
File.SetLastWriteTime(matchFile, item.Property.LastWriteTime);
}
}
if (item.Property is not null && item.Property.Id is not null && resizedFileHolder.Exists && item.Property.Width is not null && item.Property.Height is not null && item.Property.ThumbHashBytes is not null)
{
string fileName;
string c2ThumbHasherContentDirectory = Path.Combine(c2ResultsFullGroupDirectory, "()");
string c2ThumbHasherSingletonDirectory = Path.Combine(c2ResultsFullGroupDirectory, "{}");
if (!Directory.Exists(c2ThumbHasherContentDirectory))
_ = Directory.CreateDirectory(c2ThumbHasherContentDirectory);
if (!Directory.Exists(c2ThumbHasherSingletonDirectory))
_ = Directory.CreateDirectory(c2ThumbHasherSingletonDirectory);
MemoryStream memoryStream = _IThumbHasher.GetMemoryStream(item.Property.ThumbHashBytes, item.Property.Width.Value, item.Property.Height.Value);
string thumbHashJson = JsonSerializer.Serialize(item.Property.ThumbHashBytes)[1..^1];
if (!Regex.Matches(thumbHashJson, @"[\\,\/,\:,\*,\?,\"",\<,\>,\|]").Any())
fileName = Path.Combine(c2ThumbHasherSingletonDirectory, $"{thumbHashJson}.png");
else
{
// string thumbHash = BitConverter.ToString(item.Property.ThumbHashBytes).Replace("-", string.Empty);
// fileName = Path.Combine(c2ThumbHasherContentDirectory, $"{thumbHash}.png");
fileName = Path.Combine(c2ThumbHasherContentDirectory, $"{resizedFileHolder.NameWithoutExtension}.png");
}
if (!File.Exists(fileName))
{
using FileStream fileStream = new(fileName, FileMode.CreateNew);
memoryStream.WriteTo(fileStream);
memoryStream.Dispose();
if (resizedFileHolder.LastWriteTime is not null)
File.SetLastWriteTime(fileName, resizedFileHolder.LastWriteTime.Value);
}
}
if (item.Property is not null && item.Property.Id is not null && !item.Any())
{
property = item.Property;
@ -427,7 +467,7 @@ public partial class DlibDotNet
_Log.Information(string.Concat("LastWriteTimeChanged <", item.ImageFileHolder.FullName, '>'));
else if (item.Moved.HasValue && item.Moved.Value)
_Log.Information(string.Concat("Moved <", item.ImageFileHolder.FullName, '>'));
property = propertyLogic.GetProperty(_IBlurHasher, item, subFileTuples, parseExceptions);
property = propertyLogic.GetProperty(_IThumbHasher, item, subFileTuples, parseExceptions);
item.Update(property);
if (propertyHashCode is null)
{
@ -442,7 +482,6 @@ public partial class DlibDotNet
}
if (property is null || item.Property is null)
throw new NullReferenceException(nameof(property));
FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(item);
item.SetResizedFileHolder(_Resize.FileNameExtension, resizedFileHolder);
string facesDirectory = _Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution) ? _Faces.GetFacesDirectory(dResultsFullGroupDirectory, item) : string.Empty;
string facePartsDirectory = _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution) ? _FaceParts.GetFacePartsDirectory(_Configuration.PropertyConfiguration, dResultsFullGroupDirectory, item, includeNameWithoutExtension: true) : string.Empty;
@ -485,7 +524,7 @@ public partial class DlibDotNet
if ((_Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch)
&& _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
&& collection is not null && faceCollection.All(l => !l.Saved))
_Distance.LookForMatchFacesAndPossiblyRename(_Faces.FileNameExtension, eDistanceContentDirectory, mappingFromItem, faces, collection);
_Distance.LookForMatchFacesAndPossiblyRename(_Faces.FileNameExtension, mappingFromItem, faces, collection);
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
{
bool saveRotated = false;
@ -509,10 +548,9 @@ public partial class DlibDotNet
MapLogic mapLogic,
string outputResolution,
string cResultsFullGroupDirectory,
string dResultsDateGroupDirectory,
string c2ResultsFullGroupDirectory,
string dResultsFullGroupDirectory,
string d2ResultsFullGroupDirectory,
string eDistanceContentDirectory,
List<Tuple<string, DateTime>> sourceDirectoryChanges,
Dictionary<string, List<MappingFromPhotoPrism>> fileNameToCollection,
Container container,
@ -540,9 +578,8 @@ public partial class DlibDotNet
mapLogic,
outputResolution,
cResultsFullGroupDirectory,
dResultsDateGroupDirectory,
c2ResultsFullGroupDirectory,
dResultsFullGroupDirectory,
eDistanceContentDirectory,
sourceDirectoryChanges,
fileNameToCollection,
container,
@ -620,7 +657,7 @@ public partial class DlibDotNet
return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory);
}
private (string, string, string) GetResultsFullGroupDirectories(string outputResolution)
private (string, string, string, string) GetResultsFullGroupDirectories(string outputResolution)
{
string cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
_Configuration.PropertyConfiguration,
@ -629,6 +666,13 @@ public partial class DlibDotNet
includeResizeGroup: true,
includeModel: false,
includePredictorModel: false);
string c2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
_Configuration.PropertyConfiguration,
nameof(ThumbHash.Models.C2_ThumbHasher),
outputResolution,
includeResizeGroup: true,
includeModel: false,
includePredictorModel: false);
string dResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
_Configuration.PropertyConfiguration,
nameof(D_Face),
@ -643,10 +687,10 @@ public partial class DlibDotNet
includeResizeGroup: true,
includeModel: true,
includePredictorModel: true);
return new(cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory);
return new(cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory);
}
private void FullDoWork(string argZero, string propertyRoot, long ticks, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, int t, Container[] containers, A_Property propertyLogic, B_Metadata metadata, string eDistanceContentDirectory, Dictionary<string, List<MappingFromPhotoPrism>> fileNameToCollection, ReadOnlyDictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers, MapLogic mapLogic)
private void FullDoWork(string argZero, string propertyRoot, long ticks, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, int t, Container[] containers, A_Property propertyLogic, B_Metadata metadata, Dictionary<string, List<MappingFromPhotoPrism>> fileNameToCollection, ReadOnlyDictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers, MapLogic mapLogic)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
@ -659,6 +703,7 @@ public partial class DlibDotNet
bool anyNullOrNoIsUniqueFileName;
string cResultsFullGroupDirectory;
string dResultsFullGroupDirectory;
string c2ResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
int containersLength = containers.Length;
List<Tuple<string, DateTime>> sourceDirectoryChanges = new();
@ -667,7 +712,7 @@ public partial class DlibDotNet
foreach (string outputResolution in _Configuration.OutputResolutions)
{
total = 0;
(cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
(cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
for (int i = 0; i < containers.Length; i++)
{
container = containers[i];
@ -695,10 +740,9 @@ public partial class DlibDotNet
mapLogic,
outputResolution,
cResultsFullGroupDirectory,
dResultsDateGroupDirectory,
c2ResultsFullGroupDirectory,
dResultsFullGroupDirectory,
d2ResultsFullGroupDirectory,
eDistanceContentDirectory,
sourceDirectoryChanges,
fileNameToCollection,
container,
@ -911,13 +955,14 @@ public partial class DlibDotNet
string? directoryName;
string cResultsFullGroupDirectory;
string dResultsFullGroupDirectory;
string c2ResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
List<int> distinctFilteredIds = Shared.Models.Stateless.Methods.IContainer.GetFilteredDistinctIds(_Configuration.PropertyConfiguration, containers);
LookForAbandoned(idToLocationContainers, distinctFilteredIds);
LookForAbandoned(bResultsFullGroupDirectory, distinctFilteredIds);
foreach (string outputResolution in _Configuration.OutputResolutions)
{
(cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
(cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
directories = Directory.GetDirectories(cResultsFullGroupDirectory, "*", SearchOption.TopDirectoryOnly);
foreach (string directory in directories)
{
@ -1101,6 +1146,7 @@ public partial class DlibDotNet
string bResultsFullGroupDirectory;
string cResultsFullGroupDirectory;
string dResultsFullGroupDirectory;
string c2ResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
string fPhotoPrismContentDirectory;
string fPhotoPrismSingletonDirectory;
@ -1152,7 +1198,7 @@ public partial class DlibDotNet
Shared.Models.Stateless.Methods.IGenealogicalDataCommunication.CreateTree(_Configuration.MappingDefaultName, _Configuration.PersonBirthdayFormat, _Configuration.PropertyConfiguration.ResultAllInOne, _PersonContainers, _GenealogicalDataCommunicationHeaderLines, _GenealogicalDataCommunicationFooterLines, ticks, a2PeopleContentDirectory, personKeyToIds);
ReadOnlyDictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers = mapLogic.GetIdToLocationContainers();
fileNameToCollection = !Directory.Exists(fPhotoPrismSingletonDirectory) ? fileNameToCollection = new() : F_PhotoPrism.GetFileNameToCollection(fPhotoPrismSingletonDirectory);
FullDoWork(argZero, propertyRoot, ticks, aResultsFullGroupDirectory, bResultsFullGroupDirectory, t, containers, propertyLogic, metadata, eDistanceContentDirectory, fileNameToCollection, idToLocationContainers, mapLogic);
FullDoWork(argZero, propertyRoot, ticks, aResultsFullGroupDirectory, bResultsFullGroupDirectory, t, containers, propertyLogic, metadata, fileNameToCollection, idToLocationContainers, mapLogic);
if (_Configuration.LookForAbandoned)
LookForAbandoned(bResultsFullGroupDirectory, containers, idToLocationContainers);
_Distance.Clear();
@ -1175,7 +1221,7 @@ public partial class DlibDotNet
mapLogic.SaveShortcutsForOutputResolutionsPreMapLogic(eDistanceContentDirectory, personKeyToIds, distinctFilteredMappingCollection);
if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution))
mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, _PersonContainers, a2PeopleContentDirectory, personKeyToIds, distinctFilteredMappingCollection, totalNotMapped);
(cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
(cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
if (_ArgZeroIsConfigurationRootDirectory
&& _Configuration.SaveResizedSubfiles
&& outputResolution == _Configuration.OutputResolutions[0]