phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Re-Write without checking mapped

Browse Source
This commit is contained in:
Mike Phares
2022-09-14 19:00:57 -07:00
parent 73de1070b8
commit ad28ab2d38
70 changed files with 1596 additions and 2342 deletions
Download Patch File Download Diff File Expand all files Collapse all files

457
Instance/DlibDotNet.cs
View File

@ -26,7 +26,6 @@ public partial class DlibDotNet
private readonly A2_People _People;
private readonly E3_Rename _Rename;
private readonly B_Metadata _Metadata;
private readonly E_Distance _Distance;
private readonly Serilog.ILogger? _Log;
private readonly D2_FaceParts _FaceParts;
private readonly AppSettings _AppSettings;
@ -58,7 +57,6 @@ public partial class DlibDotNet
_Random = new F_Random(configuration);
_People = new A2_People(configuration);
_Rename = new E3_Rename(configuration);
_Distance = new E_Distance(configuration);
if (configuration.IgnoreExtensions is null)
throw new NullReferenceException(nameof(configuration.IgnoreExtensions));
string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(propertyConfiguration, nameof(A_Property), create: false);
@ -115,14 +113,6 @@ public partial class DlibDotNet
_Log.Information(message);
if (_Exceptions.Count != 0)
throw new Exception(message);
if (!_FirstRun && configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions.Any())
{
long ticks = DateTime.Now.Ticks;
foreach (string outputResolution in configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions)
_Distance.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions(propertyConfiguration, model, predictorModel, outputResolution);
if (appSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(E_Distance.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions));
}
if (_FirstRun)
_Log.Information("First run completed. Run again if wanted");
}
@ -210,8 +200,6 @@ public partial class DlibDotNet
throw new NullReferenceException($"{nameof(configuration.OutputResolutions)} must be a valid outputResolution!");
if ((from l in configuration.OutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
throw new Exception($"One or more {nameof(configuration.OutputResolutions)} are not in the ValidResolutions list!");
if ((from l in configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions)} are not in the ValidResolutions list!");
if ((from l in configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions)} are not in the ValidResolutions list!");
if ((from l in configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
@ -262,14 +250,14 @@ public partial class DlibDotNet
throw new Exception("Input directory should be the source and not a resized directory!");
}
private void FullParallelForWork(A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string eResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Face>?> imageFaceCollections, List<bool> faceEncodingRequired, Container container, int index, Item item)
private void FullParallelForWork(A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Face>?> imageFaceCollections, Container container, int index, Item item)
{
if (item.ImageFileHolder is null)
throw new NullReferenceException(nameof(item.ImageFileHolder));
Shared.Models.Property property;
List<Face>? faceCollection;
string original = "Original";
FileHolder? resizedFileHolder;
Shared.Models.Property property;
long ticks = DateTime.Now.Ticks;
DateTime dateTime = DateTime.Now;
List<string> parseExceptions = new();
@ -332,12 +320,12 @@ public partial class DlibDotNet
{
bool saveRotated = _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution);
_FaceParts.SaveFaceLandmarkImages(d2ResultsFullGroupDirectory, container, subFileTuples, parseExceptions, item, faceCollection, saveRotated);
string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(_Configuration.PropertyConfiguration, container.SourceDirectory);
string facesDirectory = Path.GetFullPath(Path.Combine($"{Path.Combine(d2ResultsFullGroupDirectory, "()")}{sourceDirectorySegment}", item.ImageFileHolder.NameWithoutExtension));
_FaceParts.SaveFaceLandmarkImages(facesDirectory, subFileTuples, parseExceptions, item, faceCollection, saveRotated);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D2_FaceParts.SaveFaceLandmarkImages));
}
if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_Distance.GetFaceDistances(eResultsFullGroupDirectory, faceEncodingRequired, subFileTuples, parseExceptions, item, faceCollection);
}
lock (sourceDirectoryChanges)
{
@ -350,11 +338,11 @@ public partial class DlibDotNet
}
}
private int FullParallelWork(long ticks, A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string eResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Face>?> imageFaceCollections, List<bool> faceEncodingRequired, int containersCount, Container container, Item[] filteredItems)
private int FullParallelWork(long ticks, A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Face>?> imageFaceCollections, int containersCount, Container container, Item[] filteredItems)
{
int result = 0;
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
int result = 0;
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism };
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
if (imageFaceCollections.Count != filteredItems.Length || metadataCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || propertyCollection.Count != filteredItems.Length)
@ -370,25 +358,23 @@ public partial class DlibDotNet
}
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string message = $"{container.R:000}.{container.G} / {containersCount:000}) {filteredItems.Length:000} file(s) - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}";
using (ProgressBar progressBar = new(filteredItems.Length, message, options))
{
_ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) =>
using ProgressBar progressBar = new(filteredItems.Length, message, options);
_ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) =>
{
try
{
try
{
FullParallelForWork(propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, faceEncodingRequired, container, index: i, filteredItems[i]);
if (i == 0 || sourceDirectoryChanges.Any())
progressBar.Tick();
}
catch (Exception ex)
{
result += 1;
_Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, ex.Message, Environment.NewLine, ex.StackTrace), ex);
if (result == filteredItems.Length)
throw new Exception(string.Concat("All in [", container.SourceDirectory, "] failed!"));
}
});
}
FullParallelForWork(propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, container, index: i, filteredItems[i]);
if (i == 0 || sourceDirectoryChanges.Any())
progressBar.Tick();
}
catch (Exception ex)
{
result += 1;
_Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, ex.Message, Environment.NewLine, ex.StackTrace), ex);
if (result == filteredItems.Length)
throw new Exception(string.Concat("All in [", container.SourceDirectory, "] failed!"));
}
});
return result;
}
@ -524,188 +510,64 @@ public partial class DlibDotNet
return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory);
}
private void SetAngleBracketCollections(Property.Models.Configuration configuration, A_Property propertyLogic, string outputResolution, Container container, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string eResultsFullGroupDirectory, bool onlyE)
private void SetAngleBracketCollections(Property.Models.Configuration configuration, A_Property propertyLogic, string outputResolution, Container container, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string eResultsFullGroupDirectory)
{
_Faces.AngleBracketCollection.Clear();
_Resize.AngleBracketCollection.Clear();
_Metadata.AngleBracketCollection.Clear();
_Distance.AngleBracketCollection.Clear();
propertyLogic.AngleBracketCollection.Clear();
if (!onlyE)
{
propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
aResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Properties for each image",
collectionDescription: string.Empty,
converted: false));
_Metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
bResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Metadata as key value pairs",
collectionDescription: string.Empty,
converted: true));
_Resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
cResultsFullGroupDirectory,
contentDescription: "Resized image",
singletonDescription: "Resize dimensions for each resolution",
collectionDescription: string.Empty,
converted: true));
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
_Faces.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
aResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Properties for each image",
collectionDescription: string.Empty,
converted: false));
_Metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
bResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Metadata as key value pairs",
collectionDescription: string.Empty,
dResultsFullGroupDirectory,
contentDescription: "n png file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: "For each image a json file with all faces found",
converted: true));
_Resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
_ = Property.Models.Stateless.IResult.GetDirectoryInfoCollection(
_Configuration.PropertyConfiguration,
container.SourceDirectory,
cResultsFullGroupDirectory,
contentDescription: "Resized image",
singletonDescription: "Resize dimensions for each resolution",
d2ResultsFullGroupDirectory,
contentDescription: "n x 2 gif file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: string.Empty,
converted: true));
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
_Faces.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
dResultsFullGroupDirectory,
contentDescription: "n png file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: "For each image a json file with all faces found",
converted: true));
}
converted: false);
if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_Distance.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
_ = Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
eResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "n json file(s) for each face found",
collectionDescription: string.Empty,
converted: true));
}
private (Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)[] GetValues(string argZero, Container[] containers)
{
(Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)[] results;
Sorting sorting;
Item[] filteredItems;
List<(Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)> collection = new();
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = (from l in container.Items where l.ImageFileHolder is not null && (l.Abandoned is null || !l.Abandoned.Value) && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l).ToArray();
if (!filteredItems.Any())
continue;
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null || item.ImageFileHolder is null || item.ResizedFileHolder is null)
continue;
if (!item.Faces.Any())
continue;
foreach (Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location?.Confidence is null || face.Location?.NormalizedPixelPercentage is null)
continue;
if (!face.Tags.Any() || face.Tags[0] is not FaceRecognitionDotNet.FaceEncoding faceEncoding)
continue;
sorting = new(face.Location.Confidence, item.Property.Id.Value, face.Location.NormalizedPixelPercentage.Value);
collection.Add(new(sorting, faceEncoding));
face.Tags.Clear();
}
}
}
results = (from l in collection orderby l.FaceEncoding is not null, l.Sorting.Id select l).ToArray();
if (results.Any() && results[0].FaceEncoding is null)
throw new Exception("Sorting failed!");
return results;
}
private void SaveFaceNumbers(string eResultsFullGroupDirectory, Item item, (Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)[] inputCollection, List<FaceRecognitionDotNet.FaceEncoding> faceEncodings)
{
Sorting sorting;
List<double> doubles;
List<int[]> faceNumbers;
List<Sorting> sortingOutputCollection;
FaceRecognitionDotNet.FaceEncoding faceEncoding;
if (item.Property?.Id is null || item.ImageFileHolder is null || item.ResizedFileHolder is null)
throw new Exception();
foreach (Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location?.Confidence is null || face.Location?.NormalizedPixelPercentage is null)
continue;
sortingOutputCollection = new();
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
doubles = FaceRecognition.FaceDistances(faceEncodings, faceEncoding);
for (int j = 0; j < doubles.Count; j++)
{
sorting = new(inputCollection[j].Sorting, doubles[j]);
lock (sortingOutputCollection)
sortingOutputCollection.Add(sorting);
}
faceNumbers = Shared.Models.Stateless.Methods.ISorting.GetFaceNumbers(sortingOutputCollection);
lock (face)
face.SetFaceNumbers(faceNumbers);
_Distance.SaveFaceNumbers(eResultsFullGroupDirectory, item, face, faceNumbers);
}
}
private void SaveFaceNumbers(string argZero, A_Property propertyLogic, string outputResolution, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string eResultsFullGroupDirectory, Container[] containers, (Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)[] collection)
{
if (collection.Any() && collection[0].FaceEncoding is null)
throw new Exception("Sorting failed!");
Item[] filteredItems;
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism };
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = (from l in container.Items where l.ImageFileHolder is not null && (l.Abandoned is null || !l.Abandoned.Value) && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l).ToArray();
if (!filteredItems.Any())
continue;
List<FaceRecognitionDotNet.FaceEncoding> faceEncodings = new();
foreach ((Sorting _, FaceRecognitionDotNet.FaceEncoding faceEncoding) in collection)
{
if (faceEncoding is null)
continue;
faceEncodings.Add(faceEncoding);
}
SetAngleBracketCollections(_Configuration.PropertyConfiguration, propertyLogic, outputResolution, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, eResultsFullGroupDirectory, onlyE: true);
_ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) => SaveFaceNumbers(eResultsFullGroupDirectory, filteredItems[i], collection, faceEncodings));
}
}
private static void SetFaceTags(Item item)
{
FaceRecognitionDotNet.FaceEncoding faceEncoding;
foreach (Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location?.Confidence is null || face.Location?.NormalizedPixelPercentage is null)
continue;
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
lock (face.Tags)
face.Tags.Add(faceEncoding);
}
}
private void SetFaceTags(string argZero, List<bool> faceEncodingRequired, Container[] containers)
{
Item[] filteredItems;
faceEncodingRequired.Clear();
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism };
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = (from l in container.Items where l.ImageFileHolder is not null && (l.Abandoned is null || !l.Abandoned.Value) && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l).ToArray();
if (!filteredItems.Any())
continue;
_ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) => SetFaceTags(filteredItems[i]));
}
converted: true);
}
private void FullDoWork(string argZero, Model? model, PredictorModel? predictorModel, string propertyRoot, long ticks, MapLogic mapLogic, A_Property propertyLogic, Container[] containers)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
int[] ids;
bool needToSave;
int distinctCount;
int exceptionCount;
Item[] filteredItems;
string aResultsFullGroupDirectory;
@ -714,7 +576,6 @@ public partial class DlibDotNet
string dResultsFullGroupDirectory;
string eResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
List<bool> faceEncodingRequired = new();
Shared.Models.Property[] propertyCollection;
List<List<Face>?> imageFaceCollections = new();
List<FileHolder?> propertyFileHolderCollection = new();
@ -726,7 +587,6 @@ public partial class DlibDotNet
foreach (string outputResolution in _Configuration.OutputResolutions)
{
_FileKeyValuePairs.Clear();
faceEncodingRequired.Clear();
_FilePropertiesKeyValuePairs.Clear();
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory) = GetResultsFullGroupDirectories(_Configuration.PropertyConfiguration, model, predictorModel, outputResolution);
foreach (Container container in containers)
@ -744,14 +604,8 @@ public partial class DlibDotNet
sourceDirectoryChanges.Clear();
nullablePropertyCollection.Clear();
propertyFileHolderCollection.Clear();
SetAngleBracketCollections(_Configuration.PropertyConfiguration, propertyLogic, outputResolution, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, eResultsFullGroupDirectory, onlyE: false);
exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, nullablePropertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, faceEncodingRequired, containers.Length, container, filteredItems);
#pragma warning disable
ids = (from l in filteredItems where l.Property?.Id is not null select l.Property.Id.Value).ToArray();
#pragma warning restore
distinctCount = ids.Distinct().Count();
if (ids.Length != distinctCount)
_Log.Information($"{ids.Length} != {distinctCount} <{container.SourceDirectory}>");
SetAngleBracketCollections(_Configuration.PropertyConfiguration, propertyLogic, outputResolution, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory);
exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, nullablePropertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, containers.Length, container, filteredItems);
if (metadataCollection.Count != filteredItems.Length || nullablePropertyCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || imageFaceCollections.Count != filteredItems.Length)
throw new Exception("Counts don't match!");
if (exceptionCount != 0)
@ -761,8 +615,6 @@ public partial class DlibDotNet
propertyCollection = (from l in nullablePropertyCollection where l is not null select l).ToArray();
if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0)
WriteGroup(_Configuration.PropertyConfiguration, propertyLogic, propertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, outputResolution, container, filteredItems);
if (false && exceptionCount == 0 && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_Distance.LoadOrCreateThenSaveDistanceResults(_Configuration.PropertyConfiguration, eResultsFullGroupDirectory, outputResolution, container, sourceDirectoryChanges, filteredItems);
if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any())
{
for (int y = 0; y < int.MaxValue; y++)
@ -786,79 +638,9 @@ public partial class DlibDotNet
// if (isSilent && container.SourceDirectory.EndsWith("Texas 2015")) //46#.4
// break;
}
if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
{
needToSave = faceEncodingRequired.Any();
if (needToSave)
SetFaceTags(argZero, faceEncodingRequired, containers);
if (faceEncodingRequired.Any())
throw new Exception("Why!");
if (needToSave)
{
(Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)[] collection = GetValues(argZero, containers);
SaveFaceNumbers(argZero, propertyLogic, outputResolution, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, eResultsFullGroupDirectory, containers, collection);
}
}
}
}
private static List<(string, int, Mapping, DateTime, bool?, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>)> Convert(Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> keyValuePairs)
{
List<(string, int, Mapping, DateTime, bool?, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>)> results = new();
MappingContainer mc;
foreach (KeyValuePair<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> keyValuePair in keyValuePairs)
{
foreach ((FaceRecognitionDotNet.FaceEncoding _, MappingContainer mappingContainer) in keyValuePair.Value)
{
mc = mappingContainer;
results.Add(new(mc.Key, mc.Id, mc.Mapping, mc.MinimumDateTime, mc.IsWrongYear, keyValuePair.Value));
}
}
return results;
}
private static Dictionary<int, List<MappingContainer>> Strip(Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> keyValuePairs)
{
Dictionary<int, List<MappingContainer>> results = new();
foreach (KeyValuePair<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> keyValuePair in keyValuePairs)
{
foreach ((FaceRecognitionDotNet.FaceEncoding _, MappingContainer mappingContainer) in keyValuePair.Value)
{
if (!results.ContainsKey(mappingContainer.Id))
results.Add(mappingContainer.Id, new());
results[mappingContainer.Id].Add(mappingContainer);
}
}
return results;
}
private void MapLogic(long ticks, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, MapLogic mapLogic, string outputResolution, Item[] items, int totalNotMapped)
{
mapLogic.UseKeyValuePairsSaveFaceEncoding(items);
if (_Configuration.SaveResizedSubfiles)
{
string dFacesContentDirectory;
dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()");
mapLogic.SaveNotMappedTicks();
_ = LogDeltaInMinutes(ticks, nameof(mapLogic.SaveNotMappedTicks));
Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> keyValuePairs = _Distance.ParallelWork(_AppSettings.MaxDegreeOfParallelism, ticks, mapLogic, items, totalNotMapped);
_ = LogDeltaInSeconds(ticks, nameof(E_Distance.ParallelWork));
Dictionary<int, List<MappingContainer>> strippedKeyValuePairs = Strip(keyValuePairs);
List<(string, int, Mapping, DateTime, bool?, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>)> collection = Convert(keyValuePairs);
mapLogic.SaveMapping(items, dFacesContentDirectory, d2ResultsFullGroupDirectory);
_ = LogDeltaInMinutes(ticks, nameof(mapLogic.SaveMapping));
E_Distance.AddToFaceDistance(_AppSettings.MaxDegreeOfParallelism, mapLogic, items, collection);
_ = LogDeltaInSeconds(ticks, nameof(_Distance.AddToFaceDistance));
mapLogic.AddToClosest(_AppSettings.MaxDegreeOfParallelism, items);
_ = LogDeltaInMinutes(ticks, nameof(mapLogic.AddToClosest));
mapLogic.SaveClosest(items, dFacesContentDirectory, d2ResultsFullGroupDirectory);
_ = LogDeltaInMinutes(ticks, nameof(mapLogic.SaveClosest));
mapLogic.SavePropertyHolders(items);
_ = LogDeltaInSeconds(ticks, nameof(mapLogic.SavePropertyHolders));
}
_ = LogDeltaInSeconds(ticks, nameof(MapLogic));
}
private string GetMapLogicResultsFullGroupDirectory(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel)
{
const int zero = 0;
@ -868,6 +650,79 @@ public partial class DlibDotNet
return zResultsFullGroupDirectory;
}
private List<Face> SetMappingThenGetDistinctFilteredFacesWithMapping(string argZero, Container[] containers)
{
List<Face> results = new();
Mapping mapping;
bool? isWrongYear;
Item[] filteredItems;
DateTime minimumDateTime;
List<int> distinctCheck = new();
string deterministicHashCodeKeyDisplay;
MappingFromItem mappingFromItem;
MappingFromPerson mappingFromPerson;
MappingFromLocation mappingFromLocation;
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = (from l in container.Items where l.ImageFileHolder is not null && (l.Abandoned is null || !l.Abandoned.Value) && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l).ToArray();
if (!filteredItems.Any())
continue;
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null || item.ImageFileHolder is null || item.ResizedFileHolder is null)
continue;
if (!item.Faces.Any())
continue;
foreach (Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
continue;
if (distinctCheck.Contains(item.Property.Id.Value))
continue;
minimumDateTime = Shared.Models.Stateless.Methods.IProperty.GetMinimumDateTime(item.Property);
(isWrongYear, _) = item.Property.IsWrongYear(item.ImageFileHolder.FullName, minimumDateTime);
mappingFromItem = new(item.Property.Id.Value, item.ImageFileHolder, isWrongYear, minimumDateTime, item.ResizedFileHolder);
mappingFromPerson = new(approximateYears: null, by: null, displayDirectoryName: string.Empty, personBirthday: null, segmentB: string.Empty);
deterministicHashCodeKeyDisplay = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKeyDisplay(item.Property.Id.Value, face.Location.NormalizedPixelPercentage.Value);
mappingFromLocation = new(face.Location.Confidence, deterministicHashCodeKeyDisplay, face.Location.NormalizedPixelPercentage.Value);
mapping = new(mappingFromItem, mappingFromLocation, mappingFromPerson);
face.SetMapping(mapping);
results.Add(face);
}
}
}
return results;
}
private Item[] GetIdSortedFilteredItems(string argZero, Container[] containers)
{
Item[] results;
List<Item> collection = new();
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
collection.AddRange(from l in container.Items where l.ImageFileHolder is not null && (l.Abandoned is null || !l.Abandoned.Value) && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l);
}
results = (from l in collection orderby l.Property?.Id is not null, l.Property?.Id select l).ToArray();
return results;
}
private void OldMapLogic(long ticks, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, MapLogic mapLogic, string outputResolution, Item[] filteredItems)
{
string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()");
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
mapLogic.SaveShortcuts(_Configuration.JuliePhares, filteredItems);
mapLogic.SavePropertyHolders(filteredItems);
_ = LogDeltaInSeconds(ticks, nameof(mapLogic.SavePropertyHolders));
}
private void Search(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string argZero, string propertyRoot, Person[] people)
{
Container[] containers;
@ -882,30 +737,36 @@ public partial class DlibDotNet
string peopleDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(A2_People));
MapLogic mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FilenameExtension, _Faces.FilenameExtension, _Faces.HiddenFilenameExtension, _FaceParts.FilenameExtension, ticks, people, peopleDateGroupDirectory, zResultsFullGroupDirectory);
A_Property propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FilenameExtension, _Configuration.Reverse, model, predictorModel, mapLogic.IndicesFromNew, mapLogic.KeyValuePairs);
if (string.IsNullOrEmpty(configuration.RootDirectory))
containers = A_Property.Get(configuration, propertyLogic);
else
containers = Property.Models.Stateless.Container.GetContainers(configuration, _FirstRun, propertyLogic);
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string message = $") Building Container(s) - {totalSeconds} total second(s)";
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
using (ProgressBar progressBar = new(1, message, options))
{
if (string.IsNullOrEmpty(configuration.RootDirectory))
containers = A_Property.Get(configuration, propertyLogic);
else
containers = Property.Models.Stateless.Container.GetContainers(configuration, _FirstRun, propertyLogic);
}
FullDoWork(argZero, model, predictorModel, propertyRoot, ticks, mapLogic, propertyLogic, containers);
foreach (string outputResolution in _Configuration.OutputResolutions)
{
if (_FirstRun)
break;
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory) = GetResultsFullGroupDirectories(configuration, model, predictorModel, outputResolution);
if (_ArgZeroIsConfigurationRootDirectory && _Exceptions.Count == 0 && outputResolution == _Configuration.OutputResolutions[0])
if (_ArgZeroIsConfigurationRootDirectory && _Configuration.SaveResizedSubfiles && outputResolution == _Configuration.OutputResolutions[0] && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution) && _Exceptions.Count == 0)
{
int totalNotMapped = 0;
DateTime dateTime = new(ticks);
List<(int Id, Item Item)> idAndItemCollection = new();
foreach (Container container in containers)
{
totalNotMapped += mapLogic.AddToMapping(ticks, container.Items);
Map.Models.MapLogic.AppendToItems(container.Items, idAndItemCollection);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
mapLogic.SaveShortcuts(_Configuration.JuliePhares, container.Items);
}
Item[] items = (from l in idAndItemCollection orderby l.Id select l.Item).ToArray();
MapLogic(ticks, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, mapLogic, outputResolution, items, totalNotMapped);
if (string.IsNullOrEmpty(eResultsFullGroupDirectory))
throw new NullReferenceException(nameof(eResultsFullGroupDirectory));
string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()");
string eDistanceContentDirectory = Path.Combine(eResultsFullGroupDirectory, "([])");
if (!Directory.Exists(eDistanceContentDirectory))
_ = Directory.CreateDirectory(eDistanceContentDirectory);
string eDistanceContentFileName = Path.Combine(eDistanceContentDirectory, $"{Property.Models.Stateless.IResult.AllInOne}.tvs");
List<Face> distinctFilteredFaces = SetMappingThenGetDistinctFilteredFacesWithMapping(argZero, containers);
SortingContainer[] sortingContainers = E_Distance.SetPersonTicksAndSetFaceDistancesAndSetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, ticks, mapLogic, outputResolution, eDistanceContentFileName, distinctFilteredFaces);
mapLogic.CommonWork(_AppSettings.MaxDegreeOfParallelism, dFacesContentDirectory, d2ResultsFullGroupDirectory, distinctFilteredFaces, sortingContainers);
Item[] filteredItems = GetIdSortedFilteredItems(argZero, containers);
OldMapLogic(ticks, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, mapLogic, outputResolution, filteredItems);
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any())
break;
if (_FileKeyValuePairs.Any())
@ -926,8 +787,6 @@ public partial class DlibDotNet
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(dResultsFullGroupDirectory, "[]"));
if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(eResultsFullGroupDirectory, "[]"));
if (_Configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions.Contains(outputResolution))
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(eResultsFullGroupDirectory, "[]"));
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(d2ResultsFullGroupDirectory, "[]"));
}

4
Instance/Models/Binder/Configuration.cs
View File

@ -19,7 +19,6 @@ public class Configuration
[Display(Name = "Ignore Extensions"), Required] public string[] IgnoreExtensions { get; set; }
[Display(Name = "Ignore Relative Paths"), Required] public string[] IgnoreRelativePaths { get; set; }
[Display(Name = "Julie Phares Copy Birthdays"), Required] public string[] JuliePhares { get; set; }
[Display(Name = "Load Or Create Then Save Directory Distance Results"), Required] public string[] LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions { get; set; }
[Display(Name = "Load Or Create Then Save Distance Results"), Required] public string[] LoadOrCreateThenSaveDistanceResultsForOutputResolutions { get; set; }
[Display(Name = "Load Or Create Then Save Image Faces Results"), Required] public string[] LoadOrCreateThenSaveImageFacesResultsForOutputResolutions { get; set; }
[Display(Name = "Load Or Create Then Save Index"), Required] public bool? LoadOrCreateThenSaveIndex { get; set; }
@ -129,8 +128,6 @@ public class Configuration
throw new NullReferenceException(nameof(configuration.TestDistanceResults));
if (configuration.ValidResolutions is null)
throw new NullReferenceException(nameof(configuration.ValidResolutions));
if (configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions is null)
configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions = Array.Empty<string>();
if (configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions is null)
configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions = Array.Empty<string>();
if (configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions is null)
@ -149,7 +146,6 @@ public class Configuration
configuration.IgnoreExtensions,
configuration.IgnoreRelativePaths,
configuration.JuliePhares,
configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions,
configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions,
configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions,
configuration.LoadOrCreateThenSaveIndex.Value,

4
Instance/Models/Configuration.cs
View File

@ -18,7 +18,6 @@ public class Configuration
public string[] IgnoreExtensions { init; get; }
public string[] IgnoreRelativePaths { init; get; }
public string[] JuliePhares { init; get; }
public string[] LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions { init; get; }
public string[] LoadOrCreateThenSaveDistanceResultsForOutputResolutions { init; get; }
public string[] LoadOrCreateThenSaveImageFacesResultsForOutputResolutions { init; get; }
public bool LoadOrCreateThenSaveIndex { init; get; }
@ -53,7 +52,7 @@ public class Configuration
public string[] ValidResolutions { init; get; }
[JsonConstructor]
public Configuration(Property.Models.Configuration propertyConfiguration, bool checkJsonForDistanceResults, int crossDirectoryMaxItemsInDistanceCollection, int distanceFactor, bool forceFaceLastWriteTimeToCreationTime, bool forceMetadataLastWriteTimeToCreationTime, bool forceResizeLastWriteTimeToCreationTime, string[] ignoreExtensions, string[] ignoreRelativePaths, string[] juliePhares, string[] loadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions, string[] loadOrCreateThenSaveDistanceResultsForOutputResolutions, string[] loadOrCreateThenSaveImageFacesResultsForOutputResolutions, bool loadOrCreateThenSaveIndex, int locationConfidenceFactor, int? mappedMaxIndex, int maxItemsInDistanceCollection, string[] mixedYearRelativePaths, string modelDirectory, string modelName, int numberOfJitters, int numberOfTimesToUpsample, string outputExtension, int outputQuality, string[] outputResolutions, bool overrideForFaceImages, bool overrideForFaceLandmarkImages, bool overrideForResizeImages, int paddingLoops, string predictorModelName, bool propertiesChangedForDistance, bool propertiesChangedForFaces, bool propertiesChangedForIndex, bool propertiesChangedForMetadata, bool propertiesChangedForResize, bool reverse, string[] saveFaceLandmarkForOutputResolutions, bool saveFullYearOfRandomFiles, bool saveResizedSubfiles, string[] saveShortcutsForOutputResolutions, bool skipSearch, bool testDistanceResults, string[] validResolutions)
public Configuration(Property.Models.Configuration propertyConfiguration, bool checkJsonForDistanceResults, int crossDirectoryMaxItemsInDistanceCollection, int distanceFactor, bool forceFaceLastWriteTimeToCreationTime, bool forceMetadataLastWriteTimeToCreationTime, bool forceResizeLastWriteTimeToCreationTime, string[] ignoreExtensions, string[] ignoreRelativePaths, string[] juliePhares, string[] loadOrCreateThenSaveDistanceResultsForOutputResolutions, string[] loadOrCreateThenSaveImageFacesResultsForOutputResolutions, bool loadOrCreateThenSaveIndex, int locationConfidenceFactor, int? mappedMaxIndex, int maxItemsInDistanceCollection, string[] mixedYearRelativePaths, string modelDirectory, string modelName, int numberOfJitters, int numberOfTimesToUpsample, string outputExtension, int outputQuality, string[] outputResolutions, bool overrideForFaceImages, bool overrideForFaceLandmarkImages, bool overrideForResizeImages, int paddingLoops, string predictorModelName, bool propertiesChangedForDistance, bool propertiesChangedForFaces, bool propertiesChangedForIndex, bool propertiesChangedForMetadata, bool propertiesChangedForResize, bool reverse, string[] saveFaceLandmarkForOutputResolutions, bool saveFullYearOfRandomFiles, bool saveResizedSubfiles, string[] saveShortcutsForOutputResolutions, bool skipSearch, bool testDistanceResults, string[] validResolutions)
{
_PropertyConfiguration = propertyConfiguration;
CheckJsonForDistanceResults = checkJsonForDistanceResults;
@ -65,7 +64,6 @@ public class Configuration
IgnoreExtensions = ignoreExtensions;
IgnoreRelativePaths = ignoreRelativePaths;
JuliePhares = juliePhares;
LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions = loadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions;
LoadOrCreateThenSaveDistanceResultsForOutputResolutions = loadOrCreateThenSaveDistanceResultsForOutputResolutions;
LoadOrCreateThenSaveImageFacesResultsForOutputResolutions = loadOrCreateThenSaveImageFacesResultsForOutputResolutions;
LoadOrCreateThenSaveIndex = loadOrCreateThenSaveIndex;

22
Instance/Models/_D2_FaceParts.cs
View File

@ -118,7 +118,7 @@ internal class D2_FaceParts
#pragma warning restore CA1416
internal void SaveFaceLandmarkImages(string d2ResultsFullGroupDirectory, Container container, List<Tuple<string, DateTime>> subFileTuples, List<string> parseExceptions, Item item, List<Face> faceCollection, bool saveRotated)
internal void SaveFaceLandmarkImages(string facesDirectory, List<Tuple<string, DateTime>> subFileTuples, List<string> parseExceptions, Item item, List<Face> faceCollection, bool saveRotated)
{
if (item.ImageFileHolder is null)
throw new NullReferenceException(nameof(item.ImageFileHolder));
@ -130,33 +130,23 @@ internal class D2_FaceParts
const int pointSize = 2;
FileInfo rotatedFileInfo;
DateTime? dateTime = null;
double deterministicHashCodeKey;
long ticks = DateTime.Now.Ticks;
bool updateDateWhenMatches = false;
List<string> angleBracketCollection = new();
string deterministicHashCodeKeyDisplay;
List<(Face, string, string)> collection = new();
angleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(
_Configuration.PropertyConfiguration,
container.SourceDirectory,
d2ResultsFullGroupDirectory,
contentDescription: "n x 2 gif file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: string.Empty,
converted: false));
string facesDirectory = Path.Combine(angleBracketCollection[0].Replace("<>", "()"), item.ImageFileHolder.NameWithoutExtension);
string[] changesFrom = new string[] { nameof(Property.Models.A_Property), nameof(B_Metadata), nameof(C_Resize), nameof(D_Face) };
List<DateTime> dateTimes = (from l in subFileTuples where changesFrom.Contains(l.Item1) select l.Item2).ToList();
if (!Directory.Exists(facesDirectory))
_ = Directory.CreateDirectory(facesDirectory);
foreach (Face face in faceCollection)
{
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
if (item.Property?.Id is null || face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
{
collection.Add(new(face, string.Empty, string.Empty));
continue;
}
deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item, face);
fileInfo = new FileInfo(Path.Combine(facesDirectory, $"{deterministicHashCodeKey}{item.ImageFileHolder.ExtensionLowered}{_FilenameExtension}"));
deterministicHashCodeKeyDisplay = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKeyDisplay(item.Property.Id.Value, face.Location.NormalizedPixelPercentage.Value);
fileInfo = new FileInfo(Path.Combine(facesDirectory, $"{deterministicHashCodeKeyDisplay}{item.ImageFileHolder.ExtensionLowered}{_FilenameExtension}"));
if (!fileInfo.Exists)
{
if (fileInfo.Directory?.Parent is null)
@ -167,7 +157,7 @@ internal class D2_FaceParts
}
if (string.IsNullOrEmpty(fileInfo.DirectoryName))
continue;
rotatedFileInfo = new FileInfo(Path.Combine(fileInfo.DirectoryName, $"{deterministicHashCodeKey} - R{item.ImageFileHolder.ExtensionLowered}{_FilenameExtension}"));
rotatedFileInfo = new FileInfo(Path.Combine(fileInfo.DirectoryName, $"{deterministicHashCodeKeyDisplay} - R{item.ImageFileHolder.ExtensionLowered}{_FilenameExtension}"));
collection.Add(new(face, fileInfo.FullName, rotatedFileInfo.FullName));
if (check)
continue;

19
Instance/Models/_D_Face.cs
View File

@ -2,6 +2,7 @@ using System.Drawing;
using System.Drawing.Drawing2D;
using System.Drawing.Imaging;
using System.Text.Json;
using System.Text.Json.Serialization;
using View_by_Distance.FaceRecognitionDotNet;
using View_by_Distance.Metadata.Models;
using View_by_Distance.Property.Models;
@ -35,7 +36,7 @@ public class D_Face
private readonly EncoderParameters _EncoderParameters;
private readonly ImageCodecInfo _HiddenImageCodecInfo;
private readonly EncoderParameters _HiddenEncoderParameters;
private readonly JsonSerializerOptions _WriteIndentedJsonSerializerOptions;
private readonly JsonSerializerOptions _WriteIndentedAndWhenWritingNull;
internal D_Face(Configuration configuration, string argZero, Model model, ModelParameter modelParameter, PredictorModel predictorModel, ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension, ImageCodecInfo hiddenImageCodecInfo, EncoderParameters hiddenEncoderParameters, string hiddenFilenameExtension)
{
@ -52,7 +53,7 @@ public class D_Face
_HiddenImageCodecInfo = hiddenImageCodecInfo;
_HiddenEncoderParameters = hiddenEncoderParameters;
_HiddenFilenameExtension = hiddenFilenameExtension;
_WriteIndentedJsonSerializerOptions = new JsonSerializerOptions { WriteIndented = true };
_WriteIndentedAndWhenWritingNull = new JsonSerializerOptions { WriteIndented = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull };
}
private static void GetPointBounds(PointF[] points, out float xMinimum, out float xMaximum, out float yMinimum, out float yMaximum)
@ -321,7 +322,7 @@ public class D_Face
if (results is null)
{
results = GetFaces(item, property, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation);
json = JsonSerializer.Serialize(results, _WriteIndentedJsonSerializerOptions);
json = JsonSerializer.Serialize(results, _WriteIndentedAndWhenWritingNull);
bool updateDateWhenMatches = dateTimes.Any() && fileInfo.Exists && dateTimes.Max() > fileInfo.LastWriteTime;
DateTime? dateTime = !updateDateWhenMatches ? null : dateTimes.Max();
if (Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, json, updateDateWhenMatches, compareBeforeWrite: true, updateToWhenMatches: dateTime))
@ -334,7 +335,7 @@ public class D_Face
normalizedPixelPercentageDistinctCount = normalizedPixelPercentageCollection.Distinct().Count();
if (normalizedPixelPercentageDistinctCount != normalizedPixelPercentageCollection.Length)
throw new Exception($"Not distinct! <{fileInfo.FullName}>");
json = JsonSerializer.Serialize(results, _WriteIndentedJsonSerializerOptions);
json = JsonSerializer.Serialize(results, _WriteIndentedAndWhenWritingNull);
bool updateDateWhenMatches = dateTimes.Any() && fileInfo.Exists && dateTimes.Max() > fileInfo.LastWriteTime;
DateTime? dateTime = !updateDateWhenMatches ? null : dateTimes.Max();
if (Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, json, updateDateWhenMatches, compareBeforeWrite: true, updateToWhenMatches: dateTime))
@ -356,7 +357,7 @@ public class D_Face
FileInfo fileInfo;
bool check = false;
string parentCheck;
double deterministicHashCodeKey;
string deterministicHashCodeKeyDisplay;
List<(Face, FileInfo?, string)> collection = new();
string[] changesFrom = new string[] { nameof(A_Property), nameof(B_Metadata), nameof(C_Resize) };
string facesDirectory = Path.Combine(AngleBracketCollection[0].Replace("<>", "()"), item.ImageFileHolder.NameWithoutExtension);
@ -366,13 +367,13 @@ public class D_Face
_ = Directory.CreateDirectory(facesDirectory);
foreach (Face face in faceCollection)
{
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
if (item.Property?.Id is null || face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
{
collection.Add(new(face, null, string.Empty));
continue;
}
deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item, face);
fileInfo = new FileInfo(Path.Combine(facesDirectory, $"{deterministicHashCodeKey}{item.ImageFileHolder.ExtensionLowered}{_FilenameExtension}"));
deterministicHashCodeKeyDisplay = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKeyDisplay(item.Property.Id.Value, face.Location.NormalizedPixelPercentage.Value);
fileInfo = new FileInfo(Path.Combine(facesDirectory, $"{deterministicHashCodeKeyDisplay}{item.ImageFileHolder.ExtensionLowered}{_FilenameExtension}"));
if (!fileInfo.Exists)
{
if (fileInfo.Directory?.Parent is null)
@ -381,7 +382,7 @@ public class D_Face
if (File.Exists(parentCheck))
File.Delete(parentCheck);
}
collection.Add(new(face, fileInfo, Path.Combine(facesDirectory, $"{deterministicHashCodeKey}{item.ImageFileHolder.ExtensionLowered}{_HiddenFilenameExtension}")));
collection.Add(new(face, fileInfo, Path.Combine(facesDirectory, $"{deterministicHashCodeKeyDisplay}{item.ImageFileHolder.ExtensionLowered}{_HiddenFilenameExtension}")));
if (_Configuration.OverrideForFaceImages)
check = true;
else if (!fileInfo.Exists)

855
Instance/Models/_E_Distance.cs
View File

@ -1,8 +1,6 @@
using System.Text.Json;
using ShellProgressBar;
using View_by_Distance.FaceRecognitionDotNet;
using View_by_Distance.Metadata.Models;
using View_by_Distance.Property.Models;
using View_by_Distance.Resize.Models;
using View_by_Distance.Map.Models;
using View_by_Distance.Shared.Models;
using View_by_Distance.Shared.Models.Stateless;
@ -11,764 +9,135 @@ namespace View_by_Distance.Instance.Models;
internal class E_Distance
{
internal List<string> AngleBracketCollection { get; }
private readonly Serilog.ILogger? _Log;
private readonly Configuration _Configuration;
private readonly JsonSerializerOptions _WriteIndentedJsonSerializerOptions;
internal E_Distance(Configuration configuration)
private static void SaveFaceDistances(string eDistanceContentFileName, SortingContainer[] sortingContainers)
{
_Configuration = configuration;
AngleBracketCollection = new List<string>();
_Log = Serilog.Log.ForContext<E_Distance>();
_WriteIndentedJsonSerializerOptions = new JsonSerializerOptions { WriteIndented = true };
#pragma warning disable
string[] results = (from l in sortingContainers select string.Concat(l.Sorting.WithinRange, '\t', l.Sorting.DistancePermyriad, '\t', l.Sorting.DaysDelta, '\t', l.Sorting.Id, '\t', l.Sorting.NormalizedPixelPercentage, '\t', l.Sorting.Older, '\t', l.Face.Mapping.MappingFromItem.Id, '\t', l.Face.Mapping.MappingFromLocation.NormalizedPixelPercentage)).ToArray();
#pragma warning restore
File.WriteAllLines(eDistanceContentFileName, results);
}
public override string ToString()
private static List<Sorting> GetSortingCollection(MapLogic mapLogic, List<FaceDistance> faceDistanceEncodings, int faceDistanceContainersLength, int i, FaceDistance faceDistanceEncoding)
{
string result = JsonSerializer.Serialize(this, new JsonSerializerOptions() { WriteIndented = true });
return result;
}
private static List<(DistanceHolder, FaceRecognitionDotNet.FaceEncoding?)> GetDistanceHolder(Item[] items, List<(string JSONDirectory, string TSVDirectory)> directories)
{
List<(DistanceHolder, FaceRecognitionDotNet.FaceEncoding?)> results = new();
Item item;
const int zero = 0;
string tsvDirectory;
string jsonDirectory;
FaceRecognitionDotNet.FaceEncoding? faceEncoding;
if (items.Length != directories.Count)
throw new Exception();
for (int i = 0; i < items.Length; i++)
{
faceEncoding = null;
item = items[i];
if (item.ImageFileHolder is null || item.Property?.Id is null || !item.Faces.Any())
continue;
tsvDirectory = directories[i].TSVDirectory;
jsonDirectory = directories[i].JSONDirectory;
foreach (Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
continue;
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
results.Add(new(new(face, item.ImageFileHolder, item.Property.Id.Value, jsonDirectory, item.Faces[zero].Location, tsvDirectory), faceEncoding));
}
if (faceEncoding is null)
results.Add(new(new(item.Faces[zero], item.ImageFileHolder, item.Property.Id.Value, jsonDirectory, item.Faces[zero].Location, tsvDirectory), null));
}
List<Sorting> results;
List<FaceDistance> faceDistanceLengths = FaceRecognition.FaceDistances(faceDistanceEncodings, faceDistanceEncoding);
if (faceDistanceLengths.Count != faceDistanceContainersLength)
throw new NotSupportedException();
bool anyLowerThanTolerance = (from l in faceDistanceLengths where l.Length is not null && l.Length.Value != 0 && l.Length.Value < IFaceDistance.Tolerance select true).Any();
results = mapLogic.GetSortingCollection(i, faceDistanceEncoding, faceDistanceLengths, anyLowerThanTolerance);
return results;
}
private void WriteNoFaceCollection(bool updateDateWhenMatches, DateTime? updateToWhenMatches, List<Tuple<string, DateTime>> subFileTuples, List<DistanceHolder> distanceHolders)
private static List<SortingContainer> GetSortingContainers(Face face, FaceDistance faceDistanceEncoding, List<Sorting> sortingCollection)
{
string json;
string check;
string jsonFile;
const int zero = 0;
DistanceHolder distanceHolder;
List<Tuple<Face, string>> tupleCollection;
for (int i = 0; i < distanceHolders.Count; i++)
List<SortingContainer> results = new();
SortingContainer sortingContainer;
Sorting[] collection = Shared.Models.Stateless.Methods.ISorting.Sort(sortingCollection);
foreach (Sorting sorting in collection)
{
distanceHolder = distanceHolders[i];
if (distanceHolder.Face.Location?.NormalizedPixelPercentage is null)
if (faceDistanceEncoding.NormalizedPixelPercentage is null)
throw new NotSupportedException();
if (sorting.DistancePermyriad > IFaceDistance.Permyriad || sorting.DaysDelta > ISorting.DaysDeltaTolerance)
continue;
check = Path.Combine(distanceHolder.JSONDirectory, $"{zero} - {distanceHolder.FileHolder.NameWithoutExtension}.json");
jsonFile = Path.Combine(distanceHolder.JSONDirectory, $"{distanceHolder.Id}.{zero}{distanceHolder.FileHolder.ExtensionLowered}.json");
if (File.Exists(check))
File.Move(check, jsonFile);
tupleCollection = new() { new(distanceHolders[i].Face, string.Empty) };
for (int j = 0; j < distanceHolders.Count; j++)
{
if (j == i)
continue;
distanceHolder = distanceHolders[j];
tupleCollection.Add(new(distanceHolder.Face, string.Empty));
if (tupleCollection.Count > _Configuration.MaxItemsInDistanceCollection)
break;
}
json = JsonSerializer.Serialize(tupleCollection, _WriteIndentedJsonSerializerOptions);
if (Shared.Models.Stateless.Methods.IPath.WriteAllText(jsonFile, json, updateDateWhenMatches, compareBeforeWrite: true, updateToWhenMatches: updateToWhenMatches))
subFileTuples.Add(new Tuple<string, DateTime>(nameof(E_Distance), DateTime.Now));
}
}
private static List<FaceRecognitionDotNet.FaceEncoding> GetFaceEncodings((DistanceHolder, FaceRecognitionDotNet.FaceEncoding?)[] distanceHoldersAfterSort)
{
List<FaceRecognitionDotNet.FaceEncoding> results = new();
foreach ((DistanceHolder distanceHolder, FaceRecognitionDotNet.FaceEncoding? faceEncoding) in distanceHoldersAfterSort)
{
if (distanceHolder.Face.FaceEncoding is null || distanceHolder.Face.Location?.NormalizedPixelPercentage is null || faceEncoding is null)
continue;
results.Add(faceEncoding);
}
return results;
}
private void SaveDistanceResults(bool updateDateWhenMatches, DateTime? updateToWhenMatches, List<Tuple<string, DateTime>> subFileTuples, List<(DistanceHolder DistanceHolder, FaceRecognitionDotNet.FaceEncoding? _)> distanceHolders)
{
string json;
string check;
string jsonFile;
int locationIndex;
List<double> faceDistances;
DistanceHolder distanceHolder;
int normalizedPixelPercentage;
DistanceHolder[] sortedDistanceHolders;
List<Tuple<Face, string>> tupleCollection;
List<(int Index, double Distance)> collection;
FaceRecognitionDotNet.FaceEncoding? faceEncoding;
(DistanceHolder DistanceHolder, FaceRecognitionDotNet.FaceEncoding? FaceEncoding)[] distanceHoldersAfterSort =
distanceHolders.OrderByDescending(l => l.DistanceHolder.Face.FaceEncoding is not null && l.DistanceHolder.Face.Location?.NormalizedPixelPercentage is not null).ToArray();
List<FaceRecognitionDotNet.FaceEncoding> faceEncodings = GetFaceEncodings(distanceHoldersAfterSort);
for (int i = 0; i < distanceHoldersAfterSort.Length; i++)
{
faceEncoding = distanceHoldersAfterSort[i].FaceEncoding;
distanceHolder = distanceHoldersAfterSort[i].DistanceHolder;
collection = new();
tupleCollection = new();
distanceHolder.Sort = 0d;
if (distanceHolder.Face.LocationIndex is null)
locationIndex = 0;
else
locationIndex = distanceHolder.Face.LocationIndex.Value;
if (distanceHolder.Face.FaceEncoding is null || distanceHolder.Face.Location?.NormalizedPixelPercentage is null)
normalizedPixelPercentage = 0;
else
normalizedPixelPercentage = distanceHolder.Face.Location.NormalizedPixelPercentage.Value;
check = Path.Combine(distanceHolder.JSONDirectory, $"{locationIndex} - {distanceHolder.FileHolder.NameWithoutExtension}.json");
jsonFile = Path.Combine(distanceHolder.JSONDirectory, $"{distanceHolder.Id}.{normalizedPixelPercentage}{distanceHolder.FileHolder.ExtensionLowered}.json");
if (!Directory.Exists(distanceHolder.JSONDirectory))
_ = Directory.CreateDirectory(distanceHolder.JSONDirectory);
if (File.Exists(check))
File.Move(check, jsonFile);
if (faceEncodings.Count == 1)
faceDistances = new() { 0d };
else if (faceEncoding is null)
faceDistances = Enumerable.Repeat(9d, faceEncodings.Count).ToList();
else
faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncoding);
if (distanceHolder.Face.FaceEncoding is not null && faceEncoding is not null && faceDistances[i] != 0d)
faceDistances[i] = 0d;
for (int d = 0; d < faceDistances.Count; d++)
collection.Add(new(d, faceDistances[d]));
collection = collection.OrderBy(l => l.Distance).ToList();
foreach ((int index, double distance) in collection)
{
distanceHolder = distanceHoldersAfterSort[index].DistanceHolder;
if (distanceHolder.Location is null)
continue;
distanceHolder.Sort = ((distance * _Configuration.DistanceFactor) + (distanceHolder.Location.Confidence * _Configuration.LocationConfidenceFactor)) / 10;
}
sortedDistanceHolders = (from l in distanceHoldersAfterSort orderby l.DistanceHolder.Sort select l.DistanceHolder).ToArray();
for (int j = 0; j < sortedDistanceHolders.Length; j++)
{
distanceHolder = sortedDistanceHolders[j];
tupleCollection.Add(new(distanceHoldersAfterSort[j].DistanceHolder.Face, string.Empty));
if (tupleCollection.Count > _Configuration.MaxItemsInDistanceCollection)
break;
}
json = JsonSerializer.Serialize(tupleCollection, _WriteIndentedJsonSerializerOptions);
if (Shared.Models.Stateless.Methods.IPath.WriteAllText(jsonFile, json, updateDateWhenMatches, compareBeforeWrite: true, updateToWhenMatches: updateToWhenMatches))
subFileTuples.Add(new Tuple<string, DateTime>(nameof(E_Distance), DateTime.Now));
}
}
internal void LoadOrCreateThenSaveDistanceResults(Property.Models.Configuration configuration, string eResultsFullGroupDirectory, string outputResolution, Container container, List<Tuple<string, DateTime>> sourceDirectoryChanges, Item[] filteredItems)
{
Item item;
string json;
bool check = false;
string parentCheck;
bool hasPopulatedFace;
string usingRelativePath;
DateTime? dateTime = null;
string dCollectionDirectory;
FileInfo[] fileInfoCollection;
bool updateDateWhenMatches = false;
System.IO.DirectoryInfo directoryInfo;
System.IO.DirectoryInfo tvsDirectoryInfo;
int?[] normalizedPixelPercentageCollection;
int normalizedPixelPercentageDistinctCount;
List<(string, string)> directories = new();
string[] changesFrom = new string[] { nameof(A_Property), nameof(B_Metadata), nameof(C_Resize), nameof(D_Face) };
List<DateTime> dateTimes = (from l in sourceDirectoryChanges where changesFrom.Contains(l.Item1) select l.Item2).ToList();
List<string> directoryInfoCollection = Property.Models.Stateless.IResult.GetDirectoryInfoCollection(
configuration,
container.SourceDirectory,
eResultsFullGroupDirectory,
contentDescription: ".tvs File",
singletonDescription: string.Empty,
collectionDescription: "n json file(s) for each face found (one to many)",
converted: true);
for (int i = 0; i < filteredItems.Length; i++)
{
item = filteredItems[i];
if (item.ImageFileHolder is null || item.Property?.Id is null)
continue;
hasPopulatedFace = (from l in item.Faces where l.FaceEncoding is not null && l.Location?.NormalizedPixelPercentage is not null select true).Any();
usingRelativePath = Path.Combine(directoryInfoCollection[0].Replace("<>", "[]"), item.ImageFileHolder.NameWithoutExtension);
dCollectionDirectory = Path.Combine(eResultsFullGroupDirectory, "[]", Property.Models.Stateless.IResult.AllInOne, $"{item.Property.Id.Value}{item.ImageFileHolder.ExtensionLowered}");
directoryInfo = new System.IO.DirectoryInfo(dCollectionDirectory);
if (!directoryInfo.Exists)
{
if (Directory.Exists(usingRelativePath))
{
Directory.Move(usingRelativePath, directoryInfo.FullName);
directoryInfo.Refresh();
}
if (!Directory.Exists(dCollectionDirectory))
{
if (directoryInfo.Parent?.Parent is null)
throw new Exception();
parentCheck = Path.Combine(directoryInfo.Parent.Parent.FullName, directoryInfo.Name);
if (Directory.Exists(parentCheck))
{
foreach (string file in Directory.GetFiles(parentCheck))
File.Delete(file);
Directory.Delete(parentCheck);
}
}
}
tvsDirectoryInfo = new System.IO.DirectoryInfo(Path.Combine(directoryInfoCollection[0].Replace("<>", "()"), item.ImageFileHolder.NameWithoutExtension));
directories.Add(new(directoryInfo.FullName, tvsDirectoryInfo.FullName));
if (directoryInfo.Exists && (!check || _Configuration.CheckJsonForDistanceResults))
{
json = string.Empty;
normalizedPixelPercentageCollection = Shared.Models.Stateless.Methods.IFace.GetInts(item.Faces);
normalizedPixelPercentageDistinctCount = normalizedPixelPercentageCollection.Distinct().Count();
if (normalizedPixelPercentageDistinctCount != normalizedPixelPercentageCollection.Length)
check = true;
fileInfoCollection = directoryInfo.GetFiles($"{item.Property.Id.Value}*.json", SearchOption.TopDirectoryOnly);
if (fileInfoCollection.Length < normalizedPixelPercentageDistinctCount)
check = true;
if (!check && _Configuration.CheckJsonForDistanceResults)
{
for (int j = 0; j < fileInfoCollection.Length; j++)
{
json = Shared.Models.Stateless.Methods.IIndex.GetJson(fileInfoCollection[j].FullName, fileInfoCollection[j]);
if (!_Configuration.PropertiesChangedForDistance && Shared.Models.Stateless.Methods.IFace.GetFace(fileInfoCollection[j].FullName) is null)
{
check = true;
break;
}
}
if (!check && string.IsNullOrEmpty(json))
check = true;
}
}
if (check)
continue;
if (_Configuration.PropertiesChangedForDistance)
check = true;
else if (hasPopulatedFace && !directoryInfo.Exists)
check = true;
else if (dateTimes.Any() && dateTimes.Max() > directoryInfo.LastWriteTime)
check = true;
if (check && !updateDateWhenMatches)
{
updateDateWhenMatches = dateTimes.Any() && directoryInfo.Exists && dateTimes.Max() > directoryInfo.LastWriteTime;
dateTime = !updateDateWhenMatches ? null : dateTimes.Max();
}
}
if (check)
{
DateTime? updateToWhenMatches = dateTime;
List<Tuple<string, DateTime>> subFileTuples = new();
List<(DistanceHolder, FaceRecognitionDotNet.FaceEncoding?)> distanceHolders = GetDistanceHolder(filteredItems, directories);
SaveDistanceResults(updateDateWhenMatches, updateToWhenMatches, subFileTuples, distanceHolders);
}
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(directoryInfoCollection[0].Replace("<>", "()"));
}
private List<(string, List<KeyValuePair<string, Face[]>>)> GetFiles(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string outputResolution)
{
string json;
List<KeyValuePair<string, Face[]>>? facesKeyValuePairCollection;
List<(string, List<KeyValuePair<string, Face[]>>)> results = new();
string dFacesCollectionDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), "[[]]");
string[] dFacesCollectionFiles = Directory.GetFiles(dFacesCollectionDirectory, "*.json", SearchOption.TopDirectoryOnly);
foreach (string dFacesCollectionFile in dFacesCollectionFiles)
{
json = File.ReadAllText(dFacesCollectionFile);
facesKeyValuePairCollection = JsonSerializer.Deserialize<List<KeyValuePair<string, Face[]>>>(json);
if (facesKeyValuePairCollection is null)
continue;
results.Add(new(dFacesCollectionFile, facesKeyValuePairCollection));
}
return results;
}
private static List<(string, List<Face>, List<FaceRecognitionDotNet.FaceEncoding>)> GetMatches(List<(string, List<KeyValuePair<string, Face[]>>)> files)
{
List<(string, List<Face>, List<FaceRecognitionDotNet.FaceEncoding>)> results = new();
FaceRecognitionDotNet.FaceEncoding faceEncoding;
List<Face> faces;
List<FaceRecognitionDotNet.FaceEncoding> faceEncodings;
foreach ((string, List<KeyValuePair<string, Face[]>>) file in files)
{
faces = new();
faceEncodings = new();
foreach (KeyValuePair<string, Face[]> keyValuePair in file.Item2)
{
foreach (Face face in keyValuePair.Value)
{
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
continue;
faces.Add(face);
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
faceEncodings.Add(faceEncoding);
}
}
results.Add(new(file.Item1, faces, faceEncodings));
}
return results;
}
private static int GetIndex(double[] faceDistances)
{
int result;
List<double[]> faceDistancesWithIndex = new();
for (int y = 0; y < faceDistances.Length; y++)
faceDistancesWithIndex.Add(new double[] { faceDistances[y], y });
faceDistancesWithIndex = (from l in faceDistancesWithIndex orderby l[0] select l).ToList();
result = (int)faceDistancesWithIndex[0][1];
return result;
}
private void Save(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string outputResolution, string eDistanceCollectionDirectory, int k, string relativePath, Face face, List<Tuple<Face, string>> faceAndFaceDistanceCollection)
{
if (string.IsNullOrEmpty(eDistanceCollectionDirectory))
eDistanceCollectionDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), "[]");
string fileNameWithoutExtension = Path.GetFileNameWithoutExtension(face.RelativePath);
string jsonDirectory = string.Concat(eDistanceCollectionDirectory, Path.Combine(relativePath, fileNameWithoutExtension));
if (!Directory.Exists(jsonDirectory))
_ = Directory.CreateDirectory(jsonDirectory);
string json = JsonSerializer.Serialize(faceAndFaceDistanceCollection, _WriteIndentedJsonSerializerOptions);
string jsonFile = Path.Combine(jsonDirectory, $"{k} - {fileNameWithoutExtension}.nosj");
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(jsonFile, json, updateDateWhenMatches: true, compareBeforeWrite: true);
}
private static Tuple<Face, double> Get(FaceRecognitionDotNet.FaceEncoding faceEncoding, (string, List<Face>, List<FaceRecognitionDotNet.FaceEncoding>) match)
{
Tuple<Face, double> result;
double[] faceDistances = FaceRecognition.FaceDistances(match.Item3, faceEncoding).ToArray();
int index = GetIndex(faceDistances);
result = new(match.Item2[index], faceDistances[index]);
return result;
}
internal void LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string outputResolution)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
string? relativePath;
Face face;
ParallelOptions parallelOptions = new();
FaceRecognitionDotNet.FaceEncoding faceEncoding;
string eDistanceCollectionDirectory = string.Empty;
Tuple<Face, double> faceAndFaceDistance;
List<Tuple<Face, string>> faceAndFaceDistanceCollection;
List<(string, List<KeyValuePair<string, Face[]>>)> files = GetFiles(configuration, model, predictorModel, outputResolution);
List<(string, List<Face>, List<FaceRecognitionDotNet.FaceEncoding>)> matches = GetMatches(files);
if (files.Count != matches.Count)
throw new Exception();
int filesCount = files.Count;
for (int i = 0; i < filesCount; i++)
{
_Log.Debug(string.Concat("LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions - ", nameof(outputResolution), ' ', outputResolution, " - ", i, " of ", filesCount));
for (int j = 0; j < files[i].Item2.Count; j++)
{
if (!matches[i].Item2.Any())
continue;
for (int k = 0; k < files[i].Item2[j].Value.Length; k++)
{
face = files[i].Item2[j].Value[k];
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
continue;
faceAndFaceDistanceCollection = new(matches.Count);
relativePath = Path.GetDirectoryName(face.RelativePath);
if (string.IsNullOrEmpty(relativePath))
continue;
if (face.FaceEncoding is null)
continue;
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
_ = Parallel.For(0, matches.Count, parallelOptions, z =>
{
if (z != i && matches[z].Item2.Any())
{
faceAndFaceDistance = Get(faceEncoding, matches[z]);
// if (faceAndFaceDistance.Item2 < _Configuration.)
faceAndFaceDistanceCollection.Add(new(faceAndFaceDistance.Item1, faceAndFaceDistance.Item2.ToString("0.000")));
}
});
if (faceAndFaceDistanceCollection.Any())
{
faceAndFaceDistanceCollection = (from l in faceAndFaceDistanceCollection orderby l.Item2 select l).Take(_Configuration.CrossDirectoryMaxItemsInDistanceCollection).ToList();
Save(configuration, model, predictorModel, outputResolution, eDistanceCollectionDirectory, k, relativePath, face, faceAndFaceDistanceCollection);
}
}
}
}
}
internal void SaveFaceNumbers(string eResultsFullGroupDirectory, Item item, Face face, List<int[]> faceDistances)
{
if (item.Property?.Id is null)
throw new NullReferenceException(nameof(item.Property.Id));
if (item.ImageFileHolder is null)
throw new NullReferenceException(nameof(item.ImageFileHolder));
if (face.Location?.NormalizedPixelPercentage is null)
throw new NullReferenceException(nameof(face.Location.NormalizedPixelPercentage));
if (string.IsNullOrEmpty(eResultsFullGroupDirectory))
throw new NullReferenceException(nameof(eResultsFullGroupDirectory));
string json;
List<(Face, string, string)> collection = new();
string[] changesFrom = new string[] { nameof(A_Property), nameof(B_Metadata), nameof(C_Resize), nameof(D_Face) };
string eSingletonFile = Path.Combine(eResultsFullGroupDirectory, "{}", Property.Models.Stateless.IResult.AllInOne, $"{item.Property.Id.Value}.{face.Location.NormalizedPixelPercentage.Value}{item.ImageFileHolder.ExtensionLowered}.json");
FileInfo fileInfo = new(eSingletonFile);
json = JsonSerializer.Serialize(faceDistances, _WriteIndentedJsonSerializerOptions);
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, json, updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null);
}
internal void GetFaceDistances(string eResultsFullGroupDirectory, List<bool> faceEncodingRequired, List<Tuple<string, DateTime>> subFileTuples, List<string> parseExceptions, Item item, List<Face> faceCollection)
{
List<int[]>? results;
if (item.Property?.Id is null)
throw new NullReferenceException(nameof(item.Property.Id));
if (item.ImageFileHolder is null)
throw new NullReferenceException(nameof(item.ImageFileHolder));
if (string.IsNullOrEmpty(eResultsFullGroupDirectory))
throw new NullReferenceException(nameof(eResultsFullGroupDirectory));
string json;
FileInfo fileInfo;
string eSingletonFile;
List<(Face, string, string)> collection = new();
string[] changesFrom = new string[] { nameof(A_Property), nameof(B_Metadata), nameof(C_Resize), nameof(D_Face) };
List<DateTime> dateTimes = (from l in subFileTuples where changesFrom.Contains(l.Item1) select l.Item2).ToList();
foreach (Face face in faceCollection)
{
results = null;
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
continue;
eSingletonFile = Path.Combine(eResultsFullGroupDirectory, "{}", Property.Models.Stateless.IResult.AllInOne, $"{item.Property.Id.Value}.{face.Location.NormalizedPixelPercentage.Value}{item.ImageFileHolder.ExtensionLowered}.json");
fileInfo = new(eSingletonFile);
if (_Configuration.PropertiesChangedForDistance)
results = null;
else if (!fileInfo.Exists)
results = null;
else if (dateTimes.Any() && dateTimes.Max() > fileInfo.LastWriteTime)
results = null;
else
{
json = File.ReadAllText(fileInfo.FullName);
try
{
results = JsonSerializer.Deserialize<List<int[]>>(json);
if (results is null)
throw new NullReferenceException(nameof(results));
subFileTuples.Add(new Tuple<string, DateTime>(nameof(E_Distance), fileInfo.LastWriteTime));
lock (face)
face.SetFaceNumbers(results);
}
catch (Exception)
{
results = null;
parseExceptions.Add(nameof(E_Distance));
}
}
if (results is null && !faceEncodingRequired.Any())
faceEncodingRequired.Add(true);
}
}
public static double GetStandardDeviation(IEnumerable<double> values, double average)
{
double result = 0;
if (!values.Any())
throw new Exception("Collection must have at least one value!");
double sum = values.Sum(l => (l - average) * (l - average));
result = Math.Sqrt(sum / values.Count());
return result;
}
private static int GetSelectedIndex(int maxDegreeOfParallelism, Random random, List<FaceRecognitionDotNet.FaceEncoding> faceEncodings)
{
int? result;
int selectedIndex;
List<(int? Index, double? Sum)> faceDistanceCollections = new();
if (maxDegreeOfParallelism == 1)
{
double sum;
List<double> faceDistances;
for (int i = 0; i < faceEncodings.Count; i++)
{
faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncodings[i]);
sum = faceDistances.Sum();
faceDistanceCollections.Add(new(i, sum));
}
}
else
{
for (int i = 0; i < faceEncodings.Count; i++)
faceDistanceCollections.Add(new(null, null));
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
_ = Parallel.For(0, faceEncodings.Count, parallelOptions, (i, state) =>
{
List<double> faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncodings[i]);
double sum = faceDistances.Sum();
lock (faceDistanceCollections)
faceDistanceCollections[i] = new(i, sum);
});
}
faceDistanceCollections = faceDistanceCollections.OrderBy(l => l.Sum).ToList();
if (faceDistanceCollections.Count != faceEncodings.Count)
throw new Exception();
if (faceDistanceCollections.Count > 1000)
selectedIndex = random.Next(0, 36);
else if (faceDistanceCollections.Count > 500)
selectedIndex = random.Next(0, 31);
else if (faceDistanceCollections.Count > 200)
selectedIndex = random.Next(0, 26);
else if (faceDistanceCollections.Count > 100)
selectedIndex = random.Next(0, 21);
else if (faceDistanceCollections.Count > 50)
selectedIndex = random.Next(0, 16);
else if (faceDistanceCollections.Count > 25)
selectedIndex = random.Next(0, 11);
else if (faceDistanceCollections.Count > 10)
selectedIndex = random.Next(0, 6);
else if (faceDistanceCollections.Count > 5)
selectedIndex = random.Next(0, 3);
else
selectedIndex = 0;
result = faceDistanceCollections[selectedIndex].Index;
if (result is null)
throw new NullReferenceException(nameof(result));
return result.Value;
}
private static void SetFiltered(List<(FaceRecognitionDotNet.FaceEncoding FaceEncoding, MappingContainer MappingContainer)> collection)
{
double ucl;
bool check;
double average;
double[] doubles;
double standardDeviation;
double?[] nullableDoubles;
for (int i = 0; i < int.MaxValue; i++)
{
check = true;
nullableDoubles = (from l in collection where l.MappingContainer.Mapping.Filtered is not null && !l.MappingContainer.Mapping.Filtered.Value select l.MappingContainer.Distance).ToArray();
doubles = (from l in nullableDoubles where l.HasValue select l.Value).ToArray();
if (doubles.Length < 4)
break;
average = doubles.Average();
standardDeviation = GetStandardDeviation(doubles, average);
ucl = average + (standardDeviation * 3);
if (ucl > IClosest.Tolerance)
ucl = IClosest.Tolerance;
foreach ((FaceRecognitionDotNet.FaceEncoding _, MappingContainer mappingContainer) in collection)
{
if (mappingContainer.Mapping.Filtered is null || mappingContainer.Mapping.Filtered.Value || mappingContainer.Distance <= ucl)
continue;
if (check)
check = false;
mappingContainer.Mapping.SetFiltered();
}
if (check)
sortingContainer = new(face, sorting);
results.Add(sortingContainer);
if (results.Count >= ISorting.MaximumPerFaceShouldBeHigh)
break;
}
return results;
}
private static FaceDistance GetFaceDistanceParallelFor(Face face, FaceRecognitionDotNet.FaceEncoding faceEncoding, Mapping mapping, DateTime minimumDateTime, bool? isWrongYear, string key, FaceRecognitionDotNet.FaceEncoding[] faceEncodings)
private static SortingContainer[] GetSortingContainersThenSetFaceMappingSortingCollection(int maxDegreeOfParallelism, long ticks, MapLogic mapLogic, string outputResolution, FaceDistanceContainer[] faceDistanceContainers)
{
FaceDistance result;
if (face.Location?.NormalizedPixelPercentage is null)
throw new NullReferenceException(nameof(face.Location.NormalizedPixelPercentage));
List<double> faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncoding);
result = new(faceDistances, isWrongYear, key, mapping, minimumDateTime);
return result;
}
private static List<FaceDistance> GetFaceDistanceCollection(int maxDegreeOfParallelism, List<(string Key, int Id, Mapping Mapping, DateTime MinimumDateTime, bool? IsWrongYear, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>)> collection, Face face)
{
List<FaceDistance> results;
if (face.FaceEncoding is null)
throw new NullReferenceException(nameof(face.FaceEncoding));
FaceRecognitionDotNet.FaceEncoding faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
if (maxDegreeOfParallelism == 1)
SortingContainer[] results;
List<SortingContainer> collection = new();
List<FaceDistance> faceDistanceEncodings = new();
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
string message = $") {faceDistanceContainers.Length:000} faceDistanceContainer(s) - {totalSeconds} total second(s) - {outputResolution}";
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
foreach (FaceDistanceContainer faceDistanceContainer in faceDistanceContainers)
{
if (faceDistanceContainer.FaceDistance.Encoding is null)
continue;
faceDistanceEncodings.Add(faceDistanceContainer.FaceDistance);
}
using ProgressBar progressBar = new(faceDistanceContainers.Length, message, options);
_ = Parallel.For(0, faceDistanceContainers.Length, parallelOptions, (i, state) =>
{
progressBar.Tick();
Face face = faceDistanceContainers[i].Face;
if (face.Mapping is null)
throw new NotSupportedException();
FaceDistance faceDistanceEncoding = faceDistanceContainers[i].FaceDistance;
List<Sorting> sortingCollection = GetSortingCollection(mapLogic, faceDistanceEncodings, faceDistanceContainers.Length, i, faceDistanceEncoding);
List<SortingContainer> sortingContainers = GetSortingContainers(face, faceDistanceEncoding, sortingCollection);
lock (collection)
collection.AddRange(sortingContainers);
lock (face)
face.ReleaseFaceDistance();
});
results = Shared.Models.Stateless.Methods.ISortingContainer.Sort(collection);
return results;
}
private static FaceDistanceContainer[] GetFaceDistanceContainers(Face[] firstFilteredFaces)
{
FaceDistanceContainer[] results;
FaceDistance faceDistance;
FaceDistanceContainer faceDistanceContainer;
List<FaceDistanceContainer> collection = new();
foreach (Face face in firstFilteredFaces)
{
if (face.Mapping is null)
throw new NotSupportedException();
if (face.FaceDistance?.Encoding is not FaceRecognitionDotNet.FaceEncoding faceEncoding)
throw new NotSupportedException();
faceDistance = new(face.Mapping.MappingFromLocation.Confidence, faceEncoding, face.Mapping.MappingFromItem.Id, face.Mapping.MappingFromItem.IsWrongYear, face.Mapping.MappingFromItem.MinimumDateTime, face.Mapping.MappingFromLocation.NormalizedPixelPercentage);
faceDistanceContainer = new(face, faceDistance);
collection.Add(faceDistanceContainer);
}
results = (from l in collection orderby l.FaceDistance.Encoding is not null select l).ToArray();
if (results.Any() && results[0].FaceDistance.Encoding is null)
throw new Exception("Sorting failed!");
return results;
}
private static void SetFaceDistances(int maxDegreeOfParallelism, long ticks, string outputResolution, Face[] selectedFilteredFaces)
{
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
string message = $") {selectedFilteredFaces.Length:000} Load Face Encoding - {totalSeconds} total second(s) - {outputResolution}";
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = false };
using ProgressBar progressBar = new(selectedFilteredFaces.Length, message, options);
_ = Parallel.For(0, selectedFilteredFaces.Length, parallelOptions, (i, state) =>
{
results = new();
FaceDistance faceDistance;
List<double> faceDistances;
FaceRecognitionDotNet.FaceEncoding[] faceEncodings;
if (face.Location?.NormalizedPixelPercentage is null)
throw new NullReferenceException(nameof(face.Location.NormalizedPixelPercentage));
foreach ((string key, int id, Mapping mapping, DateTime minimumDateTime, bool? isWrongYear, List<(FaceRecognitionDotNet.FaceEncoding FaceEncoding, MappingContainer _)> faceEncodingContainers) in collection)
{
faceEncodings = (from l in faceEncodingContainers select l.FaceEncoding).ToArray();
faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncoding);
faceDistance = new(faceDistances, isWrongYear, key, mapping, minimumDateTime);
results.Add(faceDistance);
if (results.Count > IFaceDistance.MaximumPer)
break;
}
}
else
{
results = new();
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
_ = Parallel.For(0, collection.Count, parallelOptions, (i, state) =>
{
(string key, int id, Mapping mapping, DateTime minimumDateTime, bool? isWrongYear, List<(FaceRecognitionDotNet.FaceEncoding FaceEncoding, MappingContainer _)> faceEncodingContainers) = collection[i];
FaceRecognitionDotNet.FaceEncoding[] faceEncodings = (from l in faceEncodingContainers select l.FaceEncoding).ToArray();
FaceDistance? closest = GetFaceDistanceParallelFor(face, faceEncoding, mapping, minimumDateTime, isWrongYear, key, faceEncodings);
if (closest is not null)
{
lock (results)
{
results.Add(closest);
if (results.Count > IFaceDistance.MaximumPer)
state.Break();
}
}
});
}
return results;
}
private static FaceRecognitionDotNet.FaceEncoding? GetFaceEncoding(Face face)
{
FaceRecognitionDotNet.FaceEncoding? result;
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
result = null;
else
result = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
return result;
}
private static List<FaceRecognitionDotNet.FaceEncoding> GetFaceEncodingsOnly(int maxDegreeOfParallelism, List<MappingContainer> collection)
{
List<FaceRecognitionDotNet.FaceEncoding> results;
if (maxDegreeOfParallelism == 1)
{
results = new();
FaceRecognitionDotNet.FaceEncoding faceEncoding;
foreach (MappingContainer mappingContainer in collection)
{
if (mappingContainer.Face?.FaceEncoding is null || mappingContainer.Face.Location?.NormalizedPixelPercentage is null)
continue;
faceEncoding = FaceRecognition.LoadFaceEncoding(mappingContainer.Face.FaceEncoding.RawEncoding);
results.Add(faceEncoding);
}
}
else
{
results = new();
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
_ = Parallel.For(0, collection.Count, parallelOptions, (i, state) =>
{
Face? face = collection[i].Face;
if (face is null)
throw new Exception();
FaceRecognitionDotNet.FaceEncoding? faceEncoding = GetFaceEncoding(face);
if (faceEncoding is not null)
{
lock (results)
results.Add(faceEncoding);
}
});
}
Face face = selectedFilteredFaces[i];
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
throw new NotSupportedException();
if (face.Mapping is null)
throw new NotSupportedException();
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
faceDistance = new(face.Location.Confidence, faceEncoding, face.Mapping.MappingFromItem.Id, face.Mapping.MappingFromItem.IsWrongYear, face.Mapping.MappingFromItem.MinimumDateTime, face.Location.NormalizedPixelPercentage.Value);
lock (face)
face.FaceDistanceAdd(faceDistance);
});
}
internal static SortingContainer[] SetPersonTicksAndSetFaceDistancesAndSetFaceMappingSortingCollectionThenGetSortingContainers(int maxDegreeOfParallelism, long ticks, MapLogic mapLogic, string outputResolution, string eDistanceContentFileName, List<Face> distinctFilteredFaces)
{
SortingContainer[] results;
Face[] orderedFilteredFaces = (from l in distinctFilteredFaces orderby l.Mapping is not null, l.Mapping?.MappingFromItem.MinimumDateTime descending select l).ToArray();
mapLogic.SetPersonTicks(outputResolution, orderedFilteredFaces);
Face[] selectedFilteredFaces = orderedFilteredFaces.Skip(ISorting.FacesToSkipAfterSortBeforeLoad).Take(ISorting.FacesToTakeAfterSortBeforeLoad).ToArray();
SetFaceDistances(maxDegreeOfParallelism, ticks, outputResolution, selectedFilteredFaces);
FaceDistanceContainer[] faceDistanceContainers = GetFaceDistanceContainers(selectedFilteredFaces);
results = GetSortingContainersThenSetFaceMappingSortingCollection(maxDegreeOfParallelism, ticks, mapLogic, outputResolution, faceDistanceContainers);
if (results.Length < IFaceDistance.Permyriad)
SaveFaceDistances(eDistanceContentFileName, results);
return results;
}
private static void SetNonFiltered(List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)> collection)
{
foreach ((FaceRecognitionDotNet.FaceEncoding _, MappingContainer mappingContainer) in collection)
{
if (mappingContainer.Mapping.Filtered is null)
mappingContainer.Mapping.SetFiltered(value: false);
}
}
private Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> GetThreeSigmaFaceEncodings(int maxDegreeOfParallelism, long ticks, Random random, Dictionary<string, List<MappingContainer>> keyValuePairs)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> results = new();
int totalSeconds;
int selectedIndex;
List<double> faceDistances;
MappingContainer mappingContainer;
int keyValuePairsCount = keyValuePairs.Count;
FaceRecognitionDotNet.FaceEncoding faceEncoding;
List<FaceRecognitionDotNet.FaceEncoding> faceEncodings;
List<(FaceRecognitionDotNet.FaceEncoding FaceEncoding, MappingContainer MappingContainer)> collection;
foreach (KeyValuePair<string, List<MappingContainer>> keyValuePair in keyValuePairs)
{
collection = new();
faceEncodings = GetFaceEncodingsOnly(maxDegreeOfParallelism, keyValuePair.Value);
for (int i = 0; i < faceEncodings.Count; i++)
{
faceEncoding = faceEncodings[i];
mappingContainer = keyValuePair.Value[i];
collection.Add(new(faceEncoding, mappingContainer));
}
results.Add(keyValuePair.Key, collection);
if (faceEncodings.Count == 1)
selectedIndex = 0;
else
selectedIndex = GetSelectedIndex(maxDegreeOfParallelism, random, faceEncodings);
faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncodings[selectedIndex]);
for (int i = 0; i < faceEncodings.Count; i++)
collection[i].MappingContainer.SetDistance(faceDistances[i]);
if (collection.Count > 3)
SetFiltered(collection);
SetNonFiltered(collection);
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
_Log.Information($"{keyValuePairsCount:0000}) {totalSeconds} total second(s) - {keyValuePair.Key} - {collection[selectedIndex].MappingContainer.Mapping.DisplayDirectoryName}");
}
return results;
}
internal Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> ParallelWork(int maxDegreeOfParallelism, long ticks, Map.Models.MapLogic mapLogic, Item[] items, int totalNotMapped)
{
Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> results;
Random random = new((int)ticks);
mapLogic.ForceSingleImage(ticks, items, totalNotMapped, random);
Dictionary<string, List<MappingContainer>> keyValuePairs = Map.Models.Stateless.IMapLogic.GetKeyValuePairs(items);
results = GetThreeSigmaFaceEncodings(maxDegreeOfParallelism, ticks, random, keyValuePairs);
return results;
}
public static void AddToFaceDistance(int maxDegreeOfParallelism, Map.Models.MapLogic mapLogic, Item[] items, List<(string, int, Mapping, DateTime, bool?, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>)> collection)
{
Face face;
double deterministicHashCodeKey;
DateTime dateTime = DateTime.Now;
List<FaceDistance> faceDistances;
foreach (Item item in items)
{
if (item.Property?.Id is null || item.ImageFileHolder is null || item.ResizedFileHolder is null)
continue;
for (int i = 0; i < item.Faces.Count; i++)
{
face = item.Faces[i];
face.FaceDistances.Clear();
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
continue;
if ((from l in item.Mapping where l.NormalizedPixelPercentage.HasValue && l.NormalizedPixelPercentage.Value == face.Location.NormalizedPixelPercentage.Value select true).Any())
continue;
deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item, face);
if (mapLogic.Skip(deterministicHashCodeKey))
continue;
faceDistances = GetFaceDistanceCollection(maxDegreeOfParallelism, collection, face);
face.FaceDistances.AddRange(faceDistances);
}
}
}
}

3
Instance/appsettings.json
View File

@ -152,9 +152,6 @@
"1976-01-05_00",
"1982-05-02_00"
],
"LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions": [
"1920 x 1080"
],
"LoadOrCreateThenSaveDistanceResultsForOutputResolutions": [
"1920 x 1080"
],