phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SegmentB, personKey,

Browse Source 
personKeyFormatted and Sorting
This commit is contained in:
Mike Phares
2022-09-01 01:18:08 -07:00
parent d4c950e8e3
commit 73de1070b8
17 changed files with 847 additions and 573 deletions
Download Patch File Download Diff File Expand all files Collapse all files

306
Instance/DlibDotNet.cs
View File

@ -15,7 +15,7 @@ using View_by_Distance.Shared.Models.Stateless;
namespace View_by_Distance.Instance;
public class DlibDotNet
public partial class DlibDotNet
{
private readonly D_Face _Faces;
@ -262,12 +262,12 @@ public class DlibDotNet
throw new Exception("Input directory should be the source and not a resized directory!");
}
private void FullParallelForWork(A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Face>> imageFaceCollections, string sourceDirectory, int index, Item item)
private void FullParallelForWork(A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string eResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Face>?> imageFaceCollections, List<bool> faceEncodingRequired, Container container, int index, Item item)
{
if (item.ImageFileHolder is null)
throw new NullReferenceException(nameof(item.ImageFileHolder));
Shared.Models.Property property;
List<Face> faceCollection;
List<Face>? faceCollection;
string original = "Original";
FileHolder? resizedFileHolder;
long ticks = DateTime.Now.Ticks;
@ -313,7 +313,7 @@ public class DlibDotNet
File.WriteAllBytes(path, bytes);
}
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
faceCollection = new();
faceCollection = null;
else
{
int[] outputResolutionCollection = imageResizeKeyValuePairs[outputResolution];
@ -330,11 +330,14 @@ public class DlibDotNet
int normalizedPixelPercentageDistinctCount = normalizedPixelPercentageCollection.Distinct().Count();
if (normalizedPixelPercentageDistinctCount != normalizedPixelPercentageCollection.Length || _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
{
bool saveRotated = _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution);
_FaceParts.SaveFaceLandmarkImages(d2ResultsFullGroupDirectory, sourceDirectory, subFileTuples, parseExceptions, item, faceCollection, saveRotated);
_FaceParts.SaveFaceLandmarkImages(d2ResultsFullGroupDirectory, container, subFileTuples, parseExceptions, item, faceCollection, saveRotated);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D2_FaceParts.SaveFaceLandmarkImages));
}
if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_Distance.GetFaceDistances(eResultsFullGroupDirectory, faceEncodingRequired, subFileTuples, parseExceptions, item, faceCollection);
}
lock (sourceDirectoryChanges)
{
@ -347,20 +350,20 @@ public class DlibDotNet
}
}
private int FullParallelWork(long ticks, A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Face>> faceCollections, int containersCount, Container container, Item[] filteredItems)
private int FullParallelWork(long ticks, A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string eResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Face>?> imageFaceCollections, List<bool> faceEncodingRequired, int containersCount, Container container, Item[] filteredItems)
{
int result = 0;
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism };
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
if (faceCollections.Count != filteredItems.Length || metadataCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || propertyCollection.Count != filteredItems.Length)
if (imageFaceCollections.Count != filteredItems.Length || metadataCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || propertyCollection.Count != filteredItems.Length)
{
for (int i = 0; i < filteredItems.Length; i++)
{
faceCollections.Add(new());
propertyCollection.Add(null);
metadataCollection.Add(new());
imageFaceCollections.Add(null);
resizeKeyValuePairs.Add(new());
propertyFileHolderCollection.Add(null);
}
@ -373,7 +376,7 @@ public class DlibDotNet
{
try
{
FullParallelForWork(propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, container.SourceDirectory, index: i, filteredItems[i]);
FullParallelForWork(propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, faceEncodingRequired, container, index: i, filteredItems[i]);
if (i == 0 || sourceDirectoryChanges.Any())
progressBar.Tick();
}
@ -424,7 +427,7 @@ public class DlibDotNet
}
}
private void WriteGroup(Property.Models.Configuration configuration, A_Property propertyLogic, Shared.Models.Property[] propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<List<Face>> faceCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, string outputResolution, Container container, Item[] filteredItems)
private void WriteGroup(Property.Models.Configuration configuration, A_Property propertyLogic, Shared.Models.Property[] propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Face>?> imageFaceCollections, string outputResolution, Container container, Item[] filteredItems)
{
Item item;
string key;
@ -437,7 +440,7 @@ public class DlibDotNet
if (!(from l in propertyCollection where l?.Width is null select true).Any())
{
string checkDirectory;
List<KeyValuePair<string, List<Face>>> faceCollectionsKeyValuePairs = new();
List<KeyValuePair<string, List<Face>?>> imageFaceCollectionsKeyValuePairs = new();
List<KeyValuePair<string, Shared.Models.Property>> propertyCollectionKeyValuePairs = new();
List<KeyValuePair<string, Dictionary<string, int[]>>> resizeKeyValuePairsCollections = new();
List<KeyValuePair<string, List<KeyValuePair<string, string>>>> metadataCollectionKeyValuePairs = new();
@ -456,7 +459,7 @@ public class DlibDotNet
_FileKeyValuePairs.Add(new KeyValuePair<string, string>(container.SourceDirectory, key));
_FilePropertiesKeyValuePairs[container.SourceDirectory].Add(new Tuple<string, Shared.Models.Property>(key, propertyCollection[i]));
}
faceCollectionsKeyValuePairs.Add(new KeyValuePair<string, List<Face>>(key, faceCollections[i]));
imageFaceCollectionsKeyValuePairs.Add(new KeyValuePair<string, List<Face>?>(key, imageFaceCollections[i]));
propertyCollectionKeyValuePairs.Add(new KeyValuePair<string, Shared.Models.Property>(key, propertyCollection[i]));
resizeKeyValuePairsCollections.Add(new KeyValuePair<string, Dictionary<string, int[]>>(key, resizeKeyValuePairs[i]));
metadataCollectionKeyValuePairs.Add(new KeyValuePair<string, List<KeyValuePair<string, string>>>(key, metadataCollection[i]));
@ -498,7 +501,7 @@ public class DlibDotNet
if (File.Exists(checkFile))
File.Move(checkFile, Path.Combine(checkDirectory, fileName));
checkFile = Path.Combine(checkDirectory, fileName);
json = JsonSerializer.Serialize(faceCollectionsKeyValuePairs, writeIndentedJsonSerializerOptions);
json = JsonSerializer.Serialize(imageFaceCollectionsKeyValuePairs, writeIndentedJsonSerializerOptions);
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true);
}
}
@ -521,48 +524,187 @@ public class DlibDotNet
return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory);
}
private void SetAngleBracketCollections(Property.Models.Configuration configuration, A_Property propertyLogic, string outputResolution, Container container, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory)
private void SetAngleBracketCollections(Property.Models.Configuration configuration, A_Property propertyLogic, string outputResolution, Container container, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string eResultsFullGroupDirectory, bool onlyE)
{
_Faces.AngleBracketCollection.Clear();
_Resize.AngleBracketCollection.Clear();
_Metadata.AngleBracketCollection.Clear();
_Distance.AngleBracketCollection.Clear();
propertyLogic.AngleBracketCollection.Clear();
propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
aResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Properties for each image",
collectionDescription: string.Empty,
converted: false));
_Metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
bResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Metadata as key value pairs",
collectionDescription: string.Empty,
converted: true));
_Resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
cResultsFullGroupDirectory,
contentDescription: "Resized image",
singletonDescription: "Resize dimensions for each resolution",
collectionDescription: string.Empty,
converted: true));
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
_Faces.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
if (!onlyE)
{
propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
dResultsFullGroupDirectory,
contentDescription: "n png file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: "For each image a json file with all faces found",
aResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Properties for each image",
collectionDescription: string.Empty,
converted: false));
_Metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
bResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Metadata as key value pairs",
collectionDescription: string.Empty,
converted: true));
_Resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
cResultsFullGroupDirectory,
contentDescription: "Resized image",
singletonDescription: "Resize dimensions for each resolution",
collectionDescription: string.Empty,
converted: true));
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
_Faces.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
dResultsFullGroupDirectory,
contentDescription: "n png file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: "For each image a json file with all faces found",
converted: true));
}
if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_Distance.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
eResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "n json file(s) for each face found",
collectionDescription: string.Empty,
converted: true));
}
private void FullDoWork(string argZero, Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string propertyRoot, long ticks, MapLogic mapLogic, A_Property propertyLogic, Container[] containers)
private (Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)[] GetValues(string argZero, Container[] containers)
{
(Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)[] results;
Sorting sorting;
Item[] filteredItems;
List<(Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)> collection = new();
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = (from l in container.Items where l.ImageFileHolder is not null && (l.Abandoned is null || !l.Abandoned.Value) && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l).ToArray();
if (!filteredItems.Any())
continue;
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null || item.ImageFileHolder is null || item.ResizedFileHolder is null)
continue;
if (!item.Faces.Any())
continue;
foreach (Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location?.Confidence is null || face.Location?.NormalizedPixelPercentage is null)
continue;
if (!face.Tags.Any() || face.Tags[0] is not FaceRecognitionDotNet.FaceEncoding faceEncoding)
continue;
sorting = new(face.Location.Confidence, item.Property.Id.Value, face.Location.NormalizedPixelPercentage.Value);
collection.Add(new(sorting, faceEncoding));
face.Tags.Clear();
}
}
}
results = (from l in collection orderby l.FaceEncoding is not null, l.Sorting.Id select l).ToArray();
if (results.Any() && results[0].FaceEncoding is null)
throw new Exception("Sorting failed!");
return results;
}
private void SaveFaceNumbers(string eResultsFullGroupDirectory, Item item, (Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)[] inputCollection, List<FaceRecognitionDotNet.FaceEncoding> faceEncodings)
{
Sorting sorting;
List<double> doubles;
List<int[]> faceNumbers;
List<Sorting> sortingOutputCollection;
FaceRecognitionDotNet.FaceEncoding faceEncoding;
if (item.Property?.Id is null || item.ImageFileHolder is null || item.ResizedFileHolder is null)
throw new Exception();
foreach (Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location?.Confidence is null || face.Location?.NormalizedPixelPercentage is null)
continue;
sortingOutputCollection = new();
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
doubles = FaceRecognition.FaceDistances(faceEncodings, faceEncoding);
for (int j = 0; j < doubles.Count; j++)
{
sorting = new(inputCollection[j].Sorting, doubles[j]);
lock (sortingOutputCollection)
sortingOutputCollection.Add(sorting);
}
faceNumbers = Shared.Models.Stateless.Methods.ISorting.GetFaceNumbers(sortingOutputCollection);
lock (face)
face.SetFaceNumbers(faceNumbers);
_Distance.SaveFaceNumbers(eResultsFullGroupDirectory, item, face, faceNumbers);
}
}
private void SaveFaceNumbers(string argZero, A_Property propertyLogic, string outputResolution, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string eResultsFullGroupDirectory, Container[] containers, (Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)[] collection)
{
if (collection.Any() && collection[0].FaceEncoding is null)
throw new Exception("Sorting failed!");
Item[] filteredItems;
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism };
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = (from l in container.Items where l.ImageFileHolder is not null && (l.Abandoned is null || !l.Abandoned.Value) && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l).ToArray();
if (!filteredItems.Any())
continue;
List<FaceRecognitionDotNet.FaceEncoding> faceEncodings = new();
foreach ((Sorting _, FaceRecognitionDotNet.FaceEncoding faceEncoding) in collection)
{
if (faceEncoding is null)
continue;
faceEncodings.Add(faceEncoding);
}
SetAngleBracketCollections(_Configuration.PropertyConfiguration, propertyLogic, outputResolution, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, eResultsFullGroupDirectory, onlyE: true);
_ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) => SaveFaceNumbers(eResultsFullGroupDirectory, filteredItems[i], collection, faceEncodings));
}
}
private static void SetFaceTags(Item item)
{
FaceRecognitionDotNet.FaceEncoding faceEncoding;
foreach (Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location?.Confidence is null || face.Location?.NormalizedPixelPercentage is null)
continue;
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
lock (face.Tags)
face.Tags.Add(faceEncoding);
}
}
private void SetFaceTags(string argZero, List<bool> faceEncodingRequired, Container[] containers)
{
Item[] filteredItems;
faceEncodingRequired.Clear();
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism };
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = (from l in container.Items where l.ImageFileHolder is not null && (l.Abandoned is null || !l.Abandoned.Value) && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l).ToArray();
if (!filteredItems.Any())
continue;
_ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) => SetFaceTags(filteredItems[i]));
}
}
private void FullDoWork(string argZero, Model? model, PredictorModel? predictorModel, string propertyRoot, long ticks, MapLogic mapLogic, A_Property propertyLogic, Container[] containers)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
int[] ids;
bool needToSave;
int distinctCount;
int exceptionCount;
Item[] filteredItems;
@ -572,19 +714,21 @@ public class DlibDotNet
string dResultsFullGroupDirectory;
string eResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
List<List<Face>> faceCollections = new();
List<bool> faceEncodingRequired = new();
Shared.Models.Property[] propertyCollection;
List<List<Face>?> imageFaceCollections = new();
List<FileHolder?> propertyFileHolderCollection = new();
List<Dictionary<string, int[]>> resizeKeyValuePairs = new();
List<Tuple<string, DateTime>> sourceDirectoryChanges = new();
List<Shared.Models.Property?> nullablePropertyCollection = new();
List<List<KeyValuePair<string, string>>> metadataCollection = new();
string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(A_Property), "{}");
string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), "{}");
foreach (string outputResolution in _Configuration.OutputResolutions)
{
_FileKeyValuePairs.Clear();
faceEncodingRequired.Clear();
_FilePropertiesKeyValuePairs.Clear();
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory) = GetResultsFullGroupDirectories(configuration, model, predictorModel, outputResolution);
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory) = GetResultsFullGroupDirectories(_Configuration.PropertyConfiguration, model, predictorModel, outputResolution);
foreach (Container container in containers)
{
if (!container.Items.Any())
@ -594,33 +738,31 @@ public class DlibDotNet
filteredItems = (from l in container.Items where l.ImageFileHolder is not null && (l.Abandoned is null || !l.Abandoned.Value) && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l).ToArray();
if (!filteredItems.Any())
continue;
faceCollections.Clear();
metadataCollection.Clear();
resizeKeyValuePairs.Clear();
imageFaceCollections.Clear();
sourceDirectoryChanges.Clear();
nullablePropertyCollection.Clear();
propertyFileHolderCollection.Clear();
SetAngleBracketCollections(configuration, propertyLogic, outputResolution, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory);
exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, nullablePropertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, containers.Length, container, filteredItems);
SetAngleBracketCollections(_Configuration.PropertyConfiguration, propertyLogic, outputResolution, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, eResultsFullGroupDirectory, onlyE: false);
exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, nullablePropertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, faceEncodingRequired, containers.Length, container, filteredItems);
#pragma warning disable
ids = (from l in filteredItems where l.Property?.Id is not null select l.Property.Id.Value).ToArray();
#pragma warning restore
distinctCount = ids.Distinct().Count();
if (ids.Length != distinctCount)
_Log.Information($"{ids.Length} != {distinctCount} <{container.SourceDirectory}>");
if (metadataCollection.Count != filteredItems.Length || nullablePropertyCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || faceCollections.Count != filteredItems.Length)
if (metadataCollection.Count != filteredItems.Length || nullablePropertyCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || imageFaceCollections.Count != filteredItems.Length)
throw new Exception("Counts don't match!");
if (exceptionCount != 0)
_Exceptions.Add(container.SourceDirectory);
for (int i = 0; i < faceCollections.Count; i++)
filteredItems[i].Faces.AddRange(from l in faceCollections[i] select l);
for (int i = 0; i < imageFaceCollections.Count; i++)
filteredItems[i].Faces.AddRange(from l in imageFaceCollections[i] select l);
propertyCollection = (from l in nullablePropertyCollection where l is not null select l).ToArray();
if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0)
WriteGroup(configuration, propertyLogic, propertyCollection, metadataCollection, faceCollections, resizeKeyValuePairs, outputResolution, container, filteredItems);
if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0 && outputResolution == _Configuration.OutputResolutions[0])
mapLogic.AddToMapLogicAllCollection(filteredItems);
if (exceptionCount == 0 && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_Distance.LoadOrCreateThenSaveDistanceResults(configuration, eResultsFullGroupDirectory, outputResolution, container, sourceDirectoryChanges, filteredItems);
WriteGroup(_Configuration.PropertyConfiguration, propertyLogic, propertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, outputResolution, container, filteredItems);
if (false && exceptionCount == 0 && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_Distance.LoadOrCreateThenSaveDistanceResults(_Configuration.PropertyConfiguration, eResultsFullGroupDirectory, outputResolution, container, sourceDirectoryChanges, filteredItems);
if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any())
{
for (int y = 0; y < int.MaxValue; y++)
@ -644,6 +786,19 @@ public class DlibDotNet
// if (isSilent && container.SourceDirectory.EndsWith("Texas 2015")) //46#.4
// break;
}
if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
{
needToSave = faceEncodingRequired.Any();
if (needToSave)
SetFaceTags(argZero, faceEncodingRequired, containers);
if (faceEncodingRequired.Any())
throw new Exception("Why!");
if (needToSave)
{
(Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)[] collection = GetValues(argZero, containers);
SaveFaceNumbers(argZero, propertyLogic, outputResolution, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, eResultsFullGroupDirectory, containers, collection);
}
}
}
}
@ -677,36 +832,28 @@ public class DlibDotNet
return results;
}
private void MapLogic(string argZero, Container[] containers, long ticks, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, MapLogic mapLogic, string outputResolution)
private void MapLogic(long ticks, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, MapLogic mapLogic, string outputResolution, Item[] items, int totalNotMapped)
{
int totalNotMapped = 0;
mapLogic.UseKeyValuePairsSaveFaceEncoding(containers);
foreach (Container container in containers)
{
totalNotMapped += mapLogic.AddToMapping(container.Items);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
mapLogic.SaveShortcuts(_Configuration.JuliePhares, container.Items);
}
mapLogic.SaveAllCollection();
mapLogic.UseKeyValuePairsSaveFaceEncoding(items);
if (_Configuration.SaveResizedSubfiles)
{
string dFacesContentDirectory;
dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()");
mapLogic.SaveNotMappedTicks();
_ = LogDeltaInMinutes(ticks, nameof(mapLogic.SaveNotMappedTicks));
Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> keyValuePairs = _Distance.ParallelWork(_AppSettings.MaxDegreeOfParallelism, _Configuration.IgnoreRelativePaths, argZero, ticks, mapLogic, containers, totalNotMapped);
Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> keyValuePairs = _Distance.ParallelWork(_AppSettings.MaxDegreeOfParallelism, ticks, mapLogic, items, totalNotMapped);
_ = LogDeltaInSeconds(ticks, nameof(E_Distance.ParallelWork));
Dictionary<int, List<MappingContainer>> strippedKeyValuePairs = Strip(keyValuePairs);
List<(string, int, Mapping, DateTime, bool?, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>)> collection = Convert(keyValuePairs);
mapLogic.SaveMapping(argZero, containers, dFacesContentDirectory, d2ResultsFullGroupDirectory);
mapLogic.SaveMapping(items, dFacesContentDirectory, d2ResultsFullGroupDirectory);
_ = LogDeltaInMinutes(ticks, nameof(mapLogic.SaveMapping));
_Distance.AddToFaceDistance(_AppSettings.MaxDegreeOfParallelism, argZero, ticks, mapLogic, containers, outputResolution, collection);
E_Distance.AddToFaceDistance(_AppSettings.MaxDegreeOfParallelism, mapLogic, items, collection);
_ = LogDeltaInSeconds(ticks, nameof(_Distance.AddToFaceDistance));
mapLogic.AddToClosest(_AppSettings.MaxDegreeOfParallelism, argZero, containers);
mapLogic.AddToClosest(_AppSettings.MaxDegreeOfParallelism, items);
_ = LogDeltaInMinutes(ticks, nameof(mapLogic.AddToClosest));
mapLogic.SaveClosest(argZero, containers, dFacesContentDirectory, d2ResultsFullGroupDirectory);
mapLogic.SaveClosest(items, dFacesContentDirectory, d2ResultsFullGroupDirectory);
_ = LogDeltaInMinutes(ticks, nameof(mapLogic.SaveClosest));
mapLogic.SavePropertyHolders(argZero, containers);
mapLogic.SavePropertyHolders(items);
_ = LogDeltaInSeconds(ticks, nameof(mapLogic.SavePropertyHolders));
}
_ = LogDeltaInSeconds(ticks, nameof(MapLogic));
@ -739,7 +886,7 @@ public class DlibDotNet
containers = A_Property.Get(configuration, propertyLogic);
else
containers = Property.Models.Stateless.Container.GetContainers(configuration, _FirstRun, propertyLogic);
FullDoWork(argZero, configuration, model, predictorModel, propertyRoot, ticks, mapLogic, propertyLogic, containers);
FullDoWork(argZero, model, predictorModel, propertyRoot, ticks, mapLogic, propertyLogic, containers);
foreach (string outputResolution in _Configuration.OutputResolutions)
{
if (_FirstRun)
@ -747,7 +894,18 @@ public class DlibDotNet
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory) = GetResultsFullGroupDirectories(configuration, model, predictorModel, outputResolution);
if (_ArgZeroIsConfigurationRootDirectory && _Exceptions.Count == 0 && outputResolution == _Configuration.OutputResolutions[0])
{
MapLogic(argZero, containers, ticks, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, mapLogic, outputResolution);
int totalNotMapped = 0;
DateTime dateTime = new(ticks);
List<(int Id, Item Item)> idAndItemCollection = new();
foreach (Container container in containers)
{
totalNotMapped += mapLogic.AddToMapping(ticks, container.Items);
Map.Models.MapLogic.AppendToItems(container.Items, idAndItemCollection);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
mapLogic.SaveShortcuts(_Configuration.JuliePhares, container.Items);
}
Item[] items = (from l in idAndItemCollection orderby l.Id select l.Item).ToArray();
MapLogic(ticks, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, mapLogic, outputResolution, items, totalNotMapped);
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any())
break;
if (_FileKeyValuePairs.Any())

4
Instance/Models/_D2_FaceParts.cs
View File

@ -118,7 +118,7 @@ internal class D2_FaceParts
#pragma warning restore CA1416
internal void SaveFaceLandmarkImages(string d2ResultsFullGroupDirectory, string sourceDirectory, List<Tuple<string, DateTime>> subFileTuples, List<string> parseExceptions, Item item, List<Face> faceCollection, bool saveRotated)
internal void SaveFaceLandmarkImages(string d2ResultsFullGroupDirectory, Container container, List<Tuple<string, DateTime>> subFileTuples, List<string> parseExceptions, Item item, List<Face> faceCollection, bool saveRotated)
{
if (item.ImageFileHolder is null)
throw new NullReferenceException(nameof(item.ImageFileHolder));
@ -137,7 +137,7 @@ internal class D2_FaceParts
List<(Face, string, string)> collection = new();
angleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(
_Configuration.PropertyConfiguration,
sourceDirectory,
container.SourceDirectory,
d2ResultsFullGroupDirectory,
contentDescription: "n x 2 gif file(s) for each face found",
singletonDescription: string.Empty,

122
Instance/Models/_E_Distance.cs
View File

@ -11,6 +11,8 @@ namespace View_by_Distance.Instance.Models;
internal class E_Distance
{
internal List<string> AngleBracketCollection { get; }
private readonly Serilog.ILogger? _Log;
private readonly Configuration _Configuration;
private readonly JsonSerializerOptions _WriteIndentedJsonSerializerOptions;
@ -18,6 +20,7 @@ internal class E_Distance
internal E_Distance(Configuration configuration)
{
_Configuration = configuration;
AngleBracketCollection = new List<string>();
_Log = Serilog.Log.ForContext<E_Distance>();
_WriteIndentedJsonSerializerOptions = new JsonSerializerOptions { WriteIndented = true };
}
@ -411,6 +414,76 @@ internal class E_Distance
}
}
internal void SaveFaceNumbers(string eResultsFullGroupDirectory, Item item, Face face, List<int[]> faceDistances)
{
if (item.Property?.Id is null)
throw new NullReferenceException(nameof(item.Property.Id));
if (item.ImageFileHolder is null)
throw new NullReferenceException(nameof(item.ImageFileHolder));
if (face.Location?.NormalizedPixelPercentage is null)
throw new NullReferenceException(nameof(face.Location.NormalizedPixelPercentage));
if (string.IsNullOrEmpty(eResultsFullGroupDirectory))
throw new NullReferenceException(nameof(eResultsFullGroupDirectory));
string json;
List<(Face, string, string)> collection = new();
string[] changesFrom = new string[] { nameof(A_Property), nameof(B_Metadata), nameof(C_Resize), nameof(D_Face) };
string eSingletonFile = Path.Combine(eResultsFullGroupDirectory, "{}", Property.Models.Stateless.IResult.AllInOne, $"{item.Property.Id.Value}.{face.Location.NormalizedPixelPercentage.Value}{item.ImageFileHolder.ExtensionLowered}.json");
FileInfo fileInfo = new(eSingletonFile);
json = JsonSerializer.Serialize(faceDistances, _WriteIndentedJsonSerializerOptions);
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, json, updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null);
}
internal void GetFaceDistances(string eResultsFullGroupDirectory, List<bool> faceEncodingRequired, List<Tuple<string, DateTime>> subFileTuples, List<string> parseExceptions, Item item, List<Face> faceCollection)
{
List<int[]>? results;
if (item.Property?.Id is null)
throw new NullReferenceException(nameof(item.Property.Id));
if (item.ImageFileHolder is null)
throw new NullReferenceException(nameof(item.ImageFileHolder));
if (string.IsNullOrEmpty(eResultsFullGroupDirectory))
throw new NullReferenceException(nameof(eResultsFullGroupDirectory));
string json;
FileInfo fileInfo;
string eSingletonFile;
List<(Face, string, string)> collection = new();
string[] changesFrom = new string[] { nameof(A_Property), nameof(B_Metadata), nameof(C_Resize), nameof(D_Face) };
List<DateTime> dateTimes = (from l in subFileTuples where changesFrom.Contains(l.Item1) select l.Item2).ToList();
foreach (Face face in faceCollection)
{
results = null;
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
continue;
eSingletonFile = Path.Combine(eResultsFullGroupDirectory, "{}", Property.Models.Stateless.IResult.AllInOne, $"{item.Property.Id.Value}.{face.Location.NormalizedPixelPercentage.Value}{item.ImageFileHolder.ExtensionLowered}.json");
fileInfo = new(eSingletonFile);
if (_Configuration.PropertiesChangedForDistance)
results = null;
else if (!fileInfo.Exists)
results = null;
else if (dateTimes.Any() && dateTimes.Max() > fileInfo.LastWriteTime)
results = null;
else
{
json = File.ReadAllText(fileInfo.FullName);
try
{
results = JsonSerializer.Deserialize<List<int[]>>(json);
if (results is null)
throw new NullReferenceException(nameof(results));
subFileTuples.Add(new Tuple<string, DateTime>(nameof(E_Distance), fileInfo.LastWriteTime));
lock (face)
face.SetFaceNumbers(results);
}
catch (Exception)
{
results = null;
parseExceptions.Add(nameof(E_Distance));
}
}
if (results is null && !faceEncodingRequired.Any())
faceEncodingRequired.Add(true);
}
}
public static double GetStandardDeviation(IEnumerable<double> values, double average)
{
double result = 0;
@ -661,55 +734,40 @@ internal class E_Distance
return results;
}
internal Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> ParallelWork(int maxDegreeOfParallelism, string[] ignoreRelativePaths, string argZero, long ticks, Map.Models.MapLogic mapLogic, Container[] containers, int totalNotMapped)
internal Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> ParallelWork(int maxDegreeOfParallelism, long ticks, Map.Models.MapLogic mapLogic, Item[] items, int totalNotMapped)
{
Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> results;
Random random = new((int)ticks);
mapLogic.ForceSingleImage(ignoreRelativePaths, argZero, containers, totalNotMapped, random);
Dictionary<string, List<MappingContainer>> keyValuePairs = Map.Models.Stateless.IMapLogic.GetKeyValuePairs(ignoreRelativePaths, argZero, containers);
mapLogic.ForceSingleImage(ticks, items, totalNotMapped, random);
Dictionary<string, List<MappingContainer>> keyValuePairs = Map.Models.Stateless.IMapLogic.GetKeyValuePairs(items);
results = GetThreeSigmaFaceEncodings(maxDegreeOfParallelism, ticks, random, keyValuePairs);
return results;
}
public void AddToFaceDistance(int maxDegreeOfParallelism, string argZero, long ticks, Map.Models.MapLogic mapLogic, Container[] containers, string outputResolution, List<(string, int, Mapping, DateTime, bool?, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>)> collection)
public static void AddToFaceDistance(int maxDegreeOfParallelism, Map.Models.MapLogic mapLogic, Item[] items, List<(string, int, Mapping, DateTime, bool?, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>)> collection)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
Face face;
string message;
int totalSeconds;
double deterministicHashCodeKey;
DateTime dateTime = DateTime.Now;
List<FaceDistance> faceDistances;
int containersCount = containers.Length;
foreach (Container container in containers)
foreach (Item item in items)
{
if (!container.Items.Any())
if (item.Property?.Id is null || item.ImageFileHolder is null || item.ResizedFileHolder is null)
continue;
if (!container.SourceDirectory.StartsWith(argZero))
continue;
foreach (Item item in container.Items)
for (int i = 0; i < item.Faces.Count; i++)
{
if (item.ImageFileHolder is null || item.Property?.Id is null)
face = item.Faces[i];
face.FaceDistances.Clear();
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
continue;
for (int i = 0; i < item.Faces.Count; i++)
{
face = item.Faces[i];
face.FaceDistances.Clear();
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
continue;
if ((from l in item.Mapping where l.NormalizedPixelPercentage.HasValue && l.NormalizedPixelPercentage.Value == face.Location.NormalizedPixelPercentage.Value select true).Any())
continue;
deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item, face);
if (mapLogic.Skip(deterministicHashCodeKey))
continue;
faceDistances = GetFaceDistanceCollection(maxDegreeOfParallelism, collection, face);
face.FaceDistances.AddRange(faceDistances);
}
if ((from l in item.Mapping where l.NormalizedPixelPercentage.HasValue && l.NormalizedPixelPercentage.Value == face.Location.NormalizedPixelPercentage.Value select true).Any())
continue;
deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item, face);
if (mapLogic.Skip(deterministicHashCodeKey))
continue;
faceDistances = GetFaceDistanceCollection(maxDegreeOfParallelism, collection, face);
face.FaceDistances.AddRange(faceDistances);
}
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
message = $"{container.R:000}.{container.G} / {containersCount:000}) {container.Items.Count:000} file(s) - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}";
_Log.Information(message);
}
}