phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Re-Write without checking mapped

Browse Source
This commit is contained in:
Mike Phares
2022-09-14 19:00:57 -07:00
parent 73de1070b8
commit ad28ab2d38
70 changed files with 1596 additions and 2342 deletions
Download Patch File Download Diff File Expand all files Collapse all files

457
Instance/DlibDotNet.cs
View File

@ -26,7 +26,6 @@ public partial class DlibDotNet
private readonly A2_People _People;
private readonly E3_Rename _Rename;
private readonly B_Metadata _Metadata;
private readonly E_Distance _Distance;
private readonly Serilog.ILogger? _Log;
private readonly D2_FaceParts _FaceParts;
private readonly AppSettings _AppSettings;
@ -58,7 +57,6 @@ public partial class DlibDotNet
_Random = new F_Random(configuration);
_People = new A2_People(configuration);
_Rename = new E3_Rename(configuration);
_Distance = new E_Distance(configuration);
if (configuration.IgnoreExtensions is null)
throw new NullReferenceException(nameof(configuration.IgnoreExtensions));
string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(propertyConfiguration, nameof(A_Property), create: false);
@ -115,14 +113,6 @@ public partial class DlibDotNet
_Log.Information(message);
if (_Exceptions.Count != 0)
throw new Exception(message);
if (!_FirstRun && configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions.Any())
{
long ticks = DateTime.Now.Ticks;
foreach (string outputResolution in configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions)
_Distance.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions(propertyConfiguration, model, predictorModel, outputResolution);
if (appSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(E_Distance.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions));
}
if (_FirstRun)
_Log.Information("First run completed. Run again if wanted");
}
@ -210,8 +200,6 @@ public partial class DlibDotNet
throw new NullReferenceException($"{nameof(configuration.OutputResolutions)} must be a valid outputResolution!");
if ((from l in configuration.OutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
throw new Exception($"One or more {nameof(configuration.OutputResolutions)} are not in the ValidResolutions list!");
if ((from l in configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions)} are not in the ValidResolutions list!");
if ((from l in configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions)} are not in the ValidResolutions list!");
if ((from l in configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
@ -262,14 +250,14 @@ public partial class DlibDotNet
throw new Exception("Input directory should be the source and not a resized directory!");
}
private void FullParallelForWork(A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string eResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Face>?> imageFaceCollections, List<bool> faceEncodingRequired, Container container, int index, Item item)
private void FullParallelForWork(A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Face>?> imageFaceCollections, Container container, int index, Item item)
{
if (item.ImageFileHolder is null)
throw new NullReferenceException(nameof(item.ImageFileHolder));
Shared.Models.Property property;
List<Face>? faceCollection;
string original = "Original";
FileHolder? resizedFileHolder;
Shared.Models.Property property;
long ticks = DateTime.Now.Ticks;
DateTime dateTime = DateTime.Now;
List<string> parseExceptions = new();
@ -332,12 +320,12 @@ public partial class DlibDotNet
{
bool saveRotated = _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution);
_FaceParts.SaveFaceLandmarkImages(d2ResultsFullGroupDirectory, container, subFileTuples, parseExceptions, item, faceCollection, saveRotated);
string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(_Configuration.PropertyConfiguration, container.SourceDirectory);
string facesDirectory = Path.GetFullPath(Path.Combine($"{Path.Combine(d2ResultsFullGroupDirectory, "()")}{sourceDirectorySegment}", item.ImageFileHolder.NameWithoutExtension));
_FaceParts.SaveFaceLandmarkImages(facesDirectory, subFileTuples, parseExceptions, item, faceCollection, saveRotated);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D2_FaceParts.SaveFaceLandmarkImages));
}
if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_Distance.GetFaceDistances(eResultsFullGroupDirectory, faceEncodingRequired, subFileTuples, parseExceptions, item, faceCollection);
}
lock (sourceDirectoryChanges)
{
@ -350,11 +338,11 @@ public partial class DlibDotNet
}
}
private int FullParallelWork(long ticks, A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string eResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Face>?> imageFaceCollections, List<bool> faceEncodingRequired, int containersCount, Container container, Item[] filteredItems)
private int FullParallelWork(long ticks, A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Face>?> imageFaceCollections, int containersCount, Container container, Item[] filteredItems)
{
int result = 0;
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
int result = 0;
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism };
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
if (imageFaceCollections.Count != filteredItems.Length || metadataCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || propertyCollection.Count != filteredItems.Length)
@ -370,25 +358,23 @@ public partial class DlibDotNet
}
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string message = $"{container.R:000}.{container.G} / {containersCount:000}) {filteredItems.Length:000} file(s) - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}";
using (ProgressBar progressBar = new(filteredItems.Length, message, options))
{
_ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) =>
using ProgressBar progressBar = new(filteredItems.Length, message, options);
_ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) =>
{
try
{
try
{
FullParallelForWork(propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, faceEncodingRequired, container, index: i, filteredItems[i]);
if (i == 0 || sourceDirectoryChanges.Any())
progressBar.Tick();
}
catch (Exception ex)
{
result += 1;
_Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, ex.Message, Environment.NewLine, ex.StackTrace), ex);
if (result == filteredItems.Length)
throw new Exception(string.Concat("All in [", container.SourceDirectory, "] failed!"));
}
});
}
FullParallelForWork(propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, container, index: i, filteredItems[i]);
if (i == 0 || sourceDirectoryChanges.Any())
progressBar.Tick();
}
catch (Exception ex)
{
result += 1;
_Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, ex.Message, Environment.NewLine, ex.StackTrace), ex);
if (result == filteredItems.Length)
throw new Exception(string.Concat("All in [", container.SourceDirectory, "] failed!"));
}
});
return result;
}
@ -524,188 +510,64 @@ public partial class DlibDotNet
return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory);
}
private void SetAngleBracketCollections(Property.Models.Configuration configuration, A_Property propertyLogic, string outputResolution, Container container, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string eResultsFullGroupDirectory, bool onlyE)
private void SetAngleBracketCollections(Property.Models.Configuration configuration, A_Property propertyLogic, string outputResolution, Container container, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string eResultsFullGroupDirectory)
{
_Faces.AngleBracketCollection.Clear();
_Resize.AngleBracketCollection.Clear();
_Metadata.AngleBracketCollection.Clear();
_Distance.AngleBracketCollection.Clear();
propertyLogic.AngleBracketCollection.Clear();
if (!onlyE)
{
propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
aResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Properties for each image",
collectionDescription: string.Empty,
converted: false));
_Metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
bResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Metadata as key value pairs",
collectionDescription: string.Empty,
converted: true));
_Resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
cResultsFullGroupDirectory,
contentDescription: "Resized image",
singletonDescription: "Resize dimensions for each resolution",
collectionDescription: string.Empty,
converted: true));
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
_Faces.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
aResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Properties for each image",
collectionDescription: string.Empty,
converted: false));
_Metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
bResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Metadata as key value pairs",
collectionDescription: string.Empty,
dResultsFullGroupDirectory,
contentDescription: "n png file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: "For each image a json file with all faces found",
converted: true));
_Resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
_ = Property.Models.Stateless.IResult.GetDirectoryInfoCollection(
_Configuration.PropertyConfiguration,
container.SourceDirectory,
cResultsFullGroupDirectory,
contentDescription: "Resized image",
singletonDescription: "Resize dimensions for each resolution",
d2ResultsFullGroupDirectory,
contentDescription: "n x 2 gif file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: string.Empty,
converted: true));
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
_Faces.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
dResultsFullGroupDirectory,
contentDescription: "n png file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: "For each image a json file with all faces found",
converted: true));
}
converted: false);
if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_Distance.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
_ = Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
eResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "n json file(s) for each face found",
collectionDescription: string.Empty,
converted: true));
}
private (Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)[] GetValues(string argZero, Container[] containers)
{
(Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)[] results;
Sorting sorting;
Item[] filteredItems;
List<(Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)> collection = new();
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = (from l in container.Items where l.ImageFileHolder is not null && (l.Abandoned is null || !l.Abandoned.Value) && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l).ToArray();
if (!filteredItems.Any())
continue;
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null || item.ImageFileHolder is null || item.ResizedFileHolder is null)
continue;
if (!item.Faces.Any())
continue;
foreach (Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location?.Confidence is null || face.Location?.NormalizedPixelPercentage is null)
continue;
if (!face.Tags.Any() || face.Tags[0] is not FaceRecognitionDotNet.FaceEncoding faceEncoding)
continue;
sorting = new(face.Location.Confidence, item.Property.Id.Value, face.Location.NormalizedPixelPercentage.Value);
collection.Add(new(sorting, faceEncoding));
face.Tags.Clear();
}
}
}
results = (from l in collection orderby l.FaceEncoding is not null, l.Sorting.Id select l).ToArray();
if (results.Any() && results[0].FaceEncoding is null)
throw new Exception("Sorting failed!");
return results;
}
private void SaveFaceNumbers(string eResultsFullGroupDirectory, Item item, (Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)[] inputCollection, List<FaceRecognitionDotNet.FaceEncoding> faceEncodings)
{
Sorting sorting;
List<double> doubles;
List<int[]> faceNumbers;
List<Sorting> sortingOutputCollection;
FaceRecognitionDotNet.FaceEncoding faceEncoding;
if (item.Property?.Id is null || item.ImageFileHolder is null || item.ResizedFileHolder is null)
throw new Exception();
foreach (Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location?.Confidence is null || face.Location?.NormalizedPixelPercentage is null)
continue;
sortingOutputCollection = new();
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
doubles = FaceRecognition.FaceDistances(faceEncodings, faceEncoding);
for (int j = 0; j < doubles.Count; j++)
{
sorting = new(inputCollection[j].Sorting, doubles[j]);
lock (sortingOutputCollection)
sortingOutputCollection.Add(sorting);
}
faceNumbers = Shared.Models.Stateless.Methods.ISorting.GetFaceNumbers(sortingOutputCollection);
lock (face)
face.SetFaceNumbers(faceNumbers);
_Distance.SaveFaceNumbers(eResultsFullGroupDirectory, item, face, faceNumbers);
}
}
private void SaveFaceNumbers(string argZero, A_Property propertyLogic, string outputResolution, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string eResultsFullGroupDirectory, Container[] containers, (Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)[] collection)
{
if (collection.Any() && collection[0].FaceEncoding is null)
throw new Exception("Sorting failed!");
Item[] filteredItems;
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism };
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = (from l in container.Items where l.ImageFileHolder is not null && (l.Abandoned is null || !l.Abandoned.Value) && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l).ToArray();
if (!filteredItems.Any())
continue;
List<FaceRecognitionDotNet.FaceEncoding> faceEncodings = new();
foreach ((Sorting _, FaceRecognitionDotNet.FaceEncoding faceEncoding) in collection)
{
if (faceEncoding is null)
continue;
faceEncodings.Add(faceEncoding);
}
SetAngleBracketCollections(_Configuration.PropertyConfiguration, propertyLogic, outputResolution, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, eResultsFullGroupDirectory, onlyE: true);
_ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) => SaveFaceNumbers(eResultsFullGroupDirectory, filteredItems[i], collection, faceEncodings));
}
}
private static void SetFaceTags(Item item)
{
FaceRecognitionDotNet.FaceEncoding faceEncoding;
foreach (Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location?.Confidence is null || face.Location?.NormalizedPixelPercentage is null)
continue;
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
lock (face.Tags)
face.Tags.Add(faceEncoding);
}
}
private void SetFaceTags(string argZero, List<bool> faceEncodingRequired, Container[] containers)
{
Item[] filteredItems;
faceEncodingRequired.Clear();
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism };
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = (from l in container.Items where l.ImageFileHolder is not null && (l.Abandoned is null || !l.Abandoned.Value) && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l).ToArray();
if (!filteredItems.Any())
continue;
_ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) => SetFaceTags(filteredItems[i]));
}
converted: true);
}
private void FullDoWork(string argZero, Model? model, PredictorModel? predictorModel, string propertyRoot, long ticks, MapLogic mapLogic, A_Property propertyLogic, Container[] containers)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
int[] ids;
bool needToSave;
int distinctCount;
int exceptionCount;
Item[] filteredItems;
string aResultsFullGroupDirectory;
@ -714,7 +576,6 @@ public partial class DlibDotNet
string dResultsFullGroupDirectory;
string eResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
List<bool> faceEncodingRequired = new();
Shared.Models.Property[] propertyCollection;
List<List<Face>?> imageFaceCollections = new();
List<FileHolder?> propertyFileHolderCollection = new();
@ -726,7 +587,6 @@ public partial class DlibDotNet
foreach (string outputResolution in _Configuration.OutputResolutions)
{
_FileKeyValuePairs.Clear();
faceEncodingRequired.Clear();
_FilePropertiesKeyValuePairs.Clear();
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory) = GetResultsFullGroupDirectories(_Configuration.PropertyConfiguration, model, predictorModel, outputResolution);
foreach (Container container in containers)
@ -744,14 +604,8 @@ public partial class DlibDotNet
sourceDirectoryChanges.Clear();
nullablePropertyCollection.Clear();
propertyFileHolderCollection.Clear();
SetAngleBracketCollections(_Configuration.PropertyConfiguration, propertyLogic, outputResolution, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, eResultsFullGroupDirectory, onlyE: false);
exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, nullablePropertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, faceEncodingRequired, containers.Length, container, filteredItems);
#pragma warning disable
ids = (from l in filteredItems where l.Property?.Id is not null select l.Property.Id.Value).ToArray();
#pragma warning restore
distinctCount = ids.Distinct().Count();
if (ids.Length != distinctCount)
_Log.Information($"{ids.Length} != {distinctCount} <{container.SourceDirectory}>");
SetAngleBracketCollections(_Configuration.PropertyConfiguration, propertyLogic, outputResolution, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory);
exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, nullablePropertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, containers.Length, container, filteredItems);
if (metadataCollection.Count != filteredItems.Length || nullablePropertyCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || imageFaceCollections.Count != filteredItems.Length)
throw new Exception("Counts don't match!");
if (exceptionCount != 0)
@ -761,8 +615,6 @@ public partial class DlibDotNet
propertyCollection = (from l in nullablePropertyCollection where l is not null select l).ToArray();
if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0)
WriteGroup(_Configuration.PropertyConfiguration, propertyLogic, propertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, outputResolution, container, filteredItems);
if (false && exceptionCount == 0 && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_Distance.LoadOrCreateThenSaveDistanceResults(_Configuration.PropertyConfiguration, eResultsFullGroupDirectory, outputResolution, container, sourceDirectoryChanges, filteredItems);
if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any())
{
for (int y = 0; y < int.MaxValue; y++)
@ -786,79 +638,9 @@ public partial class DlibDotNet
// if (isSilent && container.SourceDirectory.EndsWith("Texas 2015")) //46#.4
// break;
}
if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
{
needToSave = faceEncodingRequired.Any();
if (needToSave)
SetFaceTags(argZero, faceEncodingRequired, containers);
if (faceEncodingRequired.Any())
throw new Exception("Why!");
if (needToSave)
{
(Sorting Sorting, FaceRecognitionDotNet.FaceEncoding FaceEncoding)[] collection = GetValues(argZero, containers);
SaveFaceNumbers(argZero, propertyLogic, outputResolution, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, eResultsFullGroupDirectory, containers, collection);
}
}
}
}
private static List<(string, int, Mapping, DateTime, bool?, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>)> Convert(Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> keyValuePairs)
{
List<(string, int, Mapping, DateTime, bool?, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>)> results = new();
MappingContainer mc;
foreach (KeyValuePair<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> keyValuePair in keyValuePairs)
{
foreach ((FaceRecognitionDotNet.FaceEncoding _, MappingContainer mappingContainer) in keyValuePair.Value)
{
mc = mappingContainer;
results.Add(new(mc.Key, mc.Id, mc.Mapping, mc.MinimumDateTime, mc.IsWrongYear, keyValuePair.Value));
}
}
return results;
}
private static Dictionary<int, List<MappingContainer>> Strip(Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> keyValuePairs)
{
Dictionary<int, List<MappingContainer>> results = new();
foreach (KeyValuePair<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> keyValuePair in keyValuePairs)
{
foreach ((FaceRecognitionDotNet.FaceEncoding _, MappingContainer mappingContainer) in keyValuePair.Value)
{
if (!results.ContainsKey(mappingContainer.Id))
results.Add(mappingContainer.Id, new());
results[mappingContainer.Id].Add(mappingContainer);
}
}
return results;
}
private void MapLogic(long ticks, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, MapLogic mapLogic, string outputResolution, Item[] items, int totalNotMapped)
{
mapLogic.UseKeyValuePairsSaveFaceEncoding(items);
if (_Configuration.SaveResizedSubfiles)
{
string dFacesContentDirectory;
dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()");
mapLogic.SaveNotMappedTicks();
_ = LogDeltaInMinutes(ticks, nameof(mapLogic.SaveNotMappedTicks));
Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> keyValuePairs = _Distance.ParallelWork(_AppSettings.MaxDegreeOfParallelism, ticks, mapLogic, items, totalNotMapped);
_ = LogDeltaInSeconds(ticks, nameof(E_Distance.ParallelWork));
Dictionary<int, List<MappingContainer>> strippedKeyValuePairs = Strip(keyValuePairs);
List<(string, int, Mapping, DateTime, bool?, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>)> collection = Convert(keyValuePairs);
mapLogic.SaveMapping(items, dFacesContentDirectory, d2ResultsFullGroupDirectory);
_ = LogDeltaInMinutes(ticks, nameof(mapLogic.SaveMapping));
E_Distance.AddToFaceDistance(_AppSettings.MaxDegreeOfParallelism, mapLogic, items, collection);
_ = LogDeltaInSeconds(ticks, nameof(_Distance.AddToFaceDistance));
mapLogic.AddToClosest(_AppSettings.MaxDegreeOfParallelism, items);
_ = LogDeltaInMinutes(ticks, nameof(mapLogic.AddToClosest));
mapLogic.SaveClosest(items, dFacesContentDirectory, d2ResultsFullGroupDirectory);
_ = LogDeltaInMinutes(ticks, nameof(mapLogic.SaveClosest));
mapLogic.SavePropertyHolders(items);
_ = LogDeltaInSeconds(ticks, nameof(mapLogic.SavePropertyHolders));
}
_ = LogDeltaInSeconds(ticks, nameof(MapLogic));
}
private string GetMapLogicResultsFullGroupDirectory(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel)
{
const int zero = 0;
@ -868,6 +650,79 @@ public partial class DlibDotNet
return zResultsFullGroupDirectory;
}
private List<Face> SetMappingThenGetDistinctFilteredFacesWithMapping(string argZero, Container[] containers)
{
List<Face> results = new();
Mapping mapping;
bool? isWrongYear;
Item[] filteredItems;
DateTime minimumDateTime;
List<int> distinctCheck = new();
string deterministicHashCodeKeyDisplay;
MappingFromItem mappingFromItem;
MappingFromPerson mappingFromPerson;
MappingFromLocation mappingFromLocation;
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = (from l in container.Items where l.ImageFileHolder is not null && (l.Abandoned is null || !l.Abandoned.Value) && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l).ToArray();
if (!filteredItems.Any())
continue;
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null || item.ImageFileHolder is null || item.ResizedFileHolder is null)
continue;
if (!item.Faces.Any())
continue;
foreach (Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
continue;
if (distinctCheck.Contains(item.Property.Id.Value))
continue;
minimumDateTime = Shared.Models.Stateless.Methods.IProperty.GetMinimumDateTime(item.Property);
(isWrongYear, _) = item.Property.IsWrongYear(item.ImageFileHolder.FullName, minimumDateTime);
mappingFromItem = new(item.Property.Id.Value, item.ImageFileHolder, isWrongYear, minimumDateTime, item.ResizedFileHolder);
mappingFromPerson = new(approximateYears: null, by: null, displayDirectoryName: string.Empty, personBirthday: null, segmentB: string.Empty);
deterministicHashCodeKeyDisplay = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKeyDisplay(item.Property.Id.Value, face.Location.NormalizedPixelPercentage.Value);
mappingFromLocation = new(face.Location.Confidence, deterministicHashCodeKeyDisplay, face.Location.NormalizedPixelPercentage.Value);
mapping = new(mappingFromItem, mappingFromLocation, mappingFromPerson);
face.SetMapping(mapping);
results.Add(face);
}
}
}
return results;
}
private Item[] GetIdSortedFilteredItems(string argZero, Container[] containers)
{
Item[] results;
List<Item> collection = new();
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
collection.AddRange(from l in container.Items where l.ImageFileHolder is not null && (l.Abandoned is null || !l.Abandoned.Value) && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l);
}
results = (from l in collection orderby l.Property?.Id is not null, l.Property?.Id select l).ToArray();
return results;
}
private void OldMapLogic(long ticks, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, MapLogic mapLogic, string outputResolution, Item[] filteredItems)
{
string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()");
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
mapLogic.SaveShortcuts(_Configuration.JuliePhares, filteredItems);
mapLogic.SavePropertyHolders(filteredItems);
_ = LogDeltaInSeconds(ticks, nameof(mapLogic.SavePropertyHolders));
}
private void Search(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string argZero, string propertyRoot, Person[] people)
{
Container[] containers;
@ -882,30 +737,36 @@ public partial class DlibDotNet
string peopleDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(A2_People));
MapLogic mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FilenameExtension, _Faces.FilenameExtension, _Faces.HiddenFilenameExtension, _FaceParts.FilenameExtension, ticks, people, peopleDateGroupDirectory, zResultsFullGroupDirectory);
A_Property propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FilenameExtension, _Configuration.Reverse, model, predictorModel, mapLogic.IndicesFromNew, mapLogic.KeyValuePairs);
if (string.IsNullOrEmpty(configuration.RootDirectory))
containers = A_Property.Get(configuration, propertyLogic);
else
containers = Property.Models.Stateless.Container.GetContainers(configuration, _FirstRun, propertyLogic);
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string message = $") Building Container(s) - {totalSeconds} total second(s)";
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
using (ProgressBar progressBar = new(1, message, options))
{
if (string.IsNullOrEmpty(configuration.RootDirectory))
containers = A_Property.Get(configuration, propertyLogic);
else
containers = Property.Models.Stateless.Container.GetContainers(configuration, _FirstRun, propertyLogic);
}
FullDoWork(argZero, model, predictorModel, propertyRoot, ticks, mapLogic, propertyLogic, containers);
foreach (string outputResolution in _Configuration.OutputResolutions)
{
if (_FirstRun)
break;
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory) = GetResultsFullGroupDirectories(configuration, model, predictorModel, outputResolution);
if (_ArgZeroIsConfigurationRootDirectory && _Exceptions.Count == 0 && outputResolution == _Configuration.OutputResolutions[0])
if (_ArgZeroIsConfigurationRootDirectory && _Configuration.SaveResizedSubfiles && outputResolution == _Configuration.OutputResolutions[0] && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution) && _Exceptions.Count == 0)
{
int totalNotMapped = 0;
DateTime dateTime = new(ticks);
List<(int Id, Item Item)> idAndItemCollection = new();
foreach (Container container in containers)
{
totalNotMapped += mapLogic.AddToMapping(ticks, container.Items);
Map.Models.MapLogic.AppendToItems(container.Items, idAndItemCollection);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
mapLogic.SaveShortcuts(_Configuration.JuliePhares, container.Items);
}
Item[] items = (from l in idAndItemCollection orderby l.Id select l.Item).ToArray();
MapLogic(ticks, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, mapLogic, outputResolution, items, totalNotMapped);
if (string.IsNullOrEmpty(eResultsFullGroupDirectory))
throw new NullReferenceException(nameof(eResultsFullGroupDirectory));
string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()");
string eDistanceContentDirectory = Path.Combine(eResultsFullGroupDirectory, "([])");
if (!Directory.Exists(eDistanceContentDirectory))
_ = Directory.CreateDirectory(eDistanceContentDirectory);
string eDistanceContentFileName = Path.Combine(eDistanceContentDirectory, $"{Property.Models.Stateless.IResult.AllInOne}.tvs");
List<Face> distinctFilteredFaces = SetMappingThenGetDistinctFilteredFacesWithMapping(argZero, containers);
SortingContainer[] sortingContainers = E_Distance.SetPersonTicksAndSetFaceDistancesAndSetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, ticks, mapLogic, outputResolution, eDistanceContentFileName, distinctFilteredFaces);
mapLogic.CommonWork(_AppSettings.MaxDegreeOfParallelism, dFacesContentDirectory, d2ResultsFullGroupDirectory, distinctFilteredFaces, sortingContainers);
Item[] filteredItems = GetIdSortedFilteredItems(argZero, containers);
OldMapLogic(ticks, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, mapLogic, outputResolution, filteredItems);
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any())
break;
if (_FileKeyValuePairs.Any())
@ -926,8 +787,6 @@ public partial class DlibDotNet
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(dResultsFullGroupDirectory, "[]"));
if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(eResultsFullGroupDirectory, "[]"));
if (_Configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions.Contains(outputResolution))
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(eResultsFullGroupDirectory, "[]"));
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(d2ResultsFullGroupDirectory, "[]"));
}