This commit is contained in:
2022-08-22 09:10:19 -07:00
parent f72fcee1db
commit bc2174b17a
150 changed files with 4323 additions and 6259 deletions
.vscode
Compare
Date-Group
FaceRecognitionDotNet
Instance
Map
Metadata
Not-Copy-Copy
PrepareForOld
Property-Compare
Property
Resize
Shared
Tests
TestsWithFaceRecognitionDotNet
View-by-Distance-MKLink-Console.slnpackage.json

@ -26,14 +26,14 @@ public class DlibDotNet
private readonly B_Metadata _Metadata;
private readonly E_Distance _Distance;
private readonly Serilog.ILogger? _Log;
private readonly D2_FaceParts _FaceParts;
private readonly AppSettings _AppSettings;
private readonly List<string> _Exceptions;
private readonly IsEnvironment _IsEnvironment;
private readonly D2_FaceLandmarks _FaceLandmarks;
private readonly Models.Configuration _Configuration;
private readonly bool _ArgZeroIsConfigurationRootDirectory;
private readonly List<KeyValuePair<string, string>> _FileKeyValuePairs;
private readonly Dictionary<string, List<Tuple<string, A_Property>>> _FilePropertiesKeyValuePairs;
private readonly Dictionary<string, List<Tuple<string, Shared.Models.Property>>> _FilePropertiesKeyValuePairs;
public DlibDotNet(List<string> args, IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, AppSettings appSettings, string workingDirectory, bool isSilent, IConsole console)
{
@ -44,8 +44,8 @@ public class DlibDotNet
_Exceptions = new List<string>();
_Log = Serilog.Log.ForContext<DlibDotNet>();
_FileKeyValuePairs = new List<KeyValuePair<string, string>>();
_FilePropertiesKeyValuePairs = new Dictionary<string, List<Tuple<string, A_Property>>>();
Property.Models.Configuration propertyConfiguration = Property.Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory);
_FilePropertiesKeyValuePairs = new Dictionary<string, List<Tuple<string, Shared.Models.Property>>>();
Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot);
_Log.Information(propertyConfiguration.RootDirectory);
Property.Models.Configuration.Verify(propertyConfiguration);
Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration);
@ -57,7 +57,6 @@ public class DlibDotNet
_People = new A2_People(configuration);
_Rename = new E3_Rename(configuration);
_Distance = new E_Distance(configuration);
_FaceLandmarks = new D2_FaceLandmarks(configuration);
if (configuration.IgnoreExtensions is null)
throw new NullReferenceException(nameof(configuration.IgnoreExtensions));
_Metadata = new(configuration.ForceMetadataLastWriteTimeToCreationTime, configuration.PropertiesChangedForMetadata);
@ -66,11 +65,8 @@ public class DlibDotNet
else
argZero = Path.GetFullPath(propertyConfiguration.RootDirectory);
_ArgZeroIsConfigurationRootDirectory = propertyConfiguration.RootDirectory == argZero;
(ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters) = C_Resize.GetTuple(configuration.OutputExtension, configuration.OutputQuality);
_Resize = new C_Resize(configuration.ForceResizeLastWriteTimeToCreationTime, configuration.OverrideForResizeImages, configuration.PropertiesChangedForResize, configuration.ValidResolutions, imageCodecInfo, encoderParameters);
_Log.Information(configuration.ModelDirectory);
(Model model, PredictorModel predictorModel, ModelParameter modelParameter) = GetModel(configuration);
_Faces = new D_Face(configuration, argZero, model, modelParameter, predictorModel);
if (!_ArgZeroIsConfigurationRootDirectory)
people = Array.Empty<Person>();
else
@ -81,6 +77,18 @@ public class DlibDotNet
e2Navigate.Navigate(propertyConfiguration, model, predictorModel, configuration.OutputResolutions[0]);
_Log.Information(propertyConfiguration.RootDirectory);
}
{
(ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetGifLowQuality();
_FaceParts = new D2_FaceParts(configuration, imageCodecInfo, encoderParameters, filenameExtension);
}
{
(ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetPngLowQuality();
_Faces = new D_Face(configuration, argZero, model, modelParameter, predictorModel, imageCodecInfo, encoderParameters, filenameExtension);
}
{
(ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetTuple(configuration.OutputExtension, configuration.OutputQuality);
_Resize = new C_Resize(configuration.ForceResizeLastWriteTimeToCreationTime, configuration.OverrideForResizeImages, configuration.PropertiesChangedForResize, configuration.ValidResolutions, imageCodecInfo, encoderParameters, filenameExtension);
}
if (!configuration.SkipSearch)
Search(propertyConfiguration, configuration.Reverse, model, predictorModel, argZero, people, isSilent);
if (_Exceptions.Count == 0 && _ArgZeroIsConfigurationRootDirectory)
@ -90,13 +98,13 @@ public class DlibDotNet
foreach (string[] directoryCollection in directoryCollections)
{
_Log.Information(string.Concat("Cleaning <", directoryCollection[0], ">"));
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(directoryCollection[0]);
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(directoryCollection[0]);
}
string d2FaceLandmarksRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(D2_FaceLandmarks));
_Log.Information(string.Concat("Cleaning <", d2FaceLandmarksRootDirectory, ">"));
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(d2FaceLandmarksRootDirectory);
string d2FacePartsRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(D2_FaceParts));
_Log.Information(string.Concat("Cleaning <", d2FacePartsRootDirectory, ">"));
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(d2FacePartsRootDirectory);
if (appSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(Property.Models.Stateless.IPath.DeleteEmptyDirectories));
ticks = LogDelta(ticks, nameof(Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories));
}
string message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}";
_Log.Information(message);
@ -112,7 +120,7 @@ public class DlibDotNet
}
}
private long LogDelta(long ticks, string methodName)
private long LogDelta(long ticks, string? methodName)
{
long result;
if (_Log is null)
@ -245,16 +253,12 @@ public class DlibDotNet
throw new Exception("Input directory should be the source and not a resized directory!");
}
private void FullParallelForWork(PropertyLogic propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<D_Face>> imageFaceCollections, string sourceDirectory, int index, Item item)
private void FullParallelForWork(A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Face>> imageFaceCollections, string sourceDirectory, int index, Item item)
{
if (item.ImageFileHolder is null)
throw new NullReferenceException(nameof(item.ImageFileHolder));
if (_Configuration?.PropertyConfiguration is null)
throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration));
if (_Configuration.PropertyConfiguration.WriteBitmapDataBytes is null)
throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration.WriteBitmapDataBytes));
A_Property property;
List<D_Face> faceCollection;
Shared.Models.Property property;
List<Face> faceCollection;
string original = "Original";
long ticks = DateTime.Now.Ticks;
DateTime dateTime = DateTime.Now;
@ -282,7 +286,7 @@ public class DlibDotNet
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection));
FileHolder resizedFileHolder = new(Path.Combine(_Resize.AngleBracketCollection[0].Replace("<>", "()"), Path.GetFileName(item.ImageFileHolder.FullName)));
item.SetResizedFileHolder(resizedFileHolder);
item.SetResizedFileHolder(_Resize.FilenameExtension, resizedFileHolder);
imageResizeKeyValuePairs = _Resize.GetResizeKeyValuePairs(cResultsFullGroupDirectory, subFileTuples, parseExceptions, original, metadataCollection, item);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(C_Resize.GetResizeKeyValuePairs));
@ -291,7 +295,7 @@ public class DlibDotNet
_Resize.SaveResizedSubfile(outputResolution, cResultsFullGroupDirectory, subFileTuples, item, original, imageResizeKeyValuePairs);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(C_Resize.SaveResizedSubfile));
item.SetResizedFileHolder(FileHolder.Refresh(resizedFileHolder));
item.SetResizedFileHolder(_Resize.FilenameExtension, Shared.Models.Stateless.Methods.IFileHolder.Refresh(resizedFileHolder));
}
else if (outputResolution == _Configuration.OutputResolutions[0] && false)
{
@ -315,11 +319,14 @@ public class DlibDotNet
_Faces.SaveFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, item, faceCollection);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D_Face.SaveFaces));
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
int?[] normalizedPixelPercentageCollection = Shared.Models.Stateless.Methods.IFace.GetInts(faceCollection);
int normalizedPixelPercentageDistinctCount = normalizedPixelPercentageCollection.Distinct().Count();
if (normalizedPixelPercentageDistinctCount != normalizedPixelPercentageCollection.Length || _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
{
_FaceLandmarks.SaveFaceLandmarkImages(d2ResultsFullGroupDirectory, sourceDirectory, subFileTuples, parseExceptions, item, faceCollection);
bool saveRotated = _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution);
_FaceParts.SaveFaceLandmarkImages(d2ResultsFullGroupDirectory, sourceDirectory, subFileTuples, parseExceptions, item, faceCollection, saveRotated);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D2_FaceLandmarks.SaveFaceLandmarkImages));
ticks = LogDelta(ticks, nameof(D2_FaceParts.SaveFaceLandmarkImages));
}
}
lock (sourceDirectoryChanges)
@ -333,7 +340,7 @@ public class DlibDotNet
}
}
private int FullParallelWork(long ticks, PropertyLogic propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<D_Face>> faceCollections, int containersCount, Container container, Item[] filteredItems)
private int FullParallelWork(long ticks, A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Face>> faceCollections, int containersCount, Container container, Item[] filteredItems)
{
int result = 0;
if (_Log is null)
@ -345,9 +352,9 @@ public class DlibDotNet
for (int i = 0; i < filteredItems.Length; i++)
{
faceCollections.Add(new());
propertyCollection.Add(null);
metadataCollection.Add(new());
resizeKeyValuePairs.Add(new());
propertyCollection.Add(new());
propertyFileHolderCollection.Add(null);
}
}
@ -401,7 +408,7 @@ public class DlibDotNet
if (metadataIdLines.Any())
{
text = string.Join(Environment.NewLine, from l in metadataIdLines orderby l.Id select l.Line);
_ = Property.Models.Stateless.IPath.WriteAllText(fileInfo.FullName, text, updateDateWhenMatches: true, compareBeforeWrite: true);
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, text, updateDateWhenMatches: true, compareBeforeWrite: true);
}
else
{
@ -410,25 +417,23 @@ public class DlibDotNet
}
}
private void WriteGroup(Property.Models.Configuration configuration, PropertyLogic propertyLogic, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<List<D_Face>> faceCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, string sourceDirectory, string outputResolution, Item[] filteredItems)
private void WriteGroup(Property.Models.Configuration configuration, A_Property propertyLogic, Shared.Models.Property[] propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<List<Face>> faceCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, string sourceDirectory, string outputResolution, Item[] filteredItems)
{
if (configuration.PropertiesChangedForProperty is null)
throw new NullReferenceException(nameof(configuration.PropertiesChangedForProperty));
Item item;
string key;
string json;
string checkFile;
int sourceDirectoryLength = sourceDirectory.Length;
_FilePropertiesKeyValuePairs.Add(sourceDirectory, new List<Tuple<string, A_Property>>());
_FilePropertiesKeyValuePairs.Add(sourceDirectory, new List<Tuple<string, Shared.Models.Property>>());
JsonSerializerOptions writeIndentedJsonSerializerOptions = new() { WriteIndented = false };
if (!(from l in propertyCollection where l?.Width is null select true).Any())
{
string checkDirectory;
List<KeyValuePair<string, List<D_Face>>> faceCollectionsKeyValuePairs = new();
List<KeyValuePair<string, A_Property>> propertyCollectionKeyValuePairs = new();
List<KeyValuePair<string, List<Face>>> faceCollectionsKeyValuePairs = new();
List<KeyValuePair<string, Shared.Models.Property>> propertyCollectionKeyValuePairs = new();
List<KeyValuePair<string, Dictionary<string, int[]>>> resizeKeyValuePairsCollections = new();
List<KeyValuePair<string, List<KeyValuePair<string, string>>>> metadataCollectionKeyValuePairs = new();
(int level, List<string> directories) = Property.Models.Stateless.IPath.Get(configuration.RootDirectory, sourceDirectory);
(int level, List<string> directories) = Shared.Models.Stateless.Methods.IPath.Get(configuration.RootDirectory, sourceDirectory);
string fileName = string.Concat(string.Join(configuration.FileNameDirectorySeparator, directories), ".json");
for (int i = 0; i < filteredItems.Length; i++)
{
@ -437,53 +442,53 @@ public class DlibDotNet
continue;
if (item.ImageFileHolder is null)
continue;
key = Property.Models.Stateless.IPath.GetRelativePath(item.ImageFileHolder.FullName, sourceDirectoryLength);
key = Shared.Models.Stateless.Methods.IPath.GetRelativePath(item.ImageFileHolder.FullName, sourceDirectoryLength);
_FileKeyValuePairs.Add(new KeyValuePair<string, string>(sourceDirectory, key));
_FilePropertiesKeyValuePairs[sourceDirectory].Add(new Tuple<string, A_Property>(key, propertyCollection[i]));
faceCollectionsKeyValuePairs.Add(new KeyValuePair<string, List<D_Face>>(key, faceCollections[i]));
propertyCollectionKeyValuePairs.Add(new KeyValuePair<string, A_Property>(key, propertyCollection[i]));
_FilePropertiesKeyValuePairs[sourceDirectory].Add(new Tuple<string, Shared.Models.Property>(key, propertyCollection[i]));
faceCollectionsKeyValuePairs.Add(new KeyValuePair<string, List<Face>>(key, faceCollections[i]));
propertyCollectionKeyValuePairs.Add(new KeyValuePair<string, Shared.Models.Property>(key, propertyCollection[i]));
resizeKeyValuePairsCollections.Add(new KeyValuePair<string, Dictionary<string, int[]>>(key, resizeKeyValuePairs[i]));
metadataCollectionKeyValuePairs.Add(new KeyValuePair<string, List<KeyValuePair<string, string>>>(key, metadataCollection[i]));
}
if (propertyLogic.AngleBracketCollection.Any())
{
checkDirectory = Property.Models.Stateless.IPath.GetDirectory(propertyLogic.AngleBracketCollection[0], level, "[{}]");
checkDirectory = Shared.Models.Stateless.Methods.IPath.GetDirectory(propertyLogic.AngleBracketCollection[0], level, "[{}]");
checkFile = Path.Combine(checkDirectory, fileName);
if (File.Exists(checkFile))
File.Move(checkFile, Path.Combine(checkDirectory, fileName));
checkFile = Path.Combine(checkDirectory, fileName);
json = JsonSerializer.Serialize(propertyCollectionKeyValuePairs, writeIndentedJsonSerializerOptions);
_ = Property.Models.Stateless.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true);
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true);
}
if (_Metadata.AngleBracketCollection.Any())
{
checkDirectory = Property.Models.Stateless.IPath.GetDirectory(_Metadata.AngleBracketCollection[0], level, "[{}]");
checkDirectory = Shared.Models.Stateless.Methods.IPath.GetDirectory(_Metadata.AngleBracketCollection[0], level, "[{}]");
checkFile = Path.Combine(checkDirectory, fileName);
if (File.Exists(checkFile))
File.Move(checkFile, Path.Combine(checkDirectory, fileName));
checkFile = Path.Combine(checkDirectory, fileName);
json = JsonSerializer.Serialize(metadataCollectionKeyValuePairs, writeIndentedJsonSerializerOptions);
_ = Property.Models.Stateless.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true);
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true);
}
if (_Resize.AngleBracketCollection.Any())
{
checkDirectory = Property.Models.Stateless.IPath.GetDirectory(_Resize.AngleBracketCollection[0], level, "[{}]");
checkDirectory = Shared.Models.Stateless.Methods.IPath.GetDirectory(_Resize.AngleBracketCollection[0], level, "[{}]");
checkFile = Path.Combine(checkDirectory, fileName);
if (File.Exists(checkFile))
File.Move(checkFile, Path.Combine(checkDirectory, fileName));
checkFile = Path.Combine(checkDirectory, fileName);
json = JsonSerializer.Serialize(resizeKeyValuePairsCollections, writeIndentedJsonSerializerOptions);
_ = Property.Models.Stateless.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true);
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true);
}
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution) && _Faces.AngleBracketCollection.Any())
{
checkDirectory = Property.Models.Stateless.IPath.GetDirectory(_Faces.AngleBracketCollection[0], level, "[{}]");
checkDirectory = Shared.Models.Stateless.Methods.IPath.GetDirectory(_Faces.AngleBracketCollection[0], level, "[{}]");
checkFile = Path.Combine(checkDirectory, fileName);
if (File.Exists(checkFile))
File.Move(checkFile, Path.Combine(checkDirectory, fileName));
checkFile = Path.Combine(checkDirectory, fileName);
json = JsonSerializer.Serialize(faceCollectionsKeyValuePairs, writeIndentedJsonSerializerOptions);
_ = Property.Models.Stateless.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true);
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true);
}
}
}
@ -499,7 +504,7 @@ public class DlibDotNet
string dResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true);
string d2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(D2_FaceLandmarks), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true);
configuration, model, predictorModel, nameof(D2_FaceParts), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true);
string eResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true);
string zResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
@ -507,7 +512,7 @@ public class DlibDotNet
return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory, zResultsFullGroupDirectory);
}
private void SetAngleBracketCollections(Property.Models.Configuration configuration, PropertyLogic propertyLogic, string outputResolution, Container container, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory)
private void SetAngleBracketCollections(Property.Models.Configuration configuration, A_Property propertyLogic, string outputResolution, Container container, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory)
{
_Faces.AngleBracketCollection.Clear();
_Resize.AngleBracketCollection.Clear();
@ -544,7 +549,7 @@ public class DlibDotNet
converted: true));
}
private void FullDoWork(bool isSilent, string argZero, Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, long ticks, Dictionary<string, List<Person>> peopleCollection, PropertyLogic propertyLogic, List<Container> containers)
private void FullDoWork(string argZero, Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, long ticks, Map.Models.MapLogic mapLogic, A_Property propertyLogic, List<Container> containers)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
@ -559,11 +564,12 @@ public class DlibDotNet
string eResultsFullGroupDirectory;
string zResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
List<List<D_Face>> faceCollections = new();
List<A_Property> propertyCollection = new();
List<List<Face>> faceCollections = new();
Shared.Models.Property[] propertyCollection;
List<FileHolder?> propertyFileHolderCollection = new();
List<Dictionary<string, int[]>> resizeKeyValuePairs = new();
List<Tuple<string, DateTime>> sourceDirectoryChanges = new();
List<Shared.Models.Property?> nullablePropertyCollection = new();
List<List<KeyValuePair<string, string>>> metadataCollection = new();
string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(A_Property), "{}");
string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(configuration, nameof(A_Property));
@ -583,28 +589,29 @@ public class DlibDotNet
continue;
faceCollections.Clear();
metadataCollection.Clear();
propertyCollection.Clear();
resizeKeyValuePairs.Clear();
sourceDirectoryChanges.Clear();
nullablePropertyCollection.Clear();
propertyFileHolderCollection.Clear();
SetAngleBracketCollections(configuration, propertyLogic, outputResolution, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory);
exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, containers.Count, container, filteredItems);
exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, nullablePropertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, containers.Count, container, filteredItems);
#pragma warning disable
ids = (from l in filteredItems where l.Property?.Id is not null select l.Property.Id.Value).ToArray();
#pragma warning restore
distinctCount = ids.Distinct().Count();
if (ids.Length != distinctCount)
_Log.Information($"{ids.Length} != {distinctCount} <{container.SourceDirectory}>");
if (metadataCollection.Count != filteredItems.Length || propertyCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || faceCollections.Count != filteredItems.Length)
if (metadataCollection.Count != filteredItems.Length || nullablePropertyCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || faceCollections.Count != filteredItems.Length)
throw new Exception("Counts don't match!");
if (exceptionCount != 0)
_Exceptions.Add(container.SourceDirectory);
for (int i = 0; i < faceCollections.Count; i++)
filteredItems[i].Faces.AddRange(from l in faceCollections[i] select l);
propertyCollection = (from l in nullablePropertyCollection where l is not null select l).ToArray();
if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0)
WriteGroup(configuration, propertyLogic, propertyCollection, metadataCollection, faceCollections, resizeKeyValuePairs, container.SourceDirectory, outputResolution, filteredItems);
if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0 && outputResolution == _Configuration.OutputResolutions[0])
propertyLogic.AddToPropertyLogicAllCollection(filteredItems);
mapLogic.AddToMapLogicAllCollection(filteredItems);
if (exceptionCount == 0 && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_Distance.LoadOrCreateThenSaveDistanceResults(configuration, eResultsFullGroupDirectory, container.SourceDirectory, outputResolution, sourceDirectoryChanges, filteredItems);
if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any())
@ -633,15 +640,6 @@ public class DlibDotNet
}
}
private PropertyLogic GetPropertyLogic(bool reverse, Model? model, PredictorModel? predictorModel)
{
PropertyLogic result;
if (_Configuration?.PropertyConfiguration is null)
throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration));
result = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, reverse, model, predictorModel);
return result;
}
private void Search(Property.Models.Configuration configuration, bool reverse, Model? model, PredictorModel? predictorModel, string argZero, Person[] people, bool isSilent)
{
if (_Log is null)
@ -656,26 +654,27 @@ public class DlibDotNet
string zResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
Dictionary<string, List<Person>> peopleCollection = A2_People.Convert(people);
PropertyLogic propertyLogic = GetPropertyLogic(reverse, model, predictorModel);
Map.Models.MapLogic mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration);
A_Property propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FilenameExtension, reverse, model, predictorModel, mapLogic.IndicesFromNew, mapLogic.KeyValuePairs);
if (string.IsNullOrEmpty(configuration.RootDirectory))
containers = Property.Models.Stateless.A_Property.Get(configuration, propertyLogic);
containers = A_Property.Get(configuration, propertyLogic);
else
containers = Property.Models.Stateless.Container.GetContainers(configuration, propertyLogic);
FullDoWork(isSilent, argZero, configuration, model, predictorModel, ticks, peopleCollection, propertyLogic, containers);
FullDoWork(argZero, configuration, model, predictorModel, ticks, mapLogic, propertyLogic, containers);
foreach (string outputResolution in _Configuration.OutputResolutions)
{
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory, zResultsFullGroupDirectory) = GetResultsFullGroupDirectories(configuration, model, predictorModel, outputResolution);
if (_ArgZeroIsConfigurationRootDirectory && _Exceptions.Count == 0 && outputResolution == _Configuration.OutputResolutions[0] && (propertyLogic.NamedFaceInfoDeterministicHashCodeKeyValuePairs.Any() || propertyLogic.NamedDeterministicHashCodeKeyValuePairs.Any()))
if (_ArgZeroIsConfigurationRootDirectory && _Exceptions.Count == 0 && outputResolution == _Configuration.OutputResolutions[0] && (mapLogic.NamedFaceInfoDeterministicHashCodeKeyValuePairs.Any() || mapLogic.NamedDeterministicHashCodeKeyValuePairs.Any()))
{
if (!string.IsNullOrEmpty(propertyLogic.DeterministicHashCodeRootDirectory) && !propertyLogic.IncorrectDeterministicHashCodeKeyValuePairs.Any())
propertyLogic.UpdateKeyValuePairs(containers);
if (!string.IsNullOrEmpty(mapLogic.DeterministicHashCodeRootDirectory) && !mapLogic.IncorrectDeterministicHashCodeKeyValuePairs.Any())
mapLogic.UpdateKeyValuePairs(containers);
foreach (Container container in containers)
{
Item.AddToNamed(propertyLogic, container.Items);
Map.Models.MapLogic.AddToNamed(mapLogic, container.Items);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
D_Face.SaveShortcuts(_Configuration.JuliePhares, dResultsFullGroupDirectory, ticks, peopleCollection, propertyLogic, container.Items);
D_Face.SaveShortcuts(_Configuration.JuliePhares, dResultsFullGroupDirectory, ticks, peopleCollection, mapLogic, container.Items);
}
propertyLogic.SaveAllCollection();
mapLogic.SaveAllCollection();
if (_Configuration.SaveResizedSubfiles)
{
string dFacesContentDirectory;
@ -687,13 +686,13 @@ public class DlibDotNet
zPropertyHolderSingletonDirectory = Path.Combine(zResultsFullGroupDirectory, "{}");
eDistanceCollectionDirectory = Path.Combine(eResultsFullGroupDirectory, $"[{ticks}]");
List<(DateTime, bool?, PersonBirthday, FaceRecognitionDotNet.FaceEncoding[])> collection;
collection = E_Distance.ParallelWork(_AppSettings.MaxDegreeOfParallelism, argZero, propertyLogic, containers);
collection = E_Distance.ParallelWork(_AppSettings.MaxDegreeOfParallelism, argZero, mapLogic, containers);
_ = LogDeltaInSeconds(ticks, nameof(E_Distance.ParallelWork));
E_Distance.SavePropertyHolders(argZero, containers, zPropertyHolderSingletonDirectory);
_ = LogDeltaInSeconds(ticks, nameof(E_Distance.SavePropertyHolders));
E_Distance.SaveThreeSigmaFaceEncodings(collection, peopleCollection, eDistanceCollectionDirectory);
_ = LogDeltaInSeconds(ticks, nameof(E_Distance.SaveThreeSigmaFaceEncodings));
E_Distance.SaveClosest(argZero, containers, peopleCollection, eDistanceContentDirectory, dFacesContentDirectory);
E_Distance.SaveClosest(argZero, containers, peopleCollection, dFacesContentDirectory, d2ResultsFullGroupDirectory, eDistanceContentDirectory);
_ = LogDeltaInMinutes(ticks, nameof(E_Distance.SaveClosest));
}
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any())
@ -708,18 +707,18 @@ public class DlibDotNet
identify.WriteAllText(configuration, _Configuration.OutputResolutions[0], identifiedCollection);
if (_Configuration.LoadOrCreateThenSaveIndex && _FilePropertiesKeyValuePairs.Any())
_Index.SetIndex(configuration, model, predictorModel, _Configuration.OutputResolutions[0], _FilePropertiesKeyValuePairs);
}
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(aResultsFullGroupDirectory, "{}"));
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(bResultsFullGroupDirectory, "{}"));
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(cResultsFullGroupDirectory, "{}"));
}
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(aResultsFullGroupDirectory, "{}"));
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(bResultsFullGroupDirectory, "{}"));
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(cResultsFullGroupDirectory, "{}"));
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(dResultsFullGroupDirectory, "[]"));
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(dResultsFullGroupDirectory, "[]"));
if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(eResultsFullGroupDirectory, "[]"));
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(eResultsFullGroupDirectory, "[]"));
if (_Configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions.Contains(outputResolution))
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(eResultsFullGroupDirectory, "[]"));
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(eResultsFullGroupDirectory, "[]"));
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(d2ResultsFullGroupDirectory, "[]"));
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(d2ResultsFullGroupDirectory, "[]"));
}
}