AppSettings and Configuration changes,

major changes to E_Distance and minor for D_Face
This commit is contained in:
2022-08-19 21:37:36 -07:00
parent 004017b9dd
commit be7a6abbf4
60 changed files with 1269 additions and 1344 deletions

View File

@ -40,8 +40,6 @@ public class DlibDotNet
string argZero;
Person[] people;
_AppSettings = appSettings;
if (appSettings.MaxDegreeOfParallelism is null)
throw new NullReferenceException(nameof(appSettings.MaxDegreeOfParallelism));
_IsEnvironment = isEnvironment;
_Exceptions = new List<string>();
_Log = Serilog.Log.ForContext<DlibDotNet>();
@ -50,7 +48,7 @@ public class DlibDotNet
Property.Models.Configuration propertyConfiguration = Property.Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory);
_Log.Information(propertyConfiguration.RootDirectory);
Property.Models.Configuration.Verify(propertyConfiguration);
Models.Configuration configuration = Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory, propertyConfiguration);
Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration);
Verify(configuration);
VerifyExtra(args, propertyConfiguration, configuration);
_Configuration = configuration;
@ -60,47 +58,31 @@ public class DlibDotNet
_Rename = new E3_Rename(configuration);
_Distance = new E_Distance(configuration);
_FaceLandmarks = new D2_FaceLandmarks(configuration);
if (configuration.ForceMetadataLastWriteTimeToCreationTime is null)
throw new NullReferenceException(nameof(configuration.ForceMetadataLastWriteTimeToCreationTime));
if (configuration.ForceResizeLastWriteTimeToCreationTime is null)
throw new NullReferenceException(nameof(configuration.ForceResizeLastWriteTimeToCreationTime));
if (configuration.IgnoreExtensions is null)
throw new NullReferenceException(nameof(configuration.IgnoreExtensions));
if (configuration.OutputQuality is null)
throw new NullReferenceException(nameof(configuration.OutputQuality));
if (configuration.OverrideForResizeImages is null)
throw new NullReferenceException(nameof(configuration.OverrideForResizeImages));
if (configuration.PropertiesChangedForMetadata is null)
throw new NullReferenceException(nameof(configuration.PropertiesChangedForMetadata));
if (configuration.PropertiesChangedForResize is null)
throw new NullReferenceException(nameof(configuration.PropertiesChangedForResize));
if (configuration.Reverse is null)
throw new NullReferenceException(nameof(configuration.Reverse));
_Metadata = new(configuration.ForceMetadataLastWriteTimeToCreationTime.Value, configuration.PropertiesChangedForMetadata.Value);
_Metadata = new(configuration.ForceMetadataLastWriteTimeToCreationTime, configuration.PropertiesChangedForMetadata);
if (args.Count > 0)
argZero = Path.GetFullPath(args[0]);
else
argZero = Path.GetFullPath(propertyConfiguration.RootDirectory);
_ArgZeroIsConfigurationRootDirectory = propertyConfiguration.RootDirectory == argZero;
(ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters) = C_Resize.GetTuple(configuration.OutputExtension, configuration.OutputQuality.Value);
_Resize = new C_Resize(configuration.ForceResizeLastWriteTimeToCreationTime.Value, configuration.OverrideForResizeImages.Value, configuration.PropertiesChangedForResize.Value, configuration.ValidResolutions, imageCodecInfo, encoderParameters);
(ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters) = C_Resize.GetTuple(configuration.OutputExtension, configuration.OutputQuality);
_Resize = new C_Resize(configuration.ForceResizeLastWriteTimeToCreationTime, configuration.OverrideForResizeImages, configuration.PropertiesChangedForResize, configuration.ValidResolutions, imageCodecInfo, encoderParameters);
_Log.Information(configuration.ModelDirectory);
(Model model, PredictorModel predictorModel, ModelParameter modelParameter) = GetModel(configuration);
_Faces = new D_Face(configuration, argZero, model, modelParameter, predictorModel);
if (configuration.SkipSearch is null)
throw new NullReferenceException(nameof(configuration.SkipSearch));
if (!_ArgZeroIsConfigurationRootDirectory)
people = Array.Empty<Person>();
else
people = _People.GetPeople(propertyConfiguration);
if (!isSilent && configuration.TestDistanceResults.HasValue && configuration.TestDistanceResults.Value)
if (!isSilent && configuration.TestDistanceResults)
{
E2_Navigate e2Navigate = new(console, configuration, argZero);
e2Navigate.Navigate(propertyConfiguration, model, predictorModel, configuration.OutputResolutions[0]);
_Log.Information(propertyConfiguration.RootDirectory);
}
if (!configuration.SkipSearch.Value)
Search(propertyConfiguration, configuration.Reverse.Value, model, predictorModel, argZero, people, isSilent);
if (!configuration.SkipSearch)
Search(propertyConfiguration, configuration.Reverse, model, predictorModel, argZero, people, isSilent);
if (_Exceptions.Count == 0 && _ArgZeroIsConfigurationRootDirectory)
{
long ticks = DateTime.Now.Ticks;
@ -113,7 +95,7 @@ public class DlibDotNet
string d2FaceLandmarksRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(D2_FaceLandmarks));
_Log.Information(string.Concat("Cleaning <", d2FaceLandmarksRootDirectory, ">"));
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(d2FaceLandmarksRootDirectory);
if (appSettings.MaxDegreeOfParallelism.Value < 2)
if (appSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(Property.Models.Stateless.IPath.DeleteEmptyDirectories));
}
string message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}";
@ -125,7 +107,7 @@ public class DlibDotNet
long ticks = DateTime.Now.Ticks;
foreach (string outputResolution in configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions)
_Distance.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions(propertyConfiguration, model, predictorModel, outputResolution);
if (appSettings.MaxDegreeOfParallelism.Value < 2)
if (appSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(E_Distance.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions));
}
}
@ -141,6 +123,17 @@ public class DlibDotNet
return result;
}
private long LogDeltaInMinutes(long ticks, string methodName)
{
long result;
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
double delta = new TimeSpan(DateTime.Now.Ticks - ticks).Minutes;
_Log.Debug($"{methodName} took {Math.Floor(delta)} minutes(s)");
result = DateTime.Now.Ticks;
return result;
}
#pragma warning disable CA1416
#pragma warning restore CA1416
@ -201,82 +194,18 @@ public class DlibDotNet
throw new Exception($"One or more {nameof(configuration.SaveShortcutsForOutputResolutions)} are not in the ValidResolutions list!");
if ((from l in configuration.SaveFaceLandmarkForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
throw new Exception($"One or more {nameof(configuration.SaveFaceLandmarkForOutputResolutions)} are not in the ValidResolutions list!");
if (configuration.CheckJsonForDistanceResults is null)
throw new NullReferenceException(nameof(configuration.CheckJsonForDistanceResults));
if (configuration.CrossDirectoryMaxItemsInDistanceCollection is null)
throw new NullReferenceException(nameof(configuration.CrossDirectoryMaxItemsInDistanceCollection));
if (configuration.DistanceFactor is null)
if (configuration.DistanceFactor + configuration.LocationConfidenceFactor != 10)
throw new NullReferenceException(nameof(configuration.DistanceFactor));
if (configuration.ForceMetadataLastWriteTimeToCreationTime is null)
throw new NullReferenceException(nameof(configuration.ForceMetadataLastWriteTimeToCreationTime));
if (configuration.ForceResizeLastWriteTimeToCreationTime is null)
throw new NullReferenceException(nameof(configuration.ForceResizeLastWriteTimeToCreationTime));
if (configuration.IgnoreExtensions is null)
throw new NullReferenceException(nameof(configuration.IgnoreExtensions));
if (configuration.IgnoreRelativePaths is null)
throw new NullReferenceException(nameof(configuration.IgnoreRelativePaths));
if (configuration.LoadOrCreateThenSaveIndex is null)
throw new NullReferenceException(nameof(configuration.LoadOrCreateThenSaveIndex));
if (configuration.LocationConfidenceFactor is null)
throw new NullReferenceException(nameof(configuration.LocationConfidenceFactor));
if (configuration.MappedMaxIndex is null)
throw new NullReferenceException(nameof(configuration.MappedMaxIndex));
if (configuration.MappedMaxIndex.HasValue && configuration.LoadOrCreateThenSaveIndex.Value && !_IsEnvironment.DebuggerWasAttachedDuringConstructor)
throw new NullReferenceException(nameof(configuration.MappedMaxIndex));
if (configuration.MaxItemsInDistanceCollection is null)
throw new NullReferenceException(nameof(configuration.MaxItemsInDistanceCollection));
if (configuration.MixedYearRelativePaths is null)
throw new NullReferenceException(nameof(configuration.MixedYearRelativePaths));
if (configuration.NumberOfJitters is null)
throw new NullReferenceException(nameof(configuration.NumberOfJitters));
if (configuration.NumberOfTimesToUpsample is null)
throw new NullReferenceException(nameof(configuration.NumberOfTimesToUpsample));
if (configuration.OutputQuality is null)
throw new NullReferenceException(nameof(configuration.OutputQuality));
if (configuration.OutputResolutions is null)
throw new NullReferenceException(nameof(configuration.OutputResolutions));
if (configuration.OverrideForFaceImages is null)
throw new NullReferenceException(nameof(configuration.OverrideForFaceImages));
if (configuration.OverrideForFaceLandmarkImages is null)
throw new NullReferenceException(nameof(configuration.OverrideForFaceLandmarkImages));
if (configuration.OverrideForResizeImages is null)
throw new NullReferenceException(nameof(configuration.OverrideForResizeImages));
if (configuration.PaddingLoops is null)
throw new NullReferenceException(nameof(configuration.PaddingLoops));
if (configuration.PropertiesChangedForDistance is null)
throw new NullReferenceException(nameof(configuration.PropertiesChangedForDistance));
if (configuration.PropertiesChangedForFaces is null)
throw new NullReferenceException(nameof(configuration.PropertiesChangedForFaces));
if (configuration.PropertiesChangedForIndex is null)
throw new NullReferenceException(nameof(configuration.PropertiesChangedForIndex));
if (configuration.PropertiesChangedForMetadata is null)
throw new NullReferenceException(nameof(configuration.PropertiesChangedForMetadata));
if (configuration.PropertiesChangedForResize is null)
throw new NullReferenceException(nameof(configuration.PropertiesChangedForResize));
if (configuration.Reverse is null)
throw new NullReferenceException(nameof(configuration.Reverse));
if (configuration.SaveFaceLandmarkForOutputResolutions is null)
throw new NullReferenceException(nameof(configuration.SaveFaceLandmarkForOutputResolutions));
if (configuration.SaveFullYearOfRandomFiles is null)
throw new NullReferenceException(nameof(configuration.SaveFullYearOfRandomFiles));
if (configuration.SaveResizedSubfiles is null)
throw new NullReferenceException(nameof(configuration.SaveResizedSubfiles));
if (configuration.SkipSearch is null)
throw new NullReferenceException(nameof(configuration.SkipSearch));
if (configuration.TestDistanceResults is null)
throw new NullReferenceException(nameof(configuration.TestDistanceResults));
if (configuration.ValidResolutions is null)
throw new NullReferenceException(nameof(configuration.ValidResolutions));
if (string.IsNullOrEmpty(configuration.ModelDirectory) || !Directory.Exists(configuration.ModelDirectory))
throw new NullReferenceException(nameof(configuration.ModelDirectory));
if (string.IsNullOrEmpty(configuration.ModelName))
throw new NullReferenceException(nameof(configuration.ModelName));
if (string.IsNullOrEmpty(configuration.OutputExtension))
throw new NullReferenceException(nameof(configuration.OutputExtension));
if (string.IsNullOrEmpty(configuration.PredictorModelName))
throw new NullReferenceException(nameof(configuration.PredictorModelName));
if (configuration.DistanceFactor.Value + configuration.LocationConfidenceFactor.Value != 10)
throw new NullReferenceException(nameof(configuration.DistanceFactor));
if (string.IsNullOrEmpty(configuration.ModelDirectory) || !Directory.Exists(configuration.ModelDirectory))
throw new NullReferenceException(nameof(configuration.ModelDirectory));
if (configuration.MappedMaxIndex.HasValue && configuration.LoadOrCreateThenSaveIndex && !_IsEnvironment.DebuggerWasAttachedDuringConstructor)
throw new NullReferenceException(nameof(configuration.MappedMaxIndex));
}
private void VerifyExtra(List<string> args, Property.Models.Configuration propertyConfiguration, Models.Configuration configuration)
@ -309,10 +238,6 @@ public class DlibDotNet
{
if (item.ImageFileHolder is null)
throw new NullReferenceException(nameof(item.ImageFileHolder));
if (_AppSettings.MaxDegreeOfParallelism is null)
throw new NullReferenceException(nameof(_AppSettings.MaxDegreeOfParallelism));
if (_Configuration.SaveResizedSubfiles is null)
throw new NullReferenceException(nameof(_Configuration.SaveResizedSubfiles));
if (_Configuration?.PropertyConfiguration is null)
throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration));
if (_Configuration.PropertyConfiguration.WriteBitmapDataBytes is null)
@ -343,24 +268,24 @@ public class DlibDotNet
}
}
(int metadataGroups, metadataCollection) = _Metadata.GetMetadataCollection(bResultsFullGroupDirectory, subFileTuples, parseExceptions, item);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection));
FileHolder resizedFileHolder = new(Path.Combine(_Resize.AngleBracketCollection[0].Replace("<>", "()"), Path.GetFileName(item.ImageFileHolder.FullName)));
item.SetResizedFileHolder(resizedFileHolder);
imageResizeKeyValuePairs = _Resize.GetResizeKeyValuePairs(cResultsFullGroupDirectory, subFileTuples, parseExceptions, original, metadataCollection, property, item);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
imageResizeKeyValuePairs = _Resize.GetResizeKeyValuePairs(cResultsFullGroupDirectory, subFileTuples, parseExceptions, original, metadataCollection, item);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(C_Resize.GetResizeKeyValuePairs));
if (_Configuration.SaveResizedSubfiles.Value)
if (_Configuration.SaveResizedSubfiles)
{
_Resize.SaveResizedSubfile(outputResolution, cResultsFullGroupDirectory, subFileTuples, item, original, property, imageResizeKeyValuePairs);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
_Resize.SaveResizedSubfile(outputResolution, cResultsFullGroupDirectory, subFileTuples, item, original, imageResizeKeyValuePairs);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(C_Resize.SaveResizedSubfile));
item.SetResizedFileHolder(FileHolder.Refresh(resizedFileHolder));
}
else if (outputResolution == _Configuration.OutputResolutions[0] && false)
{
byte[] bytes = _Resize.GetResizedBytes(outputResolution, cResultsFullGroupDirectory, subFileTuples, item, property, imageResizeKeyValuePairs);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(C_Resize.GetResizedBytes));
string path = Path.Combine(resizedFileHolder.DirectoryName, resizedFileHolder.NameWithoutExtension);
File.WriteAllBytes(path, bytes);
@ -374,15 +299,15 @@ public class DlibDotNet
int outputResolutionHeight = outputResolutionCollection[1];
int outputResolutionOrientation = outputResolutionCollection[2];
faceCollection = _Faces.GetFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, item, property, resizedFileHolder, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D_Face.GetFaces));
_Faces.SaveFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, item, faceCollection);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D_Face.SaveFaces));
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
{
_FaceLandmarks.SaveFaceLandmarkImages(d2ResultsFullGroupDirectory, sourceDirectory, subFileTuples, parseExceptions, item, faceCollection);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D2_FaceLandmarks.SaveFaceLandmarkImages));
}
}
@ -402,9 +327,7 @@ public class DlibDotNet
int result = 0;
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
if (_AppSettings.MaxDegreeOfParallelism is null)
throw new NullReferenceException(nameof(AppSettings.MaxDegreeOfParallelism));
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism.Value };
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism };
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
if (faceCollections.Count != filteredItems.Length || metadataCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || propertyCollection.Count != filteredItems.Length)
{
@ -478,14 +401,8 @@ public class DlibDotNet
private void WriteGroup(Property.Models.Configuration configuration, PropertyLogic propertyLogic, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<List<D_Face>> faceCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, string sourceDirectory, string outputResolution, Item[] filteredItems)
{
if (_Configuration.PropertiesChangedForMetadata is null)
throw new NullReferenceException(nameof(_Configuration.PropertiesChangedForMetadata));
if (configuration.PropertiesChangedForProperty is null)
throw new NullReferenceException(nameof(configuration.PropertiesChangedForProperty));
if (_Configuration.PropertiesChangedForResize is null)
throw new NullReferenceException(nameof(_Configuration.PropertiesChangedForResize));
if (_Configuration.PropertiesChangedForFaces is null)
throw new NullReferenceException(nameof(_Configuration.PropertiesChangedForFaces));
Item item;
string key;
string json;
@ -560,19 +477,70 @@ public class DlibDotNet
}
}
private void FullDoWork(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string argZero, Dictionary<string, List<Person>> peopleCollection, PropertyLogic propertyLogic, List<Container> containers, bool isSilent)
private (string, string, string, string, string, string, string) GetResultsFullGroupDirectories(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string outputResolution)
{
string aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(A_Property), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false);
string bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(B_Metadata), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false);
string cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(C_Resize), outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false);
string dResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true);
string d2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(D2_FaceLandmarks), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true);
string eResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true);
string zResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, $"Z_{nameof(Item)}", outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true);
return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory, zResultsFullGroupDirectory);
}
private void SetAngleBracketCollections(Property.Models.Configuration configuration, PropertyLogic propertyLogic, string outputResolution, Container container, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory)
{
_Faces.AngleBracketCollection.Clear();
_Resize.AngleBracketCollection.Clear();
_Metadata.AngleBracketCollection.Clear();
propertyLogic.AngleBracketCollection.Clear();
propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
aResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Properties for each image",
collectionDescription: string.Empty,
converted: false));
_Metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
bResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Metadata as key value pairs",
collectionDescription: string.Empty,
converted: true));
_Resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
cResultsFullGroupDirectory,
contentDescription: "Resized image",
singletonDescription: "Resize dimensions for each resolution",
collectionDescription: string.Empty,
converted: true));
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
_Faces.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
dResultsFullGroupDirectory,
contentDescription: "n png file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: "For each image a json file with all faces found",
converted: true));
}
private void FullDoWork(bool isSilent, string argZero, Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, long ticks, Dictionary<string, List<Person>> peopleCollection, PropertyLogic propertyLogic, List<Container> containers)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
if (_AppSettings.MaxDegreeOfParallelism is null)
throw new NullReferenceException(nameof(_AppSettings.MaxDegreeOfParallelism));
if (_Configuration.SaveResizedSubfiles is null)
throw new NullReferenceException(nameof(_Configuration.SaveResizedSubfiles));
int[] ids;
int distinctCount;
int exceptionCount;
Item[] filteredItems;
long ticks = DateTime.Now.Ticks;
string aResultsFullGroupDirectory;
string bResultsFullGroupDirectory;
string cResultsFullGroupDirectory;
@ -592,20 +560,7 @@ public class DlibDotNet
{
_FileKeyValuePairs.Clear();
_FilePropertiesKeyValuePairs.Clear();
aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(A_Property), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false);
bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(B_Metadata), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false);
cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(C_Resize), outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false);
d2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(D2_FaceLandmarks), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true);
dResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true);
eResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true);
zResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, $"Z_{nameof(Item)}", outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true);
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory, zResultsFullGroupDirectory) = GetResultsFullGroupDirectories(configuration, model, predictorModel, outputResolution);
foreach (Container container in containers)
{
if (!container.Items.Any())
@ -621,39 +576,7 @@ public class DlibDotNet
resizeKeyValuePairs.Clear();
sourceDirectoryChanges.Clear();
propertyFileHolderCollection.Clear();
_Faces.AngleBracketCollection.Clear();
_Resize.AngleBracketCollection.Clear();
_Metadata.AngleBracketCollection.Clear();
propertyLogic.AngleBracketCollection.Clear();
propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
aResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Properties for each image",
collectionDescription: string.Empty,
converted: false));
_Metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
bResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Metadata as key value pairs",
collectionDescription: string.Empty,
converted: true));
_Resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
cResultsFullGroupDirectory,
contentDescription: "Resized image",
singletonDescription: "Resize dimensions for each resolution",
collectionDescription: string.Empty,
converted: true));
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
_Faces.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
container.SourceDirectory,
dResultsFullGroupDirectory,
contentDescription: "n png file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: "For each image a json file with all faces found",
converted: true));
SetAngleBracketCollections(configuration, propertyLogic, outputResolution, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory);
exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, containers.Count, container, filteredItems);
#pragma warning disable
ids = (from l in filteredItems where l.Property?.Id is not null select l.Property.Id.Value).ToArray();
@ -665,35 +588,14 @@ public class DlibDotNet
throw new Exception("Counts don't match!");
if (exceptionCount != 0)
_Exceptions.Add(container.SourceDirectory);
for (int i = 0; i < faceCollections.Count; i++)
filteredItems[i].Faces.AddRange(from l in faceCollections[i] select l);
if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0)
WriteGroup(configuration, propertyLogic, propertyCollection, metadataCollection, faceCollections, resizeKeyValuePairs, container.SourceDirectory, outputResolution, filteredItems);
if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0 && outputResolution == _Configuration.OutputResolutions[0])
propertyLogic.AddToPropertyLogicAllCollection(filteredItems);
if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0 && propertyLogic.NamedFaceInfoDeterministicHashCodeIds.Any())
{
if (outputResolution == _Configuration.OutputResolutions[0] || _Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
{
if (isSilent && container.SourceDirectory.EndsWith("Bohdi Ray 2016")) // 7#.1
break;
// if (isSilent && container.SourceDirectory.EndsWith("Halloween 2013")) // 12#.1
// break;
// if (isSilent && container.SourceDirectory.EndsWith("zzz =2014.4 Winter Tracy Pictures")) // 30#.2
// break;
// if (isSilent && container.SourceDirectory.EndsWith("=2009.0 Winter Logan Michael")) //34#.2
// break;
// if (isSilent && container.SourceDirectory.EndsWith("Animal Kingdom")) //35#.3
// break;
// if (isSilent && container.SourceDirectory.EndsWith("Texas 2015")) //46#.4
// break;
for (int i = 0; i < faceCollections.Count; i++)
filteredItems[i].Faces.AddRange(from l in faceCollections[i] select l);
Item.AddToNamed(propertyLogic, filteredItems);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
D_Face.SaveShortcuts(_Configuration.JuliePhares, dResultsFullGroupDirectory, ticks, peopleCollection, propertyLogic, filteredItems);
}
}
if (exceptionCount == 0 && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_Distance.LoadOrCreateThenSaveDistanceResultsForOutputResolutions(configuration, eResultsFullGroupDirectory, container.SourceDirectory, outputResolution, sourceDirectoryChanges, filteredItems, faceCollections);
_Distance.LoadOrCreateThenSaveDistanceResults(configuration, eResultsFullGroupDirectory, container.SourceDirectory, outputResolution, sourceDirectoryChanges, filteredItems);
if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any())
{
for (int y = 0; y < int.MaxValue; y++)
@ -704,7 +606,54 @@ public class DlibDotNet
}
_Log.Information(". . .");
}
if (isSilent && container.SourceDirectory.EndsWith("Bohdi Ray 2016")) // 7#.1
break;
// if (isSilent && container.SourceDirectory.EndsWith("Halloween 2013")) // 12#.1
// break;
// if (isSilent && container.SourceDirectory.EndsWith("zzz =2014.4 Winter Tracy Pictures")) // 30#.2
// break;
// if (isSilent && container.SourceDirectory.EndsWith("=2009.0 Winter Logan Michael")) //34#.2
// break;
// if (isSilent && container.SourceDirectory.EndsWith("Animal Kingdom")) //35#.3
// break;
// if (isSilent && container.SourceDirectory.EndsWith("Texas 2015")) //46#.4
// break;
}
}
}
private PropertyLogic GetPropertyLogic(bool reverse, Model? model, PredictorModel? predictorModel)
{
PropertyLogic result;
if (_Configuration?.PropertyConfiguration is null)
throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration));
result = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, reverse, model, predictorModel);
return result;
}
private void Search(Property.Models.Configuration configuration, bool reverse, Model? model, PredictorModel? predictorModel, string argZero, Person[] people, bool isSilent)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
List<Container> containers;
long ticks = DateTime.Now.Ticks;
string aResultsFullGroupDirectory;
string bResultsFullGroupDirectory;
string cResultsFullGroupDirectory;
string dResultsFullGroupDirectory;
string eResultsFullGroupDirectory;
string zResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
Dictionary<string, List<Person>> peopleCollection = A2_People.Convert(people);
PropertyLogic propertyLogic = GetPropertyLogic(reverse, model, predictorModel);
if (string.IsNullOrEmpty(configuration.RootDirectory))
containers = Property.Models.Stateless.A_Property.Get(configuration, propertyLogic);
else
containers = Property.Models.Stateless.Container.GetContainers(configuration, propertyLogic);
FullDoWork(isSilent, argZero, configuration, model, predictorModel, ticks, peopleCollection, propertyLogic, containers);
foreach (string outputResolution in _Configuration.OutputResolutions)
{
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory, zResultsFullGroupDirectory) = GetResultsFullGroupDirectories(configuration, model, predictorModel, outputResolution);
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(aResultsFullGroupDirectory, "{}"));
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(bResultsFullGroupDirectory, "{}"));
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(cResultsFullGroupDirectory, "{}"));
@ -716,10 +665,18 @@ public class DlibDotNet
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(eResultsFullGroupDirectory, "[]"));
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(Path.Combine(d2ResultsFullGroupDirectory, "[]"));
if (_ArgZeroIsConfigurationRootDirectory && outputResolution == _Configuration.OutputResolutions[0])
if (_ArgZeroIsConfigurationRootDirectory && _Exceptions.Count == 0 && outputResolution == _Configuration.OutputResolutions[0] && (propertyLogic.NamedFaceInfoDeterministicHashCodeKeyValuePairs.Any() || propertyLogic.NamedDeterministicHashCodeKeyValuePairs.Any()))
{
if (!string.IsNullOrEmpty(propertyLogic.DeterministicHashCodeRootDirectory) && !propertyLogic.IncorrectDeterministicHashCodeKeyValuePairs.Any())
propertyLogic.UpdateKeyValuePairs(containers);
foreach (Container container in containers)
{
Item.AddToNamed(propertyLogic, container.Items);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
D_Face.SaveShortcuts(_Configuration.JuliePhares, dResultsFullGroupDirectory, ticks, peopleCollection, propertyLogic, container.Items);
}
propertyLogic.SaveAllCollection();
if (_Configuration.SaveResizedSubfiles.Value && propertyLogic.NamedFaceInfoDeterministicHashCodeIds.Any())
if (_Configuration.SaveResizedSubfiles)
{
string dFacesContentDirectory;
string eDistanceContentDirectory;
@ -730,57 +687,31 @@ public class DlibDotNet
zPropertyHolderSingletonDirectory = Path.Combine(zResultsFullGroupDirectory, "{}");
eDistanceCollectionDirectory = Path.Combine(eResultsFullGroupDirectory, $"[{ticks}]");
List<(DateTime, bool?, PersonBirthday, FaceRecognitionDotNet.FaceEncoding[])> collection;
collection = E_Distance.ParallelWork(_AppSettings.MaxDegreeOfParallelism.Value, argZero, containers);
_ = LogDelta(ticks, nameof(E_Distance.ParallelWork));
collection = E_Distance.ParallelWork(_AppSettings.MaxDegreeOfParallelism, argZero, propertyLogic, containers);
_ = LogDeltaInMinutes(ticks, nameof(E_Distance.ParallelWork));
E_Distance.SavePropertyHolders(argZero, containers, zPropertyHolderSingletonDirectory);
_ = LogDelta(ticks, nameof(E_Distance.SavePropertyHolders));
_ = LogDeltaInMinutes(ticks, nameof(E_Distance.SavePropertyHolders));
E_Distance.SaveThreeSigmaFaceEncodings(collection, peopleCollection, eDistanceCollectionDirectory);
_ = LogDelta(ticks, nameof(E_Distance.SaveThreeSigmaFaceEncodings));
E_Distance.SaveClosest(argZero, containers, peopleCollection, propertyLogic, eDistanceContentDirectory, dFacesContentDirectory);
_ = LogDelta(ticks, nameof(E_Distance.SaveClosest));
_ = LogDeltaInMinutes(ticks, nameof(E_Distance.SaveThreeSigmaFaceEncodings));
E_Distance.SaveClosest(argZero, containers, peopleCollection, eDistanceContentDirectory, dFacesContentDirectory);
_ = LogDeltaInMinutes(ticks, nameof(E_Distance.SaveClosest));
}
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any())
break;
if (_Exceptions.Count == 0)
{
if (_FileKeyValuePairs.Any())
_Random.Random(configuration, _Configuration.OutputResolutions[0], _FileKeyValuePairs);
if (_IsEnvironment.Development)
continue;
G2_Identify identify = new(_Configuration);
List<G2_Identify> identifiedCollection = identify.GetIdentifiedCollection(configuration, _IsEnvironment, _People);
_People.WriteAllText(configuration, _Configuration.OutputResolutions[0], identifiedCollection);
identify.WriteAllText(configuration, _Configuration.OutputResolutions[0], identifiedCollection);
if (_Configuration.LoadOrCreateThenSaveIndex.HasValue && _Configuration.LoadOrCreateThenSaveIndex.Value && _FilePropertiesKeyValuePairs.Any())
_Index.SetIndex(configuration, model, predictorModel, _Configuration.OutputResolutions[0], _FilePropertiesKeyValuePairs);
}
if (_FileKeyValuePairs.Any())
_Random.Random(configuration, _Configuration.OutputResolutions[0], _FileKeyValuePairs);
if (_IsEnvironment.Development)
continue;
G2_Identify identify = new(_Configuration);
List<G2_Identify> identifiedCollection = identify.GetIdentifiedCollection(configuration, _IsEnvironment, _People);
_People.WriteAllText(configuration, _Configuration.OutputResolutions[0], identifiedCollection);
identify.WriteAllText(configuration, _Configuration.OutputResolutions[0], identifiedCollection);
if (_Configuration.LoadOrCreateThenSaveIndex && _FilePropertiesKeyValuePairs.Any())
_Index.SetIndex(configuration, model, predictorModel, _Configuration.OutputResolutions[0], _FilePropertiesKeyValuePairs);
}
}
}
private PropertyLogic GetPropertyLogic(bool reverse, Model? model, PredictorModel? predictorModel)
{
PropertyLogic result;
if (_AppSettings.MaxDegreeOfParallelism is null)
throw new NullReferenceException(nameof(_AppSettings.MaxDegreeOfParallelism));
if (_Configuration?.PropertyConfiguration is null)
throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration));
result = new(_AppSettings.MaxDegreeOfParallelism.Value, _Configuration.PropertyConfiguration, reverse, model, predictorModel);
return result;
}
private void Search(Property.Models.Configuration configuration, bool reverse, Model? model, PredictorModel? predictorModel, string argZero, Person[] people, bool isSilent)
{
List<Container> containers;
Dictionary<string, List<Person>> peopleCollection = A2_People.Convert(people);
PropertyLogic propertyLogic = GetPropertyLogic(reverse, model, predictorModel);
if (string.IsNullOrEmpty(configuration.RootDirectory))
containers = Property.Models.Stateless.A_Property.Get(configuration, propertyLogic);
else
containers = Property.Models.Stateless.Container.GetContainers(configuration, propertyLogic);
FullDoWork(configuration, model, predictorModel, argZero, peopleCollection, propertyLogic, containers, isSilent);
}
internal void RenameQueue(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel) => _Rename.RenameQueue(configuration, model, predictorModel);
}