NullReferenceException

This commit is contained in:
2022-08-13 11:19:08 -07:00
parent 3aeab88384
commit 0392de1920
33 changed files with 1278 additions and 1030 deletions

View File

@ -41,7 +41,7 @@ public class DlibDotNet
Person[] people;
_AppSettings = appSettings;
if (appSettings.MaxDegreeOfParallelism is null)
throw new ArgumentNullException(nameof(appSettings.MaxDegreeOfParallelism));
throw new NullReferenceException(nameof(appSettings.MaxDegreeOfParallelism));
_IsEnvironment = isEnvironment;
_Exceptions = new List<string>();
_Log = Serilog.Log.ForContext<DlibDotNet>();
@ -61,21 +61,21 @@ public class DlibDotNet
_Distance = new E_Distance(configuration);
_FaceLandmarks = new D2_FaceLandmarks(configuration);
if (configuration.ForceMetadataLastWriteTimeToCreationTime is null)
throw new ArgumentNullException(nameof(configuration.ForceMetadataLastWriteTimeToCreationTime));
throw new NullReferenceException(nameof(configuration.ForceMetadataLastWriteTimeToCreationTime));
if (configuration.ForceResizeLastWriteTimeToCreationTime is null)
throw new ArgumentNullException(nameof(configuration.ForceResizeLastWriteTimeToCreationTime));
throw new NullReferenceException(nameof(configuration.ForceResizeLastWriteTimeToCreationTime));
if (configuration.IgnoreExtensions is null)
throw new ArgumentNullException(nameof(configuration.IgnoreExtensions));
throw new NullReferenceException(nameof(configuration.IgnoreExtensions));
if (configuration.OutputQuality is null)
throw new ArgumentNullException(nameof(configuration.OutputQuality));
throw new NullReferenceException(nameof(configuration.OutputQuality));
if (configuration.OverrideForResizeImages is null)
throw new ArgumentNullException(nameof(configuration.OverrideForResizeImages));
throw new NullReferenceException(nameof(configuration.OverrideForResizeImages));
if (configuration.PropertiesChangedForMetadata is null)
throw new ArgumentNullException(nameof(configuration.PropertiesChangedForMetadata));
throw new NullReferenceException(nameof(configuration.PropertiesChangedForMetadata));
if (configuration.PropertiesChangedForResize is null)
throw new ArgumentNullException(nameof(configuration.PropertiesChangedForResize));
throw new NullReferenceException(nameof(configuration.PropertiesChangedForResize));
if (configuration.Reverse is null)
throw new ArgumentNullException(nameof(configuration.Reverse));
throw new NullReferenceException(nameof(configuration.Reverse));
_Metadata = new(configuration.ForceMetadataLastWriteTimeToCreationTime.Value, configuration.PropertiesChangedForMetadata.Value);
if (args.Count > 0)
argZero = Path.GetFullPath(args[0]);
@ -88,7 +88,7 @@ public class DlibDotNet
(Model model, PredictorModel predictorModel, ModelParameter modelParameter) = GetModel(configuration);
_Faces = new D_Face(configuration, argZero, model, modelParameter, predictorModel);
if (configuration.SkipSearch is null)
throw new ArgumentNullException(nameof(configuration.SkipSearch));
throw new NullReferenceException(nameof(configuration.SkipSearch));
if (!_ArgZeroIsConfigurationRootDirectory)
people = Array.Empty<Person>();
else
@ -134,7 +134,7 @@ public class DlibDotNet
{
long result;
if (_Log is null)
throw new ArgumentNullException(nameof(_Log));
throw new NullReferenceException(nameof(_Log));
double delta = new TimeSpan(DateTime.Now.Ticks - ticks).TotalMilliseconds;
_Log.Debug($"{methodName} took {Math.Floor(delta)} millisecond(s)");
result = DateTime.Now.Ticks;
@ -188,7 +188,7 @@ public class DlibDotNet
private void Verify(Models.Configuration configuration)
{
if (!configuration.OutputResolutions.Any() || string.IsNullOrEmpty(configuration.OutputResolutions[0]) || !configuration.ValidResolutions.Contains(configuration.OutputResolutions[0]))
throw new ArgumentNullException(nameof(configuration.OutputResolutions), "must be a valid outputResolution!");
throw new NullReferenceException($"{nameof(configuration.OutputResolutions)} must be a valid outputResolution!");
if ((from l in configuration.OutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
throw new Exception($"One or more {nameof(configuration.OutputResolutions)} are not in the ValidResolutions list!");
if ((from l in configuration.LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
@ -202,81 +202,81 @@ public class DlibDotNet
if ((from l in configuration.SaveFaceLandmarkForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
throw new Exception($"One or more {nameof(configuration.SaveFaceLandmarkForOutputResolutions)} are not in the ValidResolutions list!");
if (configuration.CheckJsonForDistanceResults is null)
throw new ArgumentNullException(nameof(configuration.CheckJsonForDistanceResults));
throw new NullReferenceException(nameof(configuration.CheckJsonForDistanceResults));
if (configuration.CrossDirectoryMaxItemsInDistanceCollection is null)
throw new ArgumentNullException(nameof(configuration.CrossDirectoryMaxItemsInDistanceCollection));
throw new NullReferenceException(nameof(configuration.CrossDirectoryMaxItemsInDistanceCollection));
if (configuration.DistanceFactor is null)
throw new ArgumentNullException(nameof(configuration.DistanceFactor));
throw new NullReferenceException(nameof(configuration.DistanceFactor));
if (configuration.ForceMetadataLastWriteTimeToCreationTime is null)
throw new ArgumentNullException(nameof(configuration.ForceMetadataLastWriteTimeToCreationTime));
throw new NullReferenceException(nameof(configuration.ForceMetadataLastWriteTimeToCreationTime));
if (configuration.ForceResizeLastWriteTimeToCreationTime is null)
throw new ArgumentNullException(nameof(configuration.ForceResizeLastWriteTimeToCreationTime));
throw new NullReferenceException(nameof(configuration.ForceResizeLastWriteTimeToCreationTime));
if (configuration.IgnoreExtensions is null)
throw new ArgumentNullException(nameof(configuration.IgnoreExtensions));
throw new NullReferenceException(nameof(configuration.IgnoreExtensions));
if (configuration.IgnoreRelativePaths is null)
throw new ArgumentNullException(nameof(configuration.IgnoreRelativePaths));
throw new NullReferenceException(nameof(configuration.IgnoreRelativePaths));
if (configuration.LoadOrCreateThenSaveIndex is null)
throw new ArgumentNullException(nameof(configuration.LoadOrCreateThenSaveIndex));
throw new NullReferenceException(nameof(configuration.LoadOrCreateThenSaveIndex));
if (configuration.LocationConfidenceFactor is null)
throw new ArgumentNullException(nameof(configuration.LocationConfidenceFactor));
throw new NullReferenceException(nameof(configuration.LocationConfidenceFactor));
if (configuration.MappedMaxIndex is null)
throw new ArgumentNullException(nameof(configuration.MappedMaxIndex));
throw new NullReferenceException(nameof(configuration.MappedMaxIndex));
if (configuration.MappedMaxIndex.HasValue && configuration.LoadOrCreateThenSaveIndex.Value && !_IsEnvironment.DebuggerWasAttachedDuringConstructor)
throw new ArgumentNullException(nameof(configuration.MappedMaxIndex));
throw new NullReferenceException(nameof(configuration.MappedMaxIndex));
if (configuration.MaxItemsInDistanceCollection is null)
throw new ArgumentNullException(nameof(configuration.MaxItemsInDistanceCollection));
throw new NullReferenceException(nameof(configuration.MaxItemsInDistanceCollection));
if (configuration.MixedYearRelativePaths is null)
throw new ArgumentNullException(nameof(configuration.MixedYearRelativePaths));
throw new NullReferenceException(nameof(configuration.MixedYearRelativePaths));
if (configuration.NumberOfJitters is null)
throw new ArgumentNullException(nameof(configuration.NumberOfJitters));
throw new NullReferenceException(nameof(configuration.NumberOfJitters));
if (configuration.NumberOfTimesToUpsample is null)
throw new ArgumentNullException(nameof(configuration.NumberOfTimesToUpsample));
throw new NullReferenceException(nameof(configuration.NumberOfTimesToUpsample));
if (configuration.OutputQuality is null)
throw new ArgumentNullException(nameof(configuration.OutputQuality));
throw new NullReferenceException(nameof(configuration.OutputQuality));
if (configuration.OutputResolutions is null)
throw new ArgumentNullException(nameof(configuration.OutputResolutions));
throw new NullReferenceException(nameof(configuration.OutputResolutions));
if (configuration.OverrideForFaceImages is null)
throw new ArgumentNullException(nameof(configuration.OverrideForFaceImages));
throw new NullReferenceException(nameof(configuration.OverrideForFaceImages));
if (configuration.OverrideForFaceLandmarkImages is null)
throw new ArgumentNullException(nameof(configuration.OverrideForFaceLandmarkImages));
throw new NullReferenceException(nameof(configuration.OverrideForFaceLandmarkImages));
if (configuration.OverrideForResizeImages is null)
throw new ArgumentNullException(nameof(configuration.OverrideForResizeImages));
throw new NullReferenceException(nameof(configuration.OverrideForResizeImages));
if (configuration.PaddingLoops is null)
throw new ArgumentNullException(nameof(configuration.PaddingLoops));
throw new NullReferenceException(nameof(configuration.PaddingLoops));
if (configuration.PropertiesChangedForDistance is null)
throw new ArgumentNullException(nameof(configuration.PropertiesChangedForDistance));
throw new NullReferenceException(nameof(configuration.PropertiesChangedForDistance));
if (configuration.PropertiesChangedForFaces is null)
throw new ArgumentNullException(nameof(configuration.PropertiesChangedForFaces));
throw new NullReferenceException(nameof(configuration.PropertiesChangedForFaces));
if (configuration.PropertiesChangedForIndex is null)
throw new ArgumentNullException(nameof(configuration.PropertiesChangedForIndex));
throw new NullReferenceException(nameof(configuration.PropertiesChangedForIndex));
if (configuration.PropertiesChangedForMetadata is null)
throw new ArgumentNullException(nameof(configuration.PropertiesChangedForMetadata));
throw new NullReferenceException(nameof(configuration.PropertiesChangedForMetadata));
if (configuration.PropertiesChangedForResize is null)
throw new ArgumentNullException(nameof(configuration.PropertiesChangedForResize));
throw new NullReferenceException(nameof(configuration.PropertiesChangedForResize));
if (configuration.Reverse is null)
throw new ArgumentNullException(nameof(configuration.Reverse));
throw new NullReferenceException(nameof(configuration.Reverse));
if (configuration.SaveFaceLandmarkForOutputResolutions is null)
throw new ArgumentNullException(nameof(configuration.SaveFaceLandmarkForOutputResolutions));
throw new NullReferenceException(nameof(configuration.SaveFaceLandmarkForOutputResolutions));
if (configuration.SaveFullYearOfRandomFiles is null)
throw new ArgumentNullException(nameof(configuration.SaveFullYearOfRandomFiles));
throw new NullReferenceException(nameof(configuration.SaveFullYearOfRandomFiles));
if (configuration.SaveResizedSubfiles is null)
throw new ArgumentNullException(nameof(configuration.SaveResizedSubfiles));
throw new NullReferenceException(nameof(configuration.SaveResizedSubfiles));
if (configuration.SkipSearch is null)
throw new ArgumentNullException(nameof(configuration.SkipSearch));
throw new NullReferenceException(nameof(configuration.SkipSearch));
if (configuration.TestDistanceResults is null)
throw new ArgumentNullException(nameof(configuration.TestDistanceResults));
throw new NullReferenceException(nameof(configuration.TestDistanceResults));
if (configuration.ValidResolutions is null)
throw new ArgumentNullException(nameof(configuration.ValidResolutions));
throw new NullReferenceException(nameof(configuration.ValidResolutions));
if (string.IsNullOrEmpty(configuration.ModelDirectory) || !Directory.Exists(configuration.ModelDirectory))
throw new ArgumentNullException(nameof(configuration.ModelDirectory));
throw new NullReferenceException(nameof(configuration.ModelDirectory));
if (string.IsNullOrEmpty(configuration.ModelName))
throw new ArgumentNullException(nameof(configuration.ModelName));
throw new NullReferenceException(nameof(configuration.ModelName));
if (string.IsNullOrEmpty(configuration.OutputExtension))
throw new ArgumentNullException(nameof(configuration.OutputExtension));
throw new NullReferenceException(nameof(configuration.OutputExtension));
if (string.IsNullOrEmpty(configuration.PredictorModelName))
throw new ArgumentNullException(nameof(configuration.PredictorModelName));
throw new NullReferenceException(nameof(configuration.PredictorModelName));
if (configuration.DistanceFactor.Value + configuration.LocationConfidenceFactor.Value != 10)
throw new ArgumentNullException(nameof(configuration.DistanceFactor));
throw new NullReferenceException(nameof(configuration.DistanceFactor));
}
private void VerifyExtra(List<string> args, Property.Models.Configuration propertyConfiguration, Models.Configuration configuration)
@ -305,18 +305,18 @@ public class DlibDotNet
throw new Exception("Input directory should be the source and not a resized directory!");
}
private void FullParallelForWork(PropertyLogic propertyLogic, string outputResolution, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<D_Face>> imageFaceCollections, string sourceDirectory, int index, PropertyHolder propertyHolder)
private void FullParallelForWork(PropertyLogic propertyLogic, string outputResolution, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<D_Face>> imageFaceCollections, string sourceDirectory, int index, Item item)
{
if (propertyHolder.ImageFileHolder is null)
throw new ArgumentNullException(nameof(propertyHolder.ImageFileHolder));
if (item.ImageFileHolder is null)
throw new NullReferenceException(nameof(item.ImageFileHolder));
if (_AppSettings.MaxDegreeOfParallelism is null)
throw new ArgumentNullException(nameof(_AppSettings.MaxDegreeOfParallelism));
throw new NullReferenceException(nameof(_AppSettings.MaxDegreeOfParallelism));
if (_Configuration.SaveResizedSubfiles is null)
throw new ArgumentNullException(nameof(_Configuration.SaveResizedSubfiles));
throw new NullReferenceException(nameof(_Configuration.SaveResizedSubfiles));
if (_Configuration?.PropertyConfiguration is null)
throw new ArgumentNullException(nameof(_Configuration.PropertyConfiguration));
throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration));
if (_Configuration.PropertyConfiguration.WriteBitmapDataBytes is null)
throw new ArgumentNullException(nameof(_Configuration.PropertyConfiguration.WriteBitmapDataBytes));
throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration.WriteBitmapDataBytes));
A_Property property;
List<D_Face> faceCollection;
string original = "Original";
@ -326,40 +326,44 @@ public class DlibDotNet
Dictionary<string, int[]> imageResizeKeyValuePairs;
List<Tuple<string, DateTime>> subFileTuples = new();
List<KeyValuePair<string, string>> metadataCollection;
if (propertyHolder.Property is null)
if (item.Property is null)
{
lock (sourceDirectoryChanges)
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), DateTime.Now));
property = propertyLogic.GetProperty(propertyLogic.AngleBracketCollection[0], propertyHolder, subFileTuples, parseExceptions);
propertyHolder.Update(property);
property = propertyLogic.GetProperty(item, subFileTuples, parseExceptions);
item.Update(property);
if (subFileTuples.Any())
{
lock (sourceDirectoryChanges)
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), (from l in subFileTuples select l.Item2).Max()));
}
}
else
{
property = propertyHolder.Property;
if (propertyHolder.Changed.HasValue && propertyHolder.Changed.Value)
property = item.Property;
if (item.Changed.HasValue && item.Changed.Value)
{
lock (sourceDirectoryChanges)
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), DateTime.Now));
// lock (sourceDirectoryChanges)
// sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), DateTime.Now));
throw new NotImplementedException();
}
}
(int metadataGroups, metadataCollection) = _Metadata.GetMetadataCollection(subFileTuples, parseExceptions, propertyHolder);
(int metadataGroups, metadataCollection) = _Metadata.GetMetadataCollection(subFileTuples, parseExceptions, item);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection));
FileHolder resizedFileHolder = new(Path.Combine(_Resize.AngleBracketCollection[0].Replace("<>", "()"), Path.GetFileName(propertyHolder.ImageFileHolder.FullName)));
propertyHolder.SetResizedFileHolder(resizedFileHolder);
imageResizeKeyValuePairs = _Resize.GetResizeKeyValuePairs(subFileTuples, parseExceptions, original, metadataCollection, property, propertyHolder);
FileHolder resizedFileHolder = new(Path.Combine(_Resize.AngleBracketCollection[0].Replace("<>", "()"), Path.GetFileName(item.ImageFileHolder.FullName)));
item.SetResizedFileHolder(resizedFileHolder);
imageResizeKeyValuePairs = _Resize.GetResizeKeyValuePairs(subFileTuples, parseExceptions, original, metadataCollection, property, item);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(C_Resize.GetResizeKeyValuePairs));
if (_Configuration.SaveResizedSubfiles.Value)
{
_Resize.SaveResizedSubfile(outputResolution, subFileTuples, propertyHolder, original, property, imageResizeKeyValuePairs);
_Resize.SaveResizedSubfile(outputResolution, subFileTuples, item, original, property, imageResizeKeyValuePairs);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(C_Resize.SaveResizedSubfile));
propertyHolder.SetResizedFileHolder(FileHolder.Refresh(resizedFileHolder));
item.SetResizedFileHolder(FileHolder.Refresh(resizedFileHolder));
}
else if (outputResolution == _Configuration.OutputResolutions[0] && false)
{
byte[] bytes = _Resize.GetResizedBytes(outputResolution, subFileTuples, propertyHolder, property, imageResizeKeyValuePairs);
byte[] bytes = _Resize.GetResizedBytes(outputResolution, subFileTuples, item, property, imageResizeKeyValuePairs);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(C_Resize.GetResizedBytes));
string path = Path.Combine(resizedFileHolder.DirectoryName, resizedFileHolder.NameWithoutExtension);
@ -373,15 +377,15 @@ public class DlibDotNet
int outputResolutionWidth = outputResolutionCollection[0];
int outputResolutionHeight = outputResolutionCollection[1];
int outputResolutionOrientation = outputResolutionCollection[2];
faceCollection = _Faces.GetFaces(_Configuration.PropertyConfiguration, outputResolution, subFileTuples, parseExceptions, propertyHolder, property, resizedFileHolder, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation);
faceCollection = _Faces.GetFaces(_Configuration.PropertyConfiguration, outputResolution, subFileTuples, parseExceptions, item, property, resizedFileHolder, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(D_Face.GetFaces));
_Faces.SaveFaces(_Configuration.PropertyConfiguration, subFileTuples, parseExceptions, propertyHolder, faceCollection);
_Faces.SaveFaces(_Configuration.PropertyConfiguration, subFileTuples, parseExceptions, item, faceCollection);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(D_Face.SaveFaces));
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
{
_FaceLandmarks.SaveFaceLandmarkImages(subFileTuples, parseExceptions, propertyHolder, faceCollection);
_FaceLandmarks.SaveFaceLandmarkImages(subFileTuples, parseExceptions, item, faceCollection);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(D2_FaceLandmarks.SaveFaceLandmarkImages));
}
@ -392,23 +396,23 @@ public class DlibDotNet
imageFaceCollections[index] = faceCollection;
metadataCollections[index] = metadataCollection;
resizeKeyValuePairs[index] = imageResizeKeyValuePairs;
propertyFileHolderCollection[index] = propertyHolder.ImageFileHolder;
propertyFileHolderCollection[index] = item.ImageFileHolder;
sourceDirectoryChanges.AddRange(from l in subFileTuples where l.Item2 > dateTime select l);
}
}
private int FullParallelWork(long ticks, PropertyLogic propertyLogic, string outputResolution, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<D_Face>> faceCollections, int propertyHolderCollectionsCount, int g, string sourceDirectory, int r, PropertyHolder[] filteredPropertyHolderCollection)
private int FullParallelWork(long ticks, PropertyLogic propertyLogic, string outputResolution, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<D_Face>> faceCollections, int containersCount, Container container, Item[] filteredItems)
{
int result = 0;
if (_Log is null)
throw new ArgumentNullException(nameof(_Log));
throw new NullReferenceException(nameof(_Log));
if (_AppSettings.MaxDegreeOfParallelism is null)
throw new ArgumentNullException(nameof(AppSettings.MaxDegreeOfParallelism));
throw new NullReferenceException(nameof(AppSettings.MaxDegreeOfParallelism));
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism.Value };
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
if (faceCollections.Count != filteredPropertyHolderCollection.Length || metadataCollection.Count != filteredPropertyHolderCollection.Length || resizeKeyValuePairs.Count != filteredPropertyHolderCollection.Length || propertyCollection.Count != filteredPropertyHolderCollection.Length)
if (faceCollections.Count != filteredItems.Length || metadataCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || propertyCollection.Count != filteredItems.Length)
{
for (int i = 0; i < filteredPropertyHolderCollection.Length; i++)
for (int i = 0; i < filteredItems.Length; i++)
{
faceCollections.Add(new());
metadataCollection.Add(new());
@ -418,23 +422,23 @@ public class DlibDotNet
}
}
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string message = $"{r + 1:000}.{g} / {propertyHolderCollectionsCount:000}) {filteredPropertyHolderCollection.Length:000} file(s) - {totalSeconds} total second(s) - {outputResolution} - {sourceDirectory}";
using (ProgressBar progressBar = new(filteredPropertyHolderCollection.Length, message, options))
string message = $"{container.R + 1:000}.{container.G} / {containersCount:000}) {filteredItems.Length:000} file(s) - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}";
using (ProgressBar progressBar = new(filteredItems.Length, message, options))
{
_ = Parallel.For(0, filteredPropertyHolderCollection.Length, parallelOptions, i =>
_ = Parallel.For(0, filteredItems.Length, parallelOptions, i =>
{
try
{
FullParallelForWork(propertyLogic, outputResolution, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, sourceDirectory, index: i, filteredPropertyHolderCollection[i]);
FullParallelForWork(propertyLogic, outputResolution, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, container.SourceDirectory, index: i, filteredItems[i]);
if (i == 0 || sourceDirectoryChanges.Any())
progressBar.Tick();
}
catch (Exception ex)
{
result += 1;
_Log.Error(string.Concat(sourceDirectory, Environment.NewLine, ex.Message, Environment.NewLine, ex.StackTrace), ex);
if (result == filteredPropertyHolderCollection.Length)
throw new Exception(string.Concat("All in [", sourceDirectory, "] failed!"));
_Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, ex.Message, Environment.NewLine, ex.StackTrace), ex);
if (result == filteredItems.Length)
throw new Exception(string.Concat("All in [", container.SourceDirectory, "] failed!"));
}
});
}
@ -476,20 +480,20 @@ public class DlibDotNet
}
}
private void WriteGroup(Property.Models.Configuration configuration, PropertyLogic propertyLogic, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<List<D_Face>> faceCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, string sourceDirectory, string outputResolution, PropertyHolder[] filteredPropertyHolderCollection)
private void WriteGroup(Property.Models.Configuration configuration, PropertyLogic propertyLogic, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<List<D_Face>> faceCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, string sourceDirectory, string outputResolution, Item[] filteredItems)
{
if (_Configuration.PropertiesChangedForMetadata is null)
throw new ArgumentNullException(nameof(_Configuration.PropertiesChangedForMetadata));
throw new NullReferenceException(nameof(_Configuration.PropertiesChangedForMetadata));
if (configuration.PropertiesChangedForProperty is null)
throw new ArgumentNullException(nameof(configuration.PropertiesChangedForProperty));
throw new NullReferenceException(nameof(configuration.PropertiesChangedForProperty));
if (_Configuration.PropertiesChangedForResize is null)
throw new ArgumentNullException(nameof(_Configuration.PropertiesChangedForResize));
throw new NullReferenceException(nameof(_Configuration.PropertiesChangedForResize));
if (_Configuration.PropertiesChangedForFaces is null)
throw new ArgumentNullException(nameof(_Configuration.PropertiesChangedForFaces));
throw new NullReferenceException(nameof(_Configuration.PropertiesChangedForFaces));
Item item;
string key;
string json;
string checkFile;
PropertyHolder propertyHolder;
int sourceDirectoryLength = sourceDirectory.Length;
_FilePropertiesKeyValuePairs.Add(sourceDirectory, new List<Tuple<string, A_Property>>());
JsonSerializerOptions writeIndentedJsonSerializerOptions = new() { WriteIndented = false };
@ -502,14 +506,14 @@ public class DlibDotNet
List<KeyValuePair<string, List<KeyValuePair<string, string>>>> metadataCollectionKeyValuePairs = new();
(int level, List<string> directories) = Property.Models.Stateless.IPath.Get(configuration.RootDirectory, sourceDirectory);
string fileName = string.Concat(string.Join(configuration.FileNameDirectorySeparator, directories), ".json");
for (int i = 0; i < filteredPropertyHolderCollection.Length; i++)
for (int i = 0; i < filteredItems.Length; i++)
{
propertyHolder = filteredPropertyHolderCollection[i];
if (propertyHolder.Property is null)
item = filteredItems[i];
if (item.Property is null)
continue;
if (propertyHolder.ImageFileHolder is null)
if (item.ImageFileHolder is null)
continue;
key = Property.Models.Stateless.IPath.GetRelativePath(propertyHolder.ImageFileHolder.FullName, sourceDirectoryLength);
key = Property.Models.Stateless.IPath.GetRelativePath(item.ImageFileHolder.FullName, sourceDirectoryLength);
_FileKeyValuePairs.Add(new KeyValuePair<string, string>(sourceDirectory, key));
_FilePropertiesKeyValuePairs[sourceDirectory].Add(new Tuple<string, A_Property>(key, propertyCollection[i]));
faceCollectionsKeyValuePairs.Add(new KeyValuePair<string, List<D_Face>>(key, faceCollections[i]));
@ -560,41 +564,35 @@ public class DlibDotNet
}
}
private void FullDoWork(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string argZero, Dictionary<string, List<Person>> peopleCollection, PropertyLogic propertyLogic, List<PropertyHolder[]> propertyHolderCollections)
private void FullDoWork(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string argZero, Dictionary<string, List<Person>> peopleCollection, PropertyLogic propertyLogic, List<Container> containers)
{
if (_Log is null)
throw new ArgumentNullException(nameof(_Log));
throw new NullReferenceException(nameof(_Log));
if (_AppSettings.MaxDegreeOfParallelism is null)
throw new ArgumentNullException(nameof(_AppSettings.MaxDegreeOfParallelism));
int g;
int r;
throw new NullReferenceException(nameof(_AppSettings.MaxDegreeOfParallelism));
int exceptionCount;
string sourceDirectory;
Item[] filteredItems;
long ticks = DateTime.Now.Ticks;
string eDistanceCollectionDirectory;
List<List<D_Face>> faceCollections = new();
List<A_Property> propertyCollection = new();
PropertyHolder[] filteredPropertyHolderCollection;
List<FileHolder?> propertyFileHolderCollection = new();
List<Dictionary<string, int[]>> resizeKeyValuePairs = new();
List<Tuple<string, DateTime>> sourceDirectoryChanges = new();
List<List<KeyValuePair<string, string>>> metadataCollection = new();
List<(string[], PersonBirthday, FaceRecognitionDotNet.FaceEncoding[])> collection;
string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(A_Property), "{}");
string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(configuration, nameof(A_Property));
foreach (string outputResolution in _Configuration.OutputResolutions)
{
_FileKeyValuePairs.Clear();
_FilePropertiesKeyValuePairs.Clear();
eDistanceCollectionDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), $"[{ticks}]");
foreach (PropertyHolder[] propertyHolderCollection in propertyHolderCollections)
foreach (Container container in containers)
{
if (!propertyHolderCollection.Any())
if (!container.Items.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !propertyHolderCollection[0].SourceDirectory.StartsWith(argZero))
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredPropertyHolderCollection = (from l in propertyHolderCollection where l.ImageFileHolder is not null && l.Property is not null && l.ValidImageFormatExtension.HasValue && l.ValidImageFormatExtension.Value && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.Extension) select l).ToArray();
if (!filteredPropertyHolderCollection.Any())
filteredItems = (from l in container.Items where l.ImageFileHolder is not null && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l).ToArray();
if (!filteredItems.Any())
continue;
faceCollections.Clear();
metadataCollection.Clear();
@ -605,15 +603,12 @@ public class DlibDotNet
_Faces.AngleBracketCollection.Clear();
_Resize.AngleBracketCollection.Clear();
_Metadata.AngleBracketCollection.Clear();
g = filteredPropertyHolderCollection[0].G;
r = filteredPropertyHolderCollection[0].R;
propertyLogic.AngleBracketCollection.Clear();
_FaceLandmarks.AngleBracketCollection.Clear();
sourceDirectory = filteredPropertyHolderCollection[0].SourceDirectory;
propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
model,
predictorModel,
sourceDirectory,
container.SourceDirectory,
nameof(A_Property),
outputResolution,
includeResizeGroup: false,
@ -625,7 +620,7 @@ public class DlibDotNet
_Metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
model,
predictorModel,
sourceDirectory,
container.SourceDirectory,
nameof(B_Metadata),
outputResolution,
includeResizeGroup: false,
@ -637,7 +632,7 @@ public class DlibDotNet
_Resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
model,
predictorModel,
sourceDirectory,
container.SourceDirectory,
nameof(C_Resize),
outputResolution,
includeResizeGroup: true,
@ -650,7 +645,7 @@ public class DlibDotNet
_Faces.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
model,
predictorModel,
sourceDirectory,
container.SourceDirectory,
nameof(D_Face),
outputResolution,
includeResizeGroup: true,
@ -663,7 +658,7 @@ public class DlibDotNet
_FaceLandmarks.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
model,
predictorModel,
sourceDirectory,
container.SourceDirectory,
nameof(D2_FaceLandmarks),
outputResolution,
includeResizeGroup: true,
@ -672,28 +667,28 @@ public class DlibDotNet
contentDescription: "n x 2 png file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: string.Empty));
exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, propertyHolderCollections.Count, g, sourceDirectory, r, filteredPropertyHolderCollection);
if (metadataCollection.Count != filteredPropertyHolderCollection.Length || propertyCollection.Count != filteredPropertyHolderCollection.Length || resizeKeyValuePairs.Count != filteredPropertyHolderCollection.Length || faceCollections.Count != filteredPropertyHolderCollection.Length)
exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, containers.Count, container, filteredItems);
if (metadataCollection.Count != filteredItems.Length || propertyCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || faceCollections.Count != filteredItems.Length)
throw new Exception("Counts don't match!");
if (exceptionCount != 0)
_Exceptions.Add(sourceDirectory);
_Exceptions.Add(container.SourceDirectory);
if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0)
WriteGroup(configuration, propertyLogic, propertyCollection, metadataCollection, faceCollections, resizeKeyValuePairs, sourceDirectory, outputResolution, filteredPropertyHolderCollection);
WriteGroup(configuration, propertyLogic, propertyCollection, metadataCollection, faceCollections, resizeKeyValuePairs, container.SourceDirectory, outputResolution, filteredItems);
if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0 && outputResolution == _Configuration.OutputResolutions[0])
propertyLogic.AddToPropertyLogicAllCollection(filteredPropertyHolderCollection);
propertyLogic.AddToPropertyLogicAllCollection(filteredItems);
if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0 && propertyLogic.NamedFaceInfoDeterministicHashCodeIndices.Any())
{
if (outputResolution == _Configuration.OutputResolutions[0] || _Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
{
for (int i = 0; i < faceCollections.Count; i++)
filteredPropertyHolderCollection[i].Faces.AddRange(from l in faceCollections[i] select l);
PropertyHolder.AddToNamed(propertyLogic, filteredPropertyHolderCollection);
filteredItems[i].Faces.AddRange(from l in faceCollections[i] select l);
Item.AddToNamed(propertyLogic, filteredItems);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
D_Face.SaveShortcuts(configuration, model, predictorModel, _Configuration.JuliePhares, ticks, peopleCollection, propertyLogic, outputResolution, filteredPropertyHolderCollection);
D_Face.SaveShortcuts(configuration, model, predictorModel, _Configuration.JuliePhares, ticks, peopleCollection, propertyLogic, outputResolution, filteredItems);
}
}
if (exceptionCount == 0 && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_Distance.LoadOrCreateThenSaveDistanceResultsForOutputResolutions(configuration, model, predictorModel, sourceDirectory, outputResolution, sourceDirectoryChanges, filteredPropertyHolderCollection, faceCollections);
_Distance.LoadOrCreateThenSaveDistanceResultsForOutputResolutions(configuration, model, predictorModel, container.SourceDirectory, outputResolution, sourceDirectoryChanges, filteredItems, faceCollections);
if (_Resize.AngleBracketCollection.Any())
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(_Resize.AngleBracketCollection[0].Replace("<>", "()"));
if (_Faces.AngleBracketCollection.Any())
@ -716,9 +711,22 @@ public class DlibDotNet
propertyLogic.SaveAllCollection();
if (propertyLogic.NamedFaceInfoDeterministicHashCodeIndices.Any())
{
collection = E_Distance.GetGroupedFaceEncodings(argZero, propertyHolderCollections);
E_Distance.SaveGroupedFaceEncodings(collection, peopleCollection, eDistanceCollectionDirectory);
E_Distance.GetClosest(argZero, propertyHolderCollections, collection, peopleCollection, eDistanceCollectionDirectory);
const int maxPer = 5;
const bool skipIsWrongYear = true;
string dFacesContentDirectory;
string eDistanceContentDirectory;
string eDistanceCollectionDirectory;
string zPropertyHolderSingletonDirectory;
List<(DateTime, bool?, PersonBirthday, FaceRecognitionDotNet.FaceEncoding[])> collection;
dFacesContentDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), "()");
eDistanceContentDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), $"({ticks})");
eDistanceCollectionDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), $"[{ticks}]");
zPropertyHolderSingletonDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, $"Z_{nameof(Item)}", outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), "{}");
collection = E_Distance.GetThreeSigmaFaceEncodings(argZero, containers);
E_Distance.AddClosest(argZero, containers, collection, skipIsWrongYear, maxPer);
E_Distance.SavePropertyHolders(argZero, containers, zPropertyHolderSingletonDirectory);
E_Distance.SaveClosest(argZero, containers, eDistanceContentDirectory, dFacesContentDirectory);
E_Distance.SaveThreeSigmaFaceEncodings(collection, peopleCollection, eDistanceCollectionDirectory);
}
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any())
break;
@ -739,23 +747,27 @@ public class DlibDotNet
}
}
private PropertyLogic GetPropertyLogic()
private PropertyLogic GetPropertyLogic(bool reverse, Model? model, PredictorModel? predictorModel)
{
PropertyLogic result;
if (_AppSettings.MaxDegreeOfParallelism is null)
throw new ArgumentNullException(nameof(_AppSettings.MaxDegreeOfParallelism));
throw new NullReferenceException(nameof(_AppSettings.MaxDegreeOfParallelism));
if (_Configuration?.PropertyConfiguration is null)
throw new ArgumentNullException(nameof(_Configuration.PropertyConfiguration));
result = new(_AppSettings.MaxDegreeOfParallelism.Value, _Configuration.PropertyConfiguration);
throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration));
result = new(_AppSettings.MaxDegreeOfParallelism.Value, _Configuration.PropertyConfiguration, reverse, model, predictorModel);
return result;
}
private void Search(Property.Models.Configuration configuration, bool reverse, Model? model, PredictorModel? predictorModel, string argZero, Person[] people)
{
PropertyLogic propertyLogic = GetPropertyLogic();
List<Container> containers;
Dictionary<string, List<Person>> peopleCollection = A2_People.Convert(people);
List<PropertyHolder[]> propertyHolderCollections = Property.Models.Stateless.A_Property.Get(configuration, reverse, model, predictorModel, propertyLogic);
FullDoWork(configuration, model, predictorModel, argZero, peopleCollection, propertyLogic, propertyHolderCollections);
PropertyLogic propertyLogic = GetPropertyLogic(reverse, model, predictorModel);
if (string.IsNullOrEmpty(configuration.RootDirectory))
containers = Property.Models.Stateless.A_Property.Get(configuration, propertyLogic);
else
containers = Property.Models.Stateless.Container.GetContainers(configuration, propertyLogic);
FullDoWork(configuration, model, predictorModel, argZero, peopleCollection, propertyLogic, containers);
}
internal void RenameQueue(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel) => _Rename.RenameQueue(configuration, model, predictorModel);