phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added FileHolder

Browse Source
This commit is contained in:
Mike Phares
2022-08-08 19:23:48 -07:00
parent daf5f428b9
commit 3aeab88384
15 changed files with 448 additions and 288 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
Instance/DlibDotNet.cs
View File

@ -305,10 +305,10 @@ public class DlibDotNet
throw new Exception("Input directory should be the source and not a resized directory!");
}
private void FullParallelForWork(PropertyLogic propertyLogic, object @lock, string outputResolution, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileInfo?> propertyFileInfoCollection, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<D_Face>> imageFaceCollections, string sourceDirectory, int index, PropertyHolder propertyHolder)
private void FullParallelForWork(PropertyLogic propertyLogic, string outputResolution, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<D_Face>> imageFaceCollections, string sourceDirectory, int index, PropertyHolder propertyHolder)
{
if (propertyHolder.ImageFileInfo is null)
throw new ArgumentNullException(nameof(propertyHolder.ImageFileInfo));
if (propertyHolder.ImageFileHolder is null)
throw new ArgumentNullException(nameof(propertyHolder.ImageFileHolder));
if (_AppSettings.MaxDegreeOfParallelism is null)
throw new ArgumentNullException(nameof(_AppSettings.MaxDegreeOfParallelism));
if (_Configuration.SaveResizedSubfiles is null)
@ -328,7 +328,8 @@ public class DlibDotNet
List<KeyValuePair<string, string>> metadataCollection;
if (propertyHolder.Property is null)
{
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), propertyHolder.ImageFileInfo.LastWriteTime));
lock (sourceDirectoryChanges)
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), DateTime.Now));
property = propertyLogic.GetProperty(propertyLogic.AngleBracketCollection[0], propertyHolder, subFileTuples, parseExceptions);
propertyHolder.Update(property);
}
@ -336,13 +337,16 @@ public class DlibDotNet
{
property = propertyHolder.Property;
if (propertyHolder.Changed.HasValue && propertyHolder.Changed.Value)
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), propertyHolder.ImageFileInfo.LastWriteTime));
{
lock (sourceDirectoryChanges)
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), DateTime.Now));
}
}
(int metadataGroups, metadataCollection) = _Metadata.GetMetadataCollection(subFileTuples, parseExceptions, propertyHolder);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection));
FileInfo resizedFileInfo = new(Path.Combine(_Resize.AngleBracketCollection[0].Replace("<>", "()"), Path.GetFileName(propertyHolder.ImageFileInfo.FullName)));
propertyHolder.SetResizedFileInfo(resizedFileInfo);
FileHolder resizedFileHolder = new(Path.Combine(_Resize.AngleBracketCollection[0].Replace("<>", "()"), Path.GetFileName(propertyHolder.ImageFileHolder.FullName)));
propertyHolder.SetResizedFileHolder(resizedFileHolder);
imageResizeKeyValuePairs = _Resize.GetResizeKeyValuePairs(subFileTuples, parseExceptions, original, metadataCollection, property, propertyHolder);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(C_Resize.GetResizeKeyValuePairs));
@ -351,14 +355,14 @@ public class DlibDotNet
_Resize.SaveResizedSubfile(outputResolution, subFileTuples, propertyHolder, original, property, imageResizeKeyValuePairs);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(C_Resize.SaveResizedSubfile));
resizedFileInfo.Refresh();
propertyHolder.SetResizedFileHolder(FileHolder.Refresh(resizedFileHolder));
}
else if (outputResolution == _Configuration.OutputResolutions[0] && false)
{
byte[] bytes = _Resize.GetResizedBytes(outputResolution, subFileTuples, propertyHolder, property, imageResizeKeyValuePairs);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(C_Resize.GetResizedBytes));
string path = Path.Combine(resizedFileInfo.DirectoryName, Path.GetFileNameWithoutExtension(resizedFileInfo.Name));
string path = Path.Combine(resizedFileHolder.DirectoryName, resizedFileHolder.NameWithoutExtension);
File.WriteAllBytes(path, bytes);
}
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
@ -369,7 +373,7 @@ public class DlibDotNet
int outputResolutionWidth = outputResolutionCollection[0];
int outputResolutionHeight = outputResolutionCollection[1];
int outputResolutionOrientation = outputResolutionCollection[2];
faceCollection = _Faces.GetFaces(_Configuration.PropertyConfiguration, outputResolution, subFileTuples, parseExceptions, propertyHolder, property, resizedFileInfo, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation);
faceCollection = _Faces.GetFaces(_Configuration.PropertyConfiguration, outputResolution, subFileTuples, parseExceptions, propertyHolder, property, resizedFileHolder, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(D_Face.GetFaces));
_Faces.SaveFaces(_Configuration.PropertyConfiguration, subFileTuples, parseExceptions, propertyHolder, faceCollection);
@ -382,18 +386,18 @@ public class DlibDotNet
ticks = LogDelta(ticks, nameof(D2_FaceLandmarks.SaveFaceLandmarkImages));
}
}
lock (@lock)
lock (sourceDirectoryChanges)
{
propertyCollection[index] = property;
imageFaceCollections[index] = faceCollection;
metadataCollections[index] = metadataCollection;
resizeKeyValuePairs[index] = imageResizeKeyValuePairs;
propertyFileInfoCollection[index] = propertyHolder.ImageFileInfo;
propertyFileHolderCollection[index] = propertyHolder.ImageFileHolder;
sourceDirectoryChanges.AddRange(from l in subFileTuples where l.Item2 > dateTime select l);
}
}
private int FullParallelWork(object @lock, long ticks, PropertyLogic propertyLogic, string outputResolution, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileInfo?> propertyFileInfoCollection, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<D_Face>> faceCollections, int propertyHolderCollectionsCount, int g, string sourceDirectory, int r, PropertyHolder[] filteredPropertyHolderCollection)
private int FullParallelWork(long ticks, PropertyLogic propertyLogic, string outputResolution, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<D_Face>> faceCollections, int propertyHolderCollectionsCount, int g, string sourceDirectory, int r, PropertyHolder[] filteredPropertyHolderCollection)
{
int result = 0;
if (_Log is null)
@ -410,18 +414,19 @@ public class DlibDotNet
metadataCollection.Add(new());
resizeKeyValuePairs.Add(new());
propertyCollection.Add(new());
propertyFileInfoCollection.Add(null);
propertyFileHolderCollection.Add(null);
}
}
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
using (ProgressBar progressBar = new(filteredPropertyHolderCollection.Length, $"{r + 1:000}.{g} / {propertyHolderCollectionsCount:000}) {filteredPropertyHolderCollection.Length:000} file(s) - {totalSeconds} total second(s) - {outputResolution} - {sourceDirectory}", options))
string message = $"{r + 1:000}.{g} / {propertyHolderCollectionsCount:000}) {filteredPropertyHolderCollection.Length:000} file(s) - {totalSeconds} total second(s) - {outputResolution} - {sourceDirectory}";
using (ProgressBar progressBar = new(filteredPropertyHolderCollection.Length, message, options))
{
_ = Parallel.For(0, filteredPropertyHolderCollection.Length, parallelOptions, i =>
{
try
{
FullParallelForWork(propertyLogic, @lock, outputResolution, sourceDirectoryChanges, propertyFileInfoCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, sourceDirectory, index: i, filteredPropertyHolderCollection[i]);
if (sourceDirectoryChanges.Any())
FullParallelForWork(propertyLogic, outputResolution, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, sourceDirectory, index: i, filteredPropertyHolderCollection[i]);
if (i == 0 || sourceDirectoryChanges.Any())
progressBar.Tick();
}
catch (Exception ex)
@ -502,9 +507,9 @@ public class DlibDotNet
propertyHolder = filteredPropertyHolderCollection[i];
if (propertyHolder.Property is null)
continue;
if (propertyHolder.ImageFileInfo is null)
if (propertyHolder.ImageFileHolder is null)
continue;
key = Property.Models.Stateless.IPath.GetRelativePath(propertyHolder.ImageFileInfo.FullName, sourceDirectoryLength);
key = Property.Models.Stateless.IPath.GetRelativePath(propertyHolder.ImageFileHolder.FullName, sourceDirectoryLength);
_FileKeyValuePairs.Add(new KeyValuePair<string, string>(sourceDirectory, key));
_FilePropertiesKeyValuePairs[sourceDirectory].Add(new Tuple<string, A_Property>(key, propertyCollection[i]));
faceCollectionsKeyValuePairs.Add(new KeyValuePair<string, List<D_Face>>(key, faceCollections[i]));
@ -564,29 +569,31 @@ public class DlibDotNet
int g;
int r;
int exceptionCount;
object @lock = new();
string sourceDirectory;
long ticks = DateTime.Now.Ticks;
string eDistanceCollectionDirectory;
List<List<D_Face>> faceCollections = new();
List<A_Property> propertyCollection = new();
PropertyHolder[] filteredPropertyHolderCollection;
List<FileInfo?> propertyFileInfoCollection = new();
List<FileHolder?> propertyFileHolderCollection = new();
List<Dictionary<string, int[]>> resizeKeyValuePairs = new();
List<Tuple<string, DateTime>> sourceDirectoryChanges = new();
List<List<KeyValuePair<string, string>>> metadataCollection = new();
List<(string[], PersonBirthday, FaceRecognitionDotNet.FaceEncoding[])> collection;
string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(A_Property), "{}");
string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(configuration, nameof(A_Property));
foreach (string outputResolution in _Configuration.OutputResolutions)
{
_FileKeyValuePairs.Clear();
_FilePropertiesKeyValuePairs.Clear();
eDistanceCollectionDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), $"[{ticks}]");
foreach (PropertyHolder[] propertyHolderCollection in propertyHolderCollections)
{
if (!propertyHolderCollection.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !propertyHolderCollection[0].SourceDirectory.StartsWith(argZero))
continue;
filteredPropertyHolderCollection = (from l in propertyHolderCollection where l.ImageFileInfo is not null && l.Property is not null && l.ValidImageFormatExtension.HasValue && l.ValidImageFormatExtension.Value && !_Configuration.IgnoreExtensions.Contains(l.ImageFileInfo.Extension) select l).ToArray();
filteredPropertyHolderCollection = (from l in propertyHolderCollection where l.ImageFileHolder is not null && l.Property is not null && l.ValidImageFormatExtension.HasValue && l.ValidImageFormatExtension.Value && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.Extension) select l).ToArray();
if (!filteredPropertyHolderCollection.Any())
continue;
faceCollections.Clear();
@ -594,7 +601,7 @@ public class DlibDotNet
propertyCollection.Clear();
resizeKeyValuePairs.Clear();
sourceDirectoryChanges.Clear();
propertyFileInfoCollection.Clear();
propertyFileHolderCollection.Clear();
_Faces.AngleBracketCollection.Clear();
_Resize.AngleBracketCollection.Clear();
_Metadata.AngleBracketCollection.Clear();
@ -665,7 +672,7 @@ public class DlibDotNet
contentDescription: "n x 2 png file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: string.Empty));
exceptionCount = FullParallelWork(@lock, ticks, propertyLogic, outputResolution, sourceDirectoryChanges, propertyFileInfoCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, propertyHolderCollections.Count, g, sourceDirectory, r, filteredPropertyHolderCollection);
exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, propertyHolderCollections.Count, g, sourceDirectory, r, filteredPropertyHolderCollection);
if (metadataCollection.Count != filteredPropertyHolderCollection.Length || propertyCollection.Count != filteredPropertyHolderCollection.Length || resizeKeyValuePairs.Count != filteredPropertyHolderCollection.Length || faceCollections.Count != filteredPropertyHolderCollection.Length)
throw new Exception("Counts don't match!");
if (exceptionCount != 0)
@ -708,7 +715,11 @@ public class DlibDotNet
{
propertyLogic.SaveAllCollection();
if (propertyLogic.NamedFaceInfoDeterministicHashCodeIndices.Any())
E_Distance.SaveGroupedFaceEncodings(configuration, model, predictorModel, argZero, ticks, peopleCollection, outputResolution, propertyHolderCollections);
{
collection = E_Distance.GetGroupedFaceEncodings(argZero, propertyHolderCollections);
E_Distance.SaveGroupedFaceEncodings(collection, peopleCollection, eDistanceCollectionDirectory);
E_Distance.GetClosest(argZero, propertyHolderCollections, collection, peopleCollection, eDistanceCollectionDirectory);
}
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any())
break;
if (_Exceptions.Count == 0)