Simplified GetDirectoryInfoCollection

This commit is contained in:
2022-08-14 10:42:28 -07:00
parent 0392de1920
commit b756859b61
12 changed files with 556 additions and 360 deletions

View File

@ -100,7 +100,7 @@ public class DlibDotNet
_Log.Information(propertyConfiguration.RootDirectory);
}
if (!configuration.SkipSearch.Value)
Search(propertyConfiguration, configuration.Reverse.Value, model, predictorModel, argZero, people);
Search(propertyConfiguration, configuration.Reverse.Value, model, predictorModel, argZero, people, isSilent);
if (_Exceptions.Count == 0 && _ArgZeroIsConfigurationRootDirectory)
{
long ticks = DateTime.Now.Ticks;
@ -305,7 +305,7 @@ public class DlibDotNet
throw new Exception("Input directory should be the source and not a resized directory!");
}
private void FullParallelForWork(PropertyLogic propertyLogic, string outputResolution, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<D_Face>> imageFaceCollections, string sourceDirectory, int index, Item item)
private void FullParallelForWork(PropertyLogic propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<D_Face>> imageFaceCollections, string sourceDirectory, int index, Item item)
{
if (item.ImageFileHolder is null)
throw new NullReferenceException(nameof(item.ImageFileHolder));
@ -326,7 +326,13 @@ public class DlibDotNet
Dictionary<string, int[]> imageResizeKeyValuePairs;
List<Tuple<string, DateTime>> subFileTuples = new();
List<KeyValuePair<string, string>> metadataCollection;
if (item.Property is null)
if (item.Property is not null)
{
property = item.Property;
if ((item.Changed.HasValue && item.Changed.Value) || (item.Moved.HasValue && item.Moved.Value) || (item.Abandoned.HasValue && item.Abandoned.Value))
throw new NotImplementedException();
}
else
{
property = propertyLogic.GetProperty(item, subFileTuples, parseExceptions);
item.Update(property);
@ -336,34 +342,24 @@ public class DlibDotNet
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), (from l in subFileTuples select l.Item2).Max()));
}
}
else
{
property = item.Property;
if (item.Changed.HasValue && item.Changed.Value)
{
// lock (sourceDirectoryChanges)
// sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), DateTime.Now));
throw new NotImplementedException();
}
}
(int metadataGroups, metadataCollection) = _Metadata.GetMetadataCollection(subFileTuples, parseExceptions, item);
(int metadataGroups, metadataCollection) = _Metadata.GetMetadataCollection(bResultsFullGroupDirectory, subFileTuples, parseExceptions, item);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection));
FileHolder resizedFileHolder = new(Path.Combine(_Resize.AngleBracketCollection[0].Replace("<>", "()"), Path.GetFileName(item.ImageFileHolder.FullName)));
item.SetResizedFileHolder(resizedFileHolder);
imageResizeKeyValuePairs = _Resize.GetResizeKeyValuePairs(subFileTuples, parseExceptions, original, metadataCollection, property, item);
imageResizeKeyValuePairs = _Resize.GetResizeKeyValuePairs(cResultsFullGroupDirectory, subFileTuples, parseExceptions, original, metadataCollection, property, item);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(C_Resize.GetResizeKeyValuePairs));
if (_Configuration.SaveResizedSubfiles.Value)
{
_Resize.SaveResizedSubfile(outputResolution, subFileTuples, item, original, property, imageResizeKeyValuePairs);
_Resize.SaveResizedSubfile(outputResolution, cResultsFullGroupDirectory, subFileTuples, item, original, property, imageResizeKeyValuePairs);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(C_Resize.SaveResizedSubfile));
item.SetResizedFileHolder(FileHolder.Refresh(resizedFileHolder));
}
else if (outputResolution == _Configuration.OutputResolutions[0] && false)
{
byte[] bytes = _Resize.GetResizedBytes(outputResolution, subFileTuples, item, property, imageResizeKeyValuePairs);
byte[] bytes = _Resize.GetResizedBytes(outputResolution, cResultsFullGroupDirectory, subFileTuples, item, property, imageResizeKeyValuePairs);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(C_Resize.GetResizedBytes));
string path = Path.Combine(resizedFileHolder.DirectoryName, resizedFileHolder.NameWithoutExtension);
@ -377,15 +373,15 @@ public class DlibDotNet
int outputResolutionWidth = outputResolutionCollection[0];
int outputResolutionHeight = outputResolutionCollection[1];
int outputResolutionOrientation = outputResolutionCollection[2];
faceCollection = _Faces.GetFaces(_Configuration.PropertyConfiguration, outputResolution, subFileTuples, parseExceptions, item, property, resizedFileHolder, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation);
faceCollection = _Faces.GetFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, item, property, resizedFileHolder, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(D_Face.GetFaces));
_Faces.SaveFaces(_Configuration.PropertyConfiguration, subFileTuples, parseExceptions, item, faceCollection);
_Faces.SaveFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, item, faceCollection);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(D_Face.SaveFaces));
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
{
_FaceLandmarks.SaveFaceLandmarkImages(subFileTuples, parseExceptions, item, faceCollection);
_FaceLandmarks.SaveFaceLandmarkImages(d2ResultsFullGroupDirectory, sourceDirectory, subFileTuples, parseExceptions, item, faceCollection);
if (_AppSettings.MaxDegreeOfParallelism.Value < 2)
ticks = LogDelta(ticks, nameof(D2_FaceLandmarks.SaveFaceLandmarkImages));
}
@ -401,7 +397,7 @@ public class DlibDotNet
}
}
private int FullParallelWork(long ticks, PropertyLogic propertyLogic, string outputResolution, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<D_Face>> faceCollections, int containersCount, Container container, Item[] filteredItems)
private int FullParallelWork(long ticks, PropertyLogic propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<A_Property> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<D_Face>> faceCollections, int containersCount, Container container, Item[] filteredItems)
{
int result = 0;
if (_Log is null)
@ -429,7 +425,7 @@ public class DlibDotNet
{
try
{
FullParallelForWork(propertyLogic, outputResolution, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, container.SourceDirectory, index: i, filteredItems[i]);
FullParallelForWork(propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, container.SourceDirectory, index: i, filteredItems[i]);
if (i == 0 || sourceDirectoryChanges.Any())
progressBar.Tick();
}
@ -442,6 +438,15 @@ public class DlibDotNet
}
});
}
#pragma warning disable
int[] ids = (from l in filteredItems where l.Property?.Id is not null select l.Property.Id.Value).ToArray();
#pragma warning restore
bool hasDuplicateId = ids.Length != ids.Distinct().Count();
if (hasDuplicateId)
{
if (hasDuplicateId)
{ }
}
return result;
}
@ -564,7 +569,7 @@ public class DlibDotNet
}
}
private void FullDoWork(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string argZero, Dictionary<string, List<Person>> peopleCollection, PropertyLogic propertyLogic, List<Container> containers)
private void FullDoWork(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string argZero, Dictionary<string, List<Person>> peopleCollection, PropertyLogic propertyLogic, List<Container> containers, bool isSilent)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
@ -573,6 +578,13 @@ public class DlibDotNet
int exceptionCount;
Item[] filteredItems;
long ticks = DateTime.Now.Ticks;
string aResultsFullGroupDirectory;
string bResultsFullGroupDirectory;
string cResultsFullGroupDirectory;
string dResultsFullGroupDirectory;
string eResultsFullGroupDirectory;
string zResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
List<List<D_Face>> faceCollections = new();
List<A_Property> propertyCollection = new();
List<FileHolder?> propertyFileHolderCollection = new();
@ -585,6 +597,20 @@ public class DlibDotNet
{
_FileKeyValuePairs.Clear();
_FilePropertiesKeyValuePairs.Clear();
aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(A_Property), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false);
bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(B_Metadata), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false);
cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(C_Resize), outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false);
d2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(D2_FaceLandmarks), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true);
dResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true);
eResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true);
zResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
configuration, model, predictorModel, $"Z_{nameof(Item)}", outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true);
foreach (Container container in containers)
{
if (!container.Items.Any())
@ -604,70 +630,32 @@ public class DlibDotNet
_Resize.AngleBracketCollection.Clear();
_Metadata.AngleBracketCollection.Clear();
propertyLogic.AngleBracketCollection.Clear();
_FaceLandmarks.AngleBracketCollection.Clear();
propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
model,
predictorModel,
container.SourceDirectory,
nameof(A_Property),
outputResolution,
includeResizeGroup: false,
includeModel: false,
includePredictorModel: false,
contentDescription: string.Empty,
singletonDescription: "Properties for each image",
collectionDescription: string.Empty));
container.SourceDirectory,
aResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Properties for each image",
collectionDescription: string.Empty));
_Metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
model,
predictorModel,
container.SourceDirectory,
nameof(B_Metadata),
outputResolution,
includeResizeGroup: false,
includeModel: false,
includePredictorModel: false,
contentDescription: string.Empty,
singletonDescription: "Metadata as key value pairs",
collectionDescription: string.Empty));
container.SourceDirectory,
bResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Metadata as key value pairs",
collectionDescription: string.Empty));
_Resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
model,
predictorModel,
container.SourceDirectory,
nameof(C_Resize),
outputResolution,
includeResizeGroup: true,
includeModel: false,
includePredictorModel: false,
contentDescription: "Resized image",
singletonDescription: "Resize dimensions for each resolution",
collectionDescription: string.Empty));
container.SourceDirectory,
cResultsFullGroupDirectory,
contentDescription: "Resized image",
singletonDescription: "Resize dimensions for each resolution",
collectionDescription: string.Empty));
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
_Faces.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
model,
predictorModel,
container.SourceDirectory,
nameof(D_Face),
outputResolution,
includeResizeGroup: true,
includeModel: true,
includePredictorModel: true,
contentDescription: "n png file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: "For each image a json file with all faces found"));
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
_FaceLandmarks.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(configuration,
model,
predictorModel,
container.SourceDirectory,
nameof(D2_FaceLandmarks),
outputResolution,
includeResizeGroup: true,
includeModel: true,
includePredictorModel: true,
contentDescription: "n x 2 png file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: string.Empty));
exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, containers.Count, container, filteredItems);
container.SourceDirectory,
dResultsFullGroupDirectory,
contentDescription: "n png file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: "For each image a json file with all faces found"));
exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, propertyCollection, metadataCollection, resizeKeyValuePairs, faceCollections, containers.Count, container, filteredItems);
if (metadataCollection.Count != filteredItems.Length || propertyCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || faceCollections.Count != filteredItems.Length)
throw new Exception("Counts don't match!");
if (exceptionCount != 0)
@ -680,21 +668,27 @@ public class DlibDotNet
{
if (outputResolution == _Configuration.OutputResolutions[0] || _Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
{
if (isSilent && container.SourceDirectory.EndsWith("Bohdi Ray 2016")) // 7#.1
break;
// if (isSilent && container.SourceDirectory.EndsWith("Halloween 2013")) // 12#.1
// break;
// if (isSilent && container.SourceDirectory.EndsWith("zzz =2014.4 Winter Tracy Pictures")) // 30#.2
// break;
// if (isSilent && container.SourceDirectory.EndsWith("Texas 2015")) //46#.4
// break;
for (int i = 0; i < faceCollections.Count; i++)
filteredItems[i].Faces.AddRange(from l in faceCollections[i] select l);
Item.AddToNamed(propertyLogic, filteredItems);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
D_Face.SaveShortcuts(configuration, model, predictorModel, _Configuration.JuliePhares, ticks, peopleCollection, propertyLogic, outputResolution, filteredItems);
D_Face.SaveShortcuts(_Configuration.JuliePhares, dResultsFullGroupDirectory, ticks, peopleCollection, propertyLogic, filteredItems);
}
}
if (exceptionCount == 0 && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_Distance.LoadOrCreateThenSaveDistanceResultsForOutputResolutions(configuration, model, predictorModel, container.SourceDirectory, outputResolution, sourceDirectoryChanges, filteredItems, faceCollections);
_Distance.LoadOrCreateThenSaveDistanceResultsForOutputResolutions(configuration, eResultsFullGroupDirectory, container.SourceDirectory, outputResolution, sourceDirectoryChanges, filteredItems, faceCollections);
if (_Resize.AngleBracketCollection.Any())
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(_Resize.AngleBracketCollection[0].Replace("<>", "()"));
if (_Faces.AngleBracketCollection.Any())
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(_Faces.AngleBracketCollection[0].Replace("<>", "()"));
if (_FaceLandmarks.AngleBracketCollection.Any())
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(_FaceLandmarks.AngleBracketCollection[0].Replace("<>", "()"));
if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any())
{
for (int y = 0; y < int.MaxValue; y++)
@ -711,22 +705,19 @@ public class DlibDotNet
propertyLogic.SaveAllCollection();
if (propertyLogic.NamedFaceInfoDeterministicHashCodeIndices.Any())
{
const int maxPer = 5;
const bool skipIsWrongYear = true;
string dFacesContentDirectory;
string eDistanceContentDirectory;
string eDistanceCollectionDirectory;
string zPropertyHolderSingletonDirectory;
dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()");
eDistanceContentDirectory = Path.Combine(eResultsFullGroupDirectory, $"({ticks})");
zPropertyHolderSingletonDirectory = Path.Combine(zResultsFullGroupDirectory, "{}");
eDistanceCollectionDirectory = Path.Combine(eResultsFullGroupDirectory, $"[{ticks}]");
List<(DateTime, bool?, PersonBirthday, FaceRecognitionDotNet.FaceEncoding[])> collection;
dFacesContentDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), "()");
eDistanceContentDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), $"({ticks})");
eDistanceCollectionDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), $"[{ticks}]");
zPropertyHolderSingletonDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, $"Z_{nameof(Item)}", outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), "{}");
collection = E_Distance.GetThreeSigmaFaceEncodings(argZero, containers);
E_Distance.AddClosest(argZero, containers, collection, skipIsWrongYear, maxPer);
collection = E_Distance.ParallelWork(_AppSettings.MaxDegreeOfParallelism.Value, argZero, containers);
E_Distance.SavePropertyHolders(argZero, containers, zPropertyHolderSingletonDirectory);
E_Distance.SaveClosest(argZero, containers, eDistanceContentDirectory, dFacesContentDirectory);
E_Distance.SaveThreeSigmaFaceEncodings(collection, peopleCollection, eDistanceCollectionDirectory);
E_Distance.SaveClosest(argZero, containers, peopleCollection, eDistanceContentDirectory, dFacesContentDirectory);
}
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any())
break;
@ -758,7 +749,7 @@ public class DlibDotNet
return result;
}
private void Search(Property.Models.Configuration configuration, bool reverse, Model? model, PredictorModel? predictorModel, string argZero, Person[] people)
private void Search(Property.Models.Configuration configuration, bool reverse, Model? model, PredictorModel? predictorModel, string argZero, Person[] people, bool isSilent)
{
List<Container> containers;
Dictionary<string, List<Person>> peopleCollection = A2_People.Convert(people);
@ -767,7 +758,7 @@ public class DlibDotNet
containers = Property.Models.Stateless.A_Property.Get(configuration, propertyLogic);
else
containers = Property.Models.Stateless.Container.GetContainers(configuration, propertyLogic);
FullDoWork(configuration, model, predictorModel, argZero, peopleCollection, propertyLogic, containers);
FullDoWork(configuration, model, predictorModel, argZero, peopleCollection, propertyLogic, containers, isSilent);
}
internal void RenameQueue(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel) => _Rename.RenameQueue(configuration, model, predictorModel);