This commit is contained in:
2022-11-20 23:20:28 -07:00
parent c6aa7e8e3c
commit 3b988ba152
85 changed files with 1355 additions and 980 deletions

View File

@ -278,42 +278,20 @@ public partial class DlibDotNet
return result;
}
private static MappingFromItem GetMappingFromItem(Item item)
private void FullParallelForWork(A_Property propertyLogic, Dictionary<int, List<(string, int)>> idToMappedFaceFilesCollection, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string? eDistanceContentDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Shared.Models.Face>> imageFaceCollections, Container container, int index, Item item)
{
MappingFromItem result;
bool? isWrongYear;
DateTime minimumDateTime;
if (item.Property?.Id is null)
throw new NotSupportedException();
if (item.Property?.Id is null || item.ImageFileHolder is null || item.ResizedFileHolder is null)
throw new NotSupportedException();
minimumDateTime = Shared.Models.Stateless.Methods.IProperty.GetMinimumDateTime(item.Property);
(isWrongYear, _) = item.Property.IsWrongYear(item.ImageFileHolder, minimumDateTime);
result = new(item.Property.Id.Value, item.ImageFileHolder, isWrongYear, minimumDateTime, item.RelativePath, item.ResizedFileHolder);
return result;
}
private void FullParallelForWork(A_Property propertyLogic, Dictionary<int, List<(string, int)>> idToMappedFaceFilesCollection, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string? eDistanceContentDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Shared.Models.Face>?> imageFaceCollections, Container container, int index, Item item)
{
if (item.ImageFileHolder is null)
throw new NullReferenceException(nameof(item.ImageFileHolder));
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
string original = "Original";
FileHolder? resizedFileHolder;
List<Shared.Models.Face> faces;
Shared.Models.Property property;
long ticks = DateTime.Now.Ticks;
DateTime dateTime = DateTime.Now;
List<Shared.Models.Face>? faces;
List<string> parseExceptions = new();
Dictionary<string, int[]> imageResizeKeyValuePairs;
List<Tuple<string, DateTime>> subFileTuples = new();
List<KeyValuePair<string, string>> metadataCollection;
if (item.Property is not null)
{
property = item.Property;
if ((item.Changed.HasValue && item.Changed.Value) || (item.Moved.HasValue && item.Moved.Value) || (item.Abandoned.HasValue && item.Abandoned.Value))
throw new NotImplementedException();
}
else
if (item.Property is null || item.Property.Id is null)
{
property = propertyLogic.GetProperty(item, subFileTuples, parseExceptions);
item.Update(property);
@ -323,31 +301,48 @@ public partial class DlibDotNet
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), (from l in subFileTuples select l.Item2).Max()));
}
}
(int metadataGroups, metadataCollection) = _Metadata.GetMetadataCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, subFileTuples, parseExceptions, item);
else
{
property = item.Property;
if (item.Abandoned.HasValue && item.Abandoned.Value)
_Log.Information(string.Concat("Abandoned <", item.ImageFileHolder.FullName, '>'));
else if (item.FileSizeChanged.HasValue && item.FileSizeChanged.Value)
_Log.Information(string.Concat("FileSizeChanged <", item.ImageFileHolder.FullName, '>'));
else if (item.LastWriteTimeChanged.HasValue && item.LastWriteTimeChanged.Value)
_Log.Information(string.Concat("LastWriteTimeChanged <", item.ImageFileHolder.FullName, '>'));
else if (item.Moved.HasValue && item.Moved.Value)
_Log.Information(string.Concat("Moved <", item.ImageFileHolder.FullName, '>'));
}
if (property is null || item.Property is null)
throw new NullReferenceException(nameof(property));
FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(item);
item.SetResizedFileHolder(_Resize.FileNameExtension, resizedFileHolder);
string facesDirectory = _Faces.GetFacesDirectory(dResultsFullGroupDirectory, item);
MappingFromItem mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(item, resizedFileHolder);
(int metadataGroups, metadataCollection) = _Metadata.GetMetadataCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection));
imageResizeKeyValuePairs = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, subFileTuples, parseExceptions, original, metadataCollection, item);
imageResizeKeyValuePairs = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, subFileTuples, parseExceptions, original, metadataCollection, item.Property, mappingFromItem);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(C_Resize.GetResizeKeyValuePairs));
if (_Configuration.SaveResizedSubfiles)
{
_Resize.SaveResizedSubfile(outputResolution, cResultsFullGroupDirectory, subFileTuples, item, original, imageResizeKeyValuePairs);
_Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.Property, mappingFromItem, original, imageResizeKeyValuePairs);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(C_Resize.SaveResizedSubfile));
}
else if (outputResolution == _Configuration.OutputResolutions[0] && false)
{
byte[] bytes = _Resize.GetResizedBytes(outputResolution, cResultsFullGroupDirectory, subFileTuples, item, imageResizeKeyValuePairs);
byte[] bytes = _Resize.GetResizedBytes(outputResolution, cResultsFullGroupDirectory, subFileTuples, item.Property, mappingFromItem, imageResizeKeyValuePairs);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(C_Resize.GetResizedBytes));
string path = Path.Combine(resizedFileHolder.DirectoryName, resizedFileHolder.NameWithoutExtension);
File.WriteAllBytes(path, bytes);
}
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
faces = null;
faces = new();
else
{
MappingFromItem mappingFromItem = GetMappingFromItem(item);
int[] outputResolutionCollection = imageResizeKeyValuePairs[outputResolution];
int outputResolutionWidth = outputResolutionCollection[0];
int outputResolutionHeight = outputResolutionCollection[1];
@ -355,7 +350,7 @@ public partial class DlibDotNet
faces = _Faces.GetFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D_Face.GetFaces));
bool anyFacesSaved = _Faces.SaveFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem, faces);
bool anyFacesSaved = _Faces.SaveFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem, facesDirectory, faces);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D_Face.SaveFaces));
if (!anyFacesSaved && item.Property?.Id is not null
@ -366,7 +361,7 @@ public partial class DlibDotNet
throw new NotImplementedException();
// bool saveRotated = _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution);
// string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(_Configuration.PropertyConfiguration, container.SourceDirectory);
// string facesDirectory = Path.GetFullPath(Path.Combine($"{Path.Combine(d2FacePartsContentDirectory, "()")}{sourceDirectorySegment}", item.ImageFileHolder.NameWithoutExtension));
// string facePartsDirectoryX = Path.GetFullPath(Path.Combine($"{Path.Combine(d2FacePartsContentDirectory, "()")}{sourceDirectorySegment}", item.ImageFileHolder.NameWithoutExtension));
// _FaceParts.SaveFaceLandmarkImages(facesDirectory, subFileTuples, parseExceptions, item, faceCollection, saveRotated);
// if (_AppSettings.MaxDegreeOfParallelism < 2)
// ticks = LogDelta(ticks, nameof(D2_FaceParts.SaveFaceLandmarkImages));
@ -396,7 +391,7 @@ public partial class DlibDotNet
List<Shared.Models.Property?> propertyCollection,
List<List<KeyValuePair<string, string>>> metadataCollection,
List<Dictionary<string, int[]>> resizeKeyValuePairs,
List<List<Shared.Models.Face>?> imageFaceCollections,
List<List<Shared.Models.Face>> imageFaceCollections,
Container container,
Item[] filteredItems,
string message)
@ -415,7 +410,7 @@ public partial class DlibDotNet
{
propertyCollection.Add(null);
metadataCollection.Add(new());
imageFaceCollections.Add(null);
imageFaceCollections.Add(new());
resizeKeyValuePairs.Add(new());
propertyFileHolderCollection.Add(null);
}
@ -491,7 +486,7 @@ public partial class DlibDotNet
}
}
private void WriteGroup(A_Property propertyLogic, Shared.Models.Property[] propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Shared.Models.Face>?> imageFaceCollections, string outputResolution, Container container, Item[] filteredItems)
private void WriteGroup(A_Property propertyLogic, Shared.Models.Property[] propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Shared.Models.Face>> imageFaceCollections, string outputResolution, Container container, Item[] filteredItems)
{
Item item;
string key;
@ -504,7 +499,7 @@ public partial class DlibDotNet
string checkDirectory;
List<KeyValuePair<string, Shared.Models.Property>> propertyCollectionKeyValuePairs = new();
List<KeyValuePair<string, Dictionary<string, int[]>>> resizeKeyValuePairsCollections = new();
List<KeyValuePair<string, List<Shared.Models.Face>?>> imageFaceCollectionsKeyValuePairs = new();
List<KeyValuePair<string, List<Shared.Models.Face>>> imageFaceCollectionsKeyValuePairs = new();
List<KeyValuePair<string, List<KeyValuePair<string, string>>>> metadataCollectionKeyValuePairs = new();
(int level, List<string> directories) = Shared.Models.Stateless.Methods.IPath.Get(
_Configuration.PropertyConfiguration.RootDirectory,
@ -515,12 +510,10 @@ public partial class DlibDotNet
item = filteredItems[i];
if (item.Property is null)
continue;
if (item.ImageFileHolder is null)
continue;
key = Shared.Models.Stateless.Methods.IPath.GetRelativePath(item.ImageFileHolder.FullName, sourceDirectoryLength);
propertyCollectionKeyValuePairs.Add(new KeyValuePair<string, Shared.Models.Property>(key, propertyCollection[i]));
resizeKeyValuePairsCollections.Add(new KeyValuePair<string, Dictionary<string, int[]>>(key, resizeKeyValuePairs[i]));
imageFaceCollectionsKeyValuePairs.Add(new KeyValuePair<string, List<Shared.Models.Face>?>(key, imageFaceCollections[i]));
imageFaceCollectionsKeyValuePairs.Add(new KeyValuePair<string, List<Shared.Models.Face>>(key, imageFaceCollections[i]));
metadataCollectionKeyValuePairs.Add(new KeyValuePair<string, List<KeyValuePair<string, string>>>(key, metadataCollection[i]));
}
if (propertyLogic.AngleBracketCollection.Any())
@ -599,45 +592,11 @@ public partial class DlibDotNet
return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory);
}
private void SetAngleBracketCollections(A_Property propertyLogic, string outputResolution, Container container, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory)
private void SetAngleBracketCollections(A_Property propertyLogic, Container container, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory)
{
_Faces.AngleBracketCollection.Clear();
_Resize.AngleBracketCollection.Clear();
_Metadata.AngleBracketCollection.Clear();
propertyLogic.AngleBracketCollection.Clear();
propertyLogic.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(
_Configuration.PropertyConfiguration,
container.SourceDirectory,
aResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Properties for each image",
collectionDescription: string.Empty,
converted: false));
_Metadata.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(
_Configuration.PropertyConfiguration,
container.SourceDirectory,
bResultsFullGroupDirectory,
contentDescription: string.Empty,
singletonDescription: "Metadata as key value pairs",
collectionDescription: string.Empty,
converted: true));
_Resize.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(
_Configuration.PropertyConfiguration,
container.SourceDirectory,
cResultsFullGroupDirectory,
contentDescription: "Resized image",
singletonDescription: "Resize dimensions for each resolution",
collectionDescription: string.Empty,
converted: true));
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
_Faces.AngleBracketCollection.AddRange(Property.Models.Stateless.IResult.GetDirectoryInfoCollection(
_Configuration.PropertyConfiguration,
container.SourceDirectory,
dResultsFullGroupDirectory,
contentDescription: "n png file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: "For each image a json file with all faces found",
converted: true));
propertyLogic.SetAngleBracketCollection(aResultsFullGroupDirectory, container.SourceDirectory);
_Resize.SetAngleBracketCollection(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, container.SourceDirectory);
_Metadata.SetAngleBracketCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, container.SourceDirectory);
}
private Item[] GetFilterItems(Container container)
@ -672,10 +631,10 @@ public partial class DlibDotNet
Shared.Models.Property[] propertyCollection;
List<FileHolder?> propertyFileHolderCollection = new();
List<Dictionary<string, int[]>> resizeKeyValuePairs = new();
List<List<Shared.Models.Face>> imageFaceCollections = new();
List<Tuple<string, DateTime>> sourceDirectoryChanges = new();
List<List<Shared.Models.Face>?> imageFaceCollections = new();
List<Shared.Models.Property?> nullablePropertyCollection = new();
int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism;
List<Shared.Models.Property?> nullablePropertyCollection = new();
List<List<KeyValuePair<string, string>>> metadataCollection = new();
foreach (string outputResolution in _Configuration.OutputResolutions)
{
@ -699,7 +658,9 @@ public partial class DlibDotNet
propertyFileHolderCollection.Clear();
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
message = $"{i + 1:000}.{container.G} [{filteredItems.Length:000} files] / {containersLength:000} - {total} / {t} total files - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}";
SetAngleBracketCollections(propertyLogic, outputResolution, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory);
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
_Faces.SetAngleBracketCollection(dResultsFullGroupDirectory, container.SourceDirectory);
SetAngleBracketCollections(propertyLogic, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory);
exceptionCount = FullParallelWork(
maxDegreeOfParallelism,
propertyLogic,
@ -781,10 +742,9 @@ public partial class DlibDotNet
Shared.Models.Face[] faces;
Mapping mapping;
int faceAreaPermille;
bool? isWrongYear;
Item[] filteredItems;
int confidencePercent;
DateTime minimumDateTime;
List<int> distinct = new();
int normalizedPixelPercentage;
string deterministicHashCodeKey;
MappingFromItem mappingFromItem;
@ -803,20 +763,19 @@ public partial class DlibDotNet
continue;
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null || item.ImageFileHolder is null || item.ResizedFileHolder is null)
if (item.Property?.Id is null || item.ResizedFileHolder is null)
continue;
if (distinct.Contains(item.Property.Id.Value))
continue;
distinct.Add(item.Property.Id.Value);
if (!item.Faces.Any(l => l.FaceEncoding is not null && l.Location is not null && l.OutputResolution is not null))
{
items.Add(item);
continue;
}
minimumDateTime = Shared.Models.Stateless.Methods.IProperty.GetMinimumDateTime(item.Property);
(isWrongYear, _) = item.Property.IsWrongYear(item.ImageFileHolder, minimumDateTime);
mappingFromItem = new(item.Property.Id.Value, item.ImageFileHolder, isWrongYear, minimumDateTime, item.RelativePath, item.ResizedFileHolder);
mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(item);
foreach (Shared.Models.Face face in item.Faces)
{
if (face.RelativePath != item.RelativePath)
break;
if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
continue;
confidencePercent = Shared.Models.Stateless.Methods.ILocation.GetConfidencePercent(_Configuration.FaceConfidencePercent, _Configuration.RangeFaceConfidence, face.Location.Confidence);
@ -958,9 +917,7 @@ public partial class DlibDotNet
private List<MappingFromItem> GetMappingFromItemCollection(Container[] containers)
{
List<MappingFromItem> results = new();
bool? isWrongYear;
Item[] filteredItems;
DateTime minimumDateTime;
MappingFromItem mappingFromItem;
foreach (Container container in containers)
{
@ -973,11 +930,9 @@ public partial class DlibDotNet
continue;
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null || item.ImageFileHolder is null)
if (item.Property?.Id is null)
continue;
minimumDateTime = Shared.Models.Stateless.Methods.IProperty.GetMinimumDateTime(item.Property);
(isWrongYear, _) = item.Property.IsWrongYear(item.ImageFileHolder, minimumDateTime);
mappingFromItem = new(item.Property.Id.Value, item.ImageFileHolder, isWrongYear, minimumDateTime, item.RelativePath, item.ImageFileHolder);
mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(item);
results.Add(mappingFromItem);
}
}
@ -1033,6 +988,8 @@ public partial class DlibDotNet
if (_PropertyRootExistedBefore || container is not null)
break;
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
if (_Configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions.Contains(outputResolution))
MapLogicSupport.BeforeSaveResizedImagesByPersonKeyFormatted(_Configuration.JLinks, a2PeopleSingletonDirectory);
if (_ArgZeroIsConfigurationRootDirectory
&& _Configuration.SaveResizedSubfiles
&& outputResolution == _Configuration.OutputResolutions[0]