phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Mass push

Browse Source
This commit is contained in:
Mike Phares
2022-12-16 11:26:00 -07:00
parent ae1baaaf46
commit 33c5ce6e23
132 changed files with 5869 additions and 3712 deletions
Download Patch File Download Diff File Expand all files Collapse all files

295
Instance/DlibDotNet.cs
View File

@ -47,7 +47,6 @@ public partial class DlibDotNet
IConsole console)
{
_Console = console;
string argZero;
string message;
_AppSettings = appSettings;
_IsEnvironment = isEnvironment;
@ -56,9 +55,9 @@ public partial class DlibDotNet
PersonContainer[] personContainers;
_Log = Serilog.Log.ForContext<DlibDotNet>();
Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot);
Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration);
_Log.Information(propertyConfiguration.RootDirectory);
Property.Models.Configuration.Verify(propertyConfiguration, requireExist: false);
Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration);
Verify(configuration);
VerifyExtra(args, propertyConfiguration, configuration);
_Configuration = configuration;
@ -69,8 +68,13 @@ public partial class DlibDotNet
throw new NullReferenceException(nameof(configuration.IgnoreExtensions));
string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(propertyConfiguration, nameof(A_Property), create: false);
_PropertyRootExistedBefore = !Directory.Exists(propertyRoot);
argZero = args.Count > 0 ? Path.GetFullPath(args[0]) : Path.GetFullPath(propertyConfiguration.RootDirectory);
string argZero = args.Count > 0 ? Path.GetFullPath(args[0]) : Path.GetFullPath(propertyConfiguration.RootDirectory);
_ArgZeroIsConfigurationRootDirectory = propertyConfiguration.RootDirectory == argZero;
if (!Directory.Exists(argZero))
_ = Directory.CreateDirectory(argZero);
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(argZero);
if (!Directory.Exists(argZero))
_ = Directory.CreateDirectory(argZero);
_Log.Information(configuration.ModelDirectory);
{
(ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetPngLowQuality();
@ -89,10 +93,8 @@ public partial class DlibDotNet
imageCodecInfo,
configuration.ModelDirectory,
configuration.ModelName,
configuration.NumberOfJitters,
configuration.NumberOfTimesToUpsample,
configuration.OverrideForFaceImages,
configuration.RetryImagesWithoutAFace,
configuration.RetryImagesWithoutAFaceLocation,
configuration.PredictorModelName,
configuration.PropertiesChangedForFaces);
}
@ -102,7 +104,7 @@ public partial class DlibDotNet
}
_Metadata = new(configuration.ForceMetadataLastWriteTimeToCreationTime, configuration.PropertiesChangedForMetadata);
_MapConfiguration = Get(configuration, _Faces.FileNameExtension, _Faces.HiddenFileNameExtension, _FaceParts.FileNameExtension);
_Distance = new(configuration.DistanceMoveUnableToMatch, configuration.DistancePixelDistanceTolerance, configuration.DistanceRenameToMatch, _Configuration.FaceConfidencePercent, configuration.RangeDistanceTolerance, configuration.RangeFaceConfidence);
_Distance = new(configuration.DistanceMoveUnableToMatch, configuration.DistanceRenameToMatch, _Configuration.FaceConfidencePercent, configuration.RangeDistanceTolerance, configuration.RangeFaceConfidence);
if (_PropertyRootExistedBefore || !_ArgZeroIsConfigurationRootDirectory)
personContainers = Array.Empty<PersonContainer>();
else
@ -268,6 +270,7 @@ public partial class DlibDotNet
configuration.MappingDefaultName,
configuration.PersonBirthdayFirstYear,
configuration.PersonBirthdayFormat,
configuration.PhotoPrismDirectory,
configuration.RangeDaysDeltaTolerance,
configuration.RangeDistanceTolerance,
configuration.SortingMaximumPerKey,
@ -278,7 +281,7 @@ public partial class DlibDotNet
return result;
}
private void FullParallelForWork(A_Property propertyLogic, Dictionary<int, List<(string, int)>> idToMappedFaceFilesCollection, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string? eDistanceContentDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Shared.Models.Face>> imageFaceCollections, Container container, int index, Item item)
private void FullParallelForWork(A_Property propertyLogic, Dictionary<int, List<(string, int)>> idToMappedFaceFilesCollection, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsDateGroupDirectory, string dResultsFullGroupDirectory, string? eDistanceContentDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Shared.Models.Face>> imageFaceCollections, Container container, int index, Item item, DateTime[] containerDateTimes)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
@ -291,19 +294,18 @@ public partial class DlibDotNet
Dictionary<string, int[]> imageResizeKeyValuePairs;
List<Tuple<string, DateTime>> subFileTuples = new();
List<KeyValuePair<string, string>> metadataCollection;
if (item.Property is null || item.Property.Id is null)
Dictionary<string, List<RelativeLocation>> relativeLocations;
if (item.Property is not null && item.Property.Id is not null && !item.Any())
{
property = propertyLogic.GetProperty(item, subFileTuples, parseExceptions);
item.Update(property);
if (subFileTuples.Any())
{
lock (sourceDirectoryChanges)
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), (from l in subFileTuples select l.Item2).Max()));
}
property = item.Property;
if (item.SourceDirectoryFileHolder.LastWriteTime is not null)
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), item.SourceDirectoryFileHolder.LastWriteTime.Value));
else
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), new FileInfo(item.SourceDirectoryFileHolder.FullName).LastWriteTime));
}
else
{
property = item.Property;
int? propertyHashCode = item.Property?.GetHashCode();
if (item.Abandoned.HasValue && item.Abandoned.Value)
_Log.Information(string.Concat("Abandoned <", item.ImageFileHolder.FullName, '>'));
else if (item.FileSizeChanged.HasValue && item.FileSizeChanged.Value)
@ -312,13 +314,26 @@ public partial class DlibDotNet
_Log.Information(string.Concat("LastWriteTimeChanged <", item.ImageFileHolder.FullName, '>'));
else if (item.Moved.HasValue && item.Moved.Value)
_Log.Information(string.Concat("Moved <", item.ImageFileHolder.FullName, '>'));
property = propertyLogic.GetProperty(item, subFileTuples, parseExceptions);
item.Update(property);
if (propertyHashCode is null)
{
lock (sourceDirectoryChanges)
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), DateTime.Now));
}
else if (propertyHashCode.Value != property.GetHashCode())
{
lock (sourceDirectoryChanges)
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), DateTime.Now));
}
}
if (property is null || item.Property is null)
throw new NullReferenceException(nameof(property));
FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(item);
item.SetResizedFileHolder(_Resize.FileNameExtension, resizedFileHolder);
string facesDirectory = _Faces.GetFacesDirectory(dResultsFullGroupDirectory, item);
MappingFromItem mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(item, resizedFileHolder);
string facesDirectory = _Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution) ? _Faces.GetFacesDirectory(dResultsFullGroupDirectory, item) : string.Empty;
string facePartsDirectory = _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution) ? _FaceParts.GetFacePartsDirectory(_Configuration.PropertyConfiguration, dResultsFullGroupDirectory, item) : string.Empty;
MappingFromItem mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item, resizedFileHolder);
(int metadataGroups, metadataCollection) = _Metadata.GetMetadataCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection));
@ -343,28 +358,29 @@ public partial class DlibDotNet
faces = new();
else
{
List<(string, int)>? collection;
int[] outputResolutionCollection = imageResizeKeyValuePairs[outputResolution];
int outputResolutionWidth = outputResolutionCollection[0];
int outputResolutionHeight = outputResolutionCollection[1];
int outputResolutionOrientation = outputResolutionCollection[2];
faces = _Faces.GetFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation);
relativeLocations = _Faces.GetRelativeLocations(outputResolution, dResultsDateGroupDirectory, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation, original, imageResizeKeyValuePairs, faces);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D_Face.GetFaces));
bool anyFacesSaved = _Faces.SaveFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem, facesDirectory, faces);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D_Face.SaveFaces));
if (!anyFacesSaved && item.Property?.Id is not null
&& idToMappedFaceFilesCollection.TryGetValue(item.Property.Id.Value, out List<(string, int)>? collection))
if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
&& !anyFacesSaved && item.Property?.Id is not null
&& idToMappedFaceFilesCollection.TryGetValue(item.Property.Id.Value, out collection))
_Distance.LookForMatchFacesAndPossiblyRename(_Faces.FileNameExtension, eDistanceContentDirectory, mappingFromItem, faces, collection);
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
{
throw new NotImplementedException();
// bool saveRotated = _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution);
// string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(_Configuration.PropertyConfiguration, container.SourceDirectory);
// string facePartsDirectoryX = Path.GetFullPath(Path.Combine($"{Path.Combine(d2FacePartsContentDirectory, "()")}{sourceDirectorySegment}", item.ImageFileHolder.NameWithoutExtension));
// _FaceParts.SaveFaceLandmarkImages(facesDirectory, subFileTuples, parseExceptions, item, faceCollection, saveRotated);
// if (_AppSettings.MaxDegreeOfParallelism < 2)
// ticks = LogDelta(ticks, nameof(D2_FaceParts.SaveFaceLandmarkImages));
bool saveRotated = !_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution);
string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(_Configuration.PropertyConfiguration, container.SourceDirectory);
_FaceParts.SaveFaceLandmarkImages(_Configuration.PropertyConfiguration, facePartsDirectory, subFileTuples, parseExceptions, mappingFromItem, faces, saveRotated);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D2_FaceParts.SaveFaceLandmarkImages));
}
}
lock (sourceDirectoryChanges)
@ -384,6 +400,7 @@ public partial class DlibDotNet
string outputResolution,
string bResultsFullGroupDirectory,
string cResultsFullGroupDirectory,
string dResultsDateGroupDirectory,
string dResultsFullGroupDirectory,
string? eDistanceContentDirectory,
List<Tuple<string, DateTime>> sourceDirectoryChanges,
@ -415,6 +432,7 @@ public partial class DlibDotNet
propertyFileHolderCollection.Add(null);
}
}
DateTime[] containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems);
using ProgressBar progressBar = new(filteredItems.Length, message, options);
_ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) =>
{
@ -426,6 +444,7 @@ public partial class DlibDotNet
outputResolution,
bResultsFullGroupDirectory,
cResultsFullGroupDirectory,
dResultsDateGroupDirectory,
dResultsFullGroupDirectory,
eDistanceContentDirectory,
sourceDirectoryChanges,
@ -436,7 +455,8 @@ public partial class DlibDotNet
imageFaceCollections,
container,
index: i,
filteredItems[i]);
filteredItems[i],
containerDateTimes);
if (i == 0 || sourceDirectoryChanges.Any())
progressBar.Tick();
}
@ -559,7 +579,7 @@ public partial class DlibDotNet
}
}
private (string, string, string, string) GetResultsFullGroupDirectories(string outputResolution)
private (string, string, string, string, string) GetResultsFullGroupDirectories(string outputResolution)
{
string aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
_Configuration.PropertyConfiguration,
@ -589,7 +609,14 @@ public partial class DlibDotNet
includeResizeGroup: true,
includeModel: true,
includePredictorModel: true);
return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory);
string d2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
_Configuration.PropertyConfiguration,
nameof(D2_FaceParts),
outputResolution,
includeResizeGroup: true,
includeModel: true,
includePredictorModel: true);
return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory);
}
private void SetAngleBracketCollections(A_Property propertyLogic, Container container, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory)
@ -599,20 +626,6 @@ public partial class DlibDotNet
_Metadata.SetAngleBracketCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, container.SourceDirectory);
}
private Item[] GetFilterItems(Container container)
{
List<Item> results = new();
foreach (Item item in container.Items)
{
if (item.ImageFileHolder is not null
&& (item.Abandoned is null || !item.Abandoned.Value)
&& item.ValidImageFormatExtension
&& !_Configuration.IgnoreExtensions.Contains(item.ImageFileHolder.ExtensionLowered))
results.Add(item);
}
return results.ToArray();
}
private void FullDoWork(string argZero, string propertyRoot, long ticks, A_Property propertyLogic, int t, Container[] containers, string? eDistanceContentDirectory, Dictionary<int, List<(string, int)>> idToMappedFaceFilesCollection)
{
if (_Log is null)
@ -627,6 +640,7 @@ public partial class DlibDotNet
string bResultsFullGroupDirectory;
string cResultsFullGroupDirectory;
string dResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
int containersLength = containers.Length;
Shared.Models.Property[] propertyCollection;
List<FileHolder?> propertyFileHolderCollection = new();
@ -636,10 +650,12 @@ public partial class DlibDotNet
int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism;
List<Shared.Models.Property?> nullablePropertyCollection = new();
List<List<KeyValuePair<string, string>>> metadataCollection = new();
string dResultsDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(D_Face));
foreach (string outputResolution in _Configuration.OutputResolutions)
{
total = 0;
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
// _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(dResultsFullGroupDirectory);
for (int i = 0; i < containers.Length; i++)
{
container = containers[i];
@ -647,7 +663,7 @@ public partial class DlibDotNet
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = GetFilterItems(container);
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container);
if (!filteredItems.Any())
continue;
metadataCollection.Clear();
@ -660,6 +676,8 @@ public partial class DlibDotNet
message = $"{i + 1:000}.{container.G} [{filteredItems.Length:000} files] / {containersLength:000} - {total} / {t} total files - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}";
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
_Faces.SetAngleBracketCollection(dResultsFullGroupDirectory, container.SourceDirectory);
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
_FaceParts.SetAngleBracketCollection(_Configuration.PropertyConfiguration, d2ResultsFullGroupDirectory, container.SourceDirectory);
SetAngleBracketCollections(propertyLogic, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory);
exceptionCount = FullParallelWork(
maxDegreeOfParallelism,
@ -668,6 +686,7 @@ public partial class DlibDotNet
outputResolution,
bResultsFullGroupDirectory,
cResultsFullGroupDirectory,
dResultsDateGroupDirectory,
dResultsFullGroupDirectory,
eDistanceContentDirectory,
sourceDirectoryChanges,
@ -718,37 +737,56 @@ public partial class DlibDotNet
}
}
private bool IsIgnoreRelativePath(string directory)
private void SetMapping(string argZero, Container[] containers)
{
bool result = false;
string? checkDirectory = Path.GetFullPath(directory);
for (int i = 0; i < int.MaxValue; i++)
{
if (_Configuration.IgnoreRelativePaths.Contains(Path.GetFileName(checkDirectory)))
{
result = true;
break;
}
checkDirectory = Path.GetDirectoryName(checkDirectory);
if (string.IsNullOrEmpty(checkDirectory) || checkDirectory == _Configuration.PropertyConfiguration.RootDirectory)
break;
}
return result;
}
private (List<Item>, Shared.Models.Face[]) SetMappingThenGetDistinctFilteredFacesWithMapping(string argZero, Container[] containers)
{
List<Item> items = new();
Shared.Models.Face[] faces;
Mapping mapping;
int faceAreaPermille;
Item[] filteredItems;
int confidencePercent;
List<int> distinct = new();
int normalizedPixelPercentage;
int normalizedRectangle;
bool isIgnoreRelativePath;
DateTime[] containerDateTimes;
string deterministicHashCodeKey;
MappingFromItem mappingFromItem;
MappingFromLocation mappingFromLocation;
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container);
if (!filteredItems.Any())
continue;
containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems);
isIgnoreRelativePath = _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory);
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null || item.ResizedFileHolder is null)
continue;
mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item);
foreach (Shared.Models.Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
continue;
confidencePercent = Shared.Models.Stateless.Methods.ILocation.GetConfidencePercent(_Configuration.FaceConfidencePercent, _Configuration.RangeFaceConfidence, face.Location.Confidence);
faceAreaPermille = Shared.Models.Stateless.Methods.IMapping.GetAreaPermille(_Configuration.FaceAreaPermille, face.Location, face.OutputResolution);
normalizedRectangle = Shared.Models.Stateless.Methods.ILocation.GetNormalizedRectangle(face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution);
deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item.Property.Id.Value, face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution);
mappingFromLocation = new(faceAreaPermille, confidencePercent, deterministicHashCodeKey, normalizedRectangle);
mapping = new(mappingFromItem, mappingFromLocation);
face.SetMapping(mapping);
}
}
}
}
private Shared.Models.Face[] GetFilteredDistinctFaces(string argZero, Container[] containers)
{
Shared.Models.Face[] results;
Item[] filteredItems;
bool isIgnoreRelativePath;
List<int> distinct = new();
List<Shared.Models.Face> collection = new();
foreach (Container container in containers)
{
@ -756,11 +794,10 @@ public partial class DlibDotNet
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
if (_Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && IsIgnoreRelativePath(container.SourceDirectory))
continue;
filteredItems = GetFilterItems(container);
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container);
if (!filteredItems.Any())
continue;
isIgnoreRelativePath = _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory);
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null || item.ResizedFileHolder is null)
@ -768,38 +805,55 @@ public partial class DlibDotNet
if (distinct.Contains(item.Property.Id.Value))
continue;
distinct.Add(item.Property.Id.Value);
if (!item.Faces.Any(l => l.FaceEncoding is not null && l.Location is not null && l.OutputResolution is not null))
{
items.Add(item);
if (isIgnoreRelativePath)
continue;
}
mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(item);
foreach (Shared.Models.Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
continue;
confidencePercent = Shared.Models.Stateless.Methods.ILocation.GetConfidencePercent(_Configuration.FaceConfidencePercent, _Configuration.RangeFaceConfidence, face.Location.Confidence);
faceAreaPermille = Shared.Models.Stateless.Methods.IMapping.GetAreaPermille(_Configuration.FaceAreaPermille, face.Location, face.OutputResolution);
normalizedPixelPercentage = Shared.Models.Stateless.Methods.ILocation.GetNormalizedPixelPercentage(face.Location, Shared.Models.Stateless.ILocation.Digits, Shared.Models.Stateless.ILocation.Factor, face.OutputResolution);
deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item.Property.Id.Value, face.Location, Shared.Models.Stateless.ILocation.Digits, Shared.Models.Stateless.ILocation.Factor, face.OutputResolution);
mappingFromLocation = new(faceAreaPermille, confidencePercent, deterministicHashCodeKey, normalizedPixelPercentage);
mapping = new(mappingFromItem, mappingFromLocation);
face.SetMapping(mapping);
collection.Add(face);
}
}
}
faces = (from l in collection orderby l.Mapping?.MappingFromItem.Id select l).ToArray();
return new(items, faces);
results = (from l in collection orderby l.Mapping?.MappingFromItem.Id select l).ToArray();
return results;
}
private void MapLogic(string argZero, long ticks, PersonContainer[] personContainers, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2FacePartsContentDirectory, string eDistanceContentDirectory, string outputResolution)
private List<Item> GetItems(string argZero, Container[] containers)
{
List<Item> items = new();
Item[] filteredItems;
bool isIgnoreRelativePath;
List<Shared.Models.Face> collection = new();
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container);
if (!filteredItems.Any())
continue;
isIgnoreRelativePath = _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory);
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null || item.ResizedFileHolder is null)
continue;
items.Add(item);
}
}
return items;
}
private void MapLogic(string argZero, long ticks, PersonContainer[] personContainers, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string eDistanceContentDirectory, string outputResolution)
{
int? useFiltersCounter = null;
SortingContainer[] sortingContainers;
string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()");
(List<Item> filteredItems, Shared.Models.Face[] distinctFilteredFaces) = SetMappingThenGetDistinctFilteredFacesWithMapping(argZero, containers);
string d2FacePartsContentDirectory = Path.Combine(d2ResultsFullGroupDirectory, "()");
Shared.Models.Face[] distinctFilteredFaces = GetFilteredDistinctFaces(argZero, containers);
Mapping[] mappingCollection = MapLogicSupport.GetSelectedMappingCollection(distinctFilteredFaces);
string dFacesCollectionDirectory = Path.Combine(dResultsFullGroupDirectory, "[]", _Configuration.PropertyConfiguration.ResultAllInOne);
MapLogicSupport mapLogicSupport = new(
_Configuration.FaceConfidencePercent,
_Configuration.FaceDistancePermyriad,
@ -816,33 +870,50 @@ public partial class DlibDotNet
eDistanceContentDirectory,
mappingCollection,
mapLogicSupport);
Dictionary<int, Dictionary<int, Mapping>> idToNormalizedPixelPercentageToMapping = MapLogicSupport.GetIdToNormalizedPixelPercentageToFace(mappingCollection);
mapLogic.CopyManualFiles(dFacesContentDirectory, idToNormalizedPixelPercentageToMapping);
if (_Configuration.SaveNotMappedForOutputResolutions.Contains(outputResolution))
mapLogic.CopyNotMappedFaces(_Configuration.RangeFaceAreaPermilleTolerance, dFacesContentDirectory, idToNormalizedPixelPercentageToMapping);
(Dictionary<long, int> personKeyToCount, int totalNotMapped) = mapLogic.AddToMapping(mappingCollection);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution) || (!string.IsNullOrEmpty(_Configuration.PhotoPrismDirectory) && Directory.Exists(_Configuration.PhotoPrismDirectory)))
{
List<Item> filteredItems = GetItems(argZero, containers);
if (!string.IsNullOrEmpty(_Configuration.PhotoPrismDirectory))
{
mapLogic.SaveMarkers();
mapLogic.FindMatch(filteredItems);
mapLogic.LoadMatches();
}
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
mapLogic.SaveShortcutsForOutputResolutions(filteredItems, mappingCollection, personKeyToCount);
}
Dictionary<int, Dictionary<int, Mapping>> idToNormalizedRectangleToMapping = MapLogicSupport.GetIdToNormalizedRectangleToFace(mappingCollection);
mapLogic.CopyManualFiles(dFacesContentDirectory, idToNormalizedRectangleToMapping);
if (_Configuration.SaveNotMappedForOutputResolutions.Contains(outputResolution))
mapLogic.CopyNotMappedFaces(_Configuration.RangeFaceAreaPermilleTolerance, dFacesContentDirectory, idToNormalizedRectangleToMapping);
if (_Configuration.SaveMappedForOutputResolutions.Contains(outputResolution))
mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, mappingCollection, idToNormalizedPixelPercentageToMapping, personKeyToCount, totalNotMapped);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
mapLogic.SaveShortcuts(a2PeopleSingletonDirectory, personContainers, filteredItems, mappingCollection, personKeyToCount);
mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, mappingCollection, idToNormalizedRectangleToMapping, personKeyToCount, totalNotMapped);
if (_Configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions.Contains(outputResolution))
mapLogic.SaveResizedImagesByPersonKeyFormatted(_Configuration.JLinks, a2PeopleSingletonDirectory, personContainers, mappingCollection, personKeyToCount, totalNotMapped);
if (_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution))
{
MapLogicSupport.SetFaceDistances(_AppSettings.MaxDegreeOfParallelism, ticks, distinctFilteredFaces);
sortingContainers = mapLogicSupport.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, ticks, mapLogic, distinctFilteredFaces, useFiltersCounter);
Dictionary<int, Dictionary<int, PersonContainer[]>> missingIdThenNormalizedRectangleToPersonContainers = mapLogic.GetMissing(idToNormalizedRectangleToMapping);
List<FaceDistanceContainer> missingFaceDistanceContainers = _Distance.GetMissingFaceDistanceContainer(_AppSettings.MaxDegreeOfParallelism, ticks, dFacesCollectionDirectory, missingIdThenNormalizedRectangleToPersonContainers);
sortingContainers = mapLogicSupport.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, ticks, mapLogic, distinctFilteredFaces, missingFaceDistanceContainers, useFiltersCounter);
if (!sortingContainers.Any())
{
for (useFiltersCounter = 1; useFiltersCounter < 17; useFiltersCounter++)
{
sortingContainers = mapLogicSupport.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, ticks, mapLogic, distinctFilteredFaces, useFiltersCounter);
sortingContainers = mapLogicSupport.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, ticks, mapLogic, distinctFilteredFaces, missingFaceDistanceContainers, useFiltersCounter);
if (sortingContainers.Any())
break;
}
}
MapLogicSupport.SaveFaceDistances(_Configuration.PropertyConfiguration, sortingContainers);
if (totalNotMapped > 0)
mapLogic.UpdateFromSortingContainersThenSaveContainers(dFacesContentDirectory, d2FacePartsContentDirectory, mappingCollection, idToNormalizedPixelPercentageToMapping, useFiltersCounter, sortingContainers, totalNotMapped);
{
bool saveNullPerson = !personKeyToCount.Any();
int updated = mapLogic.UpdateFromSortingContainers(sortingContainers, saveNullPerson);
List<SaveContainer> saveContainers = mapLogic.GetSaveContainers(dFacesContentDirectory, d2FacePartsContentDirectory, mappingCollection, idToNormalizedRectangleToMapping, useFiltersCounter, saveNullPerson);
mapLogic.SaveContainers(totalNotMapped, updated, saveContainers);
}
}
if (_Configuration.SaveNotMappedForOutputResolutions.Contains(outputResolution))
mapLogic.SaveNotMappedTicks();
@ -886,6 +957,7 @@ public partial class DlibDotNet
string fileName;
Task<byte[]> task;
string relativePath;
FileHolder fileHolder;
string extensionLowered;
string sourceDirectoryFile;
int length = result.Length;
@ -907,7 +979,8 @@ public partial class DlibDotNet
extensionLowered = Path.GetExtension(uri.LocalPath);
relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFile, length, forceExtensionToLower: true);
isValidImageFormatExtension = _Configuration.PropertyConfiguration.ValidImageFormatExtensions.Contains(extensionLowered);
item = new(sourceDirectoryFile, relativePath, isValidImageFormatExtension);
fileHolder = new(sourceDirectoryFile);
item = new(fileHolder, relativePath, isValidImageFormatExtension);
container.Items.Add(item);
}
_Log.Information(". . .");
@ -918,21 +991,23 @@ public partial class DlibDotNet
{
List<MappingFromItem> results = new();
Item[] filteredItems;
DateTime[] containerDateTimes;
MappingFromItem mappingFromItem;
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (_Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && IsIgnoreRelativePath(container.SourceDirectory))
if (_Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory))
continue;
filteredItems = GetFilterItems(container);
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container);
if (!filteredItems.Any())
continue;
containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems);
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null)
continue;
mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(item);
mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item);
results.Add(mappingFromItem);
}
}
@ -951,8 +1026,8 @@ public partial class DlibDotNet
string cResultsFullGroupDirectory;
string dResultsFullGroupDirectory;
string? eDistanceContentDirectory;
string d2ResultsFullGroupDirectory;
string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "{}");
string d2FacePartsContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(D2_FaceParts), "()");
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string message = $") Building Container(s) - {totalSeconds} total second(s)";
A_Property propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse);
@ -970,24 +1045,29 @@ public partial class DlibDotNet
}
else
{
string resultsGroupDirectory;
a2PeopleContentDirectory = null;
eDistanceContentDirectory = null;
string? newRootDirectory = SaveUrlAndGetNewRootDirectory(container);
for (int i = 1; i < 10; i++)
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, string.Empty, create: true));
{
resultsGroupDirectory = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, string.Empty, create: true);
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(resultsGroupDirectory);
}
argZero = newRootDirectory;
_Configuration.PropertyConfiguration.ChangeRootDirectory(newRootDirectory);
propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false);
propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse);
}
Dictionary<int, List<(string, int)>> idToMappedFaceFilesCollection = Map.Models.Stateless.Methods.IMapLogic.DeleteEmptyDirectoriesAndGetMappedFaceFiles(_MapConfiguration, ticks, a2PeopleContentDirectory, eDistanceContentDirectory, personContainers);
containers = Shared.Models.Stateless.Methods.IContainer.SortContainers(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, _ArgZeroIsConfigurationRootDirectory, argZero, containers);
FullDoWork(argZero, propertyRoot, ticks, propertyLogic, t, containers, eDistanceContentDirectory, idToMappedFaceFilesCollection);
_Distance.Clear();
foreach (string outputResolution in _Configuration.OutputResolutions)
{
if (_PropertyRootExistedBefore || container is not null)
break;
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
if (_Configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions.Contains(outputResolution))
MapLogicSupport.BeforeSaveResizedImagesByPersonKeyFormatted(_Configuration.JLinks, a2PeopleSingletonDirectory);
if (_ArgZeroIsConfigurationRootDirectory
@ -999,7 +1079,8 @@ public partial class DlibDotNet
{
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any())
break;
MapLogic(argZero, ticks, personContainers, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2FacePartsContentDirectory, eDistanceContentDirectory, outputResolution);
SetMapping(argZero, containers);
MapLogic(argZero, ticks, personContainers, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eDistanceContentDirectory, outputResolution);
if (_IsEnvironment.Development)
continue;
List<MappingFromItem> mappingFromItemCollection = GetMappingFromItemCollection(containers);