phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Get Unable to Match Count and Rename Matches

Browse Source
This commit is contained in:
Mike Phares
2022-09-24 16:39:15 -07:00
parent e64c713926
commit 751a61529c
32 changed files with 983 additions and 668 deletions
Download Patch File Download Diff File Expand all files Collapse all files

271
Instance/DlibDotNet.cs
View File

@ -28,6 +28,7 @@ public partial class DlibDotNet
private readonly F_Random _Random;
private readonly E3_Rename _Rename;
private readonly IConsole _Console;
private readonly E_Distance _Distance;
private readonly B_Metadata _Metadata;
private readonly Serilog.ILogger? _Log;
private readonly D2_FaceParts _FaceParts;
@ -67,10 +68,19 @@ public partial class DlibDotNet
Verify(configuration);
VerifyExtra(args, propertyConfiguration, configuration);
_Configuration = configuration;
_Index = new G_Index(configuration);
_Random = new F_Random(configuration);
_Index = new(configuration);
_Random = new(configuration);
_Rename = new(configuration);
_MapConfiguration = Get(configuration);
_Rename = new E3_Rename(configuration);
_Distance = new(
configuration.DistanceMoveUnableToMatch,
configuration.DistancePixelDistanceTolerance,
configuration.FaceDistanceMinimumConfidence,
configuration.FaceDistancePermyriad,
configuration.FaceDistanceTolerance,
configuration.PropertyConfiguration.ResultAllInOne,
configuration.SortingDaysDeltaTolerance,
configuration.SortingMaximumPerFaceShouldBeHigh);
if (configuration.IgnoreExtensions is null)
throw new NullReferenceException(nameof(configuration.IgnoreExtensions));
string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(propertyConfiguration, nameof(A_Property), create: false);
@ -95,13 +105,12 @@ public partial class DlibDotNet
hiddenFileNameExtension,
hiddenImageCodecInfo,
imageCodecInfo,
configuration.LocationDigits,
configuration.LocationFactor,
model,
modelParameter,
configuration.NumberOfJitters,
configuration.NumberOfTimesToUpsample,
configuration.OverrideForFaceImages,
configuration.RetryImagesWithoutAFace,
predictorModel,
configuration.PropertiesChangedForFaces);
}
@ -323,11 +332,9 @@ public partial class DlibDotNet
configuration.FaceDistancePermyriad,
configuration.FaceDistanceMinimumConfidence,
configuration.FaceDistanceTolerance,
configuration.LocationDigits,
configuration.LocationFactor,
configuration.MapLogicSigma,
configuration.MappingDefaultName,
configuration.MappingMoveUnableToMatch,
configuration.DistanceMoveUnableToMatch,
configuration.MappingSaveNotMapped,
configuration.MappingSaveMapped,
configuration.MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping,
@ -342,22 +349,7 @@ public partial class DlibDotNet
return result;
}
private void FullParallelForWork(
A_Property propertyLogic,
string outputResolution,
string bResultsFullGroupDirectory,
string cResultsFullGroupDirectory,
string dResultsFullGroupDirectory,
string d2ResultsFullGroupDirectory,
List<Tuple<string, DateTime>> sourceDirectoryChanges,
List<FileHolder?> propertyFileHolderCollection,
List<Shared.Models.Property?> propertyCollection,
List<List<KeyValuePair<string, string>>> metadataCollections,
List<Dictionary<string, int[]>> resizeKeyValuePairs,
List<List<Shared.Models.Face>?> imageFaceCollections,
Container container,
int index,
Item item)
private void FullParallelForWork(A_Property propertyLogic, string[] mappedFaceFiles, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string eDistanceContentDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Shared.Models.Face>?> imageFaceCollections, List<string[]> duplicateMappedFaceFilesCollection, Container container, int index, Item item)
{
if (item.ImageFileHolder is null)
throw new NullReferenceException(nameof(item.ImageFileHolder));
@ -366,8 +358,8 @@ public partial class DlibDotNet
Shared.Models.Property property;
long ticks = DateTime.Now.Ticks;
DateTime dateTime = DateTime.Now;
List<Shared.Models.Face>? faces;
List<string> parseExceptions = new();
List<Shared.Models.Face>? faceCollection;
Dictionary<string, int[]> imageResizeKeyValuePairs;
List<Tuple<string, DateTime>> subFileTuples = new();
List<KeyValuePair<string, string>> metadataCollection;
@ -420,46 +412,35 @@ public partial class DlibDotNet
File.WriteAllBytes(path, bytes);
}
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
faceCollection = null;
faces = null;
else
{
int[] outputResolutionCollection = imageResizeKeyValuePairs[outputResolution];
int outputResolutionWidth = outputResolutionCollection[0];
int outputResolutionHeight = outputResolutionCollection[1];
int outputResolutionOrientation = outputResolutionCollection[2];
faceCollection = _Faces.GetFaces(
dResultsFullGroupDirectory,
subFileTuples,
parseExceptions,
item,
property,
outputResolutionWidth,
outputResolutionHeight,
outputResolutionOrientation);
faces = _Faces.GetFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, item, property, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D_Face.GetFaces));
_Faces.SaveFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, item, faceCollection);
_Faces.SaveFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, item, faces);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D_Face.SaveFaces));
int?[] normalizedPixelPercentageCollection = Shared.Models.Stateless.Methods.IFace.GetInts(faceCollection);
int normalizedPixelPercentageDistinctCount = normalizedPixelPercentageCollection.Distinct().Count();
if (normalizedPixelPercentageDistinctCount != normalizedPixelPercentageCollection.Length || _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
_ = _Distance.GetUnableToMatchCountAndRenameMatches(_Faces.FileNameExtension, eDistanceContentDirectory, duplicateMappedFaceFilesCollection, item, faces, mappedFaceFiles);
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
{
bool saveRotated = _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution);
string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(
_Configuration.PropertyConfiguration,
container.SourceDirectory);
string facesDirectory = Path.GetFullPath(Path.Combine($"{Path.Combine(d2ResultsFullGroupDirectory, "()")}{sourceDirectorySegment}", item.ImageFileHolder.NameWithoutExtension));
_FaceParts.SaveFaceLandmarkImages(facesDirectory, subFileTuples, parseExceptions, item, faceCollection, saveRotated);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D2_FaceParts.SaveFaceLandmarkImages));
throw new NotImplementedException();
// bool saveRotated = _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution);
// string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(_Configuration.PropertyConfiguration, container.SourceDirectory);
// string facesDirectory = Path.GetFullPath(Path.Combine($"{Path.Combine(d2FacePartsContentDirectory, "()")}{sourceDirectorySegment}", item.ImageFileHolder.NameWithoutExtension));
// _FaceParts.SaveFaceLandmarkImages(facesDirectory, subFileTuples, parseExceptions, item, faceCollection, saveRotated);
// if (_AppSettings.MaxDegreeOfParallelism < 2)
// ticks = LogDelta(ticks, nameof(D2_FaceParts.SaveFaceLandmarkImages));
}
}
lock (sourceDirectoryChanges)
{
imageFaceCollections[index] = faces;
propertyCollection[index] = property;
imageFaceCollections[index] = faceCollection;
metadataCollections[index] = metadataCollection;
resizeKeyValuePairs[index] = imageResizeKeyValuePairs;
propertyFileHolderCollection[index] = item.ImageFileHolder;
@ -467,23 +448,7 @@ public partial class DlibDotNet
}
}
private int FullParallelWork(
int maxDegreeOfParallelism,
A_Property propertyLogic,
string outputResolution,
string bResultsFullGroupDirectory,
string cResultsFullGroupDirectory,
string dResultsFullGroupDirectory,
string d2ResultsFullGroupDirectory,
List<Tuple<string, DateTime>> sourceDirectoryChanges,
List<FileHolder?> propertyFileHolderCollection,
List<Shared.Models.Property?> propertyCollection,
List<List<KeyValuePair<string, string>>> metadataCollection,
List<Dictionary<string, int[]>> resizeKeyValuePairs,
List<List<Shared.Models.Face>?> imageFaceCollections,
Container container,
Item[] filteredItems,
string message)
private int FullParallelWork(int maxDegreeOfParallelism, A_Property propertyLogic, string[] mappedFaceFiles, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string eDistanceContentDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollection, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Shared.Models.Face>?> imageFaceCollections, List<string[]> duplicateMappedFaceFilesCollection, Container container, Item[] filteredItems, string message)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
@ -511,17 +476,19 @@ public partial class DlibDotNet
{
FullParallelForWork(
propertyLogic,
mappedFaceFiles,
outputResolution,
bResultsFullGroupDirectory,
cResultsFullGroupDirectory,
dResultsFullGroupDirectory,
d2ResultsFullGroupDirectory,
eDistanceContentDirectory,
sourceDirectoryChanges,
propertyFileHolderCollection,
propertyCollection,
metadataCollection,
resizeKeyValuePairs,
imageFaceCollections,
duplicateMappedFaceFilesCollection,
container,
index: i,
filteredItems[i]);
@ -595,9 +562,9 @@ public partial class DlibDotNet
if (!(from l in propertyCollection where l?.Width is null select true).Any())
{
string checkDirectory;
List<KeyValuePair<string, List<Shared.Models.Face>?>> imageFaceCollectionsKeyValuePairs = new();
List<KeyValuePair<string, Shared.Models.Property>> propertyCollectionKeyValuePairs = new();
List<KeyValuePair<string, Dictionary<string, int[]>>> resizeKeyValuePairsCollections = new();
List<KeyValuePair<string, List<Shared.Models.Face>?>> imageFaceCollectionsKeyValuePairs = new();
List<KeyValuePair<string, List<KeyValuePair<string, string>>>> metadataCollectionKeyValuePairs = new();
(int level, List<string> directories) = Shared.Models.Stateless.Methods.IPath.Get(
_Configuration.PropertyConfiguration.RootDirectory,
@ -616,9 +583,9 @@ public partial class DlibDotNet
_FileKeyValuePairs.Add(new KeyValuePair<string, string>(container.SourceDirectory, key));
_FilePropertiesKeyValuePairs[container.SourceDirectory].Add(new Tuple<string, Shared.Models.Property>(key, propertyCollection[i]));
}
imageFaceCollectionsKeyValuePairs.Add(new KeyValuePair<string, List<Shared.Models.Face>?>(key, imageFaceCollections[i]));
propertyCollectionKeyValuePairs.Add(new KeyValuePair<string, Shared.Models.Property>(key, propertyCollection[i]));
resizeKeyValuePairsCollections.Add(new KeyValuePair<string, Dictionary<string, int[]>>(key, resizeKeyValuePairs[i]));
imageFaceCollectionsKeyValuePairs.Add(new KeyValuePair<string, List<Shared.Models.Face>?>(key, imageFaceCollections[i]));
metadataCollectionKeyValuePairs.Add(new KeyValuePair<string, List<KeyValuePair<string, string>>>(key, metadataCollection[i]));
}
if (propertyLogic.AngleBracketCollection.Any())
@ -664,10 +631,7 @@ public partial class DlibDotNet
}
}
private (string, string, string, string, string, string) GetResultsFullGroupDirectories(
Model? model,
PredictorModel? predictorModel,
string outputResolution)
private (string, string, string, string) GetResultsFullGroupDirectories(Model? model, PredictorModel? predictorModel, string outputResolution)
{
string aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
_Configuration.PropertyConfiguration,
@ -705,25 +669,7 @@ public partial class DlibDotNet
includeResizeGroup: true,
includeModel: true,
includePredictorModel: true);
string d2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
_Configuration.PropertyConfiguration,
model,
predictorModel,
nameof(D2_FaceParts),
outputResolution,
includeResizeGroup: true,
includeModel: true,
includePredictorModel: true);
string eResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
_Configuration.PropertyConfiguration,
model,
predictorModel,
nameof(E_Distance),
outputResolution,
includeResizeGroup: true,
includeModel: true,
includePredictorModel: true);
return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory);
return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory);
}
private void SetAngleBracketCollections(
@ -733,8 +679,7 @@ public partial class DlibDotNet
string aResultsFullGroupDirectory,
string bResultsFullGroupDirectory,
string cResultsFullGroupDirectory,
string dResultsFullGroupDirectory,
string d2ResultsFullGroupDirectory)
string dResultsFullGroupDirectory)
{
_Faces.AngleBracketCollection.Clear();
_Resize.AngleBracketCollection.Clear();
@ -773,15 +718,6 @@ public partial class DlibDotNet
singletonDescription: string.Empty,
collectionDescription: "For each image a json file with all faces found",
converted: true));
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
_ = Property.Models.Stateless.IResult.GetDirectoryInfoCollection(
_Configuration.PropertyConfiguration,
container.SourceDirectory,
d2ResultsFullGroupDirectory,
contentDescription: "n x 2 gif file(s) for each face found",
singletonDescription: string.Empty,
collectionDescription: string.Empty,
converted: false);
}
private Item[] GetFilterItems(Container container)
@ -798,15 +734,7 @@ public partial class DlibDotNet
return results.ToArray();
}
private void FullDoWork(
string argZero,
Model? model,
PredictorModel? predictorModel,
string propertyRoot,
long ticks,
A_Property propertyLogic,
int t,
Container[] containers)
private void FullDoWork(string argZero, Model? model, PredictorModel? predictorModel, string propertyRoot, long ticks, A_Property propertyLogic, int t, Container[] containers, string eDistanceContentDirectory, string[] mappedFaceFiles)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
@ -820,33 +748,23 @@ public partial class DlibDotNet
string bResultsFullGroupDirectory;
string cResultsFullGroupDirectory;
string dResultsFullGroupDirectory;
string eResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
int containersLength = containers.Length;
Shared.Models.Property[] propertyCollection;
List<List<Shared.Models.Face>?> imageFaceCollections = new();
List<FileHolder?> propertyFileHolderCollection = new();
List<string[]> duplicateMappedFaceFilesCollection = new();
List<Dictionary<string, int[]>> resizeKeyValuePairs = new();
List<Tuple<string, DateTime>> sourceDirectoryChanges = new();
List<List<Shared.Models.Face>?> imageFaceCollections = new();
List<Shared.Models.Property?> nullablePropertyCollection = new();
int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism;
List<List<KeyValuePair<string, string>>> metadataCollection = new();
string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), "{}");
foreach (string outputResolution in _Configuration.OutputResolutions)
{
total = 0;
_FileKeyValuePairs.Clear();
duplicateMappedFaceFilesCollection.Clear();
_FilePropertiesKeyValuePairs.Clear();
(aResultsFullGroupDirectory,
bResultsFullGroupDirectory,
cResultsFullGroupDirectory,
dResultsFullGroupDirectory,
d2ResultsFullGroupDirectory,
eResultsFullGroupDirectory) = GetResultsFullGroupDirectories(
model,
predictorModel,
outputResolution);
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(eResultsFullGroupDirectory, "()"));
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory) = GetResultsFullGroupDirectories(model, predictorModel, outputResolution);
for (int i = 0; i < containers.Length; i++)
{
container = containers[i];
@ -865,21 +783,23 @@ public partial class DlibDotNet
propertyFileHolderCollection.Clear();
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
message = $"{i + 1:000}.{container.G} [{filteredItems.Length:000} files] / {containersLength:000} - {total} / {t} total files - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}";
SetAngleBracketCollections(propertyLogic, outputResolution, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory);
SetAngleBracketCollections(propertyLogic, outputResolution, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory);
exceptionCount = FullParallelWork(
maxDegreeOfParallelism,
propertyLogic,
mappedFaceFiles,
outputResolution,
bResultsFullGroupDirectory,
cResultsFullGroupDirectory,
dResultsFullGroupDirectory,
d2ResultsFullGroupDirectory,
eDistanceContentDirectory,
sourceDirectoryChanges,
propertyFileHolderCollection,
nullablePropertyCollection,
metadataCollection,
resizeKeyValuePairs,
imageFaceCollections,
duplicateMappedFaceFilesCollection,
container,
filteredItems,
message);
@ -916,6 +836,11 @@ public partial class DlibDotNet
// break;
total += container.Items.Count;
}
foreach (string[] duplicateMappedFaceFiles in duplicateMappedFaceFilesCollection)
{
foreach (string duplicateMappedFaceFile in duplicateMappedFaceFiles)
_Log.Information(duplicateMappedFaceFile);
}
}
}
@ -926,9 +851,10 @@ public partial class DlibDotNet
bool? isWrongYear;
Item[] filteredItems;
DateTime minimumDateTime;
int normalizedPixelPercentage;
string deterministicHashCodeKey;
MappingFromItem mappingFromItem;
MappingFromPerson mappingFromPerson;
string deterministicHashCodeKeyDisplay;
MappingFromLocation mappingFromLocation;
foreach (Container container in containers)
{
@ -949,14 +875,15 @@ public partial class DlibDotNet
{
if (face.RelativePath != item.RelativePath)
break;
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
continue;
minimumDateTime = Shared.Models.Stateless.Methods.IProperty.GetMinimumDateTime(item.Property);
(isWrongYear, _) = item.Property.IsWrongYear(item.ImageFileHolder.FullName, minimumDateTime);
mappingFromItem = new(item.Property.Id.Value, item.ImageFileHolder, isWrongYear, minimumDateTime, item.ResizedFileHolder);
mappingFromPerson = new(approximateYears: null, by: null, displayDirectoryName: string.Empty, personBirthday: null, segmentB: string.Empty);
deterministicHashCodeKeyDisplay = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item.Property.Id.Value, face.Location.NormalizedPixelPercentage.Value);
mappingFromLocation = new(face.Location.Confidence, deterministicHashCodeKeyDisplay, face.Location.NormalizedPixelPercentage.Value);
normalizedPixelPercentage = Shared.Models.Stateless.Methods.ILocation.GetNormalizedPixelPercentage(face.Location, Shared.Models.Stateless.ILocation.Digits, Shared.Models.Stateless.ILocation.Factor, face.OutputResolution);
deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item.Property.Id.Value, face.Location, Shared.Models.Stateless.ILocation.Digits, Shared.Models.Stateless.ILocation.Factor, face.OutputResolution);
mappingFromLocation = new(face.Location.Confidence, deterministicHashCodeKey, normalizedPixelPercentage);
mapping = new(mappingFromItem, mappingFromLocation, mappingFromPerson);
face.SetMapping(mapping);
results.Add(face);
@ -972,21 +899,15 @@ public partial class DlibDotNet
PersonContainer[] personContainers,
Container[] containers,
string dResultsFullGroupDirectory,
string eResultsFullGroupDirectory,
string d2ResultsFullGroupDirectory,
string outputResolution)
{
E_Distance distance = new();
if (string.IsNullOrEmpty(eResultsFullGroupDirectory))
throw new NullReferenceException(nameof(eResultsFullGroupDirectory));
string eDistanceContentDirectory = Path.Combine(eResultsFullGroupDirectory, "()");
string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()");
List<Shared.Models.Face> distinctFilteredFaces = SetMappingThenGetDistinctFilteredFacesWithMapping(argZero, containers);
List<Shared.Models.Face> selectedFilteredFaces = E_Distance.GetSelectedFilteredFaces(distinctFilteredFaces);
Shared.Models.Face[] selectedFilteredFaces = E_Distance.GetSelectedFilteredFaces(distinctFilteredFaces);
E_Distance.SetFaceDistances(_AppSettings.MaxDegreeOfParallelism, ticks, selectedFilteredFaces);
string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(
_Configuration.PropertyConfiguration,
nameof(A2_People),
"{}");
string eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), "()");
string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "{}");
string d2FacePartsContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(D2_FaceParts), "()");
MapLogic mapLogic = new(
_AppSettings.MaxDegreeOfParallelism,
_Configuration.PropertyConfiguration,
@ -996,33 +917,32 @@ public partial class DlibDotNet
_FaceParts.FileNameExtension,
ticks,
personContainers,
eDistanceContentDirectory,
a2PeopleSingletonDirectory,
eDistanceContentDirectory,
distinctFilteredFaces,
distance);
SortingContainer[] sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortingContainers(
_Distance);
SortingContainer[] sortingContainers = _Distance.SetFaceMappingSortingCollectionThenGetSortingContainers(
_AppSettings.MaxDegreeOfParallelism,
_MapConfiguration,
ticks,
mapLogic,
selectedFilteredFaces);
E_Distance.SaveFaceDistances(_Configuration.PropertyConfiguration, eResultsFullGroupDirectory, sortingContainers);
E_Distance.SaveFaceDistances(_Configuration.PropertyConfiguration, sortingContainers);
int totalNotMapped = mapLogic.AddToMapping(distinctFilteredFaces);
if (totalNotMapped > 0)
mapLogic.ForceSingleImageThenSaveMapping(
dResultsFullGroupDirectory,
d2ResultsFullGroupDirectory,
dFacesContentDirectory,
d2FacePartsContentDirectory,
distinctFilteredFaces,
sortingContainers,
totalNotMapped);
mapLogic.CopyManualFiles(dResultsFullGroupDirectory, distinctFilteredFaces);
mapLogic.CopyManualFiles(dFacesContentDirectory, distinctFilteredFaces);
if (_MapConfiguration.MappingSaveNotMapped)
mapLogic.SaveNotMappedTicks();
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
mapLogic.SaveShortcuts(_Configuration.JuliePhares, distinctFilteredFaces);
}
private static Container? AreAllSameStartsWith(string argZero, Container[] containers)
private static Container? AreAllSameEndsWith(string argZero, Container[] containers)
{
Container? result = null;
string[] directoryNames = Shared.Models.Stateless.Methods.IPath.GetDirectoryNames(argZero);
@ -1037,7 +957,7 @@ public partial class DlibDotNet
if (container.SourceDirectory == argZero)
result = container;
directoryName = Path.GetFileName(container.SourceDirectory);
if (!directoryName.StartsWith(rootDirectoryName))
if (!directoryName.EndsWith(rootDirectoryName))
{
result = null;
break;
@ -1088,13 +1008,7 @@ public partial class DlibDotNet
return result;
}
private void Search(
long ticks,
Model? model,
PredictorModel? predictorModel,
string argZero,
string propertyRoot,
PersonContainer[] personContainers)
private void Search(long ticks, Model? model, PredictorModel? predictorModel, string argZero, string propertyRoot, PersonContainer[] personContainers)
{
int j;
int f;
@ -1104,8 +1018,13 @@ public partial class DlibDotNet
string bResultsFullGroupDirectory;
string cResultsFullGroupDirectory;
string dResultsFullGroupDirectory;
string eResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
string eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), "()");
string[] mappedFaceFiles = Map.Models.Stateless.Methods.IMapLogic.DeleteEmptyDirectoriesAndGetMappedFaceFiles(
_MapConfiguration,
_Faces.FileNameExtension,
ticks,
eDistanceContentDirectory,
personContainers);
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string message = $") Building Container(s) - {totalSeconds} total second(s)";
A_Property propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, model, predictorModel);
@ -1115,7 +1034,7 @@ public partial class DlibDotNet
progressBar.Tick();
(j, f, t, containers) = Property.Models.Stateless.Container.GetContainers(_Configuration.PropertyConfiguration, _FirstRun, propertyLogic);
}
Container? container = AreAllSameStartsWith(argZero, containers);
Container? container = AreAllSameEndsWith(argZero, containers);
if (_ArgZeroIsConfigurationRootDirectory && container is not null)
{
string? newRootDirectory = SaveUrlAndGetNewRootDirectory(container);
@ -1138,20 +1057,12 @@ public partial class DlibDotNet
model,
predictorModel);
}
FullDoWork(argZero, model, predictorModel, propertyRoot, ticks, propertyLogic, t, containers);
FullDoWork(argZero, model, predictorModel, propertyRoot, ticks, propertyLogic, t, containers, eDistanceContentDirectory, mappedFaceFiles);
foreach (string outputResolution in _Configuration.OutputResolutions)
{
if (_FirstRun || container is not null)
break;
(aResultsFullGroupDirectory,
bResultsFullGroupDirectory,
cResultsFullGroupDirectory,
dResultsFullGroupDirectory,
d2ResultsFullGroupDirectory,
eResultsFullGroupDirectory) = GetResultsFullGroupDirectories(
model,
predictorModel,
outputResolution);
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory) = GetResultsFullGroupDirectories(model, predictorModel, outputResolution);
if (_ArgZeroIsConfigurationRootDirectory
&& _Configuration.SaveResizedSubfiles
&& outputResolution == _Configuration.OutputResolutions[0]
@ -1160,15 +1071,7 @@ public partial class DlibDotNet
{
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any())
break;
DistanceThenMapLogic(
argZero,
ticks,
personContainers,
containers,
dResultsFullGroupDirectory,
eResultsFullGroupDirectory,
d2ResultsFullGroupDirectory,
outputResolution);
DistanceThenMapLogic(argZero, ticks, personContainers, containers, dResultsFullGroupDirectory, outputResolution);
if (_IsEnvironment.Development)
continue;
if (_FileKeyValuePairs.Any())
@ -1194,10 +1097,6 @@ public partial class DlibDotNet
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(cResultsFullGroupDirectory, "{}"));
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(dResultsFullGroupDirectory, "[]"));
if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(eResultsFullGroupDirectory, "[]"));
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(d2ResultsFullGroupDirectory, "[]"));
}
}
}

16
Instance/Models/Binder/Configuration.cs
View File

@ -14,6 +14,8 @@ public class Configuration
[Display(Name = "Check Json For Distance Results"), Required] public bool? CheckJsonForDistanceResults { get; set; }
[Display(Name = "CrossDirectory Max Items In Distance Collection"), Required] public int? CrossDirectoryMaxItemsInDistanceCollection { get; set; }
[Display(Name = "Distance Factor"), Required] public int? DistanceFactor { get; set; }
[Display(Name = "Distance Move Unable to Match by 1 Tick"), Required] public bool? DistanceMoveUnableToMatch { get; set; }
[Display(Name = "Distance Pixel Distance Tolerance"), Required] public int? DistancePixelDistanceTolerance { get; set; }
[Display(Name = "Face Distance Hidden Image Factor"), Required] public int? FaceDistanceHiddenImageFactor { get; set; }
[Display(Name = "Location Minimum Confidence"), Required] public double? FaceDistanceMinimumConfidence { get; set; }
[Display(Name = "Face Distance Permyriad"), Required] public int? FaceDistancePermyriad { get; set; }
@ -33,7 +35,6 @@ public class Configuration
[Display(Name = "Map Logic Sigma"), Required] public int? MapLogicSigma { get; set; }
[Display(Name = "Mapped Max Index"), Required] public int? MappedMaxIndex { get; set; }
[Display(Name = "Mapping Default Name"), Required] public string MappingDefaultName { get; set; }
[Display(Name = "Mapping Move Unable to Match by 1 Tick"), Required] public bool? MappingMoveUnableToMatch { get; set; }
[Display(Name = "Mapping Save Mapped"), Required] public bool? MappingSaveMapped { get; set; }
[Display(Name = "Mapping Save Not Mapped"), Required] public bool? MappingSaveNotMapped { get; set; }
[Display(Name = "Mapping Use Deterministic Hash Code Unknown Face Key Value Pairs for Add to Mapping"), Required] public bool? MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping { get; set; }
@ -61,6 +62,7 @@ public class Configuration
[Display(Name = "Properties Changed For Metadata"), Required] public bool? PropertiesChangedForMetadata { get; set; }
[Display(Name = "Properties Changed For Resize"), Required] public bool? PropertiesChangedForResize { get; set; }
[Display(Name = "Property Configuration"), Required] public Property.Models.Configuration PropertyConfiguration { get; set; }
[Display(Name = "Retry Images Without a Face"), Required] public bool? RetryImagesWithoutAFace { get; set; }
[Display(Name = "Reverse"), Required] public bool? Reverse { get; set; }
[Display(Name = "Save Face Landmark For Output Resolutions"), Required] public string[] SaveFaceLandmarkForOutputResolutions { get; set; }
[Display(Name = "Save Full Year Of Random Files"), Required] public bool? SaveFullYearOfRandomFiles { get; set; }
@ -93,6 +95,10 @@ public class Configuration
throw new NullReferenceException(nameof(configuration.CrossDirectoryMaxItemsInDistanceCollection));
if (configuration.DistanceFactor is null)
throw new NullReferenceException(nameof(configuration.DistanceFactor));
if (configuration.DistanceMoveUnableToMatch is null)
throw new NullReferenceException(nameof(configuration.DistanceMoveUnableToMatch));
if (configuration.DistancePixelDistanceTolerance is null)
throw new NullReferenceException(nameof(configuration.DistancePixelDistanceTolerance));
if (configuration.FaceDistanceHiddenImageFactor is null)
throw new NullReferenceException(nameof(configuration.FaceDistanceHiddenImageFactor));
if (configuration.FaceDistanceMinimumConfidence is null)
@ -127,8 +133,6 @@ public class Configuration
throw new NullReferenceException(nameof(configuration.MapLogicSigma));
if (configuration.MappingDefaultName is null)
throw new NullReferenceException(nameof(configuration.MappingDefaultName));
if (configuration.MappingMoveUnableToMatch is null)
throw new NullReferenceException(nameof(configuration.MappingMoveUnableToMatch));
if (configuration.MappingSaveNotMapped is null)
throw new NullReferenceException(nameof(configuration.MappingSaveNotMapped));
if (configuration.MappingSaveMapped is null)
@ -173,6 +177,8 @@ public class Configuration
throw new NullReferenceException(nameof(configuration.PropertiesChangedForMetadata));
if (configuration.PropertiesChangedForResize is null)
throw new NullReferenceException(nameof(configuration.PropertiesChangedForResize));
if (configuration.RetryImagesWithoutAFace is null)
throw new NullReferenceException(nameof(configuration.RetryImagesWithoutAFace));
if (configuration.Reverse is null)
throw new NullReferenceException(nameof(configuration.Reverse));
if (configuration.SaveFaceLandmarkForOutputResolutions is null)
@ -212,6 +218,8 @@ public class Configuration
configuration.CheckJsonForDistanceResults.Value,
configuration.CrossDirectoryMaxItemsInDistanceCollection.Value,
configuration.DistanceFactor.Value,
configuration.DistanceMoveUnableToMatch.Value,
configuration.DistancePixelDistanceTolerance.Value,
configuration.FaceDistanceHiddenImageFactor.Value,
configuration.FaceDistanceMinimumConfidence.Value,
configuration.FaceDistancePermyriad.Value,
@ -231,7 +239,6 @@ public class Configuration
configuration.MapLogicSigma.Value,
configuration.MappedMaxIndex,
configuration.MappingDefaultName,
configuration.MappingMoveUnableToMatch.Value,
configuration.MappingSaveNotMapped.Value,
configuration.MappingSaveMapped.Value,
configuration.MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping.Value,
@ -258,6 +265,7 @@ public class Configuration
configuration.PropertiesChangedForIndex.Value,
configuration.PropertiesChangedForMetadata.Value,
configuration.PropertiesChangedForResize.Value,
configuration.RetryImagesWithoutAFace.Value,
configuration.Reverse.Value,
configuration.SaveFaceLandmarkForOutputResolutions,
configuration.SaveFullYearOfRandomFiles.Value,

12
Instance/Models/Configuration.cs
View File

@ -13,6 +13,8 @@ public class Configuration
public bool CheckJsonForDistanceResults { init; get; }
public int CrossDirectoryMaxItemsInDistanceCollection { init; get; }
public int DistanceFactor { init; get; }
public bool DistanceMoveUnableToMatch { init; get; }
public int DistancePixelDistanceTolerance { init; get; }
public int FaceDistanceHiddenImageFactor { init; get; }
public double FaceDistanceMinimumConfidence { init; get; }
public int FaceDistancePermyriad { init; get; }
@ -32,7 +34,6 @@ public class Configuration
public int MapLogicSigma { init; get; }
public int? MappedMaxIndex { init; get; }
public string MappingDefaultName { init; get; }
public bool MappingMoveUnableToMatch { init; get; }
public bool MappingSaveNotMapped { init; get; }
public bool MappingSaveMapped { init; get; }
public bool MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping { init; get; }
@ -59,6 +60,7 @@ public class Configuration
public bool PropertiesChangedForIndex { init; get; }
public bool PropertiesChangedForMetadata { init; get; }
public bool PropertiesChangedForResize { init; get; }
public bool RetryImagesWithoutAFace { init; get; }
public bool Reverse { init; get; }
public string[] SaveFaceLandmarkForOutputResolutions { init; get; }
public bool SaveFullYearOfRandomFiles { init; get; }
@ -78,6 +80,8 @@ public class Configuration
bool checkJsonForDistanceResults,
int crossDirectoryMaxItemsInDistanceCollection,
int distanceFactor,
bool distanceMoveUnableToMatch,
int distancePixelDistanceTolerance,
int faceDistanceHiddenImageFactor,
double faceDistanceMinimumConfidence,
int faceDistancePermyriad,
@ -97,7 +101,6 @@ public class Configuration
int mapLogicSigma,
int? mappedMaxIndex,
string mappingDefaultName,
bool mappingMoveUnableToMatch,
bool mappingSaveNotMapped,
bool mappingSaveMapped,
bool mappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping,
@ -124,6 +127,7 @@ public class Configuration
bool propertiesChangedForIndex,
bool propertiesChangedForMetadata,
bool propertiesChangedForResize,
bool retryImagesWithoutAFace,
bool reverse,
string[] saveFaceLandmarkForOutputResolutions,
bool saveFullYearOfRandomFiles,
@ -142,6 +146,8 @@ public class Configuration
CheckJsonForDistanceResults = checkJsonForDistanceResults;
CrossDirectoryMaxItemsInDistanceCollection = crossDirectoryMaxItemsInDistanceCollection;
DistanceFactor = distanceFactor;
DistanceMoveUnableToMatch = distanceMoveUnableToMatch;
DistancePixelDistanceTolerance = distancePixelDistanceTolerance;
FaceDistanceHiddenImageFactor = faceDistanceHiddenImageFactor;
FaceDistanceMinimumConfidence = faceDistanceMinimumConfidence;
FaceDistancePermyriad = faceDistancePermyriad;
@ -161,7 +167,6 @@ public class Configuration
MapLogicSigma = mapLogicSigma;
MappedMaxIndex = mappedMaxIndex;
MappingDefaultName = mappingDefaultName;
MappingMoveUnableToMatch = mappingMoveUnableToMatch;
MappingSaveNotMapped = mappingSaveNotMapped;
MappingSaveMapped = mappingSaveMapped;
MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping = mappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping;
@ -188,6 +193,7 @@ public class Configuration
PropertiesChangedForIndex = propertiesChangedForIndex;
PropertiesChangedForMetadata = propertiesChangedForMetadata;
PropertiesChangedForResize = propertiesChangedForResize;
RetryImagesWithoutAFace = retryImagesWithoutAFace;
Reverse = reverse;
SaveFaceLandmarkForOutputResolutions = saveFaceLandmarkForOutputResolutions;
SaveFullYearOfRandomFiles = saveFullYearOfRandomFiles;

5
Instance/appsettings.Development.json
View File

@ -55,6 +55,8 @@
"CrossDirectoryMaxItemsInDistanceCollection": 7,
"DateGroup": "2022-09-22",
"DistanceFactor": 8,
"DistanceMoveUnableToMatch": false,
"DistancePixelDistanceTolerance": 1,
"FaceDistanceHiddenImageFactor": 2,
"FaceDistanceMinimumConfidence": 0.8,
"FaceDistancePermyriad": 10000,
@ -71,7 +73,6 @@
"MapLogicSigma": 3,
"MappedMaxIndex": 1034720,
"MappingDefaultName": "John Doe~25",
"MappingMoveUnableToMatch": false,
"MappingSaveFaceEncoding": false,
"MappingSaveMapped": false,
"MappingSaveNotMapped": false,
@ -106,9 +107,11 @@
"ResultCollection": "[]",
"ResultContent": "()",
"ResultSingleton": "{}",
"RetryImagesWithoutAFace": false,
"Reverse": false,
"xRootDirectory": "C:/Tmp/phares/Pictures",
"RootDirectory": "F:/Tmp/Phares/Compare/Images 2022-09-22 - fb1c68e - III",
"xxRootDirectory": "F:/Tmp/Phares/Compare/Images 2022-09-22 - fb1c68e - III/Facebook/=2022.3 Facebook",
"SaveFullYearOfRandomFiles": true,
"SaveResizedSubFiles": true,
"SkipSearch": false,

4
Instance/appsettings.Staging.json
View File

@ -55,6 +55,8 @@
"CrossDirectoryMaxItemsInDistanceCollection": 7,
"DateGroup": "2022-09-22",
"DistanceFactor": 8,
"DistanceMoveUnableToMatch": false,
"DistancePixelDistanceTolerance": 1,
"FaceDistanceHiddenImageFactor": 2,
"FaceDistanceMinimumConfidence": 0.8,
"FaceDistancePermyriad": 10000,
@ -71,7 +73,6 @@
"MapLogicSigma": 3,
"MappedMaxIndex": 1034720,
"MappingDefaultName": "John Doe~25",
"MappingMoveUnableToMatch": false,
"MappingSaveFaceEncoding": false,
"MappingSaveMapped": false,
"MappingSaveNotMapped": false,
@ -106,6 +107,7 @@
"ResultCollection": "[]",
"ResultContent": "()",
"ResultSingleton": "{}",
"RetryImagesWithoutAFace": false,
"Reverse": false,
"RootDirectory": "E:/Images",
"SaveFullYearOfRandomFiles": true,

4
Instance/appsettings.json
View File

@ -55,6 +55,8 @@
"CrossDirectoryMaxItemsInDistanceCollection": 7,
"DateGroup": "2022-09-22",
"DistanceFactor": 8,
"DistanceMoveUnableToMatch": false,
"DistancePixelDistanceTolerance": 1,
"FaceDistanceHiddenImageFactor": 2,
"FaceDistanceMinimumConfidence": 0.8,
"FaceDistancePermyriad": 10000,
@ -71,7 +73,6 @@
"MapLogicSigma": 3,
"MappedMaxIndex": 1034720,
"MappingDefaultName": "John Doe~25",
"MappingMoveUnableToMatch": false,
"MappingSaveFaceEncoding": false,
"MappingSaveMapped": false,
"MappingSaveNotMapped": false,
@ -106,6 +107,7 @@
"ResultCollection": "[]",
"ResultContent": "()",
"ResultSingleton": "{}",
"RetryImagesWithoutAFace": false,
"Reverse": false,
"RootDirectory": "D:/Images",
"SaveFullYearOfRandomFiles": true,