|
|
|
@ -1,13 +1,10 @@
|
|
|
|
|
using System.Text.Json;
|
|
|
|
|
using System.Text.RegularExpressions;
|
|
|
|
|
using View_by_Distance.FaceRecognitionDotNet;
|
|
|
|
|
using View_by_Distance.Metadata.Models;
|
|
|
|
|
using View_by_Distance.Property.Models;
|
|
|
|
|
using View_by_Distance.Resize.Models;
|
|
|
|
|
using View_by_Distance.Shared.Models;
|
|
|
|
|
using View_by_Distance.Shared.Models.Properties;
|
|
|
|
|
using View_by_Distance.Shared.Models.Stateless;
|
|
|
|
|
using WindowsShortcutFactory;
|
|
|
|
|
|
|
|
|
|
namespace View_by_Distance.Instance.Models;
|
|
|
|
|
|
|
|
|
@ -424,6 +421,152 @@ internal class E_Distance
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static int GetSelectedIndex(int maxDegreeOfParallelism, Random random, List<FaceRecognitionDotNet.FaceEncoding> faceEncodings)
|
|
|
|
|
{
|
|
|
|
|
int? result;
|
|
|
|
|
int selectedIndex;
|
|
|
|
|
List<(int? Index, double? Sum)> faceDistanceCollections = new();
|
|
|
|
|
if (maxDegreeOfParallelism == 1)
|
|
|
|
|
{
|
|
|
|
|
double sum;
|
|
|
|
|
List<double> faceDistances;
|
|
|
|
|
for (int i = 0; i < faceEncodings.Count; i++)
|
|
|
|
|
{
|
|
|
|
|
faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncodings[i]);
|
|
|
|
|
sum = faceDistances.Sum();
|
|
|
|
|
faceDistanceCollections.Add(new(i, sum));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
for (int i = 0; i < faceEncodings.Count; i++)
|
|
|
|
|
faceDistanceCollections.Add(new(null, null));
|
|
|
|
|
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
|
|
|
|
|
_ = Parallel.For(0, faceEncodings.Count, parallelOptions, (i, state) =>
|
|
|
|
|
{
|
|
|
|
|
List<double> faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncodings[i]);
|
|
|
|
|
double sum = faceDistances.Sum();
|
|
|
|
|
lock (faceDistanceCollections)
|
|
|
|
|
faceDistanceCollections[i] = new(i, sum);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
faceDistanceCollections = faceDistanceCollections.OrderBy(l => l.Sum).ToList();
|
|
|
|
|
if (faceDistanceCollections.Count != faceEncodings.Count)
|
|
|
|
|
throw new Exception();
|
|
|
|
|
if (faceDistanceCollections.Count > 1000)
|
|
|
|
|
selectedIndex = random.Next(0, 36);
|
|
|
|
|
else if (faceDistanceCollections.Count > 500)
|
|
|
|
|
selectedIndex = random.Next(0, 31);
|
|
|
|
|
else if (faceDistanceCollections.Count > 200)
|
|
|
|
|
selectedIndex = random.Next(0, 26);
|
|
|
|
|
else if (faceDistanceCollections.Count > 100)
|
|
|
|
|
selectedIndex = random.Next(0, 21);
|
|
|
|
|
else if (faceDistanceCollections.Count > 50)
|
|
|
|
|
selectedIndex = random.Next(0, 16);
|
|
|
|
|
else if (faceDistanceCollections.Count > 25)
|
|
|
|
|
selectedIndex = random.Next(0, 11);
|
|
|
|
|
else if (faceDistanceCollections.Count > 10)
|
|
|
|
|
selectedIndex = random.Next(0, 6);
|
|
|
|
|
else if (faceDistanceCollections.Count > 5)
|
|
|
|
|
selectedIndex = random.Next(0, 3);
|
|
|
|
|
else
|
|
|
|
|
selectedIndex = 0;
|
|
|
|
|
result = faceDistanceCollections[selectedIndex].Index;
|
|
|
|
|
if (result is null)
|
|
|
|
|
throw new NullReferenceException(nameof(result));
|
|
|
|
|
return result.Value;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static void SetFiltered(List<(FaceRecognitionDotNet.FaceEncoding FaceEncoding, MappingContainer MappingContainer)> collection)
|
|
|
|
|
{
|
|
|
|
|
double ucl;
|
|
|
|
|
bool check;
|
|
|
|
|
double average;
|
|
|
|
|
double[] doubles;
|
|
|
|
|
double standardDeviation;
|
|
|
|
|
double?[] nullableDoubles;
|
|
|
|
|
for (int i = 0; i < int.MaxValue; i++)
|
|
|
|
|
{
|
|
|
|
|
check = true;
|
|
|
|
|
nullableDoubles = (from l in collection where l.MappingContainer.Mapping.Filtered is not null && !l.MappingContainer.Mapping.Filtered.Value select l.MappingContainer.Distance).ToArray();
|
|
|
|
|
doubles = (from l in nullableDoubles where l.HasValue select l.Value).ToArray();
|
|
|
|
|
if (doubles.Length < 4)
|
|
|
|
|
break;
|
|
|
|
|
average = doubles.Average();
|
|
|
|
|
standardDeviation = GetStandardDeviation(doubles, average);
|
|
|
|
|
ucl = average + (standardDeviation * 3);
|
|
|
|
|
if (ucl > IFaceDistance.Tolerance)
|
|
|
|
|
ucl = IFaceDistance.Tolerance;
|
|
|
|
|
foreach ((FaceRecognitionDotNet.FaceEncoding _, MappingContainer mappingContainer) in collection)
|
|
|
|
|
{
|
|
|
|
|
if (mappingContainer.Mapping.Filtered is null || mappingContainer.Mapping.Filtered.Value || mappingContainer.Distance <= ucl)
|
|
|
|
|
continue;
|
|
|
|
|
if (check)
|
|
|
|
|
check = false;
|
|
|
|
|
mappingContainer.Mapping.SetFiltered();
|
|
|
|
|
}
|
|
|
|
|
if (check)
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static FaceDistance GetFaceDistanceParallelFor(Face face, FaceRecognitionDotNet.FaceEncoding faceEncoding, Mapping mapping, DateTime minimumDateTime, bool? isWrongYear, string key, FaceRecognitionDotNet.FaceEncoding[] faceEncodings)
|
|
|
|
|
{
|
|
|
|
|
FaceDistance result;
|
|
|
|
|
if (face.Location?.NormalizedPixelPercentage is null)
|
|
|
|
|
throw new NullReferenceException(nameof(face.Location.NormalizedPixelPercentage));
|
|
|
|
|
List<double> faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncoding);
|
|
|
|
|
result = new(faceDistances, isWrongYear, key, mapping, minimumDateTime);
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static List<FaceDistance> GetFaceDistanceCollection(int maxDegreeOfParallelism, List<(string Key, int Id, Mapping Mapping, DateTime MinimumDateTime, bool? IsWrongYear, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>)> collection, Face face)
|
|
|
|
|
{
|
|
|
|
|
List<FaceDistance> results;
|
|
|
|
|
if (face.FaceEncoding is null)
|
|
|
|
|
throw new NullReferenceException(nameof(face.FaceEncoding));
|
|
|
|
|
FaceRecognitionDotNet.FaceEncoding faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
|
|
|
|
|
if (maxDegreeOfParallelism == 1)
|
|
|
|
|
{
|
|
|
|
|
results = new();
|
|
|
|
|
FaceDistance faceDistance;
|
|
|
|
|
List<double> faceDistances;
|
|
|
|
|
FaceRecognitionDotNet.FaceEncoding[] faceEncodings;
|
|
|
|
|
if (face.Location?.NormalizedPixelPercentage is null)
|
|
|
|
|
throw new NullReferenceException(nameof(face.Location.NormalizedPixelPercentage));
|
|
|
|
|
foreach ((string key, int id, Mapping mapping, DateTime minimumDateTime, bool? isWrongYear, List<(FaceRecognitionDotNet.FaceEncoding FaceEncoding, MappingContainer _)> faceEncodingContainers) in collection)
|
|
|
|
|
{
|
|
|
|
|
faceEncodings = (from l in faceEncodingContainers select l.FaceEncoding).ToArray();
|
|
|
|
|
faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncoding);
|
|
|
|
|
faceDistance = new(faceDistances, isWrongYear, key, mapping, minimumDateTime);
|
|
|
|
|
results.Add(faceDistance);
|
|
|
|
|
if (results.Count > IFaceDistance.MaximumPer)
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
results = new();
|
|
|
|
|
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
|
|
|
|
|
_ = Parallel.For(0, collection.Count, parallelOptions, (i, state) =>
|
|
|
|
|
{
|
|
|
|
|
(string key, int id, Mapping mapping, DateTime minimumDateTime, bool? isWrongYear, List<(FaceRecognitionDotNet.FaceEncoding FaceEncoding, MappingContainer _)> faceEncodingContainers) = collection[i];
|
|
|
|
|
FaceRecognitionDotNet.FaceEncoding[] faceEncodings = (from l in faceEncodingContainers select l.FaceEncoding).ToArray();
|
|
|
|
|
FaceDistance? closest = GetFaceDistanceParallelFor(face, faceEncoding, mapping, minimumDateTime, isWrongYear, key, faceEncodings);
|
|
|
|
|
if (closest is not null)
|
|
|
|
|
{
|
|
|
|
|
lock (results)
|
|
|
|
|
{
|
|
|
|
|
results.Add(closest);
|
|
|
|
|
if (results.Count > IFaceDistance.MaximumPer)
|
|
|
|
|
state.Break();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
return results;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static FaceRecognitionDotNet.FaceEncoding? GetFaceEncoding(Face face)
|
|
|
|
|
{
|
|
|
|
|
FaceRecognitionDotNet.FaceEncoding? result;
|
|
|
|
@ -434,24 +577,18 @@ internal class E_Distance
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static (int index, double sum) GetIndexAndSum(int i, List<FaceRecognitionDotNet.FaceEncoding> results)
|
|
|
|
|
{
|
|
|
|
|
List<double> faceDistances = FaceRecognition.FaceDistances(results, results[i]);
|
|
|
|
|
return new(i, faceDistances.Sum());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static List<FaceRecognitionDotNet.FaceEncoding> GetFaceEncodings(int maxDegreeOfParallelism, List<(DateTime MinimumDateTime, bool? IsWrongYear, PersonBirthday PersonBirthday, Face Face)> collection)
|
|
|
|
|
private static List<FaceRecognitionDotNet.FaceEncoding> GetFaceEncodingsOnly(int maxDegreeOfParallelism, List<MappingContainer> collection)
|
|
|
|
|
{
|
|
|
|
|
List<FaceRecognitionDotNet.FaceEncoding> results;
|
|
|
|
|
if (maxDegreeOfParallelism == 1)
|
|
|
|
|
{
|
|
|
|
|
results = new();
|
|
|
|
|
FaceRecognitionDotNet.FaceEncoding faceEncoding;
|
|
|
|
|
foreach ((DateTime _, bool? _, PersonBirthday _, Face face) in collection)
|
|
|
|
|
foreach (MappingContainer mappingContainer in collection)
|
|
|
|
|
{
|
|
|
|
|
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
|
|
|
|
|
if (mappingContainer.Face?.FaceEncoding is null || mappingContainer.Face.Location?.NormalizedPixelPercentage is null)
|
|
|
|
|
continue;
|
|
|
|
|
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
|
|
|
|
|
faceEncoding = FaceRecognition.LoadFaceEncoding(mappingContainer.Face.FaceEncoding.RawEncoding);
|
|
|
|
|
results.Add(faceEncoding);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
@ -459,9 +596,12 @@ internal class E_Distance
|
|
|
|
|
{
|
|
|
|
|
results = new();
|
|
|
|
|
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
|
|
|
|
|
_ = Parallel.For(0, collection.Count, parallelOptions, i =>
|
|
|
|
|
_ = Parallel.For(0, collection.Count, parallelOptions, (i, state) =>
|
|
|
|
|
{
|
|
|
|
|
FaceRecognitionDotNet.FaceEncoding? faceEncoding = GetFaceEncoding(collection[i].Face);
|
|
|
|
|
Face? face = collection[i].Face;
|
|
|
|
|
if (face is null)
|
|
|
|
|
throw new Exception();
|
|
|
|
|
FaceRecognitionDotNet.FaceEncoding? faceEncoding = GetFaceEncoding(face);
|
|
|
|
|
if (faceEncoding is not null)
|
|
|
|
|
{
|
|
|
|
|
lock (results)
|
|
|
|
@ -469,136 +609,68 @@ internal class E_Distance
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
if (collection.Count == results.Count && results.Count > 1)
|
|
|
|
|
{
|
|
|
|
|
double sum;
|
|
|
|
|
int lowestIndex;
|
|
|
|
|
double lowestSum;
|
|
|
|
|
List<double> faceDistances;
|
|
|
|
|
if (maxDegreeOfParallelism == 1)
|
|
|
|
|
{
|
|
|
|
|
lowestIndex = 0;
|
|
|
|
|
lowestSum = double.MaxValue;
|
|
|
|
|
for (int i = 0; i < results.Count; i++)
|
|
|
|
|
{
|
|
|
|
|
faceDistances = FaceRecognition.FaceDistances(results, results[i]);
|
|
|
|
|
sum = faceDistances.Sum();
|
|
|
|
|
if (sum >= lowestSum)
|
|
|
|
|
continue;
|
|
|
|
|
lowestIndex = i;
|
|
|
|
|
lowestSum = sum;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
List<(int Index, double Sum)> indicesAndSums = new();
|
|
|
|
|
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
|
|
|
|
|
_ = Parallel.For(0, results.Count, parallelOptions, i =>
|
|
|
|
|
{
|
|
|
|
|
(int index, double sum) = GetIndexAndSum(i, results);
|
|
|
|
|
lock (indicesAndSums)
|
|
|
|
|
indicesAndSums.Add(new(index, sum));
|
|
|
|
|
});
|
|
|
|
|
(lowestIndex, lowestSum) = (from l in indicesAndSums orderby l.Sum select l).First();
|
|
|
|
|
}
|
|
|
|
|
faceDistances = FaceRecognition.FaceDistances(results, results[lowestIndex]);
|
|
|
|
|
sum = faceDistances.Sum();
|
|
|
|
|
if (sum == lowestSum)
|
|
|
|
|
{
|
|
|
|
|
double average = faceDistances.Average();
|
|
|
|
|
double standardDeviation = GetStandardDeviation(faceDistances, average);
|
|
|
|
|
double lcl = average - (standardDeviation * 3);
|
|
|
|
|
double ucl = average + (standardDeviation * 3);
|
|
|
|
|
for (int i = results.Count - 1; i > -1; i--)
|
|
|
|
|
{
|
|
|
|
|
if (faceDistances[i] < lcl || faceDistances[i] > ucl)
|
|
|
|
|
results.RemoveAt(i);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return results;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static List<(DateTime, bool?, PersonBirthday, FaceRecognitionDotNet.FaceEncoding[])> GetThreeSigmaFaceEncodings(int maxDegreeOfParallelism, Dictionary<string, List<(DateTime, bool?, PersonBirthday, Face)>> keyValuePairs)
|
|
|
|
|
private Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> GetThreeSigmaFaceEncodings(int maxDegreeOfParallelism, long ticks, Dictionary<string, List<MappingContainer>> keyValuePairs)
|
|
|
|
|
{
|
|
|
|
|
List<(DateTime, bool?, PersonBirthday, FaceRecognitionDotNet.FaceEncoding[])> results = new();
|
|
|
|
|
const int zero = 0;
|
|
|
|
|
if (_Log is null)
|
|
|
|
|
throw new NullReferenceException(nameof(_Log));
|
|
|
|
|
Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> results = new();
|
|
|
|
|
int totalSeconds;
|
|
|
|
|
int selectedIndex;
|
|
|
|
|
Random random = new();
|
|
|
|
|
List<double> faceDistances;
|
|
|
|
|
MappingContainer mappingContainer;
|
|
|
|
|
int keyValuePairsCount = keyValuePairs.Count;
|
|
|
|
|
FaceRecognitionDotNet.FaceEncoding faceEncoding;
|
|
|
|
|
List<FaceRecognitionDotNet.FaceEncoding> faceEncodings;
|
|
|
|
|
foreach (KeyValuePair<string, List<(DateTime MinimumDateTime, bool? IsWrongYear, PersonBirthday PersonBirthday, Face _)>> keyValuePair in keyValuePairs)
|
|
|
|
|
List<(FaceRecognitionDotNet.FaceEncoding FaceEncoding, MappingContainer MappingContainer)> collection;
|
|
|
|
|
foreach (KeyValuePair<string, List<MappingContainer>> keyValuePair in keyValuePairs)
|
|
|
|
|
{
|
|
|
|
|
faceEncodings = GetFaceEncodings(maxDegreeOfParallelism, keyValuePair.Value);
|
|
|
|
|
results.Add(new(keyValuePair.Value[zero].MinimumDateTime, keyValuePair.Value[zero].IsWrongYear, keyValuePair.Value[zero].PersonBirthday, faceEncodings.ToArray()));
|
|
|
|
|
}
|
|
|
|
|
return results;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static Closest? GetClosestParallelFor(DateTime minimumDateTime, bool? isWrongYear, Face face, FaceRecognitionDotNet.FaceEncoding faceEncoding, (DateTime MinimumDateTime, bool? IsWrongYear, PersonBirthday PersonBirthday, FaceRecognitionDotNet.FaceEncoding[] FaceEncodings) tuple)
|
|
|
|
|
{
|
|
|
|
|
Closest? result;
|
|
|
|
|
if (isWrongYear.HasValue && !isWrongYear.Value && minimumDateTime < tuple.PersonBirthday.Value)
|
|
|
|
|
result = null;
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
List<double> faceDistances = FaceRecognition.FaceDistances(tuple.FaceEncodings, faceEncoding);
|
|
|
|
|
result = new(face.Location?.NormalizedPixelPercentage, tuple.MinimumDateTime, tuple.IsWrongYear, tuple.PersonBirthday, faceDistances);
|
|
|
|
|
if (result.Minimum > Shared.Models.Stateless.IClosest.MaximumMinimum)
|
|
|
|
|
result = null;
|
|
|
|
|
}
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static Closest[] GetClosestCollection(int maxDegreeOfParallelism, List<(DateTime, bool?, PersonBirthday, FaceRecognitionDotNet.FaceEncoding[])> collection, DateTime itemMinimumDateTime, bool? itemIsWrongYear, Face face)
|
|
|
|
|
{
|
|
|
|
|
Closest[] results;
|
|
|
|
|
List<Closest> closestCollection;
|
|
|
|
|
if (face.FaceEncoding is null)
|
|
|
|
|
throw new NullReferenceException(nameof(face.FaceEncoding));
|
|
|
|
|
FaceRecognitionDotNet.FaceEncoding faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
|
|
|
|
|
if (maxDegreeOfParallelism == 1)
|
|
|
|
|
{
|
|
|
|
|
closestCollection = new();
|
|
|
|
|
Closest closest;
|
|
|
|
|
List<double> faceDistances;
|
|
|
|
|
foreach ((DateTime minimumDateTime, bool? isWrongYear, PersonBirthday personBirthday, FaceRecognitionDotNet.FaceEncoding[] faceEncodings) in collection)
|
|
|
|
|
collection = new();
|
|
|
|
|
faceEncodings = GetFaceEncodingsOnly(maxDegreeOfParallelism, keyValuePair.Value);
|
|
|
|
|
for (int i = 0; i < faceEncodings.Count; i++)
|
|
|
|
|
{
|
|
|
|
|
if (itemIsWrongYear.HasValue && !itemIsWrongYear.Value && itemMinimumDateTime < personBirthday.Value)
|
|
|
|
|
continue;
|
|
|
|
|
faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncoding);
|
|
|
|
|
closest = new(face.Location?.NormalizedPixelPercentage, minimumDateTime, isWrongYear, personBirthday, faceDistances);
|
|
|
|
|
if (closest.Minimum > Shared.Models.Stateless.IClosest.MaximumMinimum)
|
|
|
|
|
continue;
|
|
|
|
|
closestCollection.Add(closest);
|
|
|
|
|
faceEncoding = faceEncodings[i];
|
|
|
|
|
mappingContainer = keyValuePair.Value[i];
|
|
|
|
|
collection.Add(new(faceEncoding, mappingContainer));
|
|
|
|
|
}
|
|
|
|
|
results.Add(keyValuePair.Key, collection);
|
|
|
|
|
if (faceEncodings.Count == 1)
|
|
|
|
|
selectedIndex = 0;
|
|
|
|
|
else
|
|
|
|
|
selectedIndex = GetSelectedIndex(maxDegreeOfParallelism, random, faceEncodings);
|
|
|
|
|
faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncodings[selectedIndex]);
|
|
|
|
|
for (int i = 0; i < faceEncodings.Count; i++)
|
|
|
|
|
collection[i].MappingContainer.SetDistance(faceDistances[i]);
|
|
|
|
|
if (collection.Count > 1)
|
|
|
|
|
SetFiltered(collection);
|
|
|
|
|
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
|
|
|
|
|
_Log.Information($"{keyValuePairsCount:0000}) {totalSeconds} total second(s) - {keyValuePair.Key} - {collection[selectedIndex].MappingContainer.Mapping.DisplayDirectoryName}");
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
closestCollection = new();
|
|
|
|
|
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
|
|
|
|
|
_ = Parallel.For(0, collection.Count, parallelOptions, i =>
|
|
|
|
|
{
|
|
|
|
|
Closest? closest = GetClosestParallelFor(itemMinimumDateTime, itemIsWrongYear, face, faceEncoding, collection[i]);
|
|
|
|
|
if (closest is not null)
|
|
|
|
|
{
|
|
|
|
|
lock (closestCollection)
|
|
|
|
|
closestCollection.Add(closest);
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
results = Shared.Models.Stateless.Methods.IClosest.Get(closestCollection);
|
|
|
|
|
return results;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static void AddClosest(int maxDegreeOfParallelism, string argZero, Map.Models.MapLogic mapLogic, List<Container> containers, List<(DateTime, bool?, PersonBirthday, FaceRecognitionDotNet.FaceEncoding[])> collection)
|
|
|
|
|
internal Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> ParallelWork(int maxDegreeOfParallelism, string[] ignoreRelativePaths, string argZero, long ticks, List<Container> containers)
|
|
|
|
|
{
|
|
|
|
|
string key;
|
|
|
|
|
Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> results;
|
|
|
|
|
Dictionary<string, List<MappingContainer>> keyValuePairs = Map.Models.Stateless.IMapLogic.GetKeyValuePairs(ignoreRelativePaths, argZero, containers);
|
|
|
|
|
results = GetThreeSigmaFaceEncodings(maxDegreeOfParallelism, ticks, keyValuePairs);
|
|
|
|
|
return results;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public void AddToFaceDistance(int maxDegreeOfParallelism, string argZero, long ticks, Map.Models.MapLogic mapLogic, List<Container> containers, string outputResolution, List<(string, int, Mapping, DateTime, bool?, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>)> collection)
|
|
|
|
|
{
|
|
|
|
|
if (_Log is null)
|
|
|
|
|
throw new NullReferenceException(nameof(_Log));
|
|
|
|
|
Face face;
|
|
|
|
|
Closest closest;
|
|
|
|
|
string personKey;
|
|
|
|
|
bool? itemIsWrongYear;
|
|
|
|
|
Closest[] closestCollection;
|
|
|
|
|
DateTime? itemMinimumDateTime;
|
|
|
|
|
string message;
|
|
|
|
|
int totalSeconds;
|
|
|
|
|
double deterministicHashCodeKey;
|
|
|
|
|
Dictionary<string, int> results = new();
|
|
|
|
|
DateTime dateTime = DateTime.Now;
|
|
|
|
|
List<FaceDistance> faceDistances;
|
|
|
|
|
int containersCount = containers.Count;
|
|
|
|
|
foreach (Container container in containers)
|
|
|
|
|
{
|
|
|
|
|
if (!container.Items.Any())
|
|
|
|
@ -607,55 +679,29 @@ internal class E_Distance
|
|
|
|
|
continue;
|
|
|
|
|
foreach (Item item in container.Items)
|
|
|
|
|
{
|
|
|
|
|
if (item.ImageFileHolder is null || item.Property is null || item.Named.Any())
|
|
|
|
|
if (item.ImageFileHolder is null || item.Property?.Id is null)
|
|
|
|
|
continue;
|
|
|
|
|
itemMinimumDateTime = Shared.Models.Stateless.Methods.IProperty.GetMinimumDateTime(item.Property);
|
|
|
|
|
if (itemMinimumDateTime is null)
|
|
|
|
|
continue;
|
|
|
|
|
(itemIsWrongYear, _) = Map.Models.MapLogic.IsWrongYear(item);
|
|
|
|
|
if (Shared.Models.Stateless.IClosest.SkipIsWrongYear && itemIsWrongYear.HasValue && itemIsWrongYear.Value)
|
|
|
|
|
continue;
|
|
|
|
|
item.Closest.Clear();
|
|
|
|
|
for (int i = 0; i < item.Faces.Count; i++)
|
|
|
|
|
{
|
|
|
|
|
face = item.Faces[i];
|
|
|
|
|
closest = new(face.Location?.NormalizedPixelPercentage, itemMinimumDateTime.Value, itemIsWrongYear);
|
|
|
|
|
item.Closest.Add(closest);
|
|
|
|
|
face.FaceDistances.Clear();
|
|
|
|
|
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
|
|
|
|
|
continue;
|
|
|
|
|
deterministicHashCodeKey = Shared.Models.Stateless.Methods.INamed.GetDeterministicHashCodeKey(item, face);
|
|
|
|
|
closestCollection = GetClosestCollection(maxDegreeOfParallelism, collection, itemMinimumDateTime.Value, itemIsWrongYear, face);
|
|
|
|
|
for (int j = 0; j < closestCollection.Length; j++)
|
|
|
|
|
{
|
|
|
|
|
closest = closestCollection[j];
|
|
|
|
|
if (closest.PersonBirthday is null)
|
|
|
|
|
continue;
|
|
|
|
|
personKey = Shared.Models.Stateless.Methods.IPersonBirthday.GetFormatted(closest.PersonBirthday);
|
|
|
|
|
if (mapLogic.IsIncorrect(deterministicHashCodeKey, personKey))
|
|
|
|
|
continue;
|
|
|
|
|
key = Map.Models.MapLogic.GetKey(closest.MinimumDateTime, closest.IsWrongYear, closest.PersonBirthday);
|
|
|
|
|
if (!results.ContainsKey(key))
|
|
|
|
|
results.Add(key, 0);
|
|
|
|
|
else if (results[key] > Shared.Models.Stateless.IClosest.MaximumPer)
|
|
|
|
|
continue;
|
|
|
|
|
results[key] += 1;
|
|
|
|
|
item.Closest[0] = closest;
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
if ((from l in item.Mapping where l.NormalizedPixelPercentage.HasValue && l.NormalizedPixelPercentage.Value == face.Location.NormalizedPixelPercentage.Value select true).Any())
|
|
|
|
|
continue;
|
|
|
|
|
deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item, face);
|
|
|
|
|
if (mapLogic.Skip(deterministicHashCodeKey))
|
|
|
|
|
continue;
|
|
|
|
|
faceDistances = GetFaceDistanceCollection(maxDegreeOfParallelism, collection, face);
|
|
|
|
|
face.FaceDistances.AddRange(faceDistances);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
|
|
|
|
|
message = $"{container.R:000}.{container.G} / {containersCount:000}) {container.Items.Count:000} file(s) - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}";
|
|
|
|
|
_Log.Information(message);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
internal static List<(DateTime, bool?, PersonBirthday, FaceRecognitionDotNet.FaceEncoding[])> ParallelWork(int maxDegreeOfParallelism, string argZero, Map.Models.MapLogic mapLogic, List<Container> containers)
|
|
|
|
|
{
|
|
|
|
|
List<(DateTime, bool?, PersonBirthday, FaceRecognitionDotNet.FaceEncoding[])> results;
|
|
|
|
|
Dictionary<string, List<(DateTime, bool?, PersonBirthday, Face)>> keyValuePairs = Map.Models.MapLogic.GetKeyValuePairs(argZero, containers);
|
|
|
|
|
results = GetThreeSigmaFaceEncodings(maxDegreeOfParallelism, keyValuePairs);
|
|
|
|
|
AddClosest(maxDegreeOfParallelism, argZero, mapLogic, containers, results);
|
|
|
|
|
return results;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static void SavePropertyHolders(string argZero, List<Container> containers, string zPropertyHolderSingletonDirectory)
|
|
|
|
|
{
|
|
|
|
|
string json;
|
|
|
|
@ -672,8 +718,6 @@ internal class E_Distance
|
|
|
|
|
{
|
|
|
|
|
if (item.ImageFileHolder is null || item.Property is null || !item.Faces.Any() || !item.Closest.Any())
|
|
|
|
|
continue;
|
|
|
|
|
if (!(from l in item.Closest where l.Average.HasValue select true).Any())
|
|
|
|
|
continue;
|
|
|
|
|
json = JsonSerializer.Serialize(item, jsonSerializerOptions);
|
|
|
|
|
fileInfo = new(string.Concat(zPropertyHolderSingletonDirectory, item.RelativePath, ".json"));
|
|
|
|
|
if (fileInfo.Directory is null)
|
|
|
|
@ -685,140 +729,4 @@ internal class E_Distance
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
internal static void SaveThreeSigmaFaceEncodings(List<(DateTime, bool?, PersonBirthday, FaceRecognitionDotNet.FaceEncoding[])> collection, Dictionary<string, List<Person>> peopleCollection, string eDistanceCollectionDirectory)
|
|
|
|
|
{
|
|
|
|
|
string json;
|
|
|
|
|
string checkFile;
|
|
|
|
|
string personKey;
|
|
|
|
|
string directory;
|
|
|
|
|
List<double[]> rawEncodings;
|
|
|
|
|
Person person;
|
|
|
|
|
const string facePopulatedKey = "ThreeSigma";
|
|
|
|
|
const string pattern = @"[\\,\/,\:,\*,\?,\"",\<,\>,\|]";
|
|
|
|
|
foreach ((DateTime minimumDateTime, bool? isWrongYear, PersonBirthday personBirthday, FaceRecognitionDotNet.FaceEncoding[] faceEncodings) in collection)
|
|
|
|
|
{
|
|
|
|
|
rawEncodings = new();
|
|
|
|
|
checkFile = string.Empty;
|
|
|
|
|
personKey = Shared.Models.Stateless.Methods.IPersonBirthday.GetFormatted(personBirthday);
|
|
|
|
|
directory = Map.Models.MapLogic.GetDirectory(eDistanceCollectionDirectory, facePopulatedKey, minimumDateTime, isWrongYear, personBirthday, personKey);
|
|
|
|
|
if (!peopleCollection.ContainsKey(personKey))
|
|
|
|
|
continue;
|
|
|
|
|
person = peopleCollection[personKey][0];
|
|
|
|
|
checkFile = string.Concat(directory, " - ", Regex.Replace(Shared.Models.Stateless.Methods.IPersonName.GetFullName(person.Name), pattern, string.Empty), ".json");
|
|
|
|
|
if (string.IsNullOrEmpty(checkFile))
|
|
|
|
|
continue;
|
|
|
|
|
if (!Directory.Exists(directory))
|
|
|
|
|
_ = Directory.CreateDirectory(directory);
|
|
|
|
|
for (int i = 0; i < faceEncodings.Length; i++)
|
|
|
|
|
rawEncodings.Add(faceEncodings[i].GetRawEncoding());
|
|
|
|
|
json = JsonSerializer.Serialize(rawEncodings, new JsonSerializerOptions { WriteIndented = true });
|
|
|
|
|
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: true, compareBeforeWrite: true);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
internal static List<(IFileHolder? resizedFileHolder, string directory, FileInfo? faceFileInfo, string checkFile, string shortcutFile)> GetClosest(string argZero, List<Container> containers, Dictionary<string, List<Person>> peopleCollection, string dFacesContentDirectory, string d2ResultsFullGroupDirectory, string eDistanceContentDirectory)
|
|
|
|
|
{
|
|
|
|
|
List<(IFileHolder?, string, FileInfo?, string, string)> results = new();
|
|
|
|
|
Person person;
|
|
|
|
|
string checkFile;
|
|
|
|
|
string directory;
|
|
|
|
|
string personKey;
|
|
|
|
|
string personName;
|
|
|
|
|
string shortcutFile;
|
|
|
|
|
FileInfo faceFileInfo;
|
|
|
|
|
string? directoryName;
|
|
|
|
|
string facesDirectory;
|
|
|
|
|
string personDirectory;
|
|
|
|
|
FileInfo landmarkFileInfo;
|
|
|
|
|
string landmarksDirectory;
|
|
|
|
|
double deterministicHashCodeKey;
|
|
|
|
|
const string facePopulatedKey = nameof(Closest);
|
|
|
|
|
const string pattern = @"[\\,\/,\:,\*,\?,\"",\<,\>,\|]";
|
|
|
|
|
foreach (Container container in containers)
|
|
|
|
|
{
|
|
|
|
|
if (!container.Items.Any())
|
|
|
|
|
continue;
|
|
|
|
|
if (!container.SourceDirectory.StartsWith(argZero))
|
|
|
|
|
continue;
|
|
|
|
|
foreach (Item item in container.Items)
|
|
|
|
|
{
|
|
|
|
|
if (item.ImageFileHolder is null || item.Property?.Id is null || item.ResizedFileHolder is null || item.Named.Any())
|
|
|
|
|
continue;
|
|
|
|
|
if (!item.Closest.Any())
|
|
|
|
|
continue;
|
|
|
|
|
directoryName = Path.GetDirectoryName(item.RelativePath);
|
|
|
|
|
if (directoryName is null)
|
|
|
|
|
throw new Exception();
|
|
|
|
|
foreach (Closest closest in item.Closest)
|
|
|
|
|
{
|
|
|
|
|
if (closest.Average is null || closest.NormalizedPixelPercentage is null || closest.PersonBirthday is null)
|
|
|
|
|
continue;
|
|
|
|
|
personKey = Shared.Models.Stateless.Methods.IPersonBirthday.GetFormatted(closest.PersonBirthday);
|
|
|
|
|
directory = Map.Models.MapLogic.GetDirectory(eDistanceContentDirectory, facePopulatedKey, closest.MinimumDateTime, closest.IsWrongYear, closest.PersonBirthday, personKey);
|
|
|
|
|
if (!peopleCollection.ContainsKey(personKey))
|
|
|
|
|
personDirectory = string.Empty;
|
|
|
|
|
else
|
|
|
|
|
{
|
|
|
|
|
person = peopleCollection[personKey][0];
|
|
|
|
|
personName = Shared.Models.Stateless.Methods.IPersonName.GetFullName(person.Name);
|
|
|
|
|
personDirectory = Path.Combine(directory, Regex.Replace(personName, pattern, string.Empty), "lnk");
|
|
|
|
|
results.Add(new(null, personDirectory, null, string.Empty, string.Empty));
|
|
|
|
|
}
|
|
|
|
|
facesDirectory = string.Concat(dFacesContentDirectory, Path.Combine(directoryName, item.ImageFileHolder.NameWithoutExtension));
|
|
|
|
|
landmarksDirectory = string.Concat(d2ResultsFullGroupDirectory, Path.Combine(directoryName, item.ImageFileHolder.NameWithoutExtension));
|
|
|
|
|
deterministicHashCodeKey = Shared.Models.Stateless.Methods.INamed.GetDeterministicHashCodeKey(item, closest);
|
|
|
|
|
checkFile = Path.Combine(directory, $"{deterministicHashCodeKey}{item.ImageFileHolder.ExtensionLowered}");
|
|
|
|
|
faceFileInfo = new(Path.Combine(facesDirectory, $"{deterministicHashCodeKey}{item.ImageFileHolder.ExtensionLowered}.png"));
|
|
|
|
|
landmarkFileInfo = new(Path.Combine(landmarksDirectory, $"{deterministicHashCodeKey}{item.ImageFileHolder.ExtensionLowered}.gif"));
|
|
|
|
|
if (string.IsNullOrEmpty(personDirectory))
|
|
|
|
|
shortcutFile = string.Empty;
|
|
|
|
|
else
|
|
|
|
|
shortcutFile = Path.Combine(personDirectory, $"{deterministicHashCodeKey}{item.ImageFileHolder.ExtensionLowered}.lnk");
|
|
|
|
|
results.Add(new(item.ResizedFileHolder, directory, faceFileInfo, checkFile, shortcutFile));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return results;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
internal static void SaveClosest(string argZero, List<Container> containers, Dictionary<string, List<Person>> peopleCollection, string dFacesContentDirectory, string d2ResultsFullGroupDirectory, string eDistanceContentDirectory)
|
|
|
|
|
{
|
|
|
|
|
WindowsShortcut windowsShortcut;
|
|
|
|
|
List<(IFileHolder? resizedFileHolder, string directory, FileInfo? faceFileInfo, string checkFile, string shortcutFile)> collection = GetClosest(argZero, containers, peopleCollection, dFacesContentDirectory, d2ResultsFullGroupDirectory, eDistanceContentDirectory);
|
|
|
|
|
string[] directories = (from l in collection select l.directory).Distinct().ToArray();
|
|
|
|
|
foreach (string directory in directories)
|
|
|
|
|
{
|
|
|
|
|
if (string.IsNullOrEmpty(directory))
|
|
|
|
|
continue;
|
|
|
|
|
if (!Directory.Exists(directory))
|
|
|
|
|
_ = Directory.CreateDirectory(directory);
|
|
|
|
|
}
|
|
|
|
|
foreach ((IFileHolder? resizedFileHolder, string directory, FileInfo? faceFileInfo, string checkFile, string shortcutFile) in collection)
|
|
|
|
|
{
|
|
|
|
|
if (string.IsNullOrEmpty(directory) || string.IsNullOrEmpty(checkFile) || resizedFileHolder is null || faceFileInfo is null)
|
|
|
|
|
continue;
|
|
|
|
|
if (File.Exists(checkFile))
|
|
|
|
|
continue;
|
|
|
|
|
if (faceFileInfo.Directory is not null && faceFileInfo.Directory.Exists && faceFileInfo.Exists)
|
|
|
|
|
File.Copy(faceFileInfo.FullName, checkFile);
|
|
|
|
|
else
|
|
|
|
|
File.Copy(resizedFileHolder.FullName, checkFile);
|
|
|
|
|
}
|
|
|
|
|
foreach ((IFileHolder? resizedFileHolder, string directory, FileInfo? _, string checkFile, string shortcutFile) in collection)
|
|
|
|
|
{
|
|
|
|
|
if (string.IsNullOrEmpty(directory) || string.IsNullOrEmpty(checkFile) || resizedFileHolder is null)
|
|
|
|
|
continue;
|
|
|
|
|
if (string.IsNullOrEmpty(shortcutFile) || !resizedFileHolder.Exists)
|
|
|
|
|
continue;
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
windowsShortcut = new() { Path = resizedFileHolder.FullName };
|
|
|
|
|
windowsShortcut.Save(shortcutFile);
|
|
|
|
|
windowsShortcut.Dispose();
|
|
|
|
|
}
|
|
|
|
|
catch (Exception)
|
|
|
|
|
{ }
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|