Re-Write without checking mapped
This commit is contained in:
@ -1,8 +1,6 @@
|
||||
using System.Text.Json;
|
||||
using ShellProgressBar;
|
||||
using View_by_Distance.FaceRecognitionDotNet;
|
||||
using View_by_Distance.Metadata.Models;
|
||||
using View_by_Distance.Property.Models;
|
||||
using View_by_Distance.Resize.Models;
|
||||
using View_by_Distance.Map.Models;
|
||||
using View_by_Distance.Shared.Models;
|
||||
using View_by_Distance.Shared.Models.Stateless;
|
||||
|
||||
@ -11,764 +9,135 @@ namespace View_by_Distance.Instance.Models;
|
||||
internal class E_Distance
|
||||
{
|
||||
|
||||
internal List<string> AngleBracketCollection { get; }
|
||||
|
||||
private readonly Serilog.ILogger? _Log;
|
||||
private readonly Configuration _Configuration;
|
||||
private readonly JsonSerializerOptions _WriteIndentedJsonSerializerOptions;
|
||||
|
||||
internal E_Distance(Configuration configuration)
|
||||
private static void SaveFaceDistances(string eDistanceContentFileName, SortingContainer[] sortingContainers)
|
||||
{
|
||||
_Configuration = configuration;
|
||||
AngleBracketCollection = new List<string>();
|
||||
_Log = Serilog.Log.ForContext<E_Distance>();
|
||||
_WriteIndentedJsonSerializerOptions = new JsonSerializerOptions { WriteIndented = true };
|
||||
#pragma warning disable
|
||||
string[] results = (from l in sortingContainers select string.Concat(l.Sorting.WithinRange, '\t', l.Sorting.DistancePermyriad, '\t', l.Sorting.DaysDelta, '\t', l.Sorting.Id, '\t', l.Sorting.NormalizedPixelPercentage, '\t', l.Sorting.Older, '\t', l.Face.Mapping.MappingFromItem.Id, '\t', l.Face.Mapping.MappingFromLocation.NormalizedPixelPercentage)).ToArray();
|
||||
#pragma warning restore
|
||||
File.WriteAllLines(eDistanceContentFileName, results);
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
private static List<Sorting> GetSortingCollection(MapLogic mapLogic, List<FaceDistance> faceDistanceEncodings, int faceDistanceContainersLength, int i, FaceDistance faceDistanceEncoding)
|
||||
{
|
||||
string result = JsonSerializer.Serialize(this, new JsonSerializerOptions() { WriteIndented = true });
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<(DistanceHolder, FaceRecognitionDotNet.FaceEncoding?)> GetDistanceHolder(Item[] items, List<(string JSONDirectory, string TSVDirectory)> directories)
|
||||
{
|
||||
List<(DistanceHolder, FaceRecognitionDotNet.FaceEncoding?)> results = new();
|
||||
Item item;
|
||||
const int zero = 0;
|
||||
string tsvDirectory;
|
||||
string jsonDirectory;
|
||||
FaceRecognitionDotNet.FaceEncoding? faceEncoding;
|
||||
if (items.Length != directories.Count)
|
||||
throw new Exception();
|
||||
for (int i = 0; i < items.Length; i++)
|
||||
{
|
||||
faceEncoding = null;
|
||||
item = items[i];
|
||||
if (item.ImageFileHolder is null || item.Property?.Id is null || !item.Faces.Any())
|
||||
continue;
|
||||
tsvDirectory = directories[i].TSVDirectory;
|
||||
jsonDirectory = directories[i].JSONDirectory;
|
||||
foreach (Face face in item.Faces)
|
||||
{
|
||||
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
|
||||
continue;
|
||||
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
|
||||
results.Add(new(new(face, item.ImageFileHolder, item.Property.Id.Value, jsonDirectory, item.Faces[zero].Location, tsvDirectory), faceEncoding));
|
||||
}
|
||||
if (faceEncoding is null)
|
||||
results.Add(new(new(item.Faces[zero], item.ImageFileHolder, item.Property.Id.Value, jsonDirectory, item.Faces[zero].Location, tsvDirectory), null));
|
||||
}
|
||||
List<Sorting> results;
|
||||
List<FaceDistance> faceDistanceLengths = FaceRecognition.FaceDistances(faceDistanceEncodings, faceDistanceEncoding);
|
||||
if (faceDistanceLengths.Count != faceDistanceContainersLength)
|
||||
throw new NotSupportedException();
|
||||
bool anyLowerThanTolerance = (from l in faceDistanceLengths where l.Length is not null && l.Length.Value != 0 && l.Length.Value < IFaceDistance.Tolerance select true).Any();
|
||||
results = mapLogic.GetSortingCollection(i, faceDistanceEncoding, faceDistanceLengths, anyLowerThanTolerance);
|
||||
return results;
|
||||
}
|
||||
|
||||
private void WriteNoFaceCollection(bool updateDateWhenMatches, DateTime? updateToWhenMatches, List<Tuple<string, DateTime>> subFileTuples, List<DistanceHolder> distanceHolders)
|
||||
private static List<SortingContainer> GetSortingContainers(Face face, FaceDistance faceDistanceEncoding, List<Sorting> sortingCollection)
|
||||
{
|
||||
string json;
|
||||
string check;
|
||||
string jsonFile;
|
||||
const int zero = 0;
|
||||
DistanceHolder distanceHolder;
|
||||
List<Tuple<Face, string>> tupleCollection;
|
||||
for (int i = 0; i < distanceHolders.Count; i++)
|
||||
List<SortingContainer> results = new();
|
||||
SortingContainer sortingContainer;
|
||||
Sorting[] collection = Shared.Models.Stateless.Methods.ISorting.Sort(sortingCollection);
|
||||
foreach (Sorting sorting in collection)
|
||||
{
|
||||
distanceHolder = distanceHolders[i];
|
||||
if (distanceHolder.Face.Location?.NormalizedPixelPercentage is null)
|
||||
if (faceDistanceEncoding.NormalizedPixelPercentage is null)
|
||||
throw new NotSupportedException();
|
||||
if (sorting.DistancePermyriad > IFaceDistance.Permyriad || sorting.DaysDelta > ISorting.DaysDeltaTolerance)
|
||||
continue;
|
||||
check = Path.Combine(distanceHolder.JSONDirectory, $"{zero} - {distanceHolder.FileHolder.NameWithoutExtension}.json");
|
||||
jsonFile = Path.Combine(distanceHolder.JSONDirectory, $"{distanceHolder.Id}.{zero}{distanceHolder.FileHolder.ExtensionLowered}.json");
|
||||
if (File.Exists(check))
|
||||
File.Move(check, jsonFile);
|
||||
tupleCollection = new() { new(distanceHolders[i].Face, string.Empty) };
|
||||
for (int j = 0; j < distanceHolders.Count; j++)
|
||||
{
|
||||
if (j == i)
|
||||
continue;
|
||||
distanceHolder = distanceHolders[j];
|
||||
tupleCollection.Add(new(distanceHolder.Face, string.Empty));
|
||||
if (tupleCollection.Count > _Configuration.MaxItemsInDistanceCollection)
|
||||
break;
|
||||
}
|
||||
json = JsonSerializer.Serialize(tupleCollection, _WriteIndentedJsonSerializerOptions);
|
||||
if (Shared.Models.Stateless.Methods.IPath.WriteAllText(jsonFile, json, updateDateWhenMatches, compareBeforeWrite: true, updateToWhenMatches: updateToWhenMatches))
|
||||
subFileTuples.Add(new Tuple<string, DateTime>(nameof(E_Distance), DateTime.Now));
|
||||
}
|
||||
}
|
||||
|
||||
private static List<FaceRecognitionDotNet.FaceEncoding> GetFaceEncodings((DistanceHolder, FaceRecognitionDotNet.FaceEncoding?)[] distanceHoldersAfterSort)
|
||||
{
|
||||
List<FaceRecognitionDotNet.FaceEncoding> results = new();
|
||||
foreach ((DistanceHolder distanceHolder, FaceRecognitionDotNet.FaceEncoding? faceEncoding) in distanceHoldersAfterSort)
|
||||
{
|
||||
if (distanceHolder.Face.FaceEncoding is null || distanceHolder.Face.Location?.NormalizedPixelPercentage is null || faceEncoding is null)
|
||||
continue;
|
||||
results.Add(faceEncoding);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private void SaveDistanceResults(bool updateDateWhenMatches, DateTime? updateToWhenMatches, List<Tuple<string, DateTime>> subFileTuples, List<(DistanceHolder DistanceHolder, FaceRecognitionDotNet.FaceEncoding? _)> distanceHolders)
|
||||
{
|
||||
string json;
|
||||
string check;
|
||||
string jsonFile;
|
||||
int locationIndex;
|
||||
List<double> faceDistances;
|
||||
DistanceHolder distanceHolder;
|
||||
int normalizedPixelPercentage;
|
||||
DistanceHolder[] sortedDistanceHolders;
|
||||
List<Tuple<Face, string>> tupleCollection;
|
||||
List<(int Index, double Distance)> collection;
|
||||
FaceRecognitionDotNet.FaceEncoding? faceEncoding;
|
||||
(DistanceHolder DistanceHolder, FaceRecognitionDotNet.FaceEncoding? FaceEncoding)[] distanceHoldersAfterSort =
|
||||
distanceHolders.OrderByDescending(l => l.DistanceHolder.Face.FaceEncoding is not null && l.DistanceHolder.Face.Location?.NormalizedPixelPercentage is not null).ToArray();
|
||||
List<FaceRecognitionDotNet.FaceEncoding> faceEncodings = GetFaceEncodings(distanceHoldersAfterSort);
|
||||
for (int i = 0; i < distanceHoldersAfterSort.Length; i++)
|
||||
{
|
||||
faceEncoding = distanceHoldersAfterSort[i].FaceEncoding;
|
||||
distanceHolder = distanceHoldersAfterSort[i].DistanceHolder;
|
||||
collection = new();
|
||||
tupleCollection = new();
|
||||
distanceHolder.Sort = 0d;
|
||||
if (distanceHolder.Face.LocationIndex is null)
|
||||
locationIndex = 0;
|
||||
else
|
||||
locationIndex = distanceHolder.Face.LocationIndex.Value;
|
||||
if (distanceHolder.Face.FaceEncoding is null || distanceHolder.Face.Location?.NormalizedPixelPercentage is null)
|
||||
normalizedPixelPercentage = 0;
|
||||
else
|
||||
normalizedPixelPercentage = distanceHolder.Face.Location.NormalizedPixelPercentage.Value;
|
||||
check = Path.Combine(distanceHolder.JSONDirectory, $"{locationIndex} - {distanceHolder.FileHolder.NameWithoutExtension}.json");
|
||||
jsonFile = Path.Combine(distanceHolder.JSONDirectory, $"{distanceHolder.Id}.{normalizedPixelPercentage}{distanceHolder.FileHolder.ExtensionLowered}.json");
|
||||
if (!Directory.Exists(distanceHolder.JSONDirectory))
|
||||
_ = Directory.CreateDirectory(distanceHolder.JSONDirectory);
|
||||
if (File.Exists(check))
|
||||
File.Move(check, jsonFile);
|
||||
if (faceEncodings.Count == 1)
|
||||
faceDistances = new() { 0d };
|
||||
else if (faceEncoding is null)
|
||||
faceDistances = Enumerable.Repeat(9d, faceEncodings.Count).ToList();
|
||||
else
|
||||
faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncoding);
|
||||
if (distanceHolder.Face.FaceEncoding is not null && faceEncoding is not null && faceDistances[i] != 0d)
|
||||
faceDistances[i] = 0d;
|
||||
for (int d = 0; d < faceDistances.Count; d++)
|
||||
collection.Add(new(d, faceDistances[d]));
|
||||
collection = collection.OrderBy(l => l.Distance).ToList();
|
||||
foreach ((int index, double distance) in collection)
|
||||
{
|
||||
distanceHolder = distanceHoldersAfterSort[index].DistanceHolder;
|
||||
if (distanceHolder.Location is null)
|
||||
continue;
|
||||
distanceHolder.Sort = ((distance * _Configuration.DistanceFactor) + (distanceHolder.Location.Confidence * _Configuration.LocationConfidenceFactor)) / 10;
|
||||
}
|
||||
sortedDistanceHolders = (from l in distanceHoldersAfterSort orderby l.DistanceHolder.Sort select l.DistanceHolder).ToArray();
|
||||
for (int j = 0; j < sortedDistanceHolders.Length; j++)
|
||||
{
|
||||
distanceHolder = sortedDistanceHolders[j];
|
||||
tupleCollection.Add(new(distanceHoldersAfterSort[j].DistanceHolder.Face, string.Empty));
|
||||
if (tupleCollection.Count > _Configuration.MaxItemsInDistanceCollection)
|
||||
break;
|
||||
}
|
||||
json = JsonSerializer.Serialize(tupleCollection, _WriteIndentedJsonSerializerOptions);
|
||||
if (Shared.Models.Stateless.Methods.IPath.WriteAllText(jsonFile, json, updateDateWhenMatches, compareBeforeWrite: true, updateToWhenMatches: updateToWhenMatches))
|
||||
subFileTuples.Add(new Tuple<string, DateTime>(nameof(E_Distance), DateTime.Now));
|
||||
}
|
||||
}
|
||||
|
||||
internal void LoadOrCreateThenSaveDistanceResults(Property.Models.Configuration configuration, string eResultsFullGroupDirectory, string outputResolution, Container container, List<Tuple<string, DateTime>> sourceDirectoryChanges, Item[] filteredItems)
|
||||
{
|
||||
Item item;
|
||||
string json;
|
||||
bool check = false;
|
||||
string parentCheck;
|
||||
bool hasPopulatedFace;
|
||||
string usingRelativePath;
|
||||
DateTime? dateTime = null;
|
||||
string dCollectionDirectory;
|
||||
FileInfo[] fileInfoCollection;
|
||||
bool updateDateWhenMatches = false;
|
||||
System.IO.DirectoryInfo directoryInfo;
|
||||
System.IO.DirectoryInfo tvsDirectoryInfo;
|
||||
int?[] normalizedPixelPercentageCollection;
|
||||
int normalizedPixelPercentageDistinctCount;
|
||||
List<(string, string)> directories = new();
|
||||
string[] changesFrom = new string[] { nameof(A_Property), nameof(B_Metadata), nameof(C_Resize), nameof(D_Face) };
|
||||
List<DateTime> dateTimes = (from l in sourceDirectoryChanges where changesFrom.Contains(l.Item1) select l.Item2).ToList();
|
||||
List<string> directoryInfoCollection = Property.Models.Stateless.IResult.GetDirectoryInfoCollection(
|
||||
configuration,
|
||||
container.SourceDirectory,
|
||||
eResultsFullGroupDirectory,
|
||||
contentDescription: ".tvs File",
|
||||
singletonDescription: string.Empty,
|
||||
collectionDescription: "n json file(s) for each face found (one to many)",
|
||||
converted: true);
|
||||
for (int i = 0; i < filteredItems.Length; i++)
|
||||
{
|
||||
item = filteredItems[i];
|
||||
if (item.ImageFileHolder is null || item.Property?.Id is null)
|
||||
continue;
|
||||
hasPopulatedFace = (from l in item.Faces where l.FaceEncoding is not null && l.Location?.NormalizedPixelPercentage is not null select true).Any();
|
||||
usingRelativePath = Path.Combine(directoryInfoCollection[0].Replace("<>", "[]"), item.ImageFileHolder.NameWithoutExtension);
|
||||
dCollectionDirectory = Path.Combine(eResultsFullGroupDirectory, "[]", Property.Models.Stateless.IResult.AllInOne, $"{item.Property.Id.Value}{item.ImageFileHolder.ExtensionLowered}");
|
||||
directoryInfo = new System.IO.DirectoryInfo(dCollectionDirectory);
|
||||
if (!directoryInfo.Exists)
|
||||
{
|
||||
if (Directory.Exists(usingRelativePath))
|
||||
{
|
||||
Directory.Move(usingRelativePath, directoryInfo.FullName);
|
||||
directoryInfo.Refresh();
|
||||
}
|
||||
if (!Directory.Exists(dCollectionDirectory))
|
||||
{
|
||||
if (directoryInfo.Parent?.Parent is null)
|
||||
throw new Exception();
|
||||
parentCheck = Path.Combine(directoryInfo.Parent.Parent.FullName, directoryInfo.Name);
|
||||
if (Directory.Exists(parentCheck))
|
||||
{
|
||||
foreach (string file in Directory.GetFiles(parentCheck))
|
||||
File.Delete(file);
|
||||
Directory.Delete(parentCheck);
|
||||
}
|
||||
}
|
||||
}
|
||||
tvsDirectoryInfo = new System.IO.DirectoryInfo(Path.Combine(directoryInfoCollection[0].Replace("<>", "()"), item.ImageFileHolder.NameWithoutExtension));
|
||||
directories.Add(new(directoryInfo.FullName, tvsDirectoryInfo.FullName));
|
||||
if (directoryInfo.Exists && (!check || _Configuration.CheckJsonForDistanceResults))
|
||||
{
|
||||
json = string.Empty;
|
||||
normalizedPixelPercentageCollection = Shared.Models.Stateless.Methods.IFace.GetInts(item.Faces);
|
||||
normalizedPixelPercentageDistinctCount = normalizedPixelPercentageCollection.Distinct().Count();
|
||||
if (normalizedPixelPercentageDistinctCount != normalizedPixelPercentageCollection.Length)
|
||||
check = true;
|
||||
fileInfoCollection = directoryInfo.GetFiles($"{item.Property.Id.Value}*.json", SearchOption.TopDirectoryOnly);
|
||||
if (fileInfoCollection.Length < normalizedPixelPercentageDistinctCount)
|
||||
check = true;
|
||||
if (!check && _Configuration.CheckJsonForDistanceResults)
|
||||
{
|
||||
for (int j = 0; j < fileInfoCollection.Length; j++)
|
||||
{
|
||||
json = Shared.Models.Stateless.Methods.IIndex.GetJson(fileInfoCollection[j].FullName, fileInfoCollection[j]);
|
||||
if (!_Configuration.PropertiesChangedForDistance && Shared.Models.Stateless.Methods.IFace.GetFace(fileInfoCollection[j].FullName) is null)
|
||||
{
|
||||
check = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!check && string.IsNullOrEmpty(json))
|
||||
check = true;
|
||||
}
|
||||
}
|
||||
if (check)
|
||||
continue;
|
||||
if (_Configuration.PropertiesChangedForDistance)
|
||||
check = true;
|
||||
else if (hasPopulatedFace && !directoryInfo.Exists)
|
||||
check = true;
|
||||
else if (dateTimes.Any() && dateTimes.Max() > directoryInfo.LastWriteTime)
|
||||
check = true;
|
||||
if (check && !updateDateWhenMatches)
|
||||
{
|
||||
updateDateWhenMatches = dateTimes.Any() && directoryInfo.Exists && dateTimes.Max() > directoryInfo.LastWriteTime;
|
||||
dateTime = !updateDateWhenMatches ? null : dateTimes.Max();
|
||||
}
|
||||
}
|
||||
if (check)
|
||||
{
|
||||
DateTime? updateToWhenMatches = dateTime;
|
||||
List<Tuple<string, DateTime>> subFileTuples = new();
|
||||
List<(DistanceHolder, FaceRecognitionDotNet.FaceEncoding?)> distanceHolders = GetDistanceHolder(filteredItems, directories);
|
||||
SaveDistanceResults(updateDateWhenMatches, updateToWhenMatches, subFileTuples, distanceHolders);
|
||||
}
|
||||
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(directoryInfoCollection[0].Replace("<>", "()"));
|
||||
}
|
||||
|
||||
private List<(string, List<KeyValuePair<string, Face[]>>)> GetFiles(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string outputResolution)
|
||||
{
|
||||
string json;
|
||||
List<KeyValuePair<string, Face[]>>? facesKeyValuePairCollection;
|
||||
List<(string, List<KeyValuePair<string, Face[]>>)> results = new();
|
||||
string dFacesCollectionDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(D_Face), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), "[[]]");
|
||||
string[] dFacesCollectionFiles = Directory.GetFiles(dFacesCollectionDirectory, "*.json", SearchOption.TopDirectoryOnly);
|
||||
foreach (string dFacesCollectionFile in dFacesCollectionFiles)
|
||||
{
|
||||
json = File.ReadAllText(dFacesCollectionFile);
|
||||
facesKeyValuePairCollection = JsonSerializer.Deserialize<List<KeyValuePair<string, Face[]>>>(json);
|
||||
if (facesKeyValuePairCollection is null)
|
||||
continue;
|
||||
results.Add(new(dFacesCollectionFile, facesKeyValuePairCollection));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private static List<(string, List<Face>, List<FaceRecognitionDotNet.FaceEncoding>)> GetMatches(List<(string, List<KeyValuePair<string, Face[]>>)> files)
|
||||
{
|
||||
List<(string, List<Face>, List<FaceRecognitionDotNet.FaceEncoding>)> results = new();
|
||||
FaceRecognitionDotNet.FaceEncoding faceEncoding;
|
||||
List<Face> faces;
|
||||
List<FaceRecognitionDotNet.FaceEncoding> faceEncodings;
|
||||
foreach ((string, List<KeyValuePair<string, Face[]>>) file in files)
|
||||
{
|
||||
faces = new();
|
||||
faceEncodings = new();
|
||||
foreach (KeyValuePair<string, Face[]> keyValuePair in file.Item2)
|
||||
{
|
||||
foreach (Face face in keyValuePair.Value)
|
||||
{
|
||||
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
|
||||
continue;
|
||||
faces.Add(face);
|
||||
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
|
||||
faceEncodings.Add(faceEncoding);
|
||||
}
|
||||
}
|
||||
results.Add(new(file.Item1, faces, faceEncodings));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private static int GetIndex(double[] faceDistances)
|
||||
{
|
||||
int result;
|
||||
List<double[]> faceDistancesWithIndex = new();
|
||||
for (int y = 0; y < faceDistances.Length; y++)
|
||||
faceDistancesWithIndex.Add(new double[] { faceDistances[y], y });
|
||||
faceDistancesWithIndex = (from l in faceDistancesWithIndex orderby l[0] select l).ToList();
|
||||
result = (int)faceDistancesWithIndex[0][1];
|
||||
return result;
|
||||
}
|
||||
|
||||
private void Save(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string outputResolution, string eDistanceCollectionDirectory, int k, string relativePath, Face face, List<Tuple<Face, string>> faceAndFaceDistanceCollection)
|
||||
{
|
||||
if (string.IsNullOrEmpty(eDistanceCollectionDirectory))
|
||||
eDistanceCollectionDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), "[]");
|
||||
string fileNameWithoutExtension = Path.GetFileNameWithoutExtension(face.RelativePath);
|
||||
string jsonDirectory = string.Concat(eDistanceCollectionDirectory, Path.Combine(relativePath, fileNameWithoutExtension));
|
||||
if (!Directory.Exists(jsonDirectory))
|
||||
_ = Directory.CreateDirectory(jsonDirectory);
|
||||
string json = JsonSerializer.Serialize(faceAndFaceDistanceCollection, _WriteIndentedJsonSerializerOptions);
|
||||
string jsonFile = Path.Combine(jsonDirectory, $"{k} - {fileNameWithoutExtension}.nosj");
|
||||
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(jsonFile, json, updateDateWhenMatches: true, compareBeforeWrite: true);
|
||||
}
|
||||
|
||||
private static Tuple<Face, double> Get(FaceRecognitionDotNet.FaceEncoding faceEncoding, (string, List<Face>, List<FaceRecognitionDotNet.FaceEncoding>) match)
|
||||
{
|
||||
Tuple<Face, double> result;
|
||||
double[] faceDistances = FaceRecognition.FaceDistances(match.Item3, faceEncoding).ToArray();
|
||||
int index = GetIndex(faceDistances);
|
||||
result = new(match.Item2[index], faceDistances[index]);
|
||||
return result;
|
||||
}
|
||||
|
||||
internal void LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string outputResolution)
|
||||
{
|
||||
if (_Log is null)
|
||||
throw new NullReferenceException(nameof(_Log));
|
||||
string? relativePath;
|
||||
Face face;
|
||||
ParallelOptions parallelOptions = new();
|
||||
FaceRecognitionDotNet.FaceEncoding faceEncoding;
|
||||
string eDistanceCollectionDirectory = string.Empty;
|
||||
Tuple<Face, double> faceAndFaceDistance;
|
||||
List<Tuple<Face, string>> faceAndFaceDistanceCollection;
|
||||
List<(string, List<KeyValuePair<string, Face[]>>)> files = GetFiles(configuration, model, predictorModel, outputResolution);
|
||||
List<(string, List<Face>, List<FaceRecognitionDotNet.FaceEncoding>)> matches = GetMatches(files);
|
||||
if (files.Count != matches.Count)
|
||||
throw new Exception();
|
||||
int filesCount = files.Count;
|
||||
for (int i = 0; i < filesCount; i++)
|
||||
{
|
||||
_Log.Debug(string.Concat("LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions - ", nameof(outputResolution), ' ', outputResolution, " - ", i, " of ", filesCount));
|
||||
for (int j = 0; j < files[i].Item2.Count; j++)
|
||||
{
|
||||
if (!matches[i].Item2.Any())
|
||||
continue;
|
||||
for (int k = 0; k < files[i].Item2[j].Value.Length; k++)
|
||||
{
|
||||
face = files[i].Item2[j].Value[k];
|
||||
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
|
||||
continue;
|
||||
faceAndFaceDistanceCollection = new(matches.Count);
|
||||
relativePath = Path.GetDirectoryName(face.RelativePath);
|
||||
if (string.IsNullOrEmpty(relativePath))
|
||||
continue;
|
||||
if (face.FaceEncoding is null)
|
||||
continue;
|
||||
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
|
||||
_ = Parallel.For(0, matches.Count, parallelOptions, z =>
|
||||
{
|
||||
if (z != i && matches[z].Item2.Any())
|
||||
{
|
||||
faceAndFaceDistance = Get(faceEncoding, matches[z]);
|
||||
// if (faceAndFaceDistance.Item2 < _Configuration.)
|
||||
faceAndFaceDistanceCollection.Add(new(faceAndFaceDistance.Item1, faceAndFaceDistance.Item2.ToString("0.000")));
|
||||
}
|
||||
});
|
||||
if (faceAndFaceDistanceCollection.Any())
|
||||
{
|
||||
faceAndFaceDistanceCollection = (from l in faceAndFaceDistanceCollection orderby l.Item2 select l).Take(_Configuration.CrossDirectoryMaxItemsInDistanceCollection).ToList();
|
||||
Save(configuration, model, predictorModel, outputResolution, eDistanceCollectionDirectory, k, relativePath, face, faceAndFaceDistanceCollection);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal void SaveFaceNumbers(string eResultsFullGroupDirectory, Item item, Face face, List<int[]> faceDistances)
|
||||
{
|
||||
if (item.Property?.Id is null)
|
||||
throw new NullReferenceException(nameof(item.Property.Id));
|
||||
if (item.ImageFileHolder is null)
|
||||
throw new NullReferenceException(nameof(item.ImageFileHolder));
|
||||
if (face.Location?.NormalizedPixelPercentage is null)
|
||||
throw new NullReferenceException(nameof(face.Location.NormalizedPixelPercentage));
|
||||
if (string.IsNullOrEmpty(eResultsFullGroupDirectory))
|
||||
throw new NullReferenceException(nameof(eResultsFullGroupDirectory));
|
||||
string json;
|
||||
List<(Face, string, string)> collection = new();
|
||||
string[] changesFrom = new string[] { nameof(A_Property), nameof(B_Metadata), nameof(C_Resize), nameof(D_Face) };
|
||||
string eSingletonFile = Path.Combine(eResultsFullGroupDirectory, "{}", Property.Models.Stateless.IResult.AllInOne, $"{item.Property.Id.Value}.{face.Location.NormalizedPixelPercentage.Value}{item.ImageFileHolder.ExtensionLowered}.json");
|
||||
FileInfo fileInfo = new(eSingletonFile);
|
||||
json = JsonSerializer.Serialize(faceDistances, _WriteIndentedJsonSerializerOptions);
|
||||
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, json, updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null);
|
||||
}
|
||||
|
||||
internal void GetFaceDistances(string eResultsFullGroupDirectory, List<bool> faceEncodingRequired, List<Tuple<string, DateTime>> subFileTuples, List<string> parseExceptions, Item item, List<Face> faceCollection)
|
||||
{
|
||||
List<int[]>? results;
|
||||
if (item.Property?.Id is null)
|
||||
throw new NullReferenceException(nameof(item.Property.Id));
|
||||
if (item.ImageFileHolder is null)
|
||||
throw new NullReferenceException(nameof(item.ImageFileHolder));
|
||||
if (string.IsNullOrEmpty(eResultsFullGroupDirectory))
|
||||
throw new NullReferenceException(nameof(eResultsFullGroupDirectory));
|
||||
string json;
|
||||
FileInfo fileInfo;
|
||||
string eSingletonFile;
|
||||
List<(Face, string, string)> collection = new();
|
||||
string[] changesFrom = new string[] { nameof(A_Property), nameof(B_Metadata), nameof(C_Resize), nameof(D_Face) };
|
||||
List<DateTime> dateTimes = (from l in subFileTuples where changesFrom.Contains(l.Item1) select l.Item2).ToList();
|
||||
foreach (Face face in faceCollection)
|
||||
{
|
||||
results = null;
|
||||
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
|
||||
continue;
|
||||
eSingletonFile = Path.Combine(eResultsFullGroupDirectory, "{}", Property.Models.Stateless.IResult.AllInOne, $"{item.Property.Id.Value}.{face.Location.NormalizedPixelPercentage.Value}{item.ImageFileHolder.ExtensionLowered}.json");
|
||||
fileInfo = new(eSingletonFile);
|
||||
if (_Configuration.PropertiesChangedForDistance)
|
||||
results = null;
|
||||
else if (!fileInfo.Exists)
|
||||
results = null;
|
||||
else if (dateTimes.Any() && dateTimes.Max() > fileInfo.LastWriteTime)
|
||||
results = null;
|
||||
else
|
||||
{
|
||||
json = File.ReadAllText(fileInfo.FullName);
|
||||
try
|
||||
{
|
||||
results = JsonSerializer.Deserialize<List<int[]>>(json);
|
||||
if (results is null)
|
||||
throw new NullReferenceException(nameof(results));
|
||||
subFileTuples.Add(new Tuple<string, DateTime>(nameof(E_Distance), fileInfo.LastWriteTime));
|
||||
lock (face)
|
||||
face.SetFaceNumbers(results);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
results = null;
|
||||
parseExceptions.Add(nameof(E_Distance));
|
||||
}
|
||||
}
|
||||
if (results is null && !faceEncodingRequired.Any())
|
||||
faceEncodingRequired.Add(true);
|
||||
}
|
||||
}
|
||||
|
||||
public static double GetStandardDeviation(IEnumerable<double> values, double average)
|
||||
{
|
||||
double result = 0;
|
||||
if (!values.Any())
|
||||
throw new Exception("Collection must have at least one value!");
|
||||
double sum = values.Sum(l => (l - average) * (l - average));
|
||||
result = Math.Sqrt(sum / values.Count());
|
||||
return result;
|
||||
}
|
||||
|
||||
private static int GetSelectedIndex(int maxDegreeOfParallelism, Random random, List<FaceRecognitionDotNet.FaceEncoding> faceEncodings)
|
||||
{
|
||||
int? result;
|
||||
int selectedIndex;
|
||||
List<(int? Index, double? Sum)> faceDistanceCollections = new();
|
||||
if (maxDegreeOfParallelism == 1)
|
||||
{
|
||||
double sum;
|
||||
List<double> faceDistances;
|
||||
for (int i = 0; i < faceEncodings.Count; i++)
|
||||
{
|
||||
faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncodings[i]);
|
||||
sum = faceDistances.Sum();
|
||||
faceDistanceCollections.Add(new(i, sum));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (int i = 0; i < faceEncodings.Count; i++)
|
||||
faceDistanceCollections.Add(new(null, null));
|
||||
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
|
||||
_ = Parallel.For(0, faceEncodings.Count, parallelOptions, (i, state) =>
|
||||
{
|
||||
List<double> faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncodings[i]);
|
||||
double sum = faceDistances.Sum();
|
||||
lock (faceDistanceCollections)
|
||||
faceDistanceCollections[i] = new(i, sum);
|
||||
});
|
||||
}
|
||||
faceDistanceCollections = faceDistanceCollections.OrderBy(l => l.Sum).ToList();
|
||||
if (faceDistanceCollections.Count != faceEncodings.Count)
|
||||
throw new Exception();
|
||||
if (faceDistanceCollections.Count > 1000)
|
||||
selectedIndex = random.Next(0, 36);
|
||||
else if (faceDistanceCollections.Count > 500)
|
||||
selectedIndex = random.Next(0, 31);
|
||||
else if (faceDistanceCollections.Count > 200)
|
||||
selectedIndex = random.Next(0, 26);
|
||||
else if (faceDistanceCollections.Count > 100)
|
||||
selectedIndex = random.Next(0, 21);
|
||||
else if (faceDistanceCollections.Count > 50)
|
||||
selectedIndex = random.Next(0, 16);
|
||||
else if (faceDistanceCollections.Count > 25)
|
||||
selectedIndex = random.Next(0, 11);
|
||||
else if (faceDistanceCollections.Count > 10)
|
||||
selectedIndex = random.Next(0, 6);
|
||||
else if (faceDistanceCollections.Count > 5)
|
||||
selectedIndex = random.Next(0, 3);
|
||||
else
|
||||
selectedIndex = 0;
|
||||
result = faceDistanceCollections[selectedIndex].Index;
|
||||
if (result is null)
|
||||
throw new NullReferenceException(nameof(result));
|
||||
return result.Value;
|
||||
}
|
||||
|
||||
private static void SetFiltered(List<(FaceRecognitionDotNet.FaceEncoding FaceEncoding, MappingContainer MappingContainer)> collection)
|
||||
{
|
||||
double ucl;
|
||||
bool check;
|
||||
double average;
|
||||
double[] doubles;
|
||||
double standardDeviation;
|
||||
double?[] nullableDoubles;
|
||||
for (int i = 0; i < int.MaxValue; i++)
|
||||
{
|
||||
check = true;
|
||||
nullableDoubles = (from l in collection where l.MappingContainer.Mapping.Filtered is not null && !l.MappingContainer.Mapping.Filtered.Value select l.MappingContainer.Distance).ToArray();
|
||||
doubles = (from l in nullableDoubles where l.HasValue select l.Value).ToArray();
|
||||
if (doubles.Length < 4)
|
||||
break;
|
||||
average = doubles.Average();
|
||||
standardDeviation = GetStandardDeviation(doubles, average);
|
||||
ucl = average + (standardDeviation * 3);
|
||||
if (ucl > IClosest.Tolerance)
|
||||
ucl = IClosest.Tolerance;
|
||||
foreach ((FaceRecognitionDotNet.FaceEncoding _, MappingContainer mappingContainer) in collection)
|
||||
{
|
||||
if (mappingContainer.Mapping.Filtered is null || mappingContainer.Mapping.Filtered.Value || mappingContainer.Distance <= ucl)
|
||||
continue;
|
||||
if (check)
|
||||
check = false;
|
||||
mappingContainer.Mapping.SetFiltered();
|
||||
}
|
||||
if (check)
|
||||
sortingContainer = new(face, sorting);
|
||||
results.Add(sortingContainer);
|
||||
if (results.Count >= ISorting.MaximumPerFaceShouldBeHigh)
|
||||
break;
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private static FaceDistance GetFaceDistanceParallelFor(Face face, FaceRecognitionDotNet.FaceEncoding faceEncoding, Mapping mapping, DateTime minimumDateTime, bool? isWrongYear, string key, FaceRecognitionDotNet.FaceEncoding[] faceEncodings)
|
||||
private static SortingContainer[] GetSortingContainersThenSetFaceMappingSortingCollection(int maxDegreeOfParallelism, long ticks, MapLogic mapLogic, string outputResolution, FaceDistanceContainer[] faceDistanceContainers)
|
||||
{
|
||||
FaceDistance result;
|
||||
if (face.Location?.NormalizedPixelPercentage is null)
|
||||
throw new NullReferenceException(nameof(face.Location.NormalizedPixelPercentage));
|
||||
List<double> faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncoding);
|
||||
result = new(faceDistances, isWrongYear, key, mapping, minimumDateTime);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<FaceDistance> GetFaceDistanceCollection(int maxDegreeOfParallelism, List<(string Key, int Id, Mapping Mapping, DateTime MinimumDateTime, bool? IsWrongYear, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>)> collection, Face face)
|
||||
{
|
||||
List<FaceDistance> results;
|
||||
if (face.FaceEncoding is null)
|
||||
throw new NullReferenceException(nameof(face.FaceEncoding));
|
||||
FaceRecognitionDotNet.FaceEncoding faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
|
||||
if (maxDegreeOfParallelism == 1)
|
||||
SortingContainer[] results;
|
||||
List<SortingContainer> collection = new();
|
||||
List<FaceDistance> faceDistanceEncodings = new();
|
||||
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
|
||||
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
|
||||
string message = $") {faceDistanceContainers.Length:000} faceDistanceContainer(s) - {totalSeconds} total second(s) - {outputResolution}";
|
||||
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
|
||||
foreach (FaceDistanceContainer faceDistanceContainer in faceDistanceContainers)
|
||||
{
|
||||
if (faceDistanceContainer.FaceDistance.Encoding is null)
|
||||
continue;
|
||||
faceDistanceEncodings.Add(faceDistanceContainer.FaceDistance);
|
||||
}
|
||||
using ProgressBar progressBar = new(faceDistanceContainers.Length, message, options);
|
||||
_ = Parallel.For(0, faceDistanceContainers.Length, parallelOptions, (i, state) =>
|
||||
{
|
||||
progressBar.Tick();
|
||||
Face face = faceDistanceContainers[i].Face;
|
||||
if (face.Mapping is null)
|
||||
throw new NotSupportedException();
|
||||
FaceDistance faceDistanceEncoding = faceDistanceContainers[i].FaceDistance;
|
||||
List<Sorting> sortingCollection = GetSortingCollection(mapLogic, faceDistanceEncodings, faceDistanceContainers.Length, i, faceDistanceEncoding);
|
||||
List<SortingContainer> sortingContainers = GetSortingContainers(face, faceDistanceEncoding, sortingCollection);
|
||||
lock (collection)
|
||||
collection.AddRange(sortingContainers);
|
||||
lock (face)
|
||||
face.ReleaseFaceDistance();
|
||||
});
|
||||
results = Shared.Models.Stateless.Methods.ISortingContainer.Sort(collection);
|
||||
return results;
|
||||
}
|
||||
|
||||
private static FaceDistanceContainer[] GetFaceDistanceContainers(Face[] firstFilteredFaces)
|
||||
{
|
||||
FaceDistanceContainer[] results;
|
||||
FaceDistance faceDistance;
|
||||
FaceDistanceContainer faceDistanceContainer;
|
||||
List<FaceDistanceContainer> collection = new();
|
||||
foreach (Face face in firstFilteredFaces)
|
||||
{
|
||||
if (face.Mapping is null)
|
||||
throw new NotSupportedException();
|
||||
if (face.FaceDistance?.Encoding is not FaceRecognitionDotNet.FaceEncoding faceEncoding)
|
||||
throw new NotSupportedException();
|
||||
faceDistance = new(face.Mapping.MappingFromLocation.Confidence, faceEncoding, face.Mapping.MappingFromItem.Id, face.Mapping.MappingFromItem.IsWrongYear, face.Mapping.MappingFromItem.MinimumDateTime, face.Mapping.MappingFromLocation.NormalizedPixelPercentage);
|
||||
faceDistanceContainer = new(face, faceDistance);
|
||||
collection.Add(faceDistanceContainer);
|
||||
}
|
||||
results = (from l in collection orderby l.FaceDistance.Encoding is not null select l).ToArray();
|
||||
if (results.Any() && results[0].FaceDistance.Encoding is null)
|
||||
throw new Exception("Sorting failed!");
|
||||
return results;
|
||||
}
|
||||
|
||||
private static void SetFaceDistances(int maxDegreeOfParallelism, long ticks, string outputResolution, Face[] selectedFilteredFaces)
|
||||
{
|
||||
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
|
||||
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
|
||||
string message = $") {selectedFilteredFaces.Length:000} Load Face Encoding - {totalSeconds} total second(s) - {outputResolution}";
|
||||
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = false };
|
||||
using ProgressBar progressBar = new(selectedFilteredFaces.Length, message, options);
|
||||
_ = Parallel.For(0, selectedFilteredFaces.Length, parallelOptions, (i, state) =>
|
||||
{
|
||||
results = new();
|
||||
FaceDistance faceDistance;
|
||||
List<double> faceDistances;
|
||||
FaceRecognitionDotNet.FaceEncoding[] faceEncodings;
|
||||
if (face.Location?.NormalizedPixelPercentage is null)
|
||||
throw new NullReferenceException(nameof(face.Location.NormalizedPixelPercentage));
|
||||
foreach ((string key, int id, Mapping mapping, DateTime minimumDateTime, bool? isWrongYear, List<(FaceRecognitionDotNet.FaceEncoding FaceEncoding, MappingContainer _)> faceEncodingContainers) in collection)
|
||||
{
|
||||
faceEncodings = (from l in faceEncodingContainers select l.FaceEncoding).ToArray();
|
||||
faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncoding);
|
||||
faceDistance = new(faceDistances, isWrongYear, key, mapping, minimumDateTime);
|
||||
results.Add(faceDistance);
|
||||
if (results.Count > IFaceDistance.MaximumPer)
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
results = new();
|
||||
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
|
||||
_ = Parallel.For(0, collection.Count, parallelOptions, (i, state) =>
|
||||
{
|
||||
(string key, int id, Mapping mapping, DateTime minimumDateTime, bool? isWrongYear, List<(FaceRecognitionDotNet.FaceEncoding FaceEncoding, MappingContainer _)> faceEncodingContainers) = collection[i];
|
||||
FaceRecognitionDotNet.FaceEncoding[] faceEncodings = (from l in faceEncodingContainers select l.FaceEncoding).ToArray();
|
||||
FaceDistance? closest = GetFaceDistanceParallelFor(face, faceEncoding, mapping, minimumDateTime, isWrongYear, key, faceEncodings);
|
||||
if (closest is not null)
|
||||
{
|
||||
lock (results)
|
||||
{
|
||||
results.Add(closest);
|
||||
if (results.Count > IFaceDistance.MaximumPer)
|
||||
state.Break();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private static FaceRecognitionDotNet.FaceEncoding? GetFaceEncoding(Face face)
|
||||
{
|
||||
FaceRecognitionDotNet.FaceEncoding? result;
|
||||
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
|
||||
result = null;
|
||||
else
|
||||
result = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static List<FaceRecognitionDotNet.FaceEncoding> GetFaceEncodingsOnly(int maxDegreeOfParallelism, List<MappingContainer> collection)
|
||||
{
|
||||
List<FaceRecognitionDotNet.FaceEncoding> results;
|
||||
if (maxDegreeOfParallelism == 1)
|
||||
{
|
||||
results = new();
|
||||
FaceRecognitionDotNet.FaceEncoding faceEncoding;
|
||||
foreach (MappingContainer mappingContainer in collection)
|
||||
{
|
||||
if (mappingContainer.Face?.FaceEncoding is null || mappingContainer.Face.Location?.NormalizedPixelPercentage is null)
|
||||
continue;
|
||||
faceEncoding = FaceRecognition.LoadFaceEncoding(mappingContainer.Face.FaceEncoding.RawEncoding);
|
||||
results.Add(faceEncoding);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
results = new();
|
||||
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
|
||||
_ = Parallel.For(0, collection.Count, parallelOptions, (i, state) =>
|
||||
{
|
||||
Face? face = collection[i].Face;
|
||||
if (face is null)
|
||||
throw new Exception();
|
||||
FaceRecognitionDotNet.FaceEncoding? faceEncoding = GetFaceEncoding(face);
|
||||
if (faceEncoding is not null)
|
||||
{
|
||||
lock (results)
|
||||
results.Add(faceEncoding);
|
||||
}
|
||||
});
|
||||
}
|
||||
Face face = selectedFilteredFaces[i];
|
||||
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
|
||||
throw new NotSupportedException();
|
||||
if (face.Mapping is null)
|
||||
throw new NotSupportedException();
|
||||
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
|
||||
faceDistance = new(face.Location.Confidence, faceEncoding, face.Mapping.MappingFromItem.Id, face.Mapping.MappingFromItem.IsWrongYear, face.Mapping.MappingFromItem.MinimumDateTime, face.Location.NormalizedPixelPercentage.Value);
|
||||
lock (face)
|
||||
face.FaceDistanceAdd(faceDistance);
|
||||
});
|
||||
}
|
||||
|
||||
internal static SortingContainer[] SetPersonTicksAndSetFaceDistancesAndSetFaceMappingSortingCollectionThenGetSortingContainers(int maxDegreeOfParallelism, long ticks, MapLogic mapLogic, string outputResolution, string eDistanceContentFileName, List<Face> distinctFilteredFaces)
|
||||
{
|
||||
SortingContainer[] results;
|
||||
Face[] orderedFilteredFaces = (from l in distinctFilteredFaces orderby l.Mapping is not null, l.Mapping?.MappingFromItem.MinimumDateTime descending select l).ToArray();
|
||||
mapLogic.SetPersonTicks(outputResolution, orderedFilteredFaces);
|
||||
Face[] selectedFilteredFaces = orderedFilteredFaces.Skip(ISorting.FacesToSkipAfterSortBeforeLoad).Take(ISorting.FacesToTakeAfterSortBeforeLoad).ToArray();
|
||||
SetFaceDistances(maxDegreeOfParallelism, ticks, outputResolution, selectedFilteredFaces);
|
||||
FaceDistanceContainer[] faceDistanceContainers = GetFaceDistanceContainers(selectedFilteredFaces);
|
||||
results = GetSortingContainersThenSetFaceMappingSortingCollection(maxDegreeOfParallelism, ticks, mapLogic, outputResolution, faceDistanceContainers);
|
||||
if (results.Length < IFaceDistance.Permyriad)
|
||||
SaveFaceDistances(eDistanceContentFileName, results);
|
||||
return results;
|
||||
}
|
||||
|
||||
private static void SetNonFiltered(List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)> collection)
|
||||
{
|
||||
foreach ((FaceRecognitionDotNet.FaceEncoding _, MappingContainer mappingContainer) in collection)
|
||||
{
|
||||
if (mappingContainer.Mapping.Filtered is null)
|
||||
mappingContainer.Mapping.SetFiltered(value: false);
|
||||
}
|
||||
}
|
||||
|
||||
private Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> GetThreeSigmaFaceEncodings(int maxDegreeOfParallelism, long ticks, Random random, Dictionary<string, List<MappingContainer>> keyValuePairs)
|
||||
{
|
||||
if (_Log is null)
|
||||
throw new NullReferenceException(nameof(_Log));
|
||||
Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> results = new();
|
||||
int totalSeconds;
|
||||
int selectedIndex;
|
||||
List<double> faceDistances;
|
||||
MappingContainer mappingContainer;
|
||||
int keyValuePairsCount = keyValuePairs.Count;
|
||||
FaceRecognitionDotNet.FaceEncoding faceEncoding;
|
||||
List<FaceRecognitionDotNet.FaceEncoding> faceEncodings;
|
||||
List<(FaceRecognitionDotNet.FaceEncoding FaceEncoding, MappingContainer MappingContainer)> collection;
|
||||
foreach (KeyValuePair<string, List<MappingContainer>> keyValuePair in keyValuePairs)
|
||||
{
|
||||
collection = new();
|
||||
faceEncodings = GetFaceEncodingsOnly(maxDegreeOfParallelism, keyValuePair.Value);
|
||||
for (int i = 0; i < faceEncodings.Count; i++)
|
||||
{
|
||||
faceEncoding = faceEncodings[i];
|
||||
mappingContainer = keyValuePair.Value[i];
|
||||
collection.Add(new(faceEncoding, mappingContainer));
|
||||
}
|
||||
results.Add(keyValuePair.Key, collection);
|
||||
if (faceEncodings.Count == 1)
|
||||
selectedIndex = 0;
|
||||
else
|
||||
selectedIndex = GetSelectedIndex(maxDegreeOfParallelism, random, faceEncodings);
|
||||
faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncodings[selectedIndex]);
|
||||
for (int i = 0; i < faceEncodings.Count; i++)
|
||||
collection[i].MappingContainer.SetDistance(faceDistances[i]);
|
||||
if (collection.Count > 3)
|
||||
SetFiltered(collection);
|
||||
SetNonFiltered(collection);
|
||||
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
|
||||
_Log.Information($"{keyValuePairsCount:0000}) {totalSeconds} total second(s) - {keyValuePair.Key} - {collection[selectedIndex].MappingContainer.Mapping.DisplayDirectoryName}");
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
internal Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> ParallelWork(int maxDegreeOfParallelism, long ticks, Map.Models.MapLogic mapLogic, Item[] items, int totalNotMapped)
|
||||
{
|
||||
Dictionary<string, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>> results;
|
||||
Random random = new((int)ticks);
|
||||
mapLogic.ForceSingleImage(ticks, items, totalNotMapped, random);
|
||||
Dictionary<string, List<MappingContainer>> keyValuePairs = Map.Models.Stateless.IMapLogic.GetKeyValuePairs(items);
|
||||
results = GetThreeSigmaFaceEncodings(maxDegreeOfParallelism, ticks, random, keyValuePairs);
|
||||
return results;
|
||||
}
|
||||
|
||||
public static void AddToFaceDistance(int maxDegreeOfParallelism, Map.Models.MapLogic mapLogic, Item[] items, List<(string, int, Mapping, DateTime, bool?, List<(FaceRecognitionDotNet.FaceEncoding, MappingContainer)>)> collection)
|
||||
{
|
||||
Face face;
|
||||
double deterministicHashCodeKey;
|
||||
DateTime dateTime = DateTime.Now;
|
||||
List<FaceDistance> faceDistances;
|
||||
foreach (Item item in items)
|
||||
{
|
||||
if (item.Property?.Id is null || item.ImageFileHolder is null || item.ResizedFileHolder is null)
|
||||
continue;
|
||||
for (int i = 0; i < item.Faces.Count; i++)
|
||||
{
|
||||
face = item.Faces[i];
|
||||
face.FaceDistances.Clear();
|
||||
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
|
||||
continue;
|
||||
if ((from l in item.Mapping where l.NormalizedPixelPercentage.HasValue && l.NormalizedPixelPercentage.Value == face.Location.NormalizedPixelPercentage.Value select true).Any())
|
||||
continue;
|
||||
deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item, face);
|
||||
if (mapLogic.Skip(deterministicHashCodeKey))
|
||||
continue;
|
||||
faceDistances = GetFaceDistanceCollection(maxDegreeOfParallelism, collection, face);
|
||||
face.FaceDistances.AddRange(faceDistances);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user