After testing E_Distance.SaveGroupedFaceEncodings
This commit is contained in:
@ -128,7 +128,7 @@ internal class E_Distance
|
||||
return result;
|
||||
}
|
||||
|
||||
private void LoadOrCreateThenSaveDistanceResultsForOutputResolutionsLoop(Property.Models.Configuration configuration, List<List<D_Face>> faceCollections, int subFilesCount, int i, List<D_Face> faceCollection, List<int[]> locationIndicesCollection, List<Tuple<string, DateTime>> subFileTuples, List<FaceEncoding> faceEncodingCollection, List<FaceEncoding> faceEncodingCollections, string fileNameWithoutExtension, string jsonDirectory, string tvsDirectory)
|
||||
private void LoadOrCreateThenSaveDistanceResultsForOutputResolutionsLoop(Property.Models.Configuration configuration, List<List<D_Face>> faceCollections, int subFilesCount, int i, List<D_Face> faceCollection, List<int[]> locationIndicesCollection, List<Tuple<string, DateTime>> subFileTuples, List<FaceEncoding> faceEncodingCollection, List<FaceEncoding> faceEncodingCollections, string fileNameWithoutExtension, string jsonDirectory, string tvsDirectory, bool updateDateWhenMatches, DateTime? updateToWhenMatches)
|
||||
{
|
||||
string text;
|
||||
string json;
|
||||
@ -144,7 +144,7 @@ internal class E_Distance
|
||||
orderedFaceCollection = GetOrderedNoFaceCollection(faceCollections, i, faceCollection[j]);
|
||||
json = JsonSerializer.Serialize(orderedFaceCollection, _WriteIndentedJsonSerializerOptions);
|
||||
jsonFile = Path.Combine(jsonDirectory, $"{j} - {fileNameWithoutExtension}.json");
|
||||
if (Property.Models.Stateless.IPath.WriteAllText(jsonFile, json, compareBeforeWrite: true))
|
||||
if (Property.Models.Stateless.IPath.WriteAllText(jsonFile, json, updateDateWhenMatches, compareBeforeWrite: true, updateToWhenMatches: updateToWhenMatches))
|
||||
subFileTuples.Add(new Tuple<string, DateTime>(nameof(E_Distance), DateTime.Now));
|
||||
}
|
||||
else
|
||||
@ -162,16 +162,16 @@ internal class E_Distance
|
||||
indicesAndValues = GetValues(faceCollections, locationIndicesCollection, faceDistances);
|
||||
orderedFaceCollection = GetOrderedFaceCollection(faceCollections, locationIndicesCollection, indicesAndValues);
|
||||
text = GetText(fileNameWithoutExtension, faceCollections, locationIndicesCollection, indicesAndValues);
|
||||
if (Property.Models.Stateless.IPath.WriteAllText(tvsFile, text, compareBeforeWrite: true))
|
||||
if (Property.Models.Stateless.IPath.WriteAllText(tvsFile, text, updateDateWhenMatches, compareBeforeWrite: true))
|
||||
subFileTuples.Add(new Tuple<string, DateTime>(nameof(E_Distance), DateTime.Now));
|
||||
json = JsonSerializer.Serialize(orderedFaceCollection, _WriteIndentedJsonSerializerOptions);
|
||||
if (Property.Models.Stateless.IPath.WriteAllText(jsonFile, json, compareBeforeWrite: true))
|
||||
if (Property.Models.Stateless.IPath.WriteAllText(jsonFile, json, updateDateWhenMatches, compareBeforeWrite: true))
|
||||
subFileTuples.Add(new Tuple<string, DateTime>(nameof(E_Distance), DateTime.Now));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void LoadOrCreateThenSaveDistanceResultsForOutputResolutions(Property.Models.Configuration configuration, PropertyHolder[] filteredPropertyHolderCollection, List<List<D_Face>> faceCollections, List<string[]> directories)
|
||||
private void LoadOrCreateThenSaveDistanceResultsForOutputResolutions(Property.Models.Configuration configuration, PropertyHolder[] filteredPropertyHolderCollection, List<List<D_Face>> faceCollections, List<string[]> directories, bool updateDateWhenMatches, DateTime? updateToWhenMatches)
|
||||
{
|
||||
FileInfo? fileInfo;
|
||||
string fileNameWithoutExtension;
|
||||
@ -190,7 +190,7 @@ internal class E_Distance
|
||||
if (fileInfo is null)
|
||||
continue;
|
||||
fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileInfo.FullName);
|
||||
LoadOrCreateThenSaveDistanceResultsForOutputResolutionsLoop(configuration, faceCollections, filteredPropertyHolderCollection.Length, i, faceCollections[i], locationIndicesCollection, subFileTuples, faceEncodingCollection, faceEncodingCollections[i], fileNameWithoutExtension, directories[i][0], directories[i][1]);
|
||||
LoadOrCreateThenSaveDistanceResultsForOutputResolutionsLoop(configuration, faceCollections, filteredPropertyHolderCollection.Length, i, faceCollections[i], locationIndicesCollection, subFileTuples, faceEncodingCollection, faceEncodingCollections[i], fileNameWithoutExtension, directories[i][0], directories[i][1], updateDateWhenMatches, updateToWhenMatches);
|
||||
}
|
||||
}
|
||||
|
||||
@ -204,8 +204,10 @@ internal class E_Distance
|
||||
FileInfo? fileInfo;
|
||||
bool check = false;
|
||||
string parentCheck;
|
||||
DateTime? dateTime = null;
|
||||
FileInfo[] fileInfoCollection;
|
||||
string fileNameWithoutExtension;
|
||||
bool updateDateWhenMatches = false;
|
||||
List<string[]> directories = new();
|
||||
System.IO.DirectoryInfo directoryInfo;
|
||||
System.IO.DirectoryInfo tvsDirectoryInfo;
|
||||
@ -264,9 +266,14 @@ internal class E_Distance
|
||||
check = true;
|
||||
else if (dateTimes.Any() && dateTimes.Max() > directoryInfo.LastWriteTime)
|
||||
check = true;
|
||||
if (check && !updateDateWhenMatches)
|
||||
{
|
||||
updateDateWhenMatches = dateTimes.Any() && directoryInfo.Exists && dateTimes.Max() > directoryInfo.LastWriteTime;
|
||||
dateTime = !updateDateWhenMatches ? null : dateTimes.Max();
|
||||
}
|
||||
}
|
||||
if (check)
|
||||
LoadOrCreateThenSaveDistanceResultsForOutputResolutions(configuration, filteredPropertyHolderCollection, faceCollections, directories);
|
||||
LoadOrCreateThenSaveDistanceResultsForOutputResolutions(configuration, filteredPropertyHolderCollection, faceCollections, directories, updateDateWhenMatches, updateToWhenMatches: dateTime);
|
||||
_ = Property.Models.Stateless.IPath.DeleteEmptyDirectories(directoryInfoCollection[0].Replace("<>", "()"));
|
||||
}
|
||||
|
||||
@ -335,7 +342,7 @@ internal class E_Distance
|
||||
_ = Directory.CreateDirectory(jsonDirectory);
|
||||
string json = JsonSerializer.Serialize(faceAndFaceDistanceCollection, _WriteIndentedJsonSerializerOptions);
|
||||
string jsonFile = Path.Combine(jsonDirectory, $"{k} - {fileNameWithoutExtension}.nosj");
|
||||
_ = Property.Models.Stateless.IPath.WriteAllText(jsonFile, json, compareBeforeWrite: true);
|
||||
_ = Property.Models.Stateless.IPath.WriteAllText(jsonFile, json, updateDateWhenMatches: true, compareBeforeWrite: true);
|
||||
}
|
||||
|
||||
private static Tuple<Shared.Models.Face, double> Get(FaceEncoding faceEncoding, (string, List<Shared.Models.Face>, List<FaceEncoding>) match)
|
||||
@ -350,7 +357,7 @@ internal class E_Distance
|
||||
internal void LoadOrCreateThenSaveDirectoryDistanceResultsForOutputResolutions(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string outputResolution)
|
||||
{
|
||||
if (_Log is null)
|
||||
throw new Exception($"{nameof(_Log)} is null!");
|
||||
throw new ArgumentNullException(nameof(_Log));
|
||||
string? relativePath;
|
||||
Shared.Models.Face face;
|
||||
ParallelOptions parallelOptions = new();
|
||||
@ -401,87 +408,89 @@ internal class E_Distance
|
||||
}
|
||||
}
|
||||
|
||||
private static Dictionary<string, List<(string personKey, D_Face face)>> Convert(string argZero, List<PropertyHolder[]> propertyHolderCollections)
|
||||
public static double GetStandardDeviation(IEnumerable<double> values, double average)
|
||||
{
|
||||
Dictionary<string, List<(string personKey, D_Face face)>> results = new();
|
||||
string key;
|
||||
string personKey;
|
||||
bool? isWrongYear;
|
||||
TimeSpan? timeSpan;
|
||||
string isWrongYearFlag;
|
||||
foreach (PropertyHolder[] propertyHolderCollection in propertyHolderCollections)
|
||||
{
|
||||
if (!propertyHolderCollection.Any())
|
||||
continue;
|
||||
if (!propertyHolderCollection[0].SourceDirectory.StartsWith(argZero))
|
||||
continue;
|
||||
foreach (PropertyHolder propertyHolder in propertyHolderCollection)
|
||||
{
|
||||
if (propertyHolder.ImageFileInfo is null || propertyHolder.Property is null || !propertyHolder.Named.Any())
|
||||
continue;
|
||||
if (propertyHolder.Named.Count != 1 || propertyHolder.Faces.Count != 1)
|
||||
continue;
|
||||
for (int i = 0; i < propertyHolder.Named.Count; i++)
|
||||
{
|
||||
if (propertyHolder.MinimumDateTime is null)
|
||||
continue;
|
||||
if (propertyHolder.Faces[i] is not D_Face face)
|
||||
continue;
|
||||
timeSpan = propertyHolder.Named[i].TimeSpan;
|
||||
personKey = propertyHolder.Named[i].PersonKey;
|
||||
isWrongYear = propertyHolder.Named[i].IsWrongYear;
|
||||
isWrongYearFlag = PropertyHolder.GetWrongYearFlag(isWrongYear);
|
||||
if (timeSpan is null)
|
||||
key = $"{personKey}\t{isWrongYearFlag}{propertyHolder.MinimumDateTime.Value:yyyy}";
|
||||
else if (timeSpan.Value.Ticks < 0)
|
||||
key = $"{personKey}\t{isWrongYearFlag}!---";
|
||||
else
|
||||
key = $"{personKey}\t^{Math.Floor(timeSpan.Value.TotalDays / 365):000}";
|
||||
if (!results.ContainsKey(key))
|
||||
results.Add(key, new());
|
||||
results[key].Add(new(personKey, face));
|
||||
}
|
||||
}
|
||||
}
|
||||
return results;
|
||||
double result = 0;
|
||||
if (!values.Any())
|
||||
throw new Exception("Collection must have at least one value!");
|
||||
double sum = values.Sum(l => (l - average) * (l - average));
|
||||
result = Math.Sqrt(sum / values.Count());
|
||||
return result;
|
||||
}
|
||||
|
||||
internal static void SaveGroupedFaceEncodings(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string argZero, Dictionary<string, List<Shared.Models.Person>> peopleCollection, string outputResolution, List<PropertyHolder[]> propertyHolderCollections)
|
||||
internal static void SaveGroupedFaceEncodings(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string argZero, long ticks, Dictionary<string, List<Shared.Models.Person>> peopleCollection, string outputResolution, List<PropertyHolder[]> propertyHolderCollections)
|
||||
{
|
||||
double lcl;
|
||||
double ucl;
|
||||
string json;
|
||||
double average;
|
||||
int lowestIndex;
|
||||
string checkFile;
|
||||
string directory;
|
||||
string personKey;
|
||||
double lowestAverage;
|
||||
double standardDeviation;
|
||||
FaceEncoding faceEncoding;
|
||||
List<double> faceDistances;
|
||||
List<double[]> rawEncodings;
|
||||
Shared.Models.Person person;
|
||||
List<FaceEncoding> faceEncodings;
|
||||
List<string> checkDirectories = new();
|
||||
List<Shared.Models.Properties.IFace> collection;
|
||||
const string pattern = @"[\\,\/,\:,\*,\?,\"",\<,\>,\|]";
|
||||
Dictionary<string, List<(string personKey, D_Face face)>> keyValuePairs = Convert(argZero, propertyHolderCollections);
|
||||
string eDistanceCollectionDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), "[_]");
|
||||
foreach (KeyValuePair<string, List<(string personKey, D_Face face)>> keyValuePair in keyValuePairs)
|
||||
string eDistanceCollectionDirectory = Path.Combine(Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(configuration, model, predictorModel, nameof(E_Distance), outputResolution, includeResizeGroup: true, includeModel: true, includePredictorModel: true), $"[{ticks}]");
|
||||
List<(string, Shared.Models.PersonBirthday, Shared.Models.Properties.IFace)[]> collection = PropertyHolder.GetCollection(argZero, propertyHolderCollections, eDistanceCollectionDirectory);
|
||||
foreach ((string, Shared.Models.PersonBirthday, Shared.Models.Properties.IFace)[] group in collection)
|
||||
{
|
||||
collection = new();
|
||||
lowestIndex = 0;
|
||||
rawEncodings = new();
|
||||
faceEncodings = new();
|
||||
checkDirectories.Clear();
|
||||
checkFile = string.Empty;
|
||||
foreach ((string personKey, D_Face face) in keyValuePair.Value)
|
||||
lowestAverage = double.MaxValue;
|
||||
foreach ((string directory, Shared.Models.PersonBirthday personBirthday, Shared.Models.Properties.IFace @interface) in group)
|
||||
{
|
||||
personKey = Shared.Models.Stateless.Methods.IPersonBirthday.GetFormatted(personBirthday);
|
||||
if (string.IsNullOrEmpty(personKey) || !peopleCollection.ContainsKey(personKey))
|
||||
continue;
|
||||
if (@interface is not D_Face face || !face.Populated)
|
||||
continue;
|
||||
person = peopleCollection[personKey][0];
|
||||
directory = Path.Combine(eDistanceCollectionDirectory, $"{personKey}{keyValuePair.Key}");
|
||||
checkFile = Path.Combine(directory, Regex.Replace($"{Shared.Models.Stateless.Methods.IPersonName.GetFullName(person.Name)}.json", pattern, string.Empty));
|
||||
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
|
||||
checkFile = string.Concat(directory, " - ", Regex.Replace(Shared.Models.Stateless.Methods.IPersonName.GetFullName(person.Name), pattern, string.Empty), ".json");
|
||||
checkDirectories.Add(directory);
|
||||
collection.Add(face);
|
||||
faceEncodings.Add(faceEncoding);
|
||||
}
|
||||
if (!string.IsNullOrEmpty(checkFile) && checkDirectories.Any())
|
||||
if (string.IsNullOrEmpty(checkFile) || !checkDirectories.Any() || faceEncodings.Count < 2)
|
||||
continue;
|
||||
foreach (string checkDirectory in checkDirectories.Distinct())
|
||||
{
|
||||
foreach (string checkDirectory in checkDirectories)
|
||||
{
|
||||
if (!Directory.Exists(checkDirectory))
|
||||
_ = Directory.CreateDirectory(checkDirectory);
|
||||
}
|
||||
json = JsonSerializer.Serialize(collection, new JsonSerializerOptions { WriteIndented = true });
|
||||
_ = Property.Models.Stateless.IPath.WriteAllText(checkFile, json, compareBeforeWrite: true);
|
||||
if (!Directory.Exists(checkDirectory))
|
||||
_ = Directory.CreateDirectory(checkDirectory);
|
||||
}
|
||||
for (int i = 0; i < faceEncodings.Count; i++)
|
||||
{
|
||||
faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncodings[i]);
|
||||
average = faceDistances.Average();
|
||||
if (average > lowestAverage)
|
||||
continue;
|
||||
lowestIndex = i;
|
||||
lowestAverage = average;
|
||||
}
|
||||
faceDistances = FaceRecognition.FaceDistances(faceEncodings, faceEncodings[lowestIndex]);
|
||||
average = faceDistances.Average();
|
||||
if (average != lowestAverage)
|
||||
continue;
|
||||
standardDeviation = GetStandardDeviation(faceDistances, average);
|
||||
lcl = average - (standardDeviation * 3);
|
||||
ucl = average + (standardDeviation * 3);
|
||||
for (int i = 0; i < faceEncodings.Count; i++)
|
||||
{
|
||||
if (faceDistances[i] < lcl || faceDistances[i] > ucl)
|
||||
continue;
|
||||
rawEncodings.Add(faceEncodings[i].GetRawEncoding());
|
||||
}
|
||||
// outOfControl = faceEncodings.Count - rawEncodings.Count;
|
||||
json = JsonSerializer.Serialize(rawEncodings, new JsonSerializerOptions { WriteIndented = true });
|
||||
_ = Property.Models.Stateless.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: true, compareBeforeWrite: true);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user