Distance bug fix

This commit is contained in:
2022-09-29 13:13:41 -07:00
parent 152612bacb
commit 4568d3fbc6
21 changed files with 933 additions and 879 deletions

View File

@ -10,21 +10,35 @@ namespace View_by_Distance.Distance.Models;
public class E_Distance : Shared.Models.Methods.IFaceDistance
{
private readonly List<string> _Moved;
private readonly List<double?> _Debug;
private readonly List<string> _Renamed;
private readonly Serilog.ILogger? _Log;
private readonly string _ResultAllInOne;
private readonly int _FaceDistancePermyriad;
private readonly bool _DistanceRenameToMatch;
private readonly double _FaceDistanceTolerance;
private readonly int _SortingDaysDeltaTolerance;
private readonly bool _DistanceMoveUnableToMatch;
private readonly List<string> _AllMappedFaceFiles;
private readonly int _DistancePixelDistanceTolerance;
private readonly List<string> _AllMappedFaceFileNames;
private readonly double _FaceDistanceMinimumConfidence;
private readonly List<string> _DuplicateMappedFaceFiles;
private readonly int _FaceDistanceAreaPermilleTolerance;
private readonly int _SortingMaximumPerFaceShouldBeHigh;
public E_Distance(bool distanceMoveUnableToMatch, int distancePixelDistanceTolerance, int faceDistanceAreaPermilleTolerance, double faceDistanceMinimumConfidence, int faceDistancePermyriad, double faceDistanceTolerance, string resultAllInOne, int sortingDaysDeltaTolerance, int sortingMaximumPerFaceShouldBeHigh)
public E_Distance(bool distanceMoveUnableToMatch, int distancePixelDistanceTolerance, bool distanceRenameToMatch, int faceDistanceAreaPermilleTolerance, double faceDistanceMinimumConfidence, int faceDistancePermyriad, double faceDistanceTolerance, string resultAllInOne, int sortingDaysDeltaTolerance, int sortingMaximumPerFaceShouldBeHigh)
{
_Debug = new();
_Moved = new();
_Renamed = new();
_AllMappedFaceFiles = new();
_AllMappedFaceFileNames = new();
_ResultAllInOne = resultAllInOne;
_DuplicateMappedFaceFiles = new();
_Log = Serilog.Log.ForContext<E_Distance>();
_DistanceRenameToMatch = distanceRenameToMatch;
_FaceDistancePermyriad = faceDistancePermyriad;
_FaceDistanceTolerance = faceDistanceTolerance;
_DistanceMoveUnableToMatch = distanceMoveUnableToMatch;
@ -314,17 +328,16 @@ public class E_Distance : Shared.Models.Methods.IFaceDistance
}
}
private void MoveUnableToMatch(string eDistanceContentDirectory, string file)
private void MoveUnableToMatch(string eDistanceContentDirectory, string mappedFaceFile, string mappedFaceFileName)
{
bool result;
string? fileName = Path.GetFileName(file);
string? directoryName = Path.GetDirectoryName(file);
if (fileName is null || directoryName is null)
result = false;
bool check;
string? directoryName = Path.GetDirectoryName(mappedFaceFile);
if (mappedFaceFileName is null || directoryName is null)
check = false;
else
{
if (string.IsNullOrEmpty(directoryName) || string.IsNullOrEmpty(directoryName) || !directoryName.Contains(eDistanceContentDirectory))
result = false;
check = false;
else
{
List<string> directoryNames = new();
@ -342,8 +355,8 @@ public class E_Distance : Shared.Models.Methods.IFaceDistance
}
if (string.IsNullOrEmpty(checkDirectoryName) || !directoryNames.Any() || !long.TryParse(directoryNames[^1][1..^1], out long directoryTicks))
{
result = false;
File.Delete(file);
check = false;
File.Delete(mappedFaceFile);
}
else
{
@ -352,45 +365,13 @@ public class E_Distance : Shared.Models.Methods.IFaceDistance
checkDirectoryName = Path.Combine(checkDirectoryName, directoryNames[i]);
if (!Directory.Exists(checkDirectoryName))
_ = Directory.CreateDirectory(checkDirectoryName);
File.Move(file, Path.Combine(checkDirectoryName, fileName));
result = true;
File.Move(mappedFaceFile, Path.Combine(checkDirectoryName, mappedFaceFileName));
check = true;
}
}
}
if (result)
{ }
}
private static string[] GetMatchingDuplicates(string[] mappedFaceFiles, List<string> duplicateMappedFaceFiles, string mappedFaceFile)
{
string[] results;
string checkFile;
FileInfo fileInfo = new(mappedFaceFile);
List<(long Length, string FullName)> collection = new();
if (fileInfo.Exists)
collection.Add(new(fileInfo.Length, fileInfo.FullName));
string fileName = Path.GetFileName(mappedFaceFile);
foreach (string file in mappedFaceFiles)
{
if (duplicateMappedFaceFiles.Contains(file))
continue;
if (file == mappedFaceFile || !file.EndsWith(fileName))
continue;
fileInfo = new(file);
if (!fileInfo.Exists)
continue;
collection.Add(new(fileInfo.Length, fileInfo.FullName));
}
collection = collection.OrderBy(l => l.Length).ToList();
for (int i = 0; i < collection.Count - 1; i++)
{
checkFile = string.Concat(collection[i].FullName, ".dup");
if (File.Exists(checkFile))
continue;
File.Move(collection[i].FullName, checkFile);
}
results = (from l in collection select l.FullName).ToArray();
return results;
if (check)
_Moved.Add(mappedFaceFile);
}
public static string? GetFaceEncoding(string file)
@ -419,6 +400,32 @@ public class E_Distance : Shared.Models.Methods.IFaceDistance
return result;
}
public static string? GetFaceLocation(string file)
{
string? result;
List<string> results = new();
const string artist = "Artist: ";
if (File.Exists(file))
{
IReadOnlyList<MetadataExtractor.Directory> directories = MetadataExtractor.ImageMetadataReader.ReadMetadata(file);
foreach (MetadataExtractor.Directory directory in directories)
{
if (directory.Name != "PNG-tEXt")
continue;
foreach (MetadataExtractor.Tag tag in directory.Tags)
{
if (tag.Name != "Textual Data" || string.IsNullOrEmpty(tag.Description))
continue;
if (!tag.Description.StartsWith(artist))
continue;
results.Add(tag.Description);
}
}
}
result = results.Any() ? results[0][artist.Length..] : null;
return result;
}
private static FaceDistanceContainer[] GetOrderedFaceDistanceContainers(MappingFromItem mappingFromItem, List<Face> faces)
{
FaceDistanceContainer[] results;
@ -447,9 +454,9 @@ public class E_Distance : Shared.Models.Methods.IFaceDistance
return results;
}
private List<(Face Face, double Length)> GetValues(MappingFromItem mappingFromItem, List<Face> faces, string json)
private static List<(Face Face, double? Length)> GetValues(MappingFromItem mappingFromItem, List<Face> faces, string json)
{
List<(Face Face, double Length)> results = new();
List<(Face Face, double? Length)> results = new();
Face face;
FaceDistance faceDistanceLength;
Shared.Models.FaceEncoding? modelsFaceEncoding = JsonSerializer.Deserialize<Shared.Models.FaceEncoding>(json);
@ -473,64 +480,99 @@ public class E_Distance : Shared.Models.Methods.IFaceDistance
faceDistanceLength = faceDistanceLengths[i];
if (faceDistanceLength.Length is null)
throw new NotSupportedException();
if (faceDistanceLength.Length.Value > _FaceDistanceTolerance)
continue;
results.Add(new(face, faceDistanceLength.Length.Value));
}
return results;
}
private Face[] GetMatchingFaces(MappingFromItem mappingFromItem, List<Face> faces, string json)
private (Face, double?)[] GetClosestFaceByDistanceIgnoringTolerance(MappingFromItem mappingFromItem, List<Face> faces, string json)
{
Face[] results;
List<(Face Face, double Length)> collection = GetValues(mappingFromItem, faces, json);
if (!collection.Any())
results = Array.Empty<Face>();
else
results = (from l in collection orderby l.Length select l.Face).Take(1).ToArray();
(Face, double?)[] results;
List<(Face Face, double? Length)> collection = GetValues(mappingFromItem, faces, json);
results = (from l in collection orderby l.Length select l).Take(1).ToArray();
(Face face, double? length) = results.First();
lock (_Debug)
_Debug.Add(length);
return results;
}
private static Face[] GetMatchingFaces(int pixelDistanceTolerance, List<Face> faces)
static (int?, int?) GetXY(int normalizedPixelPercentage, OutputResolution? outputResolution)
{
Face[] results;
int? x;
int? y;
if (outputResolution is null)
{
x = null;
y = null;
}
else
{
string normalizedPixelPercentagePadded = Shared.Models.Stateless.Methods.ILocation.GetLeftPadded(Shared.Models.Stateless.ILocation.Digits, normalizedPixelPercentage);
(x, y) = Shared.Models.Stateless.Methods.ILocation.GetXY(Shared.Models.Stateless.ILocation.Digits, Shared.Models.Stateless.ILocation.Factor, outputResolution.Width, outputResolution.Height, normalizedPixelPercentagePadded);
}
return new(x, y);
}
private (Face, double?)[] GetClosestFaceByPixel(List<Face> faces, int? x1, int? y1)
{
(Face, double?)[] results;
int? x2;
int? y2;
double distance;
double center = 2f;
double xCenterValue;
double yCenterValue;
int normalizedPixelPercentage;
int normalizedPixelPercentageLoop;
string normalizedPixelPercentagePadded;
List<(double Order, Face Face)> collection = new();
List<(Face Face, double? Order)> collection = new();
if (x1 is null || y1 is null)
throw new NotSupportedException();
foreach (Face face in faces)
{
if (face.Location is null || face.OutputResolution is null)
throw new NotSupportedException();
xCenterValue = (face.Location.Left + face.Location.Right) / center;
yCenterValue = (face.Location.Top + face.Location.Bottom) / center;
if (xCenterValue < face.Location.Left || xCenterValue > face.Location.Right)
throw new Exception();
if (yCenterValue < face.Location.Top || yCenterValue > face.Location.Bottom)
throw new Exception();
normalizedPixelPercentage = Shared.Models.Stateless.Methods.ILocation.GetNormalizedPixelPercentage(face.Location, Shared.Models.Stateless.ILocation.Digits, Shared.Models.Stateless.ILocation.Factor, face.OutputResolution);
normalizedPixelPercentagePadded = Shared.Models.Stateless.Methods.ILocation.GetLeftPadded(Shared.Models.Stateless.ILocation.Digits, normalizedPixelPercentage);
(x, y) = Shared.Models.Stateless.Methods.ILocation.GetXY(Shared.Models.Stateless.ILocation.Digits, Shared.Models.Stateless.ILocation.Factor, face.OutputResolution.Width, face.OutputResolution.Height, normalizedPixelPercentagePadded);
if (x is null || y is null)
normalizedPixelPercentageLoop = Shared.Models.Stateless.Methods.ILocation.GetNormalizedPixelPercentage(face.Location, Shared.Models.Stateless.ILocation.Digits, Shared.Models.Stateless.ILocation.Factor, face.OutputResolution);
normalizedPixelPercentagePadded = Shared.Models.Stateless.Methods.ILocation.GetLeftPadded(Shared.Models.Stateless.ILocation.Digits, normalizedPixelPercentageLoop);
(x2, y2) = Shared.Models.Stateless.Methods.ILocation.GetXY(Shared.Models.Stateless.ILocation.Digits, Shared.Models.Stateless.ILocation.Factor, face.OutputResolution.Width, face.OutputResolution.Height, normalizedPixelPercentagePadded);
if (x2 is null || y2 is null)
throw new NotSupportedException();
distance = Math.Sqrt(Math.Pow(xCenterValue - x.Value, 2) + Math.Pow(yCenterValue - y.Value, 2));
collection.Add(new(distance, face));
distance = Math.Sqrt(Math.Pow(x1.Value - x2.Value, 2) + Math.Pow(y1.Value - y2.Value, 2));
collection.Add(new(face, distance));
}
if (!collection.Any())
results = Array.Empty<Face>();
else
results = (from l in collection orderby l.Order where l.Order < pixelDistanceTolerance select l.Face).Take(1).ToArray();
results = (from l in collection orderby l.Order where l.Order < _DistancePixelDistanceTolerance select l).Take(1).ToArray();
return results;
}
private static List<Face> GetMatchingFaces(List<Face> faces, string? json)
private (Face, double?)[] GetClosestFaceByPixel(List<Face> faces, int normalizedPixelPercentage)
{
List<Face> results = new();
if (!faces.Any())
throw new NotSupportedException();
(Face, double?)[] results;
const int zero = 0;
OutputResolution? outputResolution = faces[zero].OutputResolution;
(int? x1, int? y1) = GetXY(normalizedPixelPercentage, outputResolution);
results = GetClosestFaceByPixel(faces, x1, y1);
return results;
}
private (Face, double?)[] GetClosestFaceByPixel(List<Face> faces, string json)
{
if (!faces.Any())
throw new NotSupportedException();
(Face, double?)[] results;
const int zero = 0;
OutputResolution? outputResolution = faces[zero].OutputResolution;
if (outputResolution is null)
throw new NullReferenceException(nameof(outputResolution));
Location? location = JsonSerializer.Deserialize<Location>(json);
if (location is null)
throw new NullReferenceException(nameof(location));
int normalizedPixelPercentage = Shared.Models.Stateless.Methods.ILocation.GetNormalizedPixelPercentage(location, Shared.Models.Stateless.ILocation.Digits, Shared.Models.Stateless.ILocation.Factor, outputResolution);
(int? x1, int? y1) = GetXY(normalizedPixelPercentage, outputResolution);
results = GetClosestFaceByPixel(faces, x1, y1);
return results;
}
private static List<(Face, double?)> GetMatchingFacesByFaceEncoding(List<Face> faces, string? json)
{
List<(Face, double?)> results = new();
string check;
foreach (Face face in faces)
{
@ -541,18 +583,18 @@ public class E_Distance : Shared.Models.Methods.IFaceDistance
check = JsonSerializer.Serialize(face.FaceEncoding);
if (check != json)
continue;
results.Add(face);
results.Add(new(face, 0));
}
return results;
}
private static FileInfo? CheckFileThenGetFileInfo(string facesFileNameExtension, MappingFromItem mappingFromItem, string mappedFaceFile, List<Face> checkFaces)
private static FileInfo? CheckFileThenGetFileInfo(string facesFileNameExtension, MappingFromItem mappingFromItem, string mappedFaceFile, List<(Face, double?)> checkFaces)
{
FileInfo? result = null;
string checkFile;
string? mappedFaceDirectory;
string deterministicHashCodeKey;
foreach (Face face in checkFaces)
foreach ((Face face, _) in checkFaces)
{
if (checkFaces.Count != 1)
break;
@ -566,112 +608,127 @@ public class E_Distance : Shared.Models.Methods.IFaceDistance
if (checkFile == mappedFaceFile)
continue;
result = new FileInfo(checkFile);
if (!result.Exists)
continue;
File.Delete(result.FullName);
result = null;
}
return result;
}
private static List<Face> GetMatchingFaces(List<Face> faces, string[] mappedFaceFiles, List<int> debugChecks, int normalizedPixelPercentageValue, List<int> normalizedPixelPercentages, List<string> duplicateMappedFaceFiles, string mappedFaceFile)
private void AppendMatchingDuplicates(string mappedFaceFile, string[] matches)
{
List<Face> results = new();
bool check;
int normalizedPixelPercentage;
foreach (Face face in faces)
string checkFile;
FileInfo fileInfo = new(mappedFaceFile);
List<(long Length, string FullName)> collection = new();
if (fileInfo.Exists)
collection.Add(new(fileInfo.Length, fileInfo.FullName));
lock (_DuplicateMappedFaceFiles)
_DuplicateMappedFaceFiles.Add(mappedFaceFile);
foreach (string match in matches)
{
if (face.Location is null || face.OutputResolution is null)
throw new NotSupportedException();
normalizedPixelPercentage = Shared.Models.Stateless.Methods.ILocation.GetNormalizedPixelPercentage(face.Location, Shared.Models.Stateless.ILocation.Digits, Shared.Models.Stateless.ILocation.Factor, face.OutputResolution);
debugChecks.Add(normalizedPixelPercentage);
if (normalizedPixelPercentage != normalizedPixelPercentageValue)
fileInfo = new(match);
if (!fileInfo.Exists)
continue;
if (normalizedPixelPercentages.Contains(normalizedPixelPercentage))
{
duplicateMappedFaceFiles.AddRange(GetMatchingDuplicates(mappedFaceFiles, duplicateMappedFaceFiles, mappedFaceFile));
continue;
}
check = true;
results.Add(face);
if (!check)
debugChecks.Add(normalizedPixelPercentage);
collection.Add(new(fileInfo.Length, fileInfo.FullName));
break;
}
collection = collection.OrderBy(l => l.Length).ToList();
for (int i = 0; i < collection.Count - 1; i++)
{
checkFile = string.Concat(collection[i].FullName, ".dup");
if (File.Exists(checkFile))
continue;
File.Move(collection[i].FullName, checkFile);
}
return results;
}
public (int, int, int) GetUnableToMatchCountAndRenameMatches(string facesFileNameExtension, string? eDistanceContentDirectory, MappingFromItem mappingFromItem, List<Face> faces, List<(string MappedFaceFile, int normalizedPixelPercentage)> collection)
public void LookForMatchFacesAndPossiblyRename(string facesFileNameExtension, string? eDistanceContentDirectory, MappingFromItem mappingFromItem, List<Face> faces, List<(string MappedFaceFile, int normalizedPixelPercentage)> collection)
{
int result = 0;
string? json;
int renamed = 0;
string[] matches;
FileInfo? fileInfo;
List<Face> checkFaces = new();
List<int> debugChecks = new();
List<int> normalizedPixelPercentages = new();
List<string> duplicateMappedFaceFiles = new();
string[] mappedFaceFiles = (from l in collection select l.MappedFaceFile).ToArray();
string mappedFaceFileName;
List<(Face, double?)> checkFaces = new();
foreach ((string mappedFaceFile, int normalizedPixelPercentage) in collection)
{
if (duplicateMappedFaceFiles.Contains(mappedFaceFile))
mappedFaceFileName = Path.GetFileName(mappedFaceFile);
if (_DuplicateMappedFaceFiles.Contains(mappedFaceFileName))
continue;
json = null;
checkFaces.Clear();
debugChecks.Clear();
checkFaces.AddRange(GetMatchingFaces(faces, mappedFaceFiles, debugChecks, normalizedPixelPercentage, normalizedPixelPercentages, duplicateMappedFaceFiles, mappedFaceFile));
if (checkFaces.Count != 1)
json = GetFaceEncoding(mappedFaceFile);
if (json is null)
{
checkFaces.Clear();
json = GetFaceEncoding(mappedFaceFile);
if (json is null)
{
result++;
if (!string.IsNullOrEmpty(eDistanceContentDirectory) && _DistanceMoveUnableToMatch)
MoveUnableToMatch(eDistanceContentDirectory, mappedFaceFile);
continue;
}
checkFaces.AddRange(GetMatchingFaces(faces, json));
if (!string.IsNullOrEmpty(eDistanceContentDirectory) && _DistanceMoveUnableToMatch)
MoveUnableToMatch(eDistanceContentDirectory, mappedFaceFile, mappedFaceFileName);
continue;
}
checkFaces.AddRange(GetMatchingFacesByFaceEncoding(faces, json));
if (checkFaces.Count == 1)
_Debug.Add(0);
if (checkFaces.Count != 1 && !string.IsNullOrEmpty(json))
{
checkFaces.Clear();
if (json is null)
throw new NotSupportedException();
checkFaces.AddRange(GetMatchingFaces(mappingFromItem, faces, json));
checkFaces.AddRange(GetClosestFaceByDistanceIgnoringTolerance(mappingFromItem, faces, json));
}
if (checkFaces.Count != 1 && _DistancePixelDistanceTolerance > 0)
{
checkFaces.Clear();
checkFaces.AddRange(GetMatchingFaces(_DistancePixelDistanceTolerance, faces));
json = GetFaceLocation(mappedFaceFile);
if (json is not null)
checkFaces.AddRange(GetClosestFaceByPixel(faces, json));
else
checkFaces.AddRange(GetClosestFaceByPixel(faces, normalizedPixelPercentage));
throw new NotImplementedException("Without a tolerance this should not ever occur!");
}
if (!checkFaces.Any() && faces.Count == 1)
checkFaces.AddRange(faces);
if (!checkFaces.Any())
{
result++;
if (!string.IsNullOrEmpty(eDistanceContentDirectory) && _DistanceMoveUnableToMatch)
MoveUnableToMatch(eDistanceContentDirectory, mappedFaceFile);
MoveUnableToMatch(eDistanceContentDirectory, mappedFaceFile, mappedFaceFileName);
continue;
}
if (checkFaces.Count != 1)
{
result++;
if (!string.IsNullOrEmpty(eDistanceContentDirectory) && _DistanceMoveUnableToMatch)
MoveUnableToMatch(eDistanceContentDirectory, mappedFaceFile);
MoveUnableToMatch(eDistanceContentDirectory, mappedFaceFile, mappedFaceFileName);
continue;
}
normalizedPixelPercentages.Add(normalizedPixelPercentage);
fileInfo = CheckFileThenGetFileInfo(facesFileNameExtension, mappingFromItem, mappedFaceFile, checkFaces);
if (fileInfo is null)
if (fileInfo is not null)
{
if (_DistanceRenameToMatch && fileInfo is not null)
{
if (fileInfo.Exists)
File.Delete(fileInfo.FullName);
File.Move(mappedFaceFile, fileInfo.FullName);
_Renamed.Add(mappedFaceFile);
}
continue;
File.Move(mappedFaceFile, fileInfo.FullName);
renamed++;
}
if (_AllMappedFaceFileNames.Contains(mappedFaceFileName))
{
lock (_AllMappedFaceFiles)
matches = (from l in _AllMappedFaceFiles where l != mappedFaceFile && Path.GetFileName(l) == mappedFaceFileName select l).ToArray();
if (matches.Any())
AppendMatchingDuplicates(mappedFaceFile, matches);
}
lock (_AllMappedFaceFiles)
_AllMappedFaceFiles.Add(mappedFaceFile);
lock (_AllMappedFaceFileNames)
_AllMappedFaceFileNames.Add(mappedFaceFileName);
}
if (duplicateMappedFaceFiles.Any())
{
duplicateMappedFaceFiles.Sort();
}
return new(result, duplicateMappedFaceFiles.Count, renamed);
}
public void Clear()
{
double?[] debug = (from l in _Debug where l is null or not 0 select l).ToArray();
string debugMessage = $"{_Debug.Count - debug.Length} - {debug.Min()} - {_Debug.Max()}";
if (_Moved.Any() || _Renamed.Any() || _DuplicateMappedFaceFiles.Any())
throw new NotImplementedException("Restart!");
_Debug.Clear();
_Moved.Clear();
_Renamed.Clear();
_AllMappedFaceFiles.Clear();
_AllMappedFaceFileNames.Clear();
_DuplicateMappedFaceFiles.Clear();
}
}