Get Unable to Match Count and Rename Matches

This commit is contained in:
2022-09-24 16:39:15 -07:00
parent e64c713926
commit 751a61529c
32 changed files with 983 additions and 668 deletions

View File

@ -46,7 +46,7 @@ internal abstract class MapLogic
{
if (!personDisplayDirectoryAllFile.EndsWith(facesFileNameExtension))
continue;
(id, normalizedPixelPercentage, _) = IMapping.GetReversedDeterministicHashCodeKey(configuration.LocationDigits, facesFileNameExtension, personDisplayDirectoryAllFile);
(id, normalizedPixelPercentage, _) = IMapping.GetReversedDeterministicHashCodeKey(Shared.Models.Stateless.ILocation.Digits, facesFileNameExtension, personDisplayDirectoryAllFile);
if (id is null || normalizedPixelPercentage is null)
continue;
if (!skipCollection.ContainsKey(id.Value))
@ -73,7 +73,7 @@ internal abstract class MapLogic
}
}
internal static List<(string, string[], string)> DeleteEmptyDirectoriesAndGetCollection(Configuration configuration, string facesFileNameExtension, long ticks, string eDistanceContentDirectory, List<string> personKeyFormattedCollection)
internal static List<(string, string[], string)> DeleteEmptyDirectoriesAndGetCollection(Configuration configuration, string facesFileNameExtension, List<string> personKeyFormattedCollection, string[] ticksDirectories, string message)
{
List<(string, string[], string)> results = new();
int? id;
@ -91,10 +91,7 @@ internal abstract class MapLogic
string? personFirstInitialDirectory;
string[] personDisplayDirectoryNames;
string manualCopyHumanized = nameof(IMapLogic.ManualCopy).Humanize(LetterCasing.Title);
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string forceSingleImageHumanized = nameof(IMapLogic.ForceSingleImage).Humanize(LetterCasing.Title);
string[] ticksDirectories = Directory.GetDirectories(eDistanceContentDirectory, "*", SearchOption.TopDirectoryOnly);
string message = $") {ticksDirectories.Length:000} compile from and clean ticks Director(ies) - A - {totalSeconds} total second(s)";
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
using ProgressBar progressBar = new(ticksDirectories.Length, message, options);
foreach (string ticksDirectory in ticksDirectories)
@ -151,7 +148,7 @@ internal abstract class MapLogic
{
if (file.EndsWith(".lnk") || file.EndsWith(".json"))
continue;
(id, normalizedPixelPercentage, _) = IMapping.GetReversedDeterministicHashCodeKey(configuration.LocationDigits, facesFileNameExtension, file);
(id, normalizedPixelPercentage, _) = IMapping.GetReversedDeterministicHashCodeKey(Shared.Models.Stateless.ILocation.Digits, facesFileNameExtension, file);
if (id is null || normalizedPixelPercentage is null)
continue;
results.Add(new(personKeyFormatted, personDisplayDirectoryNames, file));
@ -180,6 +177,35 @@ internal abstract class MapLogic
return results;
}
internal static string[] DeleteEmptyDirectoriesAndGetMappedFaceFiles(Configuration configuration, string facesFileNameExtension, long ticks, string eDistanceContentDirectory, PersonContainer[] personContainers)
{
string[] results;
string personKeyFormatted;
List<string> personKeyFormattedCollection = new();
_ = GetDistinctCollection(configuration, personContainers.ToList(), new());
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string[] ticksDirectories = Directory.GetDirectories(eDistanceContentDirectory, "*", SearchOption.TopDirectoryOnly);
string message = $") {ticksDirectories.Length:000} collect from and clean ticks Director(ies) - A - {totalSeconds} total second(s)";
foreach (PersonContainer personContainer in personContainers)
{
if (personContainer.Person is null || personContainer.Key is null || personContainer.Birthdays is null || !personContainer.Birthdays.Any())
continue;
foreach (PersonBirthday personBirthday in personContainer.Birthdays)
{
personKeyFormatted = IPersonBirthday.GetFormatted(configuration.PersonBirthdayFormat, personBirthday);
personKeyFormattedCollection.Add(personKeyFormatted);
}
}
List<(string PersonKeyFormatted, string[] PersonDisplayDirectoryNames, string File)> collection = DeleteEmptyDirectoriesAndGetCollection(
configuration,
facesFileNameExtension,
personKeyFormattedCollection,
ticksDirectories,
message);
results = (from l in collection select l.File).ToArray();
return results;
}
private static PersonContainer[] GetDistinctPersonContainers(List<PersonContainer> personContainers)
{
List<PersonContainer> results = new();
@ -195,32 +221,14 @@ internal abstract class MapLogic
private static void SetKeyValuePairs(Configuration configuration, long ticks, List<PersonContainer> personContainers, List<Face> distinctFilteredFaces, List<(string, string[], int, int)> personKeyFormattedIdThenNormalizedPixelPercentageCollection, List<(string, int, int)> incorrectPersonKeyFormattedIdThenNormalizedPixelPercentageCollection, Dictionary<long, PersonContainer> personKeyToPersonContainer, Dictionary<int, Dictionary<int, PersonContainer[]>> idThenNormalizedPixelPercentageToPersonContainers, List<(string[], PersonContainer)> possiblyNewPersonDisplayDirectoryNamesAndPersonContainer, Dictionary<int, Dictionary<int, PersonContainer[]>> incorrectIdThenNormalizedPixelPercentageToPersonContainers, Dictionary<long, (long LCL, long Minimum, long Maximum, long UCL)> personKeyToRanges)
{
const int zero = 0;
PersonBirthday? personBirthday;
string newestPersonKeyFormatted;
PersonContainer[] distinctPersonContainers;
Dictionary<string, PersonContainer> personKeyFormattedToPersonContainer = new();
Dictionary<long, List<PersonContainer>> personKeyToPersonContainerCollection = new();
Dictionary<int, Dictionary<int, List<PersonContainer>>> idThenNormalizedPixelPercentageToPersonContainerCollection = new();
Dictionary<int, Dictionary<int, List<PersonContainer>>> incorrectIdThenNormalizedPixelPercentageToPersonContainerCollection = new();
foreach (PersonContainer personContainer in personContainers)
{
if (personContainer.Key is null)
continue;
if (!personKeyToPersonContainerCollection.ContainsKey(personContainer.Key.Value))
personKeyToPersonContainerCollection.Add(personContainer.Key.Value, new());
personKeyToPersonContainerCollection[personContainer.Key.Value].Add(personContainer);
newestPersonKeyFormatted = IPersonBirthday.GetFormatted(configuration.PersonBirthdayFormat, personContainer.Key.Value);
if (!personKeyFormattedToPersonContainer.ContainsKey(newestPersonKeyFormatted))
personKeyFormattedToPersonContainer.Add(newestPersonKeyFormatted, personContainer);
}
foreach (KeyValuePair<long, List<PersonContainer>> keyValuePair in personKeyToPersonContainerCollection)
{
if (keyValuePair.Value.Count != 1 && (from l in keyValuePair.Value select l.DisplayDirectoryName).Distinct().Count() != 1)
throw new NotImplementedException();
personKeyToPersonContainer.Add(keyValuePair.Key, keyValuePair.Value[zero]);
}
List<(long, PersonContainer)> collection = GetDistinctCollection(configuration, personContainers, personKeyFormattedToPersonContainer);
foreach ((long personKey, PersonContainer personContainer) in collection)
personKeyToPersonContainer.Add(personKey, personContainer);
if (personKeyFormattedIdThenNormalizedPixelPercentageCollection.Any())
{
string personDisplayDirectory;
@ -291,104 +299,60 @@ internal abstract class MapLogic
}
}
private static string? GetCheckFile(Configuration configuration, string facesFileNameExtension, string file, int id, int normalizedPixelPercentage)
private static List<(long, PersonContainer)> GetDistinctCollection(Configuration configuration, List<PersonContainer> personContainers, Dictionary<string, PersonContainer> personKeyFormattedToPersonContainer)
{
string? result;
string fileName = Path.GetFileName(file);
string? directoryName = Path.GetDirectoryName(file);
(string? Id, string? NormalizedPixelPercentage, string? ExtensionLowered, bool? Check) segments = IMapping.GetSegments(configuration.LocationDigits, facesFileNameExtension, fileName);
if (string.IsNullOrEmpty(directoryName) || string.IsNullOrEmpty(segments.Id) || string.IsNullOrEmpty(segments.NormalizedPixelPercentage) || string.IsNullOrEmpty(segments.ExtensionLowered))
result = null;
else
result = Path.Combine(directoryName, $"{IMapping.GetDeterministicHashCodeKey(id, normalizedPixelPercentage)}{segments.ExtensionLowered}.json");
return result;
}
private static void MoveUnableToMatch(Configuration configuration, string eDistanceContentDirectory, string file, string jsonFile)
{
bool result;
string? fileName = Path.GetFileName(file);
string? jsonFileName = Path.GetFileName(jsonFile);
if (fileName is null || jsonFileName is null)
result = false;
else
List<(long, PersonContainer)> results = new();
const int zero = 0;
string newestPersonKeyFormatted;
Dictionary<long, List<PersonContainer>> personKeyToPersonContainerCollection = new();
foreach (PersonContainer personContainer in personContainers)
{
string? directoryName = Path.GetDirectoryName(jsonFile);
string? jsonDirectoryName = Path.GetDirectoryName(jsonFile);
if (string.IsNullOrEmpty(directoryName) || string.IsNullOrEmpty(directoryName) || directoryName != jsonDirectoryName || !directoryName.Contains(eDistanceContentDirectory))
result = false;
else
{
List<string> directoryNames = new();
string? checkDirectoryName = directoryName;
for (int i = 0; i < int.MaxValue; i++)
{
if (string.IsNullOrEmpty(checkDirectoryName))
continue;
directoryNames.Add(Path.GetFileName(checkDirectoryName));
checkDirectoryName = Path.GetDirectoryName(checkDirectoryName);
if (string.IsNullOrEmpty(checkDirectoryName))
continue;
if (checkDirectoryName == eDistanceContentDirectory)
break;
}
if (string.IsNullOrEmpty(checkDirectoryName) || !directoryNames.Any() || !long.TryParse(directoryNames[^1][1..^1], out long directoryTicks))
{
result = false;
File.Delete(jsonFile);
File.Delete(file);
}
else
{
bool jsonFileExists = File.Exists(jsonFile);
if (!jsonFileExists)
checkDirectoryName = Path.Combine(checkDirectoryName, $"({directoryTicks}.00)");
else
checkDirectoryName = Path.Combine(checkDirectoryName, $"({directoryTicks}{configuration.FaceDistanceTolerance.ToString()[1..]})");
for (int i = directoryNames.Count - 1 - 1; i > -1; i--)
checkDirectoryName = Path.Combine(checkDirectoryName, directoryNames[i]);
if (!Directory.Exists(checkDirectoryName))
_ = Directory.CreateDirectory(checkDirectoryName);
File.Move(file, Path.Combine(checkDirectoryName, fileName));
if (jsonFileExists)
File.Move(jsonFile, Path.Combine(checkDirectoryName, jsonFileName));
result = true;
}
}
if (personContainer.Key is null)
continue;
if (!personKeyToPersonContainerCollection.ContainsKey(personContainer.Key.Value))
personKeyToPersonContainerCollection.Add(personContainer.Key.Value, new());
personKeyToPersonContainerCollection[personContainer.Key.Value].Add(personContainer);
newestPersonKeyFormatted = IPersonBirthday.GetFormatted(configuration.PersonBirthdayFormat, personContainer.Key.Value);
if (!personKeyFormattedToPersonContainer.ContainsKey(newestPersonKeyFormatted))
personKeyFormattedToPersonContainer.Add(newestPersonKeyFormatted, personContainer);
}
if (result)
{ }
foreach (KeyValuePair<long, List<PersonContainer>> keyValuePair in personKeyToPersonContainerCollection)
{
if (keyValuePair.Value.Count != 1 && (from l in keyValuePair.Value select l.DisplayDirectoryName).Distinct().Count() != 1)
throw new NotSupportedException(keyValuePair.Value[zero].DisplayDirectoryName);
results.Add(new(keyValuePair.Key, keyValuePair.Value[zero]));
}
return results;
}
private static int SetCollectionsAndGetUnableToMatchCount(Configuration configuration, string facesFileNameExtension, long ticks, string eDistanceContentDirectory, Shared.Models.Methods.IFaceDistance? distance, Dictionary<int, List<Face>> idToFaces, Dictionary<string, string> personKeyFormattedToNewestPersonKeyFormatted, List<(string, string[], int, int)> personKeyFormattedIdThenNormalizedPixelPercentageCollection, List<(string, int, int)> incorrectPersonKeyFormattedIdThenNormalizedPixelPercentageCollection, List<(string, string[], string)> collection)
private static (int, int) SetCollectionsAndGetUnableToMatchCount(string facesFileNameExtension, long ticks, Dictionary<int, List<Face>> idToFaces, Dictionary<string, string> personKeyFormattedToNewestPersonKeyFormatted, List<(string, string[], int, int)> personKeyFormattedIdThenNormalizedPixelPercentageCollection, List<(string, int, int)> incorrectPersonKeyFormattedIdThenNormalizedPixelPercentageCollection, List<(string, string[], string)> collection)
{
int? id;
int result = 0;
int? id;
bool debugCheck;
string? checkFile;
List<Face>? faces;
List<int> debugChecks = new();
List<Face> checkFaces = new();
int? normalizedPixelPercentage;
string newestPersonKeyFormatted;
List<string> duplicates = new();
string personDisplayDirectoryName;
bool idToFacesAny = idToFaces.Any();
List<int> normalizedPixelPercentages;
List<string> duplicateMappedFaceFiles = new();
Dictionary<int, List<int>> idToNormalizedPixelPercentages = new();
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string message = $") {collection.Count:000} join from ticks Director(ies) - B - {totalSeconds} total second(s)";
string message = $") {collection.Count:000} join from ticks Director(ies) - C - {totalSeconds} total second(s)";
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
using ProgressBar progressBar = new(collection.Count, message, options);
foreach ((string personKeyFormatted, string[] personDisplayDirectoryNames, string file) in collection)
foreach ((string personKeyFormatted, string[] personDisplayDirectoryNames, string mappedFaceFile) in collection)
{
progressBar.Tick();
(id, normalizedPixelPercentage, faces) = IMapping.GetReversedDeterministicHashCodeKey(
configuration.LocationDigits,
Shared.Models.Stateless.ILocation.Digits,
facesFileNameExtension,
idToFacesAny,
idToFaces,
file);
mappedFaceFile);
if (id is null || normalizedPixelPercentage is null)
{
result++;
@ -397,9 +361,6 @@ internal abstract class MapLogic
if (!idToNormalizedPixelPercentages.ContainsKey(id.Value))
idToNormalizedPixelPercentages.Add(id.Value, new());
normalizedPixelPercentages = idToNormalizedPixelPercentages[id.Value];
checkFile = GetCheckFile(configuration, facesFileNameExtension, file, id.Value, normalizedPixelPercentage.Value);
if (string.IsNullOrEmpty(checkFile))
throw new NotSupportedException();
if (faces is null)
{
result++;
@ -417,35 +378,25 @@ internal abstract class MapLogic
continue;
if (normalizedPixelPercentages.Contains(face.Mapping.MappingFromLocation.NormalizedPixelPercentage))
{
duplicates.Add(string.Concat(id.Value, '.', normalizedPixelPercentage.Value, ".jpg", facesFileNameExtension));
duplicateMappedFaceFiles.Add(mappedFaceFile);
continue;
}
debugCheck = true;
checkFaces.Add(face);
if (!debugCheck)
debugChecks.Add(normalizedPixelPercentage.Value);
debugChecks.Add(face.Mapping.MappingFromLocation.NormalizedPixelPercentage);
}
if (checkFaces.Count != 1 && distance is not null && File.Exists(checkFile))
{
checkFaces.Clear();
checkFaces.AddRange(distance.GetMatchingFaces(configuration.FaceDistanceTolerance, checkFile, faces));
}
if (!checkFaces.Any() && faces.Count == 1)
checkFaces.AddRange(faces);
if (!checkFaces.Any())
{
result++;
if (configuration.MappingMoveUnableToMatch)
MoveUnableToMatch(configuration, eDistanceContentDirectory, file, checkFile);
continue;
}
if (checkFaces.Count != 1)
{
result++;
if (configuration.MappingMoveUnableToMatch)
MoveUnableToMatch(configuration, eDistanceContentDirectory, file, checkFile);
continue;
}
normalizedPixelPercentages.Add(normalizedPixelPercentage.Value);
idToNormalizedPixelPercentages[id.Value].Add(normalizedPixelPercentage.Value);
if (!personKeyFormattedToNewestPersonKeyFormatted.ContainsKey(personKeyFormatted))
newestPersonKeyFormatted = personKeyFormatted;
@ -457,13 +408,13 @@ internal abstract class MapLogic
else
personKeyFormattedIdThenNormalizedPixelPercentageCollection.Add(new(newestPersonKeyFormatted, personDisplayDirectoryNames, id.Value, normalizedPixelPercentage.Value));
}
if (duplicates.Any())
if (duplicateMappedFaceFiles.Any())
{
duplicates.Sort();
if (duplicates.Any())
duplicateMappedFaceFiles.Sort();
if (duplicateMappedFaceFiles.Any())
{ }
}
return result;
return new(result, duplicateMappedFaceFiles.Count);
}
private static double GetStandardDeviation(IEnumerable<long> values, double average)
@ -616,6 +567,8 @@ internal abstract class MapLogic
{
if (configuration is null)
throw new NullReferenceException(nameof(configuration));
string message;
int totalSeconds;
List<long> personKeys = new();
List<long?> nullablePersonKeyCollection = new();
List<string> personKeyFormattedCollection = new();
@ -636,7 +589,7 @@ internal abstract class MapLogic
personContainers.AddRange(AddToPersonKeysThenGetNonSpecificPeopleCollection(configuration, personKeys));
foreach (Face face in distinctFilteredFaces)
{
if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null)
if (face.FaceEncoding is null || face.Location is null)
throw new NotSupportedException();
if (face.Mapping is null)
throw new NotSupportedException();
@ -644,17 +597,18 @@ internal abstract class MapLogic
keyValuePairs.Add(face.Mapping.MappingFromItem.Id, new());
keyValuePairs[face.Mapping.MappingFromItem.Id].Add(face);
}
List<(string, string[], string)> collection = DeleteEmptyDirectoriesAndGetCollection(configuration,
facesFileNameExtension,
ticks,
eDistanceContentDirectory,
personKeyFormattedCollection);
int unableToMatchCount = SetCollectionsAndGetUnableToMatchCount(
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string[] ticksDirectories = Directory.GetDirectories(eDistanceContentDirectory, "*", SearchOption.TopDirectoryOnly);
message = $") {ticksDirectories.Length:000} compile from and clean ticks Director(ies) - B - {totalSeconds} total second(s)";
List<(string, string[], string)> collection = DeleteEmptyDirectoriesAndGetCollection(
configuration,
facesFileNameExtension,
personKeyFormattedCollection,
ticksDirectories,
message);
(int unableToMatchCount, int duplicateCount) = SetCollectionsAndGetUnableToMatchCount(
facesFileNameExtension,
ticks,
eDistanceContentDirectory,
faceDistance,
keyValuePairs,
personKeyFormattedToNewestPersonKeyFormatted,
personKeyFormattedIdThenNormalizedPixelPercentageCollection,
@ -672,8 +626,8 @@ internal abstract class MapLogic
possiblyNewPersonDisplayDirectoryNamesAndPersonContainer,
incorrectIdThenNormalizedPixelPercentageToPersonContainers,
personKeyToRanges);
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string message = $") {collection.Count:000} message from ticks Director(ies) - C - {unableToMatchCount} Unable To Match Count / {collection.Count} Collection - {totalSeconds} total second(s)";
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
message = $") {collection.Count:000} message from ticks Director(ies) - D - {duplicateCount} Duplicate Count {unableToMatchCount} Unable To Match Count / {collection.Count} Collection - {totalSeconds} total second(s)";
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
using (ProgressBar progressBar = new(collection.Count, message, options))
{