phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Moved population of idToLocationContainers in

Browse Source 
preparation to change to a class over dictionary
This commit is contained in:
Mike Phares
2023-04-14 02:58:03 -07:00
parent 544b008e07
commit b97a939c58
5 changed files with 214 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
Map/Models/Configuration.cs
View File

@ -6,11 +6,15 @@ namespace View_by_Distance.Map.Models;
public class Configuration
{
public bool DeletePossibleDuplicates { get; internal set; }
public bool DistanceMoveUnableToMatch { init; get; }
public bool DistanceRenameToMatch { init; get; }
public int FaceConfidencePercent { init; get; }
public int FaceDistancePermyriad { init; get; }
public string FacePartsFileNameExtension { init; get; }
public string FacesFileNameExtension { init; get; }
public string FacesHiddenFileNameExtension { init; get; }
public int LocationDigits { init; get; }
public string MappingDefaultName { init; get; }
public int PersonBirthdayFirstYear { init; get; }
public string PersonBirthdayFormat { init; get; }
@ -23,8 +27,12 @@ public class Configuration
public int SortingMinimumToUseSigma { init; get; }
[JsonConstructor]
public Configuration(int faceConfidencePercent,
public Configuration(bool deletePossibleDuplicates,
bool distanceMoveUnableToMatch,
bool distanceRenameToMatch,
int faceConfidencePercent,
int faceDistancePermyriad,
int locationDigits,
string mappingDefaultName,
int personBirthdayFirstYear,
string personBirthdayFormat,
@ -39,18 +47,22 @@ public class Configuration
string facesHiddenFileNameExtension,
string facePartsFileNameExtension)
{
LocationDigits = locationDigits;
PersonCharacters = personCharacters;
MappingDefaultName = mappingDefaultName;
PersonBirthdayFormat = personBirthdayFormat;
SortingMaximumPerKey = sortingMaximumPerKey;
DistanceRenameToMatch = distanceRenameToMatch;
FaceConfidencePercent = faceConfidencePercent;
FaceDistancePermyriad = faceDistancePermyriad;
FacesFileNameExtension = facesFileNameExtension;
SkipNotSkipDirectories = skipNotSkipDirectories;
PersonBirthdayFirstYear = personBirthdayFirstYear;
RangeDistanceTolerance = rangeDistanceTolerance[1];
DeletePossibleDuplicates = deletePossibleDuplicates;
SaveSortingWithoutPerson = saveSortingWithoutPerson;
SortingMinimumToUseSigma = sortingMinimumToUseSigma;
DistanceMoveUnableToMatch = distanceMoveUnableToMatch;
RangeDaysDeltaTolerance = rangeDaysDeltaTolerance[1];
FacePartsFileNameExtension = facePartsFileNameExtension;
FacesHiddenFileNameExtension = facesHiddenFileNameExtension;

14
Map/Models/MapLogic.cs
View File

@ -23,24 +23,22 @@ public class MapLogic : Shared.Models.Methods.IMapLogic
private readonly long _Ticks;
private readonly Serilog.ILogger? _Log;
private readonly int _MaxDegreeOfParallelism;
private readonly Configuration? _Configuration;
private readonly string _EDistanceContentTicksDirectory;
private readonly Shared.Models.Properties.IPropertyConfiguration _PropertyConfiguration;
public MapLogic(int maxDegreeOfParallelism, Shared.Models.Properties.IPropertyConfiguration propertyConfiguration, Configuration? configuration, PersonContainer[] personContainers, long ticks, string a2PeopleSingletonDirectory, string eDistanceContentDirectory)
public MapLogic(int maxDegreeOfParallelism, Shared.Models.Properties.IPropertyConfiguration propertyConfiguration, Configuration? configuration, PersonContainer[] personContainers, long ticks, string a2PeopleSingletonDirectory, string eDistanceContentDirectory, Dictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers)
{
_Ticks = ticks;
_Configuration = configuration;
_Log = Serilog.Log.ForContext<MapLogic>();
_PropertyConfiguration = propertyConfiguration;
_MaxDegreeOfParallelism = maxDegreeOfParallelism;
if (_Log is null)
{ }
if (idToLocationContainers is null)
throw new NullReferenceException(nameof(idToLocationContainers));
if (propertyConfiguration.VerifyToSeason is null || !propertyConfiguration.VerifyToSeason.Any())
throw new Exception();
if (_MaxDegreeOfParallelism == 0)
{ }
throw new NullReferenceException(nameof(propertyConfiguration.VerifyToSeason));
string json;
string fullPath;
List<KeyValuePair<int, int[]>>? collection;
@ -58,7 +56,8 @@ public class MapLogic : Shared.Models.Methods.IMapLogic
if (configuration is not null)
{
List<PersonContainer> personContainerCollection = new(personContainers);
Stateless.MapLogic.Set(configuration,
Stateless.MapLogic.Set(maxDegreeOfParallelism,
configuration,
ticks,
personContainerCollection,
a2PeopleSingletonDirectory,
@ -67,6 +66,7 @@ public class MapLogic : Shared.Models.Methods.IMapLogic
notMappedPersonContainers,
skipCollection,
skipNotSkipCollection,
idToLocationContainers,
idThenNormalizedRectangleToPersonContainers);
if (personContainerCollection.Count == personContainers.Length)
throw new NotSupportedException();

273
Map/Models/Stateless/MapLogic.cs
View File

@ -1,5 +1,7 @@
using Humanizer;
using ShellProgressBar;
using System.Diagnostics;
using System.Drawing;
using System.Text.Json;
using View_by_Distance.Shared.Models;
using View_by_Distance.Shared.Models.Stateless.Methods;
@ -162,7 +164,7 @@ internal abstract class MapLogic
personDisplayDirectory = Path.Combine(matchDirectory, alphaDirectoryName);
if (!Directory.Exists(personDisplayDirectory) || !Directory.Exists(matchDirectory))
continue;
_ = System.Diagnostics.Process.Start("explorer", matchDirectory);
_ = Process.Start("explorer", matchDirectory);
for (int i = 0; i < int.MaxValue; i++)
{
Thread.Sleep(500);
@ -301,67 +303,6 @@ internal abstract class MapLogic
return results;
}
private static List<string> GetPersonKeyFormattedCollection(Configuration configuration, PersonContainer[] personContainers, List<(long? PersonKey, string Line)> lines)
{
List<string> results = new();
string personKeyFormatted;
foreach (PersonContainer personContainer in personContainers)
{
lines.AddRange(IPersonContainer.GetDisplay(configuration.PersonBirthdayFormat, personContainer));
if (personContainer.Person is null || personContainer.Key is null || personContainer.Birthdays is null || !personContainer.Birthdays.Any())
continue;
foreach (PersonBirthday personBirthday in personContainer.Birthdays)
{
personKeyFormatted = IPersonBirthday.GetFormatted(configuration.PersonBirthdayFormat, personBirthday);
results.Add(personKeyFormatted);
}
}
return results;
}
internal static List<(long, string)> DeleteEmptyDirectoriesAndGetMappedFaceFiles(Configuration configuration, PersonContainer[] personContainers, long ticks, string? a2PeopleContentDirectory, string eDistanceContentDirectory)
{
List<(long, string)> results = new();
List<(long? PersonKey, string Line)> lines = new();
_ = GetDistinctCollection(configuration, personContainers, new(), new());
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string[] ticksDirectories;
if (string.IsNullOrEmpty(eDistanceContentDirectory))
ticksDirectories = Array.Empty<string>();
else
{
if (!Directory.Exists(eDistanceContentDirectory))
_ = Directory.CreateDirectory(eDistanceContentDirectory);
else
{
_ = IPath.DeleteEmptyDirectories(eDistanceContentDirectory);
if (!Directory.Exists(eDistanceContentDirectory))
_ = Directory.CreateDirectory(eDistanceContentDirectory);
}
ticksDirectories = Directory.GetDirectories(eDistanceContentDirectory, "*", SearchOption.TopDirectoryOnly);
}
long personKey;
List<string> distinct = new();
PersonBirthday? personBirthday;
string message = $") {ticksDirectories.Length:000} collect from and clean ticks Director(ies) - A - {totalSeconds} total second(s)";
List<string> personKeyFormattedCollection = GetPersonKeyFormattedCollection(configuration, personContainers, lines);
if (!string.IsNullOrEmpty(a2PeopleContentDirectory))
File.WriteAllLines(Path.Combine(a2PeopleContentDirectory, "People - C.tsv"), from l in lines select l.Line);
List<(string, string[], string)> collection = DeleteEmptyDirectoriesAndGetCollection(configuration, personKeyFormattedCollection, ticksDirectories, message);
foreach ((string personKeyFormatted, _, string mappedFaceFile) in collection)
{
personBirthday = IPersonBirthday.GetPersonBirthday(configuration.PersonBirthdayFormat, personKeyFormatted);
if (personBirthday is null)
continue;
if (distinct.Contains(mappedFaceFile))
continue;
distinct.Add(mappedFaceFile);
personKey = personBirthday.Value.Ticks;
results.Add(new(personKey, mappedFaceFile));
}
return results;
}
private static PersonContainer[] GetDistinctPersonContainers(List<PersonContainer> personContainers)
{
List<PersonContainer> results = new();
@ -660,10 +601,190 @@ internal abstract class MapLogic
return results;
}
internal static void Set(Configuration? configuration, long ticks, List<PersonContainer> personContainers, string? a2PeopleSingletonDirectory, string eDistanceContentDirectory, Dictionary<long, PersonContainer> personKeyToPersonContainer, List<PersonContainer> notMappedPersonContainers, Dictionary<int, List<int>> skipCollection, Dictionary<int, List<int>> skipNotSkipCollection, Dictionary<int, Dictionary<int, PersonContainer[]>> idThenNormalizedRectangleToPersonContainers)
private static List<(long, string)> GetDisplayDirectoryAllFiles(string fileNameExtension, List<PersonContainer> personContainers)
{
List<(long, string)> results = new();
List<string> distinct = new();
foreach (PersonContainer personContainer in personContainers)
{
if (personContainer.Key is null)
continue;
foreach (string displayDirectoryAllFile in personContainer.DisplayDirectoryAllFiles)
{
if (!displayDirectoryAllFile.EndsWith(fileNameExtension))
continue;
if (distinct.Contains(displayDirectoryAllFile))
continue;
distinct.Add(displayDirectoryAllFile);
results.Add(new(personContainer.Key.Value, displayDirectoryAllFile));
}
}
return results;
}
private static List<(long PersonKey, string File)> GetCollection(Configuration configuration, List<PersonContainer> personContainers, List<(string, string[], string)> collection)
{
List<(long PersonKey, string File)> results = new();
string file;
long personKey;
List<string> distinct = new();
PersonBirthday? personBirthday;
results.AddRange(GetDisplayDirectoryAllFiles(configuration.FacesFileNameExtension, personContainers));
foreach ((string personKeyFormatted, _, string mappedFaceFile) in collection)
{
personBirthday = IPersonBirthday.GetPersonBirthday(configuration.PersonBirthdayFormat, personKeyFormatted);
if (personBirthday is null)
continue;
if (distinct.Contains(mappedFaceFile))
continue;
distinct.Add(mappedFaceFile);
personKey = personBirthday.Value.Ticks;
results.Add(new(personKey, mappedFaceFile));
}
for (int i = results.Count - 1; i > -1; i--)
{
file = results[i].File;
if (file.EndsWith(".old"))
{
results.RemoveAt(i);
continue;
}
if (!file.EndsWith(".dup") && !file.EndsWith(".unk") && !file.EndsWith(".abd"))
continue;
if (!File.Exists(file))
continue;
File.Move(file, file[..^4]);
results[i] = new(results[i].PersonKey, file[..^4]);
}
return results;
}
private static void ParallelFor(Configuration configuration, string eDistanceContentDirectory, List<LocationContainer<MetadataExtractor.Directory>> collection, long personKey, string file)
{
const string lnk = ".lnk";
int? id, normalizedRectangle;
IReadOnlyList<MetadataExtractor.Directory> directories;
bool fromDistanceContent = !file.EndsWith(lnk) && file.Contains(eDistanceContentDirectory);
if (!file.EndsWith(lnk))
(id, normalizedRectangle) = IMapping.GetConverted(configuration.FacesFileNameExtension, file);
else
(id, normalizedRectangle) = IMapping.GetConverted(configuration.FacesFileNameExtension, file[..^4]);
if (id is null || normalizedRectangle is null)
return;
if (file.EndsWith(lnk) || (!configuration.DistanceMoveUnableToMatch && !configuration.DistanceRenameToMatch))
directories = new List<MetadataExtractor.Directory>();
else
directories = MetadataExtractor.ImageMetadataReader.ReadMetadata(file);
Rectangle? rectangle = ILocation.GetNormalizedRectangle(configuration.LocationDigits, normalizedRectangle.Value);
lock (collection)
collection.Add(new(fromDistanceContent, file, personKey, id.Value, normalizedRectangle.Value, directories, rectangle, null));
}
private static void OpenPossibleDuplicates(Configuration configuration, List<(long, int, string, double?)> duplicates)
{
string personKeyFormatted;
foreach ((long personKey, int id, string file, double? percent) in duplicates)
{
if (percent is null)
continue;
_ = Process.Start("explorer.exe", string.Concat("\"", Path.GetDirectoryName(file), "\""));
personKeyFormatted = IPersonBirthday.GetFormatted(configuration.PersonBirthdayFormat, personKey);
}
foreach ((long personKey, int id, string file, double? percent) in duplicates)
{
if (percent is not null && percent.Value == 0)
continue;
_ = Process.Start("explorer.exe", string.Concat("\"", Path.GetDirectoryName(file), "\""));
personKeyFormatted = IPersonBirthday.GetFormatted(configuration.PersonBirthdayFormat, personKey);
}
foreach ((long personKey, int id, string file, double? percent) in duplicates)
{
if (percent is not null && percent.Value > 0)
continue;
_ = Process.Start("explorer.exe", string.Concat("\"", Path.GetDirectoryName(file), "\""));
personKeyFormatted = IPersonBirthday.GetFormatted(configuration.PersonBirthdayFormat, personKey);
}
}
private static void LookForPossibleDuplicates(Configuration configuration, List<LocationContainer<MetadataExtractor.Directory>> collection)
{
string key;
double? percent;
Rectangle? rectangle;
List<string> delete = new();
Rectangle intersectRectangle;
(string File, int NormalizedRectangle) item;
Dictionary<string, (string, int)> distinct = new();
List<(long, int, string, double?)> duplicates = new();
foreach (LocationContainer<MetadataExtractor.Directory> locationContainer in collection)
{
key = string.Concat(locationContainer.PersonKey, locationContainer.Id);
if (distinct.TryGetValue(key, out item))
{
if (item.NormalizedRectangle == locationContainer.NormalizedRectangle)
continue;
if (locationContainer.Rectangle is null)
percent = null;
else
{
rectangle = ILocation.GetNormalizedRectangle(configuration.LocationDigits, item.NormalizedRectangle);
if (locationContainer.Rectangle is null || rectangle is null)
percent = null;
else
{
intersectRectangle = Rectangle.Intersect(locationContainer.Rectangle.Value, rectangle.Value);
percent = intersectRectangle.Width * intersectRectangle.Height;
}
}
delete.Add(item.File);
delete.Add(locationContainer.File);
duplicates.Add(new(locationContainer.PersonKey, locationContainer.Id, locationContainer.File, percent));
continue;
}
distinct.Add(key, new(locationContainer.File, locationContainer.NormalizedRectangle));
}
if (!configuration.DeletePossibleDuplicates && duplicates.Any())
OpenPossibleDuplicates(configuration, duplicates);
else
{
if (delete.Count > 5)
throw new Exception("Something maybe wrong!");
foreach (string file in delete)
{
if (File.Exists(file))
File.Delete(file);
}
}
}
private static void SetLocationContainers(int maxDegreeOfParallelism, Configuration configuration, long ticks, List<PersonContainer> personContainers, string eDistanceContentDirectory, Dictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers, List<(string, string[], string)> sourceCollection)
{
List<LocationContainer<MetadataExtractor.Directory>> results = new();
List<(long PersonKey, string File)> collection = GetCollection(configuration, personContainers, sourceCollection);
if (collection.Any() && (configuration.DistanceMoveUnableToMatch || configuration.DistanceRenameToMatch))
{
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string message = $") Building Mapped Face Files Collection - {totalSeconds} total second(s)";
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
using ProgressBar progressBar = new(collection.Count, message, options);
_ = Parallel.For(0, collection.Count, parallelOptions, (i, state) =>
{
progressBar.Tick();
ParallelFor(configuration, eDistanceContentDirectory, results, collection[i].PersonKey, collection[i].File);
});
}
LookForPossibleDuplicates(configuration, results);
foreach (LocationContainer<MetadataExtractor.Directory> locationContainer in results)
{
if (!idToLocationContainers.ContainsKey(locationContainer.Id))
idToLocationContainers.Add(locationContainer.Id, new());
idToLocationContainers[locationContainer.Id].Add(locationContainer);
}
}
internal static void Set(int maxDegreeOfParallelism, Configuration configuration, long ticks, List<PersonContainer> personContainers, string? a2PeopleSingletonDirectory, string eDistanceContentDirectory, Dictionary<long, PersonContainer> personKeyToPersonContainer, List<PersonContainer> notMappedPersonContainers, Dictionary<int, List<int>> skipCollection, Dictionary<int, List<int>> skipNotSkipCollection, Dictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers, Dictionary<int, Dictionary<int, PersonContainer[]>> idThenNormalizedRectangleToPersonContainers)
{
if (configuration is null)
throw new NullReferenceException(nameof(configuration));
string message;
int totalSeconds;
List<long> personKeys = new();
@ -680,6 +801,7 @@ internal abstract class MapLogic
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
message = $") {ticksDirectories.Length:000} compile from and clean ticks Director(ies) - B - {totalSeconds} total second(s)";
List<(string, string[], string)> collection = DeleteEmptyDirectoriesAndGetCollection(configuration, personKeyFormattedCollection, ticksDirectories, message);
SetLocationContainers(maxDegreeOfParallelism, configuration, ticks, personContainers, eDistanceContentDirectory, idToLocationContainers, collection);
(int unableToMatchCount, int duplicateCount) = SetCollectionsAndGetUnableToConvertCount(configuration, ticks, personKeyFormattedToNewestPersonKeyFormatted, personKeyFormattedIdThenNormalizedRectangleCollection, collection);
SetKeyValuePairs(configuration, personContainers, personKeyToPersonContainerCollection, personKeyFormattedToPersonContainer, personKeyFormattedIdThenNormalizedRectangleCollection, personKeyToPersonContainer, idThenNormalizedRectangleToPersonContainers, possiblyNewPersonDisplayDirectoryNamesAndPersonContainer);
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
@ -900,27 +1022,6 @@ internal abstract class MapLogic
}
}
internal static List<(long, string)> GetDisplayDirectoryAllFiles(string fileNameExtension, PersonContainer[] personContainers)
{
List<(long, string)> results = new();
List<string> distinct = new();
foreach (PersonContainer personContainer in personContainers)
{
if (personContainer.Key is null)
continue;
foreach (string displayDirectoryAllFile in personContainer.DisplayDirectoryAllFiles)
{
if (!displayDirectoryAllFile.EndsWith(fileNameExtension))
continue;
if (distinct.Contains(displayDirectoryAllFile))
continue;
distinct.Add(displayDirectoryAllFile);
results.Add(new(personContainer.Key.Value, displayDirectoryAllFile));
}
}
return results;
}
internal static Dictionary<int, List<long>> GetIdToPersonKeys(Dictionary<long, List<int>> personKeyToIds)
{
Dictionary<int, List<long>> results = new();

10
Map/Models/Stateless/Methods/IMapLogic.cs
View File

@ -8,11 +8,6 @@ public interface IMapLogic
static Dictionary<int, List<long>> GetIdToPersonKeys(Dictionary<long, List<int>> personKeyToIds) =>
MapLogic.GetIdToPersonKeys(personKeyToIds);
List<(long, string)> TestStatic_GetDisplayDirectoryAllFiles(string fileNameExtension, Shared.Models.PersonContainer[] personContainers) =>
GetDisplayDirectoryAllFiles(fileNameExtension, personContainers);
static List<(long, string)> GetDisplayDirectoryAllFiles(string fileNameExtension, Shared.Models.PersonContainer[] personContainers) =>
MapLogic.GetDisplayDirectoryAllFiles(fileNameExtension, personContainers);
Shared.Models.Mapping[] TestStatic_GetSelectedMappingCollection(List<Shared.Models.Face> distinctFilteredFaces) =>
GetSelectedMappingCollection(distinctFilteredFaces);
static Shared.Models.Mapping[] GetSelectedMappingCollection(List<Shared.Models.Face> distinctFilteredFaces) =>
@ -23,9 +18,4 @@ public interface IMapLogic
static Dictionary<int, Dictionary<int, Shared.Models.Mapping>> GetIdToNormalizedRectangleToFace(Shared.Models.Mapping[] mappingCollection) =>
MapLogic.GetIdToNormalizedRectangleToFace(mappingCollection);
List<(long, string)> TestStatic_DeleteEmptyDirectoriesAndGetMappedFaceFiles(Configuration configuration, Shared.Models.PersonContainer[] personContainers, long ticks, string? a2PeopleContentDirectory, string eDistanceContentDirectory) =>
DeleteEmptyDirectoriesAndGetMappedFaceFiles(configuration, personContainers, ticks, a2PeopleContentDirectory, eDistanceContentDirectory);
static List<(long, string)> DeleteEmptyDirectoriesAndGetMappedFaceFiles(Configuration configuration, Shared.Models.PersonContainer[] personContainers, long ticks, string? a2PeopleContentDirectory, string eDistanceContentDirectory) =>
MapLogic.DeleteEmptyDirectoriesAndGetMappedFaceFiles(configuration, personContainers, ticks, a2PeopleContentDirectory, eDistanceContentDirectory);
}