From b97a939c58cc6c43fea9ac6e3ce2abdd63580dde Mon Sep 17 00:00:00 2001 From: Mike Phares Date: Fri, 14 Apr 2023 02:58:03 -0700 Subject: [PATCH] Moved population of idToLocationContainers in preparation to change to a class over dictionary --- Instance/DlibDotNet.cs | 165 +------------ Map/Models/Configuration.cs | 14 +- Map/Models/MapLogic.cs | 14 +- Map/Models/Stateless/MapLogic.cs | 273 +++++++++++++++------- Map/Models/Stateless/Methods/IMapLogic.cs | 10 - 5 files changed, 214 insertions(+), 262 deletions(-) diff --git a/Instance/DlibDotNet.cs b/Instance/DlibDotNet.cs index eacef44..a86f3cd 100644 --- a/Instance/DlibDotNet.cs +++ b/Instance/DlibDotNet.cs @@ -1,8 +1,6 @@ using Microsoft.Extensions.Configuration; using Phares.Shared; using ShellProgressBar; -using System.Diagnostics; -using System.Drawing; using System.Drawing.Imaging; using View_by_Distance.Distance.Models; using View_by_Distance.Face.Models; @@ -250,8 +248,12 @@ public partial class DlibDotNet private static Map.Models.Configuration Get(Models.Configuration configuration, string facesFileNameExtension, string facesHiddenFileNameExtension, string facePartsFileNameExtension) { Map.Models.Configuration result = new( + configuration.DeletePossibleDuplicates, + configuration.DistanceMoveUnableToMatch, + configuration.DistanceRenameToMatch, configuration.FaceConfidencePercent, configuration.FaceDistancePermyriad, + configuration.LocationDigits, configuration.MappingDefaultName, configuration.PersonBirthdayFirstYear, configuration.PersonBirthdayFormat, @@ -631,7 +633,7 @@ public partial class DlibDotNet sourceDirectoryChanges.Clear(); totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); anyNullOrNoIsUniqueFileName = filteredItems.Any(l => l.IsUniqueFileName is null || !l.IsUniqueFileName.Value); - message = $"{i + 1:000} [{filteredItems.Length:000} collection] / {containersLength:000} - {total} / {t} total collection - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}"; + message = $"{i + 1:000} [{filteredItems.Length:000} collectionB] / {containersLength:000} - {total} / {t} total collectionB - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}"; if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution)) _Faces.SetAngleBracketCollection(dResultsFullGroupDirectory, container.SourceDirectory); if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution)) @@ -857,159 +859,6 @@ public partial class DlibDotNet return result; } - private void ParallelFor(string eDistanceContentDirectory, List> collection, long personKey, string file) - { - const string lnk = ".lnk"; - int? id, normalizedRectangle; - IReadOnlyList directories; - bool fromDistanceContent = !file.EndsWith(lnk) && file.Contains(eDistanceContentDirectory); - if (!file.EndsWith(lnk)) - (id, normalizedRectangle) = Shared.Models.Stateless.Methods.IMapping.GetConverted(_MapConfiguration.FacesFileNameExtension, file); - else - (id, normalizedRectangle) = Shared.Models.Stateless.Methods.IMapping.GetConverted(_MapConfiguration.FacesFileNameExtension, file[..^4]); - if (id is null || normalizedRectangle is null) - return; - if (file.EndsWith(lnk) || (!_Configuration.DistanceMoveUnableToMatch && !_Configuration.DistanceRenameToMatch)) - directories = new List(); - else - directories = MetadataExtractor.ImageMetadataReader.ReadMetadata(file); - Rectangle? rectangle = Shared.Models.Stateless.Methods.ILocation.GetNormalizedRectangle(_Configuration.LocationDigits, normalizedRectangle.Value); - lock (collection) - collection.Add(new(fromDistanceContent, file, personKey, id.Value, normalizedRectangle.Value, directories, rectangle, null)); - } - - private void OpenPossibleDuplicates(List<(long, int, string, double?)> duplicates) - { - string personKeyFormatted; - foreach ((long personKey, int id, string file, double? percent) in duplicates) - { - if (percent is null) - continue; - _ = Process.Start("explorer.exe", string.Concat("\"", Path.GetDirectoryName(file), "\"")); - personKeyFormatted = Shared.Models.Stateless.Methods.IPersonBirthday.GetFormatted(_Configuration.PersonBirthdayFormat, personKey); - } - foreach ((long personKey, int id, string file, double? percent) in duplicates) - { - if (percent is not null && percent.Value == 0) - continue; - _ = Process.Start("explorer.exe", string.Concat("\"", Path.GetDirectoryName(file), "\"")); - personKeyFormatted = Shared.Models.Stateless.Methods.IPersonBirthday.GetFormatted(_Configuration.PersonBirthdayFormat, personKey); - } - foreach ((long personKey, int id, string file, double? percent) in duplicates) - { - if (percent is not null && percent.Value > 0) - continue; - _ = Process.Start("explorer.exe", string.Concat("\"", Path.GetDirectoryName(file), "\"")); - personKeyFormatted = Shared.Models.Stateless.Methods.IPersonBirthday.GetFormatted(_Configuration.PersonBirthdayFormat, personKey); - } - } - - private void LookForPossibleDuplicates(List> collection) - { - string key; - double? percent; - Rectangle? rectangle; - List delete = new(); - Rectangle intersectRectangle; - (string File, int NormalizedRectangle) item; - Dictionary distinct = new(); - List<(long, int, string, double?)> duplicates = new(); - foreach (LocationContainer locationContainer in collection) - { - key = string.Concat(locationContainer.PersonKey, locationContainer.Id); - if (distinct.TryGetValue(key, out item)) - { - if (item.NormalizedRectangle == locationContainer.NormalizedRectangle) - continue; - if (locationContainer.Rectangle is null) - percent = null; - else - { - rectangle = Shared.Models.Stateless.Methods.ILocation.GetNormalizedRectangle(_Configuration.LocationDigits, item.NormalizedRectangle); - if (locationContainer.Rectangle is null || rectangle is null) - percent = null; - else - { - intersectRectangle = Rectangle.Intersect(locationContainer.Rectangle.Value, rectangle.Value); - percent = intersectRectangle.Width * intersectRectangle.Height; - } - } - delete.Add(item.File); - delete.Add(locationContainer.File); - duplicates.Add(new(locationContainer.PersonKey, locationContainer.Id, locationContainer.File, percent)); - continue; - } - distinct.Add(key, new(locationContainer.File, locationContainer.NormalizedRectangle)); - } - if (!_Configuration.DeletePossibleDuplicates) - { - if (duplicates.Any() && _IsEnvironment.Development) - OpenPossibleDuplicates(duplicates); - } - else - { - if (delete.Count > 5) - throw new Exception("Something maybe wrong!"); - foreach (string file in delete) - { - if (File.Exists(file)) - File.Delete(file); - } - } - } - - private List> GetCollection(long ticks, string? a2PeopleContentDirectory, string eDistanceContentDirectory) - { - string file; - List<(long PersonKey, string File)> collection = new(); - List> results = new(); - collection.AddRange(Map.Models.Stateless.Methods.IMapLogic.GetDisplayDirectoryAllFiles(_Faces.FileNameExtension, _PersonContainers)); - collection.AddRange(Map.Models.Stateless.Methods.IMapLogic.DeleteEmptyDirectoriesAndGetMappedFaceFiles(_MapConfiguration, _PersonContainers, ticks, a2PeopleContentDirectory, eDistanceContentDirectory)); - for (int i = collection.Count - 1; i > -1; i--) - { - file = collection[i].File; - if (file.EndsWith(".old")) - { - collection.RemoveAt(i); - continue; - } - if (!file.EndsWith(".dup") && !file.EndsWith(".unk") && !file.EndsWith(".abd")) - continue; - if (!File.Exists(file)) - continue; - File.Move(file, file[..^4]); - collection[i] = new(collection[i].PersonKey, file[..^4]); - } - if (collection.Any() && (_Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch)) - { - int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); - string message = $") Building Mapped Face Files Collection - {totalSeconds} total second(s)"; - ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism }; - ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; - using ProgressBar progressBar = new(collection.Count, message, options); - _ = Parallel.For(0, collection.Count, parallelOptions, (i, state) => - { - progressBar.Tick(); - ParallelFor(eDistanceContentDirectory, results, collection[i].PersonKey, collection[i].File); - }); - } - LookForPossibleDuplicates(results); - return results; - } - - private Dictionary>> GetDictionary(long ticks, string? a2PeopleContentDirectory, string eDistanceContentDirectory) - { - Dictionary>> results = new(); - List> collection = GetCollection(ticks, a2PeopleContentDirectory, eDistanceContentDirectory); - foreach (LocationContainer locationContainer in collection) - { - if (!results.ContainsKey(locationContainer.Id)) - results.Add(locationContainer.Id, new()); - results[locationContainer.Id].Add(locationContainer); - } - return results; - } - private static void LookForAbandoned(Dictionary>> idToLocationContainers, List distinctFilteredIds) { List renameCollection = new(); @@ -1057,13 +906,14 @@ public partial class DlibDotNet Dictionary> personKeyToIds; Dictionary> fileNameToCollection; (aResultsFullGroupDirectory, bResultsFullGroupDirectory) = GetResultsFullGroupDirectories(); + Dictionary>> idToLocationContainers = new(); string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "{}"); a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "([])"); eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), "()"); fPhotoPrismContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), "()"); fPhotoPrismSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), "{}"); propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory); - MapLogic mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _PersonContainers, ticks, a2PeopleSingletonDirectory, eDistanceContentDirectory); + MapLogic mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _PersonContainers, ticks, a2PeopleSingletonDirectory, eDistanceContentDirectory, idToLocationContainers); int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); string message = $") Building Container(s) - {totalSeconds} total second(s)"; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; @@ -1100,7 +950,6 @@ public partial class DlibDotNet if (!string.IsNullOrEmpty(_Configuration.GenealogicalDataCommunicationFile) && !string.IsNullOrEmpty(a2PeopleContentDirectory) && _GenealogicalDataCommunicationHeaderLines is not null && _GenealogicalDataCommunicationFooterLines is not null && _GenealogicalDataCommunicationHeaderLines.Any() && _GenealogicalDataCommunicationFooterLines.Any()) Shared.Models.Stateless.Methods.IGenealogicalDataCommunication.CreateTree(_Configuration.MappingDefaultName, _Configuration.PersonBirthdayFormat, _Configuration.PropertyConfiguration.ResultAllInOne, _PersonContainers, _GenealogicalDataCommunicationHeaderLines, _GenealogicalDataCommunicationFooterLines, ticks, a2PeopleContentDirectory, personKeyToIds); fileNameToCollection = !Directory.Exists(fPhotoPrismSingletonDirectory) ? fileNameToCollection = new() : F_PhotoPrism.GetFileNameToCollection(fPhotoPrismSingletonDirectory); - Dictionary>> idToLocationContainers = GetDictionary(ticks, a2PeopleContentDirectory, eDistanceContentDirectory); FullDoWork(argZero, propertyRoot, ticks, aResultsFullGroupDirectory, bResultsFullGroupDirectory, t, containers, propertyLogic, metadata, eDistanceContentDirectory, fileNameToCollection, idToLocationContainers, mapLogic); _Distance.Clear(); if (!personKeyToIds.Any()) diff --git a/Map/Models/Configuration.cs b/Map/Models/Configuration.cs index da085be..8f38bd0 100644 --- a/Map/Models/Configuration.cs +++ b/Map/Models/Configuration.cs @@ -6,11 +6,15 @@ namespace View_by_Distance.Map.Models; public class Configuration { + public bool DeletePossibleDuplicates { get; internal set; } + public bool DistanceMoveUnableToMatch { init; get; } + public bool DistanceRenameToMatch { init; get; } public int FaceConfidencePercent { init; get; } public int FaceDistancePermyriad { init; get; } public string FacePartsFileNameExtension { init; get; } public string FacesFileNameExtension { init; get; } public string FacesHiddenFileNameExtension { init; get; } + public int LocationDigits { init; get; } public string MappingDefaultName { init; get; } public int PersonBirthdayFirstYear { init; get; } public string PersonBirthdayFormat { init; get; } @@ -23,8 +27,12 @@ public class Configuration public int SortingMinimumToUseSigma { init; get; } [JsonConstructor] - public Configuration(int faceConfidencePercent, + public Configuration(bool deletePossibleDuplicates, + bool distanceMoveUnableToMatch, + bool distanceRenameToMatch, + int faceConfidencePercent, int faceDistancePermyriad, + int locationDigits, string mappingDefaultName, int personBirthdayFirstYear, string personBirthdayFormat, @@ -39,18 +47,22 @@ public class Configuration string facesHiddenFileNameExtension, string facePartsFileNameExtension) { + LocationDigits = locationDigits; PersonCharacters = personCharacters; MappingDefaultName = mappingDefaultName; PersonBirthdayFormat = personBirthdayFormat; SortingMaximumPerKey = sortingMaximumPerKey; + DistanceRenameToMatch = distanceRenameToMatch; FaceConfidencePercent = faceConfidencePercent; FaceDistancePermyriad = faceDistancePermyriad; FacesFileNameExtension = facesFileNameExtension; SkipNotSkipDirectories = skipNotSkipDirectories; PersonBirthdayFirstYear = personBirthdayFirstYear; RangeDistanceTolerance = rangeDistanceTolerance[1]; + DeletePossibleDuplicates = deletePossibleDuplicates; SaveSortingWithoutPerson = saveSortingWithoutPerson; SortingMinimumToUseSigma = sortingMinimumToUseSigma; + DistanceMoveUnableToMatch = distanceMoveUnableToMatch; RangeDaysDeltaTolerance = rangeDaysDeltaTolerance[1]; FacePartsFileNameExtension = facePartsFileNameExtension; FacesHiddenFileNameExtension = facesHiddenFileNameExtension; diff --git a/Map/Models/MapLogic.cs b/Map/Models/MapLogic.cs index a491aa7..c026b3a 100644 --- a/Map/Models/MapLogic.cs +++ b/Map/Models/MapLogic.cs @@ -23,24 +23,22 @@ public class MapLogic : Shared.Models.Methods.IMapLogic private readonly long _Ticks; private readonly Serilog.ILogger? _Log; - private readonly int _MaxDegreeOfParallelism; private readonly Configuration? _Configuration; private readonly string _EDistanceContentTicksDirectory; private readonly Shared.Models.Properties.IPropertyConfiguration _PropertyConfiguration; - public MapLogic(int maxDegreeOfParallelism, Shared.Models.Properties.IPropertyConfiguration propertyConfiguration, Configuration? configuration, PersonContainer[] personContainers, long ticks, string a2PeopleSingletonDirectory, string eDistanceContentDirectory) + public MapLogic(int maxDegreeOfParallelism, Shared.Models.Properties.IPropertyConfiguration propertyConfiguration, Configuration? configuration, PersonContainer[] personContainers, long ticks, string a2PeopleSingletonDirectory, string eDistanceContentDirectory, Dictionary>> idToLocationContainers) { _Ticks = ticks; _Configuration = configuration; _Log = Serilog.Log.ForContext(); _PropertyConfiguration = propertyConfiguration; - _MaxDegreeOfParallelism = maxDegreeOfParallelism; if (_Log is null) { } + if (idToLocationContainers is null) + throw new NullReferenceException(nameof(idToLocationContainers)); if (propertyConfiguration.VerifyToSeason is null || !propertyConfiguration.VerifyToSeason.Any()) - throw new Exception(); - if (_MaxDegreeOfParallelism == 0) - { } + throw new NullReferenceException(nameof(propertyConfiguration.VerifyToSeason)); string json; string fullPath; List>? collection; @@ -58,7 +56,8 @@ public class MapLogic : Shared.Models.Methods.IMapLogic if (configuration is not null) { List personContainerCollection = new(personContainers); - Stateless.MapLogic.Set(configuration, + Stateless.MapLogic.Set(maxDegreeOfParallelism, + configuration, ticks, personContainerCollection, a2PeopleSingletonDirectory, @@ -67,6 +66,7 @@ public class MapLogic : Shared.Models.Methods.IMapLogic notMappedPersonContainers, skipCollection, skipNotSkipCollection, + idToLocationContainers, idThenNormalizedRectangleToPersonContainers); if (personContainerCollection.Count == personContainers.Length) throw new NotSupportedException(); diff --git a/Map/Models/Stateless/MapLogic.cs b/Map/Models/Stateless/MapLogic.cs index 343dddc..9c399cb 100644 --- a/Map/Models/Stateless/MapLogic.cs +++ b/Map/Models/Stateless/MapLogic.cs @@ -1,5 +1,7 @@ using Humanizer; using ShellProgressBar; +using System.Diagnostics; +using System.Drawing; using System.Text.Json; using View_by_Distance.Shared.Models; using View_by_Distance.Shared.Models.Stateless.Methods; @@ -162,7 +164,7 @@ internal abstract class MapLogic personDisplayDirectory = Path.Combine(matchDirectory, alphaDirectoryName); if (!Directory.Exists(personDisplayDirectory) || !Directory.Exists(matchDirectory)) continue; - _ = System.Diagnostics.Process.Start("explorer", matchDirectory); + _ = Process.Start("explorer", matchDirectory); for (int i = 0; i < int.MaxValue; i++) { Thread.Sleep(500); @@ -301,67 +303,6 @@ internal abstract class MapLogic return results; } - private static List GetPersonKeyFormattedCollection(Configuration configuration, PersonContainer[] personContainers, List<(long? PersonKey, string Line)> lines) - { - List results = new(); - string personKeyFormatted; - foreach (PersonContainer personContainer in personContainers) - { - lines.AddRange(IPersonContainer.GetDisplay(configuration.PersonBirthdayFormat, personContainer)); - if (personContainer.Person is null || personContainer.Key is null || personContainer.Birthdays is null || !personContainer.Birthdays.Any()) - continue; - foreach (PersonBirthday personBirthday in personContainer.Birthdays) - { - personKeyFormatted = IPersonBirthday.GetFormatted(configuration.PersonBirthdayFormat, personBirthday); - results.Add(personKeyFormatted); - } - } - return results; - } - - internal static List<(long, string)> DeleteEmptyDirectoriesAndGetMappedFaceFiles(Configuration configuration, PersonContainer[] personContainers, long ticks, string? a2PeopleContentDirectory, string eDistanceContentDirectory) - { - List<(long, string)> results = new(); - List<(long? PersonKey, string Line)> lines = new(); - _ = GetDistinctCollection(configuration, personContainers, new(), new()); - int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); - string[] ticksDirectories; - if (string.IsNullOrEmpty(eDistanceContentDirectory)) - ticksDirectories = Array.Empty(); - else - { - if (!Directory.Exists(eDistanceContentDirectory)) - _ = Directory.CreateDirectory(eDistanceContentDirectory); - else - { - _ = IPath.DeleteEmptyDirectories(eDistanceContentDirectory); - if (!Directory.Exists(eDistanceContentDirectory)) - _ = Directory.CreateDirectory(eDistanceContentDirectory); - } - ticksDirectories = Directory.GetDirectories(eDistanceContentDirectory, "*", SearchOption.TopDirectoryOnly); - } - long personKey; - List distinct = new(); - PersonBirthday? personBirthday; - string message = $") {ticksDirectories.Length:000} collect from and clean ticks Director(ies) - A - {totalSeconds} total second(s)"; - List personKeyFormattedCollection = GetPersonKeyFormattedCollection(configuration, personContainers, lines); - if (!string.IsNullOrEmpty(a2PeopleContentDirectory)) - File.WriteAllLines(Path.Combine(a2PeopleContentDirectory, "People - C.tsv"), from l in lines select l.Line); - List<(string, string[], string)> collection = DeleteEmptyDirectoriesAndGetCollection(configuration, personKeyFormattedCollection, ticksDirectories, message); - foreach ((string personKeyFormatted, _, string mappedFaceFile) in collection) - { - personBirthday = IPersonBirthday.GetPersonBirthday(configuration.PersonBirthdayFormat, personKeyFormatted); - if (personBirthday is null) - continue; - if (distinct.Contains(mappedFaceFile)) - continue; - distinct.Add(mappedFaceFile); - personKey = personBirthday.Value.Ticks; - results.Add(new(personKey, mappedFaceFile)); - } - return results; - } - private static PersonContainer[] GetDistinctPersonContainers(List personContainers) { List results = new(); @@ -660,10 +601,190 @@ internal abstract class MapLogic return results; } - internal static void Set(Configuration? configuration, long ticks, List personContainers, string? a2PeopleSingletonDirectory, string eDistanceContentDirectory, Dictionary personKeyToPersonContainer, List notMappedPersonContainers, Dictionary> skipCollection, Dictionary> skipNotSkipCollection, Dictionary> idThenNormalizedRectangleToPersonContainers) + private static List<(long, string)> GetDisplayDirectoryAllFiles(string fileNameExtension, List personContainers) + { + List<(long, string)> results = new(); + List distinct = new(); + foreach (PersonContainer personContainer in personContainers) + { + if (personContainer.Key is null) + continue; + foreach (string displayDirectoryAllFile in personContainer.DisplayDirectoryAllFiles) + { + if (!displayDirectoryAllFile.EndsWith(fileNameExtension)) + continue; + if (distinct.Contains(displayDirectoryAllFile)) + continue; + distinct.Add(displayDirectoryAllFile); + results.Add(new(personContainer.Key.Value, displayDirectoryAllFile)); + } + } + return results; + } + + private static List<(long PersonKey, string File)> GetCollection(Configuration configuration, List personContainers, List<(string, string[], string)> collection) + { + List<(long PersonKey, string File)> results = new(); + string file; + long personKey; + List distinct = new(); + PersonBirthday? personBirthday; + results.AddRange(GetDisplayDirectoryAllFiles(configuration.FacesFileNameExtension, personContainers)); + foreach ((string personKeyFormatted, _, string mappedFaceFile) in collection) + { + personBirthday = IPersonBirthday.GetPersonBirthday(configuration.PersonBirthdayFormat, personKeyFormatted); + if (personBirthday is null) + continue; + if (distinct.Contains(mappedFaceFile)) + continue; + distinct.Add(mappedFaceFile); + personKey = personBirthday.Value.Ticks; + results.Add(new(personKey, mappedFaceFile)); + } + for (int i = results.Count - 1; i > -1; i--) + { + file = results[i].File; + if (file.EndsWith(".old")) + { + results.RemoveAt(i); + continue; + } + if (!file.EndsWith(".dup") && !file.EndsWith(".unk") && !file.EndsWith(".abd")) + continue; + if (!File.Exists(file)) + continue; + File.Move(file, file[..^4]); + results[i] = new(results[i].PersonKey, file[..^4]); + } + return results; + } + + private static void ParallelFor(Configuration configuration, string eDistanceContentDirectory, List> collection, long personKey, string file) + { + const string lnk = ".lnk"; + int? id, normalizedRectangle; + IReadOnlyList directories; + bool fromDistanceContent = !file.EndsWith(lnk) && file.Contains(eDistanceContentDirectory); + if (!file.EndsWith(lnk)) + (id, normalizedRectangle) = IMapping.GetConverted(configuration.FacesFileNameExtension, file); + else + (id, normalizedRectangle) = IMapping.GetConverted(configuration.FacesFileNameExtension, file[..^4]); + if (id is null || normalizedRectangle is null) + return; + if (file.EndsWith(lnk) || (!configuration.DistanceMoveUnableToMatch && !configuration.DistanceRenameToMatch)) + directories = new List(); + else + directories = MetadataExtractor.ImageMetadataReader.ReadMetadata(file); + Rectangle? rectangle = ILocation.GetNormalizedRectangle(configuration.LocationDigits, normalizedRectangle.Value); + lock (collection) + collection.Add(new(fromDistanceContent, file, personKey, id.Value, normalizedRectangle.Value, directories, rectangle, null)); + } + + private static void OpenPossibleDuplicates(Configuration configuration, List<(long, int, string, double?)> duplicates) + { + string personKeyFormatted; + foreach ((long personKey, int id, string file, double? percent) in duplicates) + { + if (percent is null) + continue; + _ = Process.Start("explorer.exe", string.Concat("\"", Path.GetDirectoryName(file), "\"")); + personKeyFormatted = IPersonBirthday.GetFormatted(configuration.PersonBirthdayFormat, personKey); + } + foreach ((long personKey, int id, string file, double? percent) in duplicates) + { + if (percent is not null && percent.Value == 0) + continue; + _ = Process.Start("explorer.exe", string.Concat("\"", Path.GetDirectoryName(file), "\"")); + personKeyFormatted = IPersonBirthday.GetFormatted(configuration.PersonBirthdayFormat, personKey); + } + foreach ((long personKey, int id, string file, double? percent) in duplicates) + { + if (percent is not null && percent.Value > 0) + continue; + _ = Process.Start("explorer.exe", string.Concat("\"", Path.GetDirectoryName(file), "\"")); + personKeyFormatted = IPersonBirthday.GetFormatted(configuration.PersonBirthdayFormat, personKey); + } + } + + private static void LookForPossibleDuplicates(Configuration configuration, List> collection) + { + string key; + double? percent; + Rectangle? rectangle; + List delete = new(); + Rectangle intersectRectangle; + (string File, int NormalizedRectangle) item; + Dictionary distinct = new(); + List<(long, int, string, double?)> duplicates = new(); + foreach (LocationContainer locationContainer in collection) + { + key = string.Concat(locationContainer.PersonKey, locationContainer.Id); + if (distinct.TryGetValue(key, out item)) + { + if (item.NormalizedRectangle == locationContainer.NormalizedRectangle) + continue; + if (locationContainer.Rectangle is null) + percent = null; + else + { + rectangle = ILocation.GetNormalizedRectangle(configuration.LocationDigits, item.NormalizedRectangle); + if (locationContainer.Rectangle is null || rectangle is null) + percent = null; + else + { + intersectRectangle = Rectangle.Intersect(locationContainer.Rectangle.Value, rectangle.Value); + percent = intersectRectangle.Width * intersectRectangle.Height; + } + } + delete.Add(item.File); + delete.Add(locationContainer.File); + duplicates.Add(new(locationContainer.PersonKey, locationContainer.Id, locationContainer.File, percent)); + continue; + } + distinct.Add(key, new(locationContainer.File, locationContainer.NormalizedRectangle)); + } + if (!configuration.DeletePossibleDuplicates && duplicates.Any()) + OpenPossibleDuplicates(configuration, duplicates); + else + { + if (delete.Count > 5) + throw new Exception("Something maybe wrong!"); + foreach (string file in delete) + { + if (File.Exists(file)) + File.Delete(file); + } + } + } + + private static void SetLocationContainers(int maxDegreeOfParallelism, Configuration configuration, long ticks, List personContainers, string eDistanceContentDirectory, Dictionary>> idToLocationContainers, List<(string, string[], string)> sourceCollection) + { + List> results = new(); + List<(long PersonKey, string File)> collection = GetCollection(configuration, personContainers, sourceCollection); + if (collection.Any() && (configuration.DistanceMoveUnableToMatch || configuration.DistanceRenameToMatch)) + { + int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); + string message = $") Building Mapped Face Files Collection - {totalSeconds} total second(s)"; + ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; + ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; + using ProgressBar progressBar = new(collection.Count, message, options); + _ = Parallel.For(0, collection.Count, parallelOptions, (i, state) => + { + progressBar.Tick(); + ParallelFor(configuration, eDistanceContentDirectory, results, collection[i].PersonKey, collection[i].File); + }); + } + LookForPossibleDuplicates(configuration, results); + foreach (LocationContainer locationContainer in results) + { + if (!idToLocationContainers.ContainsKey(locationContainer.Id)) + idToLocationContainers.Add(locationContainer.Id, new()); + idToLocationContainers[locationContainer.Id].Add(locationContainer); + } + } + + internal static void Set(int maxDegreeOfParallelism, Configuration configuration, long ticks, List personContainers, string? a2PeopleSingletonDirectory, string eDistanceContentDirectory, Dictionary personKeyToPersonContainer, List notMappedPersonContainers, Dictionary> skipCollection, Dictionary> skipNotSkipCollection, Dictionary>> idToLocationContainers, Dictionary> idThenNormalizedRectangleToPersonContainers) { - if (configuration is null) - throw new NullReferenceException(nameof(configuration)); string message; int totalSeconds; List personKeys = new(); @@ -680,6 +801,7 @@ internal abstract class MapLogic totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); message = $") {ticksDirectories.Length:000} compile from and clean ticks Director(ies) - B - {totalSeconds} total second(s)"; List<(string, string[], string)> collection = DeleteEmptyDirectoriesAndGetCollection(configuration, personKeyFormattedCollection, ticksDirectories, message); + SetLocationContainers(maxDegreeOfParallelism, configuration, ticks, personContainers, eDistanceContentDirectory, idToLocationContainers, collection); (int unableToMatchCount, int duplicateCount) = SetCollectionsAndGetUnableToConvertCount(configuration, ticks, personKeyFormattedToNewestPersonKeyFormatted, personKeyFormattedIdThenNormalizedRectangleCollection, collection); SetKeyValuePairs(configuration, personContainers, personKeyToPersonContainerCollection, personKeyFormattedToPersonContainer, personKeyFormattedIdThenNormalizedRectangleCollection, personKeyToPersonContainer, idThenNormalizedRectangleToPersonContainers, possiblyNewPersonDisplayDirectoryNamesAndPersonContainer); totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); @@ -900,27 +1022,6 @@ internal abstract class MapLogic } } - internal static List<(long, string)> GetDisplayDirectoryAllFiles(string fileNameExtension, PersonContainer[] personContainers) - { - List<(long, string)> results = new(); - List distinct = new(); - foreach (PersonContainer personContainer in personContainers) - { - if (personContainer.Key is null) - continue; - foreach (string displayDirectoryAllFile in personContainer.DisplayDirectoryAllFiles) - { - if (!displayDirectoryAllFile.EndsWith(fileNameExtension)) - continue; - if (distinct.Contains(displayDirectoryAllFile)) - continue; - distinct.Add(displayDirectoryAllFile); - results.Add(new(personContainer.Key.Value, displayDirectoryAllFile)); - } - } - return results; - } - internal static Dictionary> GetIdToPersonKeys(Dictionary> personKeyToIds) { Dictionary> results = new(); diff --git a/Map/Models/Stateless/Methods/IMapLogic.cs b/Map/Models/Stateless/Methods/IMapLogic.cs index e43d041..6947c9b 100644 --- a/Map/Models/Stateless/Methods/IMapLogic.cs +++ b/Map/Models/Stateless/Methods/IMapLogic.cs @@ -8,11 +8,6 @@ public interface IMapLogic static Dictionary> GetIdToPersonKeys(Dictionary> personKeyToIds) => MapLogic.GetIdToPersonKeys(personKeyToIds); - List<(long, string)> TestStatic_GetDisplayDirectoryAllFiles(string fileNameExtension, Shared.Models.PersonContainer[] personContainers) => - GetDisplayDirectoryAllFiles(fileNameExtension, personContainers); - static List<(long, string)> GetDisplayDirectoryAllFiles(string fileNameExtension, Shared.Models.PersonContainer[] personContainers) => - MapLogic.GetDisplayDirectoryAllFiles(fileNameExtension, personContainers); - Shared.Models.Mapping[] TestStatic_GetSelectedMappingCollection(List distinctFilteredFaces) => GetSelectedMappingCollection(distinctFilteredFaces); static Shared.Models.Mapping[] GetSelectedMappingCollection(List distinctFilteredFaces) => @@ -23,9 +18,4 @@ public interface IMapLogic static Dictionary> GetIdToNormalizedRectangleToFace(Shared.Models.Mapping[] mappingCollection) => MapLogic.GetIdToNormalizedRectangleToFace(mappingCollection); - List<(long, string)> TestStatic_DeleteEmptyDirectoriesAndGetMappedFaceFiles(Configuration configuration, Shared.Models.PersonContainer[] personContainers, long ticks, string? a2PeopleContentDirectory, string eDistanceContentDirectory) => - DeleteEmptyDirectoriesAndGetMappedFaceFiles(configuration, personContainers, ticks, a2PeopleContentDirectory, eDistanceContentDirectory); - static List<(long, string)> DeleteEmptyDirectoriesAndGetMappedFaceFiles(Configuration configuration, Shared.Models.PersonContainer[] personContainers, long ticks, string? a2PeopleContentDirectory, string eDistanceContentDirectory) => - MapLogic.DeleteEmptyDirectoriesAndGetMappedFaceFiles(configuration, personContainers, ticks, a2PeopleContentDirectory, eDistanceContentDirectory); - } \ No newline at end of file