From 9ec5da1e7ad5555906eb4cfc3a312b1473d7dc98 Mon Sep 17 00:00:00 2001 From: Mike Phares Date: Sat, 17 Sep 2022 11:25:02 -0700 Subject: [PATCH] Person Require People File --- Compare/Compare.cs | 3 +- Instance/DlibDotNet.cs | 118 +++++++++------------ Instance/Models/Binder/Configuration.cs | 4 + Instance/Models/Configuration.cs | 3 + Instance/Models/_A2_People.cs | 2 +- Instance/appsettings.Development.json | 1 + Instance/appsettings.Staging.json | 1 + Instance/appsettings.json | 1 + Map/Models/MapLogic.cs | 62 +++++------ Map/Models/Stateless/ByRef.cs | 28 +++-- Shared/Models/Stateless/Methods/IPerson.cs | 8 +- Shared/Models/Stateless/Methods/Person.cs | 21 ++-- 12 files changed, 125 insertions(+), 127 deletions(-) diff --git a/Compare/Compare.cs b/Compare/Compare.cs index 188cc77..db4c96b 100644 --- a/Compare/Compare.cs +++ b/Compare/Compare.cs @@ -56,8 +56,9 @@ public class Compare PredictorModel? predictorModel = null; string peopleDateGroupDirectory = string.Empty; string zResultsFullGroupDirectory = string.Empty; + Map.Models.Configuration? mapConfiguration = null; Shared.Models.Person[] people = Array.Empty(); - Map.Models.MapLogic mapLogic = new(_AppSettings.MaxDegreeOfParallelism, propertyConfiguration, outputExtension, ticks, people, peopleDateGroupDirectory, zResultsFullGroupDirectory); + Map.Models.MapLogic mapLogic = new(_AppSettings.MaxDegreeOfParallelism, propertyConfiguration, mapConfiguration, outputExtension, ticks, people, peopleDateGroupDirectory, zResultsFullGroupDirectory); A_Property propertyLogic = GetPropertyLogic(reverse, model, outputExtension, predictorModel, mapLogic); foreach (string spelling in configuration.Spelling) { diff --git a/Instance/DlibDotNet.cs b/Instance/DlibDotNet.cs index 7e1fdca..2f6bdc1 100644 --- a/Instance/DlibDotNet.cs +++ b/Instance/DlibDotNet.cs @@ -33,15 +33,18 @@ public partial class DlibDotNet private readonly IsEnvironment _IsEnvironment; private readonly Models.Configuration _Configuration; private readonly bool _ArgZeroIsConfigurationRootDirectory; + private readonly Map.Models.Configuration _MapConfiguration; private readonly List> _FileKeyValuePairs; private readonly Dictionary>> _FilePropertiesKeyValuePairs; public DlibDotNet(List args, IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, AppSettings appSettings, string workingDirectory, bool isSilent, IConsole console) { string argZero; + string message; Person[] people; _AppSettings = appSettings; _IsEnvironment = isEnvironment; + long ticks = DateTime.Now.Ticks; _Exceptions = new List(); _Log = Serilog.Log.ForContext(); _FileKeyValuePairs = new List>(); @@ -55,6 +58,7 @@ public partial class DlibDotNet _Configuration = configuration; _Index = new G_Index(configuration); _Random = new F_Random(configuration); + _MapConfiguration = Get(configuration); _People = new A2_People(configuration); _Rename = new E3_Rename(configuration); if (configuration.IgnoreExtensions is null) @@ -72,7 +76,14 @@ public partial class DlibDotNet if (_FirstRun || !_ArgZeroIsConfigurationRootDirectory) people = Array.Empty(); else + { + int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); + message = $") Building People Collection - {totalSeconds} total second(s)"; + ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; + using ProgressBar progressBar = new(1, message, options); + progressBar.Tick(); people = _People.GetPeople(propertyConfiguration); + } if (!isSilent && configuration.TestDistanceResults) { E2_Navigate e2Navigate = new(console, configuration, argZero); @@ -93,10 +104,9 @@ public partial class DlibDotNet _Resize = new C_Resize(configuration.ForceResizeLastWriteTimeToCreationTime, configuration.OverrideForResizeImages, configuration.PropertiesChangedForResize, configuration.ValidResolutions, imageCodecInfo, encoderParameters, filenameExtension); } if (!configuration.SkipSearch) - Search(model, predictorModel, argZero, propertyRoot, people); + Search(ticks, model, predictorModel, argZero, propertyRoot, people); if (!_FirstRun && !_IsEnvironment.Development && _Exceptions.Count == 0 && _ArgZeroIsConfigurationRootDirectory) { - long ticks = DateTime.Now.Ticks; List directoryCollections = _Rename.GetDirectoryRenameCollections(propertyConfiguration, model, predictorModel, relativePath: string.Empty, newDirectoryName: string.Empty, jsonFiles4InfoAny: false); foreach (string[] directoryCollection in directoryCollections) { @@ -109,7 +119,7 @@ public partial class DlibDotNet if (appSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories)); } - string message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}"; + message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}"; _Log.Information(message); if (_Exceptions.Count != 0) throw new Exception(message); @@ -254,6 +264,32 @@ public partial class DlibDotNet throw new Exception("Configuration has to match interface!"); } + private static Map.Models.Configuration Get(Models.Configuration configuration) + { + Map.Models.Configuration result = new(configuration.FaceDistanceHiddenImageFactor, + configuration.FaceDistancePermyriad, + configuration.FaceDistanceMinimumConfidence, + configuration.FaceDistanceTolerance, + configuration.LocationDigits, + configuration.LocationFactor, + configuration.MapLogicSigma, + configuration.MappingSaveFaceEncoding, + configuration.MappingSaveNotMapped, + configuration.MappingSaveMapped, + configuration.MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping, + configuration.MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping, + configuration.PersonBirthdayFirstYear, + configuration.PersonBirthdayFormat, + configuration.PersonKeyFormat, + configuration.SortingDaysDeltaTolerance, + configuration.SortingFacesToSkipAfterSortBeforeLoad, + configuration.SortingFacesToTakeAfterSortBeforeLoad, + configuration.SortingMaximumPerFaceShouldBeHigh, + configuration.SortingMaximumPerKey, + configuration.SortingSigma); + return result; + } + private void FullParallelForWork(A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List> sourceDirectoryChanges, List propertyFileHolderCollection, List propertyCollection, List>> metadataCollections, List> resizeKeyValuePairs, List?> imageFaceCollections, Container container, int index, Item item) { if (item.ImageFileHolder is null) @@ -654,24 +690,13 @@ public partial class DlibDotNet } } - private string GetMapLogicResultsFullGroupDirectory(Model? model, PredictorModel? predictorModel) - { - const int zero = 0; - string outputResolution = _Configuration.OutputResolutions[zero]; - string zResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( - _Configuration.PropertyConfiguration, model, predictorModel, $"Z_{nameof(Item)}", outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false); - return zResultsFullGroupDirectory; - } - private List SetMappingThenGetDistinctFilteredFacesWithMapping(string argZero, Container[] containers) { List results = new(); - bool added; Mapping mapping; bool? isWrongYear; Item[] filteredItems; DateTime minimumDateTime; - List distinctCheck = new(); MappingFromItem mappingFromItem; MappingFromPerson mappingFromPerson; string deterministicHashCodeKeyDisplay; @@ -691,7 +716,6 @@ public partial class DlibDotNet continue; if (!item.Faces.Any()) continue; - added = false; foreach (Face face in item.Faces) { if (face.RelativePath != item.RelativePath) @@ -707,32 +731,13 @@ public partial class DlibDotNet mapping = new(mappingFromItem, mappingFromLocation, mappingFromPerson); face.SetMapping(mapping); results.Add(face); - added = true; } - if (added) - distinctCheck.Add(item.Property.Id.Value); } } return results; } - private Item[] GetIdSortedFilteredItems(string argZero, Container[] containers) - { - Item[] results; - List collection = new(); - foreach (Container container in containers) - { - if (!container.Items.Any()) - continue; - if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero)) - continue; - collection.AddRange(from l in container.Items where l.ImageFileHolder is not null && (l.Abandoned is null || !l.Abandoned.Value) && l.ValidImageFormatExtension && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l); - } - results = (from l in collection orderby l.Property?.Id is not null, l.Property?.Id select l).ToArray(); - return results; - } - - private void DistanceThenMapLogic(string argZero, long ticks, Person[] people, Container[] containers, MapLogic? mapLogic, string dResultsFullGroupDirectory, string eResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string zResultsFullGroupDirectory, string peopleDateGroupDirectory, string outputResolution) + private void DistanceThenMapLogic(string argZero, long ticks, Person[] people, Container[] containers, MapLogic? mapLogic, string dResultsFullGroupDirectory, string eResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string peopleDateGroupDirectory, string outputResolution) { E_Distance distance = new(); if (string.IsNullOrEmpty(eResultsFullGroupDirectory)) @@ -741,48 +746,25 @@ public partial class DlibDotNet string eDistanceContentDirectory = Path.Combine(eResultsFullGroupDirectory, "([])"); if (!Directory.Exists(eDistanceContentDirectory)) _ = Directory.CreateDirectory(eDistanceContentDirectory); - Map.Models.Configuration configuration = new(_Configuration.FaceDistanceHiddenImageFactor, - _Configuration.FaceDistancePermyriad, - _Configuration.FaceDistanceMinimumConfidence, - _Configuration.FaceDistanceTolerance, - _Configuration.LocationDigits, - _Configuration.LocationFactor, - _Configuration.MapLogicSigma, - _Configuration.MappingSaveFaceEncoding, - _Configuration.MappingSaveNotMapped, - _Configuration.MappingSaveMapped, - _Configuration.MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForAddToMapping, - _Configuration.MappingUseDeterministicHashCodeUnknownFaceKeyValuePairsForSaveMapping, - _Configuration.PersonBirthdayFirstYear, - _Configuration.PersonBirthdayFormat, - _Configuration.PersonKeyFormat, - _Configuration.SortingDaysDeltaTolerance, - _Configuration.SortingFacesToSkipAfterSortBeforeLoad, - _Configuration.SortingFacesToTakeAfterSortBeforeLoad, - _Configuration.SortingMaximumPerFaceShouldBeHigh, - _Configuration.SortingMaximumPerKey, - _Configuration.SortingSigma); - if (mapLogic is not null) - mapLogic.Update(configuration); string eDistanceContentFileName = Path.Combine(eDistanceContentDirectory, $"{_Configuration.PropertyConfiguration.ResultAllInOne}.tvs"); List distinctFilteredFaces = SetMappingThenGetDistinctFilteredFacesWithMapping(argZero, containers); - List selectedFilteredFaces = E_Distance.GetSelectedFilteredFaces(configuration, distinctFilteredFaces); + List selectedFilteredFaces = E_Distance.GetSelectedFilteredFaces(_MapConfiguration, distinctFilteredFaces); E_Distance.SetFaceDistances(_AppSettings.MaxDegreeOfParallelism, ticks, selectedFilteredFaces); if (mapLogic is null) - mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FilenameExtension, _Faces.FilenameExtension, _Faces.HiddenFilenameExtension, _FaceParts.FilenameExtension, ticks, people, peopleDateGroupDirectory, zResultsFullGroupDirectory, configuration, distinctFilteredFaces, distance); + mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _Resize.FilenameExtension, _Faces.FilenameExtension, _Faces.HiddenFilenameExtension, _FaceParts.FilenameExtension, ticks, people, peopleDateGroupDirectory, eDistanceContentDirectory, distinctFilteredFaces, distance); mapLogic.SetPersonTicks(distinctFilteredFaces); - SortingContainer[] sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, configuration, ticks, mapLogic, selectedFilteredFaces); + SortingContainer[] sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, selectedFilteredFaces); E_Distance.SaveFaceDistances(eDistanceContentFileName, sortingContainers); int totalNotMapped = mapLogic.AddToMapping(distinctFilteredFaces); if (totalNotMapped > 0) mapLogic.ForceSingleImageThenSaveMapping(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, dFacesContentDirectory, d2ResultsFullGroupDirectory, distinctFilteredFaces, sortingContainers, totalNotMapped); - if (configuration.MappingSaveNotMapped) + if (_MapConfiguration.MappingSaveNotMapped) mapLogic.SaveNotMappedTicks(_Configuration.PropertyConfiguration); if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution)) mapLogic.SaveShortcuts(_Configuration.JuliePhares, distinctFilteredFaces); } - private void Search(Model? model, PredictorModel? predictorModel, string argZero, string propertyRoot, Person[] people) + private void Search(long ticks, Model? model, PredictorModel? predictorModel, string argZero, string propertyRoot, Person[] people) { int j; int f; @@ -790,15 +772,13 @@ public partial class DlibDotNet MapLogic? mapLogic; Container[] containers; A_Property propertyLogic; - long ticks = DateTime.Now.Ticks; string aResultsFullGroupDirectory; string bResultsFullGroupDirectory; string cResultsFullGroupDirectory; string dResultsFullGroupDirectory; string eResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; - string zResultsFullGroupDirectory = GetMapLogicResultsFullGroupDirectory(model, predictorModel); - string peopleDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People)); + string a2PeopleDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People)); if (!_FirstRun) { mapLogic = null; @@ -806,7 +786,9 @@ public partial class DlibDotNet } else { - mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FilenameExtension, _Faces.FilenameExtension, _Faces.HiddenFilenameExtension, _FaceParts.FilenameExtension, ticks, people, peopleDateGroupDirectory, zResultsFullGroupDirectory); + string outputResolution = _Configuration.OutputResolutions[^1]; + (aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory) = GetResultsFullGroupDirectories(model, predictorModel, outputResolution); + mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _Resize.FilenameExtension, _Faces.FilenameExtension, _Faces.HiddenFilenameExtension, _FaceParts.FilenameExtension, ticks, people, a2PeopleDateGroupDirectory, eResultsFullGroupDirectory); propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FilenameExtension, _Configuration.Reverse, model, predictorModel, mapLogic.IndicesFromNew, mapLogic.KeyValuePairs); } int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); @@ -830,7 +812,7 @@ public partial class DlibDotNet { if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any()) break; - DistanceThenMapLogic(argZero, ticks, people, containers, mapLogic, dResultsFullGroupDirectory, eResultsFullGroupDirectory, d2ResultsFullGroupDirectory, zResultsFullGroupDirectory, peopleDateGroupDirectory, outputResolution); + DistanceThenMapLogic(argZero, ticks, people, containers, mapLogic, dResultsFullGroupDirectory, eResultsFullGroupDirectory, d2ResultsFullGroupDirectory, a2PeopleDateGroupDirectory, outputResolution); if (_IsEnvironment.Development) continue; if (_FileKeyValuePairs.Any()) diff --git a/Instance/Models/Binder/Configuration.cs b/Instance/Models/Binder/Configuration.cs index 7d7bb50..f8defa8 100644 --- a/Instance/Models/Binder/Configuration.cs +++ b/Instance/Models/Binder/Configuration.cs @@ -52,6 +52,7 @@ public class Configuration [Display(Name = "Person Birthday First Year"), Required] public int? PersonBirthdayFirstYear { get; set; } [Display(Name = "Person Birthday Format"), Required] public string PersonBirthdayFormat { get; set; } [Display(Name = "PersonKey Format"), Required] public string PersonKeyFormat { get; set; } + [Display(Name = "Person Require People File"), Required] public bool? PersonRequirePeopleFile { get; set; } [Display(Name = "Predictor Model Name"), Required] public string PredictorModelName { get; set; } [Display(Name = "Properties Changed For Distance"), Required] public bool? PropertiesChangedForDistance { get; set; } [Display(Name = "Properties Changed For Faces"), Required] public bool? PropertiesChangedForFaces { get; set; } @@ -159,6 +160,8 @@ public class Configuration throw new NullReferenceException(nameof(configuration.PersonBirthdayFormat)); if (configuration.PersonKeyFormat is null) throw new NullReferenceException(nameof(configuration.PersonKeyFormat)); + if (configuration.PersonRequirePeopleFile is null) + throw new NullReferenceException(nameof(configuration.PersonRequirePeopleFile)); if (configuration.PropertiesChangedForDistance is null) throw new NullReferenceException(nameof(configuration.PropertiesChangedForDistance)); if (configuration.PropertiesChangedForFaces is null) @@ -250,6 +253,7 @@ public class Configuration configuration.PersonBirthdayFirstYear.Value, configuration.PersonBirthdayFormat, configuration.PersonKeyFormat, + configuration.PersonRequirePeopleFile.Value, configuration.PredictorModelName, configuration.PropertiesChangedForDistance.Value, configuration.PropertiesChangedForFaces.Value, diff --git a/Instance/Models/Configuration.cs b/Instance/Models/Configuration.cs index f806214..3e8370a 100644 --- a/Instance/Models/Configuration.cs +++ b/Instance/Models/Configuration.cs @@ -58,6 +58,7 @@ public class Configuration public bool PropertiesChangedForMetadata { init; get; } public bool PropertiesChangedForResize { init; get; } public bool Reverse { init; get; } + public bool PersonRequirePeopleFile { init; get; } public string[] SaveFaceLandmarkForOutputResolutions { init; get; } public bool SaveFullYearOfRandomFiles { init; get; } public bool SaveResizedSubfiles { init; get; } @@ -116,6 +117,7 @@ public class Configuration int personBirthdayFirstYear, string personBirthdayFormat, string personKeyFormat, + bool personRequirePeopleFile, string predictorModelName, bool propertiesChangedForDistance, bool propertiesChangedForFaces, @@ -180,6 +182,7 @@ public class Configuration PersonBirthdayFirstYear = personBirthdayFirstYear; PersonBirthdayFormat = personBirthdayFormat; PersonKeyFormat = personKeyFormat; + PersonRequirePeopleFile = personRequirePeopleFile; PredictorModelName = predictorModelName; PropertiesChangedForDistance = propertiesChangedForDistance; PropertiesChangedForFaces = propertiesChangedForFaces; diff --git a/Instance/Models/_A2_People.cs b/Instance/Models/_A2_People.cs index 1a74f86..41b08ea 100644 --- a/Instance/Models/_A2_People.cs +++ b/Instance/Models/_A2_People.cs @@ -69,7 +69,7 @@ internal class A2_People if (rootResultsDirectory is null) throw new Exception(); Storage storage = new(rootDirectory, rootResultsDirectory, peopleRootDirectory); - results = Shared.Models.Stateless.Methods.IPerson.GetPeople(storage, _Configuration.PersonBirthdayFirstYear, _Configuration.PersonBirthdayFormat, _Configuration.PersonKeyFormat); + results = Shared.Models.Stateless.Methods.IPerson.GetPeople(storage, _Configuration.PersonBirthdayFirstYear, _Configuration.PersonBirthdayFormat, _Configuration.PersonKeyFormat, _Configuration.PersonRequirePeopleFile); return results.ToArray(); } diff --git a/Instance/appsettings.Development.json b/Instance/appsettings.Development.json index 12fc0d3..bfb4619 100644 --- a/Instance/appsettings.Development.json +++ b/Instance/appsettings.Development.json @@ -90,6 +90,7 @@ "PersonBirthdayFirstYear": 1500, "PersonBirthdayFormat": "yyyy-MM-dd_HH", "PersonKeyFormat": "yyyy-MM-dd_HH", + "PersonRequirePeopleFile": true, "PopulatePropertyId": true, "PredictorModelName": "Large", "PropertiesChangedForDistance": false, diff --git a/Instance/appsettings.Staging.json b/Instance/appsettings.Staging.json index 83e8d4a..9aed48a 100644 --- a/Instance/appsettings.Staging.json +++ b/Instance/appsettings.Staging.json @@ -90,6 +90,7 @@ "PersonBirthdayFirstYear": 1500, "PersonBirthdayFormat": "yyyy-MM-dd_HH", "PersonKeyFormat": "yyyy-MM-dd_HH", + "PersonRequirePeopleFile": false, "PopulatePropertyId": true, "PredictorModelName": "Large", "PropertiesChangedForDistance": false, diff --git a/Instance/appsettings.json b/Instance/appsettings.json index 9416724..e35e9a3 100644 --- a/Instance/appsettings.json +++ b/Instance/appsettings.json @@ -90,6 +90,7 @@ "PersonBirthdayFirstYear": 1500, "PersonBirthdayFormat": "yyyy-MM-dd_HH", "PersonKeyFormat": "yyyy-MM-dd_HH", + "PersonRequirePeopleFile": false, "PopulatePropertyId": true, "PredictorModelName": "Large", "PropertiesChangedForDistance": false, diff --git a/Map/Models/MapLogic.cs b/Map/Models/MapLogic.cs index 5f7269d..6eea518 100644 --- a/Map/Models/MapLogic.cs +++ b/Map/Models/MapLogic.cs @@ -24,17 +24,17 @@ public class MapLogic private readonly long _Ticks; private const int _Mapping = 1; private const int _Sorting = 2; - private Configuration? _Configuration; private readonly Serilog.ILogger? _Log; private const int _ForceSingleImage = 3; private readonly int _MaxDegreeOfParallelism; + private readonly Configuration? _Configuration; private readonly string _FacesFilenameExtension; private readonly string _ResizeFilenameExtension; private readonly string _FacePartsFilenameExtension; private readonly string _FacesHiddenFilenameExtension; - private readonly string _ZPropertyHolderContentTicksDirectory; + private readonly string _EDistanceContentTicksDirectory; - public MapLogic(int maxDegreeOfParallelism, Property.Models.Configuration propertyConfiguration, string resizeFilenameExtension, string facesFilenameExtension, string facesHiddenFilenameExtension, string facePartsFilenameExtension, long ticks, Person[] people, string peopleDateGroupDirectory, string zResultsFullGroupDirectory, Configuration? configuration, List distinctFilteredFaces, Shared.Models.Methods.IFaceDistance? distance) + public MapLogic(int maxDegreeOfParallelism, Property.Models.Configuration propertyConfiguration, Configuration? configuration, string resizeFilenameExtension, string facesFilenameExtension, string facesHiddenFilenameExtension, string facePartsFilenameExtension, long ticks, Person[] people, string a2PeopleDateGroupDirectory, string eResultsFullGroupDirectory, List distinctFilteredFaces, Shared.Models.Methods.IFaceDistance? distance) { _Ticks = ticks; _PersonKeysRanges = new(); @@ -55,30 +55,32 @@ public class MapLogic string[] files; string fullPath; const int zero = 0; - List notMappedTicks = new(); List skipCollection = new(); Dictionary? keyValuePairs; + List notMappedPersonKeys = new(); List>? collection; Dictionary indicesFromNew = new(); + string a2PeopleContentDirectory = Path.Combine(a2PeopleDateGroupDirectory, "()"); + string eDistanceContentDirectory = Path.Combine(eResultsFullGroupDirectory, "()"); Dictionary peopleKeyValuePairs = new(); - string zPropertyHolderContentDirectory = Path.Combine(zResultsFullGroupDirectory, "()"); string? rootDirectoryParent = Path.GetDirectoryName(propertyConfiguration.RootDirectory); + string eDistanceContentTicksDirectory = Path.Combine(eDistanceContentDirectory, $"({ticks})"); + string a2PeopleContentKnownPeopleDirectory = Path.Combine(a2PeopleContentDirectory, "(KnownPeople)"); Dictionary> idThenNormalizedPixelPercentageKeyValuePairs = new(); - string zPropertyHolderContentTicksDirectory = Path.Combine(zPropertyHolderContentDirectory, $"({ticks})"); - string zPropertyHolderPeopleContentDirectory = Path.Combine(peopleDateGroupDirectory, "()", "(KnownPeople)"); Dictionary> incorrectIdThenNormalizedPixelPercentageKeyValuePairs = new(); for (int i = 1; i < 5; i++) - _ = IPath.DeleteEmptyDirectories(zPropertyHolderContentDirectory); - _ = IPath.DeleteEmptyDirectories(zPropertyHolderContentDirectory); + _ = IPath.DeleteEmptyDirectories(eDistanceContentDirectory); + _ = IPath.DeleteEmptyDirectories(eDistanceContentDirectory); if (string.IsNullOrEmpty(rootDirectoryParent)) throw new NullReferenceException(nameof(rootDirectoryParent)); - if (!Directory.Exists(zPropertyHolderContentDirectory)) - _ = Directory.CreateDirectory(zPropertyHolderContentDirectory); - if (!Directory.Exists(zPropertyHolderPeopleContentDirectory)) - _ = Directory.CreateDirectory(zPropertyHolderPeopleContentDirectory); - Stateless.ByRef.Set(propertyConfiguration, configuration, ticks, _ResizeFilenameExtension, people, zPropertyHolderContentDirectory, zPropertyHolderPeopleContentDirectory, distinctFilteredFaces, distance, skipCollection, peopleKeyValuePairs, notMappedTicks, idThenNormalizedPixelPercentageKeyValuePairs, incorrectIdThenNormalizedPixelPercentageKeyValuePairs); - if (!Directory.Exists(zPropertyHolderContentTicksDirectory)) - _ = Directory.CreateDirectory(zPropertyHolderContentTicksDirectory); + if (!Directory.Exists(eDistanceContentDirectory)) + _ = Directory.CreateDirectory(eDistanceContentDirectory); + if (!Directory.Exists(a2PeopleContentKnownPeopleDirectory)) + _ = Directory.CreateDirectory(a2PeopleContentKnownPeopleDirectory); + if (configuration is not null) + Stateless.ByRef.Set(propertyConfiguration, configuration, ticks, _ResizeFilenameExtension, people, eDistanceContentDirectory, a2PeopleContentDirectory, distinctFilteredFaces, distance, skipCollection, peopleKeyValuePairs, notMappedPersonKeys, idThenNormalizedPixelPercentageKeyValuePairs, incorrectIdThenNormalizedPixelPercentageKeyValuePairs); + if (!Directory.Exists(eDistanceContentTicksDirectory)) + _ = Directory.CreateDirectory(eDistanceContentTicksDirectory); files = Directory.GetFiles(rootDirectoryParent, "*keyValuePairs-6*.json", SearchOption.TopDirectoryOnly); if (files.Length != 1) keyValuePairs = new(); @@ -110,19 +112,19 @@ public class MapLogic _KeyValuePairs = keyValuePairs; _IndicesFromNew = indicesFromNew; _SkipCollection = skipCollection; - _NotMappedPersonKeys = notMappedTicks; + _NotMappedPersonKeys = notMappedPersonKeys; _PeopleKeyValuePairs = peopleKeyValuePairs; - _ZPropertyHolderContentTicksDirectory = zPropertyHolderContentTicksDirectory; + _EDistanceContentTicksDirectory = eDistanceContentTicksDirectory; _IdThenNormalizedPixelPercentageKeyValuePairs = idThenNormalizedPixelPercentageKeyValuePairs; _IncorrectIdThenNormalizedPixelPercentageKeyValuePairs = incorrectIdThenNormalizedPixelPercentageKeyValuePairs; } - public MapLogic(int maxDegreeOfParallelism, Property.Models.Configuration propertyConfiguration, string resizeFilenameExtension, string facesFilenameExtension, string facesHiddenFilenameExtension, string facePartsFilenameExtension, long ticks, Person[] people, string peopleDateGroupDirectory, string zResultsFullGroupDirectory) : - this(maxDegreeOfParallelism, propertyConfiguration, resizeFilenameExtension, facesFilenameExtension, facesHiddenFilenameExtension, facePartsFilenameExtension, ticks, people, peopleDateGroupDirectory, zResultsFullGroupDirectory, null, new(), null) + public MapLogic(int maxDegreeOfParallelism, Property.Models.Configuration propertyConfiguration, Configuration? configuration, string resizeFilenameExtension, string facesFilenameExtension, string facesHiddenFilenameExtension, string facePartsFilenameExtension, long ticks, Person[] people, string peopleDateGroupDirectory, string zResultsFullGroupDirectory) : + this(maxDegreeOfParallelism, propertyConfiguration, configuration, resizeFilenameExtension, facesFilenameExtension, facesHiddenFilenameExtension, facePartsFilenameExtension, ticks, people, peopleDateGroupDirectory, zResultsFullGroupDirectory, new(), null) { } - public MapLogic(int maxDegreeOfParallelism, Property.Models.Configuration propertyConfiguration, string outputExtension, long ticks, Person[] people, string peopleDateGroupDirectory, string zResultsFullGroupDirectory) : - this(maxDegreeOfParallelism, propertyConfiguration, outputExtension, outputExtension, outputExtension, outputExtension, ticks, people, peopleDateGroupDirectory, zResultsFullGroupDirectory, null, new(), null) + public MapLogic(int maxDegreeOfParallelism, Property.Models.Configuration propertyConfiguration, Configuration? configuration, string outputExtension, long ticks, Person[] people, string peopleDateGroupDirectory, string zResultsFullGroupDirectory) : + this(maxDegreeOfParallelism, propertyConfiguration, configuration, outputExtension, outputExtension, outputExtension, outputExtension, ticks, people, peopleDateGroupDirectory, zResultsFullGroupDirectory, new(), null) { } public override string ToString() @@ -131,8 +133,6 @@ public class MapLogic return result; } - public void Update(Configuration configuration) => _Configuration = configuration; - public bool Skip(double deterministicHashCodeKey) => _SkipCollection.Contains(deterministicHashCodeKey); private List<(long, long, long, long)> GetPersonKeysRangesCollection(PersonBirthday[] personBirthdays) @@ -398,7 +398,7 @@ public class MapLogic { personBirthday = IPersonBirthday.GetPersonBirthday(personKey); personKeyFormatted = IPersonBirthday.GetFormatted(_Configuration.PersonBirthdayFormat, personBirthday); - directory = Path.Combine(_ZPropertyHolderContentTicksDirectory, $"{facePopulatedKey}NotMapped", personKeyFormatted, propertyConfiguration.ResultAllInOne); + directory = Path.Combine(_EDistanceContentTicksDirectory, $"{facePopulatedKey}NotMapped", personKeyFormatted, propertyConfiguration.ResultAllInOne); saveContainer = new(directory); saveContainers.Add(saveContainer); } @@ -438,7 +438,7 @@ public class MapLogic if (!_IdThenNormalizedPixelPercentageKeyValuePairs.ContainsKey(face.Mapping.MappingFromItem.Id)) { personKey = null; - directory = Path.Combine(_ZPropertyHolderContentTicksDirectory, $"Unnamed{relativePath[2..]}"); + directory = Path.Combine(_EDistanceContentTicksDirectory, $"Unnamed{relativePath[2..]}"); } else { @@ -451,7 +451,7 @@ public class MapLogic if (personBirthdays.Length != 1) { personKey = null; - directory = Path.Combine(_ZPropertyHolderContentTicksDirectory, $"Not Supported{relativePath[2..]}", subDirectoryName); + directory = Path.Combine(_EDistanceContentTicksDirectory, $"Not Supported{relativePath[2..]}", subDirectoryName); } else { @@ -469,11 +469,11 @@ public class MapLogic } } personKeyFormatted = IPersonBirthday.GetFormatted(_Configuration.PersonBirthdayFormat, personBirthday); - directory = Path.Combine(_ZPropertyHolderContentTicksDirectory, "Shortcuts", personKeyFormatted, subDirectoryName); + directory = Path.Combine(_EDistanceContentTicksDirectory, "Shortcuts", personKeyFormatted, subDirectoryName); if (face.FaceEncoding is not null && face.Location?.NormalizedPixelPercentage is not null) - copyDirectory = Path.Combine(_ZPropertyHolderContentTicksDirectory, "Images", personKeyFormatted, subDirectoryName); + copyDirectory = Path.Combine(_EDistanceContentTicksDirectory, "Images", personKeyFormatted, subDirectoryName); else - copyDirectory = Path.Combine(_ZPropertyHolderContentTicksDirectory, "ImagesBut", personKeyFormatted, subDirectoryName); + copyDirectory = Path.Combine(_EDistanceContentTicksDirectory, "ImagesBut", personKeyFormatted, subDirectoryName); copyFileName = Path.Combine(copyDirectory, $"{face.Mapping.MappingFromItem.Id}{face.Mapping.MappingFromItem.ResizedFileHolder.ExtensionLowered}"); } } @@ -624,7 +624,7 @@ public class MapLogic _ => throw new NotImplementedException() }; } - directory = Path.Combine(_ZPropertyHolderContentTicksDirectory, by[1..], personKeyFormatted, face.Mapping.MappingFromPerson.SegmentB); + directory = Path.Combine(_EDistanceContentTicksDirectory, by[1..], personKeyFormatted, face.Mapping.MappingFromPerson.SegmentB); if (!keyValuePairs.ContainsKey(directory)) keyValuePairs.Add(directory, 0); keyValuePairs[directory]++; diff --git a/Map/Models/Stateless/ByRef.cs b/Map/Models/Stateless/ByRef.cs index 264048f..dc18aa0 100644 --- a/Map/Models/Stateless/ByRef.cs +++ b/Map/Models/Stateless/ByRef.cs @@ -8,7 +8,7 @@ namespace View_by_Distance.Map.Models.Stateless; public class ByRef { - private static void SetOther(Property.Models.Configuration propertyConfiguration, Configuration configuration, string resizeFilenameExtension, Person[] people, string deterministicHashCodePeopleDirectory, List skipCollection, Dictionary personKeyAliases, List<(string, int?, PersonBirthday[], long)> peopleCollection) + private static void SetOther(Property.Models.Configuration propertyConfiguration, Configuration configuration, string resizeFilenameExtension, Person[] people, string a2PeopleContentDirectory, List skipCollection, Dictionary personKeyAliases, List<(string, int?, PersonBirthday[], long)> peopleCollection) { long pK; string json; @@ -37,7 +37,7 @@ public class ByRef break; personKeyValuePairs.Add(personKeyFormatted, person); } - string[] groupDirectories = Directory.GetDirectories(deterministicHashCodePeopleDirectory, "*", SearchOption.TopDirectoryOnly); + string[] groupDirectories = Directory.GetDirectories(a2PeopleContentDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string groupDirectory in groupDirectories) { groupDirectoryName = Path.GetFileName(groupDirectory); @@ -138,7 +138,7 @@ public class ByRef } } - internal static List<(string, char, string, int?, int?, List?)> DeleteEmptyDirectoriesAndGetCollection(Configuration configuration, long ticks, string zPropertyHolderContentDirectory, Dictionary> keyValuePairs) + internal static List<(string, char, string, int?, int?, List?)> DeleteEmptyDirectoriesAndGetCollection(Configuration configuration, long ticks, string eDistanceContentDirectory, Dictionary> keyValuePairs) { List<(string, char, string, int?, int?, List?)> results = new(); int? id; @@ -156,7 +156,7 @@ public class ByRef string? personFirstInitialDirectory; bool keyValuePairsAny = keyValuePairs.Any(); int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); - string[] ticksDirectories = Directory.GetDirectories(zPropertyHolderContentDirectory, "*", SearchOption.TopDirectoryOnly); + string[] ticksDirectories = Directory.GetDirectories(eDistanceContentDirectory, "*", SearchOption.TopDirectoryOnly); string message = $") {ticksDirectories.Length:000} ticks Director(ies) - A - {totalSeconds} total second(s)"; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; using ProgressBar progressBar = new(ticksDirectories.Length, message, options); @@ -397,17 +397,19 @@ public class ByRef return result; } - internal static void Set(Property.Models.Configuration propertyConfiguration, Configuration? configuration, long ticks, string resizeFilenameExtension, Person[] people, string zPropertyHolderContentDirectory, string zPropertyHolderPeopleContentDirectory, List distinctFilteredFaces, Shared.Models.Methods.IFaceDistance? distance, List skipCollection, Dictionary peopleKeyValuePairs, List notMappedPersonKeys, Dictionary> idThenNormalizedPixelPercentageKeyValuePairs, Dictionary> incorrectIdThenNormalizedPixelPercentageKeyValuePairs) + internal static void Set(Property.Models.Configuration propertyConfiguration, Configuration? configuration, long ticks, string resizeFilenameExtension, Person[] people, string eDistanceContentDirectory, string a2PeopleContentDirectory, List distinctFilteredFaces, Shared.Models.Methods.IFaceDistance? distance, List skipCollection, Dictionary peopleKeyValuePairs, List notMappedPersonKeys, Dictionary> idThenNormalizedPixelPercentageKeyValuePairs, Dictionary> incorrectIdThenNormalizedPixelPercentageKeyValuePairs) { if (configuration is null) throw new NullReferenceException(nameof(configuration)); Dictionary personKeyAliases = new(); Dictionary> keyValuePairs = new(); + List notMappedAndNotNamedPersonKeys = new(); + List notMappedAndWithNamedPersonKeys = new(); List idThenNormalizedPixelPercentagePersonKeys = new(); List<(string, int?, PersonBirthday[], long)> peopleCollection = new(); List<(PersonBirthday, int, int)> idThenNormalizedPixelPercentageCollection = new(); List<(PersonBirthday, int, int)> incorrectIdThenNormalizedPixelPercentageCollection = new(); - SetOther(propertyConfiguration, configuration, resizeFilenameExtension, people, zPropertyHolderPeopleContentDirectory, skipCollection, personKeyAliases, peopleCollection); + SetOther(propertyConfiguration, configuration, resizeFilenameExtension, people, a2PeopleContentDirectory, skipCollection, personKeyAliases, peopleCollection); foreach (Face face in distinctFilteredFaces) { if (face.FaceEncoding is null || face.Location?.NormalizedPixelPercentage is null) @@ -418,7 +420,7 @@ public class ByRef keyValuePairs.Add(face.Mapping.MappingFromItem.Id, new()); keyValuePairs[face.Mapping.MappingFromItem.Id].Add(face); } - List<(string, char, string, int?, int?, List?)> collection = DeleteEmptyDirectoriesAndGetCollection(configuration, ticks, zPropertyHolderContentDirectory, keyValuePairs); + List<(string, char, string, int?, int?, List?)> collection = DeleteEmptyDirectoriesAndGetCollection(configuration, ticks, eDistanceContentDirectory, keyValuePairs); int unableToMatchCount = SetCollectionsAndGet(configuration, ticks, distance, personKeyAliases, idThenNormalizedPixelPercentageCollection, incorrectIdThenNormalizedPixelPercentageCollection, collection); SetKeyValuePairs(configuration, idThenNormalizedPixelPercentageCollection, incorrectIdThenNormalizedPixelPercentageCollection, idThenNormalizedPixelPercentageKeyValuePairs, incorrectIdThenNormalizedPixelPercentageKeyValuePairs); int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); @@ -442,12 +444,18 @@ public class ByRef continue; throw new NotImplementedException(); } - if (!idThenNormalizedPixelPercentagePersonKeys.Contains(personKey)) - notMappedPersonKeys.Add(personKey); - else + if (idThenNormalizedPixelPercentagePersonKeys.Contains(personKey)) peopleKeyValuePairs.Add(personKey, new(displayDirectoryName, approximateYears, personBirthdays, personKey)); + else if (string.IsNullOrEmpty(displayDirectoryName) || displayDirectoryName == propertyConfiguration.ResultAllInOne) + notMappedAndNotNamedPersonKeys.Add(personKey); + else + notMappedAndWithNamedPersonKeys.Add(personKey); } } + notMappedAndNotNamedPersonKeys.Sort(); + notMappedAndWithNamedPersonKeys.Sort(); + notMappedPersonKeys.AddRange(notMappedAndNotNamedPersonKeys); + notMappedPersonKeys.AddRange(notMappedAndWithNamedPersonKeys); if (idThenNormalizedPixelPercentagePersonKeys.Any()) { int? approximateYears = null; diff --git a/Shared/Models/Stateless/Methods/IPerson.cs b/Shared/Models/Stateless/Methods/IPerson.cs index d2577b6..17d203e 100644 --- a/Shared/Models/Stateless/Methods/IPerson.cs +++ b/Shared/Models/Stateless/Methods/IPerson.cs @@ -10,10 +10,10 @@ public interface IPerson static Dictionary Split(int personBirthdayFirstYear, string personKeyFormat, string knownPeopleFile) => Person.Split(personBirthdayFirstYear, personKeyFormat, knownPeopleFile); - Models.Person[] TestStatic_GetPeople(Properties.IStorage storage, int personBirthdayFirstYear, string personBirthdayFormat, string personKeyFormat) => - GetPeople(storage, personBirthdayFirstYear, personBirthdayFormat, personKeyFormat); - static Models.Person[] GetPeople(Properties.IStorage storage, int personBirthdayFirstYear, string personBirthdayFormat, string personKeyFormat) => - Person.GetPeople(storage, personBirthdayFirstYear, personBirthdayFormat, personKeyFormat); + Models.Person[] TestStatic_GetPeople(Properties.IStorage storage, int personBirthdayFirstYear, string personBirthdayFormat, string personKeyFormat, bool personRequirePeopleFile) => + GetPeople(storage, personBirthdayFirstYear, personBirthdayFormat, personKeyFormat, personRequirePeopleFile); + static Models.Person[] GetPeople(Properties.IStorage storage, int personBirthdayFirstYear, string personBirthdayFormat, string personKeyFormat, bool personRequirePeopleFile) => + Person.GetPeople(storage, personBirthdayFirstYear, personBirthdayFormat, personKeyFormat, personRequirePeopleFile); void TestStatic_SavePerson(Properties.IStorage storage, string personBirthdayFormat, Models.Person person) => SavePerson(storage, personBirthdayFormat, person); diff --git a/Shared/Models/Stateless/Methods/Person.cs b/Shared/Models/Stateless/Methods/Person.cs index 3dd5412..f1bc8d0 100644 --- a/Shared/Models/Stateless/Methods/Person.cs +++ b/Shared/Models/Stateless/Methods/Person.cs @@ -196,7 +196,7 @@ internal abstract class Person return results; } - internal static Models.Person[] GetPeople(Properties.IStorage storage, int personBirthdayFirstYear, string personBirthdayFormat, string personKeyFormat) + internal static Models.Person[] GetPeople(Properties.IStorage storage, int personBirthdayFirstYear, string personBirthdayFormat, string personKeyFormat, bool personRequirePeopleFile) { List results = new(); string json; @@ -210,18 +210,15 @@ internal abstract class Person if (!Directory.Exists(peopleSingletonDirectory)) _ = Directory.CreateDirectory(peopleSingletonDirectory); if (!Directory.Exists(peopleContentDirectory)) - localKnownPeopleFile = string.Empty; - else - { - files = Directory.GetFiles(peopleContentDirectory, "*People*.txt", SearchOption.TopDirectoryOnly); - if (files.Any()) - localKnownPeopleFile = files[0]; - else - localKnownPeopleFile = string.Empty; - } - files = Directory.GetFiles(peopleSingletonDirectory, "*.json", SearchOption.TopDirectoryOnly); - if (!files.Any() && string.IsNullOrEmpty(localKnownPeopleFile)) + _ = Directory.CreateDirectory(peopleContentDirectory); + files = Directory.GetFiles(peopleContentDirectory, "*People*.txt", SearchOption.TopDirectoryOnly); + if (!files.Any() && personRequirePeopleFile) throw new Exception("Copy \"KnownPeople.txt\" file from server!"); + if (files.Any()) + localKnownPeopleFile = files[0]; + else + localKnownPeopleFile = string.Empty; + files = Directory.GetFiles(peopleSingletonDirectory, "*.json", SearchOption.TopDirectoryOnly); foreach (string file in files) { fileInfo = new(file);