Sorting without ... improvements

This commit is contained in:
2022-12-21 22:54:50 -07:00
parent 44d8eaf5fb
commit 2da3db7aa5
25 changed files with 353 additions and 358 deletions

View File

@ -212,8 +212,8 @@ public partial class DlibDotNet
throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions)} are not in the ValidResolutions list!");
if ((from l in configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions)} are not in the ValidResolutions list!");
if ((from l in configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
throw new Exception($"One or more {nameof(configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions)} are not in the ValidResolutions list!");
if ((from l in configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
throw new Exception($"One or more {nameof(configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions)} are not in the ValidResolutions list!");
if ((from l in configuration.SaveShortcutsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
throw new Exception($"One or more {nameof(configuration.SaveShortcutsForOutputResolutions)} are not in the ValidResolutions list!");
if ((from l in configuration.SaveFaceLandmarkForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
@ -281,7 +281,7 @@ public partial class DlibDotNet
return result;
}
private void FullParallelForWork(A_Property propertyLogic, Dictionary<int, List<(string, int)>> idToMappedFaceFilesCollection, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsDateGroupDirectory, string dResultsFullGroupDirectory, string? eDistanceContentDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Shared.Models.Face>> imageFaceCollections, Container container, int index, Item item, DateTime[] containerDateTimes)
private void FullParallelForWork(A_Property propertyLogic, Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> idToMappedFaceFilesWithCollection, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsDateGroupDirectory, string dResultsFullGroupDirectory, string? eDistanceContentDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Shared.Models.Face>> imageFaceCollections, Container container, int index, Item item, DateTime[] containerDateTimes)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
@ -358,7 +358,7 @@ public partial class DlibDotNet
faces = new();
else
{
List<(string, int)>? collection;
List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>? collection;
int[] outputResolutionCollection = imageResizeKeyValuePairs[outputResolution];
int outputResolutionWidth = outputResolutionCollection[0];
int outputResolutionHeight = outputResolutionCollection[1];
@ -370,9 +370,10 @@ public partial class DlibDotNet
bool anyFacesSaved = _Faces.SaveFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem, facesDirectory, faces);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D_Face.SaveFaces));
if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
if ((_Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch)
&& _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
&& !anyFacesSaved && item.Property?.Id is not null
&& idToMappedFaceFilesCollection.TryGetValue(item.Property.Id.Value, out collection))
&& idToMappedFaceFilesWithCollection.TryGetValue(item.Property.Id.Value, out collection))
_Distance.LookForMatchFacesAndPossiblyRename(_Faces.FileNameExtension, eDistanceContentDirectory, mappingFromItem, faces, collection);
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
{
@ -396,7 +397,7 @@ public partial class DlibDotNet
private int FullParallelWork(int maxDegreeOfParallelism,
A_Property propertyLogic,
Dictionary<int, List<(string, int)>> idToMappedFaceFilesCollection,
Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> idToMappedFaceFilesWithCollection,
string outputResolution,
string bResultsFullGroupDirectory,
string cResultsFullGroupDirectory,
@ -440,7 +441,7 @@ public partial class DlibDotNet
{
FullParallelForWork(
propertyLogic,
idToMappedFaceFilesCollection,
idToMappedFaceFilesWithCollection,
outputResolution,
bResultsFullGroupDirectory,
cResultsFullGroupDirectory,
@ -626,7 +627,7 @@ public partial class DlibDotNet
_Metadata.SetAngleBracketCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, container.SourceDirectory);
}
private void FullDoWork(string argZero, string propertyRoot, long ticks, A_Property propertyLogic, int t, Container[] containers, string? eDistanceContentDirectory, Dictionary<int, List<(string, int)>> idToMappedFaceFilesCollection)
private void FullDoWork(string argZero, string propertyRoot, long ticks, PersonContainer[] personContainers, A_Property propertyLogic, int t, Container[] containers, string? a2PeopleContentDirectory, string eDistanceContentDirectory)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
@ -650,6 +651,7 @@ public partial class DlibDotNet
int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism;
List<Shared.Models.Property?> nullablePropertyCollection = new();
List<List<KeyValuePair<string, string>>> metadataCollection = new();
Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> idToMappedFaceFilesWithCollection = GetCollection(ticks, personContainers, a2PeopleContentDirectory, eDistanceContentDirectory);
string dResultsDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(D_Face));
foreach (string outputResolution in _Configuration.OutputResolutions)
{
@ -682,7 +684,7 @@ public partial class DlibDotNet
exceptionCount = FullParallelWork(
maxDegreeOfParallelism,
propertyLogic,
idToMappedFaceFilesCollection,
idToMappedFaceFilesWithCollection,
outputResolution,
bResultsFullGroupDirectory,
cResultsFullGroupDirectory,
@ -845,7 +847,7 @@ public partial class DlibDotNet
return items;
}
private void MapLogic(string argZero, long ticks, PersonContainer[] personContainers, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string eDistanceContentDirectory, string outputResolution)
private void MapLogic(string argZero, long ticks, PersonContainer[] personContainers, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, MapLogicSupport mapLogicSupport, MapLogic mapLogic, string outputResolution)
{
int? useFiltersCounter = null;
SortingContainer[] sortingContainers;
@ -854,22 +856,6 @@ public partial class DlibDotNet
Shared.Models.Face[] distinctFilteredFaces = GetFilteredDistinctFaces(argZero, containers);
Mapping[] mappingCollection = MapLogicSupport.GetSelectedMappingCollection(distinctFilteredFaces);
string dFacesCollectionDirectory = Path.Combine(dResultsFullGroupDirectory, "[]", _Configuration.PropertyConfiguration.ResultAllInOne);
MapLogicSupport mapLogicSupport = new(
_Configuration.FaceConfidencePercent,
_Configuration.FaceDistancePermyriad,
_Configuration.RangeDaysDeltaTolerance,
_Configuration.RangeFaceAreaPermilleTolerance,
_Configuration.SortingMaximumPerFaceShouldBeHigh);
MapLogic mapLogic = new(
_AppSettings.MaxDegreeOfParallelism,
_Configuration.PropertyConfiguration,
_MapConfiguration,
ticks,
personContainers,
a2PeopleSingletonDirectory,
eDistanceContentDirectory,
mappingCollection,
mapLogicSupport);
(Dictionary<long, int> personKeyToCount, int totalNotMapped) = mapLogic.AddToMapping(mappingCollection);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution) || (!string.IsNullOrEmpty(_Configuration.PhotoPrismDirectory) && Directory.Exists(_Configuration.PhotoPrismDirectory)))
{
@ -889,10 +875,11 @@ public partial class DlibDotNet
mapLogic.CopyNotMappedFaces(_Configuration.RangeFaceAreaPermilleTolerance, dFacesContentDirectory, idToNormalizedRectangleToMapping);
if (_Configuration.SaveMappedForOutputResolutions.Contains(outputResolution))
mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, mappingCollection, idToNormalizedRectangleToMapping, personKeyToCount, totalNotMapped);
if (_Configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions.Contains(outputResolution))
mapLogic.SaveResizedImagesByPersonKeyFormatted(_Configuration.JLinks, a2PeopleSingletonDirectory, personContainers, mappingCollection, personKeyToCount, totalNotMapped);
if (_Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution))
mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, a2PeopleSingletonDirectory, personContainers, mappingCollection, personKeyToCount, totalNotMapped);
if (_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution))
{
mapLogic.UpdatedPersonKeyToRanges(_MapConfiguration, ticks, mappingCollection);
MapLogicSupport.SetFaceDistances(_AppSettings.MaxDegreeOfParallelism, ticks, distinctFilteredFaces);
Dictionary<int, Dictionary<int, PersonContainer[]>> missingIdThenNormalizedRectangleToPersonContainers = mapLogic.GetMissing(idToNormalizedRectangleToMapping);
List<FaceDistanceContainer> missingFaceDistanceContainers = _Distance.GetMissingFaceDistanceContainer(_AppSettings.MaxDegreeOfParallelism, ticks, dFacesCollectionDirectory, missingIdThenNormalizedRectangleToPersonContainers);
@ -909,7 +896,7 @@ public partial class DlibDotNet
MapLogicSupport.SaveFaceDistances(_Configuration.PropertyConfiguration, sortingContainers);
if (totalNotMapped > 0)
{
bool saveNullPerson = !personKeyToCount.Any();
bool saveNullPerson = true; // !personKeyToCount.Any();
int updated = mapLogic.UpdateFromSortingContainers(sortingContainers, saveNullPerson);
List<SaveContainer> saveContainers = mapLogic.GetSaveContainers(dFacesContentDirectory, d2FacePartsContentDirectory, mappingCollection, idToNormalizedRectangleToMapping, useFiltersCounter, saveNullPerson);
mapLogic.SaveContainers(totalNotMapped, updated, saveContainers);
@ -1014,6 +1001,48 @@ public partial class DlibDotNet
return results;
}
private void ParallelForAdd(Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> results, (string PersonKeyFormatted, string[] PersonDisplayDirectoryNames, string MappedFaceFile) value)
{
(int? id, int? normalizedRectangle) = Shared.Models.Stateless.Methods.IMapping.GetConverted(_Faces.FileNameExtension, value.MappedFaceFile);
if (id is null || normalizedRectangle is null || !results.TryGetValue(id.Value, out List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>? collection))
return;
IReadOnlyList<MetadataExtractor.Directory> directories = MetadataExtractor.ImageMetadataReader.ReadMetadata(value.MappedFaceFile);
lock (collection)
collection.Add(new(value.MappedFaceFile, normalizedRectangle.Value, directories));
}
private Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> GetCollection(long ticks, PersonContainer[] personContainers, string? a2PeopleContentDirectory, string? eDistanceContentDirectory)
{
Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> results = new();
int? id;
int? normalizedRectangle;
List<(string, string[], string)> collection = Map.Models.Stateless.Methods.IMapLogic.DeleteEmptyDirectoriesAndGetMappedFaceFiles(_MapConfiguration, ticks, a2PeopleContentDirectory, eDistanceContentDirectory, personContainers);
foreach ((string personKeyFormatted, string[] personDisplayDirectoryNames, string mappedFaceFile) in collection)
{
(id, normalizedRectangle) = Shared.Models.Stateless.Methods.IMapping.GetConverted(_Faces.FileNameExtension, mappedFaceFile);
if (id is null || normalizedRectangle is null)
continue;
if (!results.ContainsKey(id.Value))
results.Add(id.Value, new());
if (!_Configuration.DistanceMoveUnableToMatch && !_Configuration.DistanceRenameToMatch)
results[id.Value].Add(new(mappedFaceFile, normalizedRectangle.Value, null));
}
if (results.Any() && (_Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch))
{
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string message = $") Building Mapped Face Files Collection - {totalSeconds} total second(s)";
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism };
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
using ProgressBar progressBar = new(collection.Count, message, options);
_ = Parallel.For(0, collection.Count, parallelOptions, (i, state) =>
{
ParallelForAdd(results, collection[i]);
progressBar.Tick();
});
}
return results;
}
private void Search(long ticks, string argZero, string propertyRoot, PersonContainer[] personContainers)
{
int j;
@ -1047,7 +1076,9 @@ public partial class DlibDotNet
{
string resultsGroupDirectory;
a2PeopleContentDirectory = null;
eDistanceContentDirectory = null;
eDistanceContentDirectory = Path.GetPathRoot(argZero);
if (eDistanceContentDirectory is null)
throw new NullReferenceException(nameof(eDistanceContentDirectory));
string? newRootDirectory = SaveUrlAndGetNewRootDirectory(container);
for (int i = 1; i < 10; i++)
{
@ -1059,17 +1090,19 @@ public partial class DlibDotNet
propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false);
propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse);
}
Dictionary<int, List<(string, int)>> idToMappedFaceFilesCollection = Map.Models.Stateless.Methods.IMapLogic.DeleteEmptyDirectoriesAndGetMappedFaceFiles(_MapConfiguration, ticks, a2PeopleContentDirectory, eDistanceContentDirectory, personContainers);
containers = Shared.Models.Stateless.Methods.IContainer.SortContainers(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, _ArgZeroIsConfigurationRootDirectory, argZero, containers);
FullDoWork(argZero, propertyRoot, ticks, propertyLogic, t, containers, eDistanceContentDirectory, idToMappedFaceFilesCollection);
MapLogicSupport mapLogicSupport = new(_Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeFaceAreaPermilleTolerance, _Configuration.SortingMaximumPerFaceShouldBeHigh);
MapLogic? mapLogic = _Configuration.DistanceMoveUnableToMatch ? null : new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, personContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory, mapLogicSupport);
FullDoWork(argZero, propertyRoot, ticks, personContainers, propertyLogic, t, containers, a2PeopleContentDirectory, eDistanceContentDirectory);
_Distance.Clear();
mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, personContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory, mapLogicSupport);
foreach (string outputResolution in _Configuration.OutputResolutions)
{
if (_PropertyRootExistedBefore || container is not null)
break;
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
if (_Configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions.Contains(outputResolution))
MapLogicSupport.BeforeSaveResizedImagesByPersonKeyFormatted(_Configuration.JLinks, a2PeopleSingletonDirectory);
if (_Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution))
MapLogicSupport.BeforeSaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, a2PeopleSingletonDirectory);
if (_ArgZeroIsConfigurationRootDirectory
&& _Configuration.SaveResizedSubfiles
&& outputResolution == _Configuration.OutputResolutions[0]
@ -1080,7 +1113,7 @@ public partial class DlibDotNet
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any())
break;
SetMapping(argZero, containers);
MapLogic(argZero, ticks, personContainers, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eDistanceContentDirectory, outputResolution);
MapLogic(argZero, ticks, personContainers, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, mapLogicSupport, mapLogic, outputResolution);
if (_IsEnvironment.Development)
continue;
List<MappingFromItem> mappingFromItemCollection = GetMappingFromItemCollection(containers);

View File

@ -69,7 +69,7 @@ public class Configuration
[Display(Name = "Save Full Year Of Random Files"), Required] public bool? SaveFullYearOfRandomFiles { get; set; }
[Display(Name = "Save Mapped"), Required] public string[] SaveMappedForOutputResolutions { get; set; }
[Display(Name = "Save Not Mapped"), Required] public string[] SaveNotMappedForOutputResolutions { get; set; }
[Display(Name = "Save Resized Images by Person Key Formatted"), Required] public string[] SaveResizedImagesByPersonKeyFormattedForOutputResolutions { get; set; }
[Display(Name = "Save Resized Images by Person Key Formatted"), Required] public string[] SaveFilteredOriginalImagesFromJLinksForOutputResolutions { get; set; }
[Display(Name = "Save Resized Subfiles"), Required] public bool? SaveResizedSubfiles { get; set; }
[Display(Name = "Save Shortcuts"), Required] public string[] SaveShortcutsForOutputResolutions { get; set; }
[Display(Name = "Skip Search"), Required] public bool? SkipSearch { get; set; }
@ -190,7 +190,7 @@ public class Configuration
throw new NullReferenceException(nameof(configuration.SaveFullYearOfRandomFiles));
configuration.SaveMappedForOutputResolutions ??= Array.Empty<string>();
configuration.SaveNotMappedForOutputResolutions ??= Array.Empty<string>();
configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions ??= Array.Empty<string>();
configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions ??= Array.Empty<string>();
if (configuration.SaveResizedSubfiles is null)
throw new NullReferenceException(nameof(configuration.SaveResizedSubfiles));
configuration.SaveShortcutsForOutputResolutions ??= Array.Empty<string>();
@ -265,7 +265,7 @@ public class Configuration
configuration.SaveFullYearOfRandomFiles.Value,
configuration.SaveMappedForOutputResolutions,
configuration.SaveNotMappedForOutputResolutions,
configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions,
configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions,
configuration.SaveResizedSubfiles.Value,
configuration.SaveShortcutsForOutputResolutions,
configuration.SkipSearch.Value,

View File

@ -65,7 +65,7 @@ public class Configuration
public bool SaveFullYearOfRandomFiles { init; get; }
public string[] SaveMappedForOutputResolutions { init; get; }
public string[] SaveNotMappedForOutputResolutions { init; get; }
public string[] SaveResizedImagesByPersonKeyFormattedForOutputResolutions { init; get; }
public string[] SaveFilteredOriginalImagesFromJLinksForOutputResolutions { init; get; }
public bool SaveResizedSubfiles { init; get; }
public string[] SaveShortcutsForOutputResolutions { init; get; }
public bool SkipSearch { init; get; }
@ -133,7 +133,7 @@ public class Configuration
bool saveFullYearOfRandomFiles,
string[] saveMappedForOutputResolutions,
string[] saveNotMappedForOutputResolutions,
string[] saveResizedImagesByPersonKeyFormattedForOutputResolutions,
string[] saveFilteredOriginalImagesFromJLinksForOutputResolutions,
bool saveResizedSubfiles,
string[] saveShortcutsForOutputResolutions,
bool skipSearch,
@ -200,7 +200,7 @@ public class Configuration
SaveFullYearOfRandomFiles = saveFullYearOfRandomFiles;
SaveMappedForOutputResolutions = saveMappedForOutputResolutions;
SaveNotMappedForOutputResolutions = saveNotMappedForOutputResolutions;
SaveResizedImagesByPersonKeyFormattedForOutputResolutions = saveResizedImagesByPersonKeyFormattedForOutputResolutions;
SaveFilteredOriginalImagesFromJLinksForOutputResolutions = saveFilteredOriginalImagesFromJLinksForOutputResolutions;
SaveResizedSubfiles = saveResizedSubfiles;
SaveShortcutsForOutputResolutions = saveShortcutsForOutputResolutions;
SkipSearch = skipSearch;