Sorting without ... improvements
This commit is contained in:
@ -212,8 +212,8 @@ public partial class DlibDotNet
|
||||
throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions)} are not in the ValidResolutions list!");
|
||||
if ((from l in configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
||||
throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions)} are not in the ValidResolutions list!");
|
||||
if ((from l in configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
||||
throw new Exception($"One or more {nameof(configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions)} are not in the ValidResolutions list!");
|
||||
if ((from l in configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
||||
throw new Exception($"One or more {nameof(configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions)} are not in the ValidResolutions list!");
|
||||
if ((from l in configuration.SaveShortcutsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
||||
throw new Exception($"One or more {nameof(configuration.SaveShortcutsForOutputResolutions)} are not in the ValidResolutions list!");
|
||||
if ((from l in configuration.SaveFaceLandmarkForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
||||
@ -281,7 +281,7 @@ public partial class DlibDotNet
|
||||
return result;
|
||||
}
|
||||
|
||||
private void FullParallelForWork(A_Property propertyLogic, Dictionary<int, List<(string, int)>> idToMappedFaceFilesCollection, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsDateGroupDirectory, string dResultsFullGroupDirectory, string? eDistanceContentDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Shared.Models.Face>> imageFaceCollections, Container container, int index, Item item, DateTime[] containerDateTimes)
|
||||
private void FullParallelForWork(A_Property propertyLogic, Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> idToMappedFaceFilesWithCollection, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsDateGroupDirectory, string dResultsFullGroupDirectory, string? eDistanceContentDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Shared.Models.Face>> imageFaceCollections, Container container, int index, Item item, DateTime[] containerDateTimes)
|
||||
{
|
||||
if (_Log is null)
|
||||
throw new NullReferenceException(nameof(_Log));
|
||||
@ -358,7 +358,7 @@ public partial class DlibDotNet
|
||||
faces = new();
|
||||
else
|
||||
{
|
||||
List<(string, int)>? collection;
|
||||
List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>? collection;
|
||||
int[] outputResolutionCollection = imageResizeKeyValuePairs[outputResolution];
|
||||
int outputResolutionWidth = outputResolutionCollection[0];
|
||||
int outputResolutionHeight = outputResolutionCollection[1];
|
||||
@ -370,9 +370,10 @@ public partial class DlibDotNet
|
||||
bool anyFacesSaved = _Faces.SaveFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem, facesDirectory, faces);
|
||||
if (_AppSettings.MaxDegreeOfParallelism < 2)
|
||||
ticks = LogDelta(ticks, nameof(D_Face.SaveFaces));
|
||||
if (_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
|
||||
if ((_Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch)
|
||||
&& _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
|
||||
&& !anyFacesSaved && item.Property?.Id is not null
|
||||
&& idToMappedFaceFilesCollection.TryGetValue(item.Property.Id.Value, out collection))
|
||||
&& idToMappedFaceFilesWithCollection.TryGetValue(item.Property.Id.Value, out collection))
|
||||
_Distance.LookForMatchFacesAndPossiblyRename(_Faces.FileNameExtension, eDistanceContentDirectory, mappingFromItem, faces, collection);
|
||||
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
|
||||
{
|
||||
@ -396,7 +397,7 @@ public partial class DlibDotNet
|
||||
|
||||
private int FullParallelWork(int maxDegreeOfParallelism,
|
||||
A_Property propertyLogic,
|
||||
Dictionary<int, List<(string, int)>> idToMappedFaceFilesCollection,
|
||||
Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> idToMappedFaceFilesWithCollection,
|
||||
string outputResolution,
|
||||
string bResultsFullGroupDirectory,
|
||||
string cResultsFullGroupDirectory,
|
||||
@ -440,7 +441,7 @@ public partial class DlibDotNet
|
||||
{
|
||||
FullParallelForWork(
|
||||
propertyLogic,
|
||||
idToMappedFaceFilesCollection,
|
||||
idToMappedFaceFilesWithCollection,
|
||||
outputResolution,
|
||||
bResultsFullGroupDirectory,
|
||||
cResultsFullGroupDirectory,
|
||||
@ -626,7 +627,7 @@ public partial class DlibDotNet
|
||||
_Metadata.SetAngleBracketCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, container.SourceDirectory);
|
||||
}
|
||||
|
||||
private void FullDoWork(string argZero, string propertyRoot, long ticks, A_Property propertyLogic, int t, Container[] containers, string? eDistanceContentDirectory, Dictionary<int, List<(string, int)>> idToMappedFaceFilesCollection)
|
||||
private void FullDoWork(string argZero, string propertyRoot, long ticks, PersonContainer[] personContainers, A_Property propertyLogic, int t, Container[] containers, string? a2PeopleContentDirectory, string eDistanceContentDirectory)
|
||||
{
|
||||
if (_Log is null)
|
||||
throw new NullReferenceException(nameof(_Log));
|
||||
@ -650,6 +651,7 @@ public partial class DlibDotNet
|
||||
int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism;
|
||||
List<Shared.Models.Property?> nullablePropertyCollection = new();
|
||||
List<List<KeyValuePair<string, string>>> metadataCollection = new();
|
||||
Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> idToMappedFaceFilesWithCollection = GetCollection(ticks, personContainers, a2PeopleContentDirectory, eDistanceContentDirectory);
|
||||
string dResultsDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(D_Face));
|
||||
foreach (string outputResolution in _Configuration.OutputResolutions)
|
||||
{
|
||||
@ -682,7 +684,7 @@ public partial class DlibDotNet
|
||||
exceptionCount = FullParallelWork(
|
||||
maxDegreeOfParallelism,
|
||||
propertyLogic,
|
||||
idToMappedFaceFilesCollection,
|
||||
idToMappedFaceFilesWithCollection,
|
||||
outputResolution,
|
||||
bResultsFullGroupDirectory,
|
||||
cResultsFullGroupDirectory,
|
||||
@ -845,7 +847,7 @@ public partial class DlibDotNet
|
||||
return items;
|
||||
}
|
||||
|
||||
private void MapLogic(string argZero, long ticks, PersonContainer[] personContainers, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string eDistanceContentDirectory, string outputResolution)
|
||||
private void MapLogic(string argZero, long ticks, PersonContainer[] personContainers, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, MapLogicSupport mapLogicSupport, MapLogic mapLogic, string outputResolution)
|
||||
{
|
||||
int? useFiltersCounter = null;
|
||||
SortingContainer[] sortingContainers;
|
||||
@ -854,22 +856,6 @@ public partial class DlibDotNet
|
||||
Shared.Models.Face[] distinctFilteredFaces = GetFilteredDistinctFaces(argZero, containers);
|
||||
Mapping[] mappingCollection = MapLogicSupport.GetSelectedMappingCollection(distinctFilteredFaces);
|
||||
string dFacesCollectionDirectory = Path.Combine(dResultsFullGroupDirectory, "[]", _Configuration.PropertyConfiguration.ResultAllInOne);
|
||||
MapLogicSupport mapLogicSupport = new(
|
||||
_Configuration.FaceConfidencePercent,
|
||||
_Configuration.FaceDistancePermyriad,
|
||||
_Configuration.RangeDaysDeltaTolerance,
|
||||
_Configuration.RangeFaceAreaPermilleTolerance,
|
||||
_Configuration.SortingMaximumPerFaceShouldBeHigh);
|
||||
MapLogic mapLogic = new(
|
||||
_AppSettings.MaxDegreeOfParallelism,
|
||||
_Configuration.PropertyConfiguration,
|
||||
_MapConfiguration,
|
||||
ticks,
|
||||
personContainers,
|
||||
a2PeopleSingletonDirectory,
|
||||
eDistanceContentDirectory,
|
||||
mappingCollection,
|
||||
mapLogicSupport);
|
||||
(Dictionary<long, int> personKeyToCount, int totalNotMapped) = mapLogic.AddToMapping(mappingCollection);
|
||||
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution) || (!string.IsNullOrEmpty(_Configuration.PhotoPrismDirectory) && Directory.Exists(_Configuration.PhotoPrismDirectory)))
|
||||
{
|
||||
@ -889,10 +875,11 @@ public partial class DlibDotNet
|
||||
mapLogic.CopyNotMappedFaces(_Configuration.RangeFaceAreaPermilleTolerance, dFacesContentDirectory, idToNormalizedRectangleToMapping);
|
||||
if (_Configuration.SaveMappedForOutputResolutions.Contains(outputResolution))
|
||||
mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, mappingCollection, idToNormalizedRectangleToMapping, personKeyToCount, totalNotMapped);
|
||||
if (_Configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions.Contains(outputResolution))
|
||||
mapLogic.SaveResizedImagesByPersonKeyFormatted(_Configuration.JLinks, a2PeopleSingletonDirectory, personContainers, mappingCollection, personKeyToCount, totalNotMapped);
|
||||
if (_Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution))
|
||||
mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, a2PeopleSingletonDirectory, personContainers, mappingCollection, personKeyToCount, totalNotMapped);
|
||||
if (_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution))
|
||||
{
|
||||
mapLogic.UpdatedPersonKeyToRanges(_MapConfiguration, ticks, mappingCollection);
|
||||
MapLogicSupport.SetFaceDistances(_AppSettings.MaxDegreeOfParallelism, ticks, distinctFilteredFaces);
|
||||
Dictionary<int, Dictionary<int, PersonContainer[]>> missingIdThenNormalizedRectangleToPersonContainers = mapLogic.GetMissing(idToNormalizedRectangleToMapping);
|
||||
List<FaceDistanceContainer> missingFaceDistanceContainers = _Distance.GetMissingFaceDistanceContainer(_AppSettings.MaxDegreeOfParallelism, ticks, dFacesCollectionDirectory, missingIdThenNormalizedRectangleToPersonContainers);
|
||||
@ -909,7 +896,7 @@ public partial class DlibDotNet
|
||||
MapLogicSupport.SaveFaceDistances(_Configuration.PropertyConfiguration, sortingContainers);
|
||||
if (totalNotMapped > 0)
|
||||
{
|
||||
bool saveNullPerson = !personKeyToCount.Any();
|
||||
bool saveNullPerson = true; // !personKeyToCount.Any();
|
||||
int updated = mapLogic.UpdateFromSortingContainers(sortingContainers, saveNullPerson);
|
||||
List<SaveContainer> saveContainers = mapLogic.GetSaveContainers(dFacesContentDirectory, d2FacePartsContentDirectory, mappingCollection, idToNormalizedRectangleToMapping, useFiltersCounter, saveNullPerson);
|
||||
mapLogic.SaveContainers(totalNotMapped, updated, saveContainers);
|
||||
@ -1014,6 +1001,48 @@ public partial class DlibDotNet
|
||||
return results;
|
||||
}
|
||||
|
||||
private void ParallelForAdd(Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> results, (string PersonKeyFormatted, string[] PersonDisplayDirectoryNames, string MappedFaceFile) value)
|
||||
{
|
||||
(int? id, int? normalizedRectangle) = Shared.Models.Stateless.Methods.IMapping.GetConverted(_Faces.FileNameExtension, value.MappedFaceFile);
|
||||
if (id is null || normalizedRectangle is null || !results.TryGetValue(id.Value, out List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>? collection))
|
||||
return;
|
||||
IReadOnlyList<MetadataExtractor.Directory> directories = MetadataExtractor.ImageMetadataReader.ReadMetadata(value.MappedFaceFile);
|
||||
lock (collection)
|
||||
collection.Add(new(value.MappedFaceFile, normalizedRectangle.Value, directories));
|
||||
}
|
||||
|
||||
private Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> GetCollection(long ticks, PersonContainer[] personContainers, string? a2PeopleContentDirectory, string? eDistanceContentDirectory)
|
||||
{
|
||||
Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> results = new();
|
||||
int? id;
|
||||
int? normalizedRectangle;
|
||||
List<(string, string[], string)> collection = Map.Models.Stateless.Methods.IMapLogic.DeleteEmptyDirectoriesAndGetMappedFaceFiles(_MapConfiguration, ticks, a2PeopleContentDirectory, eDistanceContentDirectory, personContainers);
|
||||
foreach ((string personKeyFormatted, string[] personDisplayDirectoryNames, string mappedFaceFile) in collection)
|
||||
{
|
||||
(id, normalizedRectangle) = Shared.Models.Stateless.Methods.IMapping.GetConverted(_Faces.FileNameExtension, mappedFaceFile);
|
||||
if (id is null || normalizedRectangle is null)
|
||||
continue;
|
||||
if (!results.ContainsKey(id.Value))
|
||||
results.Add(id.Value, new());
|
||||
if (!_Configuration.DistanceMoveUnableToMatch && !_Configuration.DistanceRenameToMatch)
|
||||
results[id.Value].Add(new(mappedFaceFile, normalizedRectangle.Value, null));
|
||||
}
|
||||
if (results.Any() && (_Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch))
|
||||
{
|
||||
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
|
||||
string message = $") Building Mapped Face Files Collection - {totalSeconds} total second(s)";
|
||||
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism };
|
||||
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
|
||||
using ProgressBar progressBar = new(collection.Count, message, options);
|
||||
_ = Parallel.For(0, collection.Count, parallelOptions, (i, state) =>
|
||||
{
|
||||
ParallelForAdd(results, collection[i]);
|
||||
progressBar.Tick();
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private void Search(long ticks, string argZero, string propertyRoot, PersonContainer[] personContainers)
|
||||
{
|
||||
int j;
|
||||
@ -1047,7 +1076,9 @@ public partial class DlibDotNet
|
||||
{
|
||||
string resultsGroupDirectory;
|
||||
a2PeopleContentDirectory = null;
|
||||
eDistanceContentDirectory = null;
|
||||
eDistanceContentDirectory = Path.GetPathRoot(argZero);
|
||||
if (eDistanceContentDirectory is null)
|
||||
throw new NullReferenceException(nameof(eDistanceContentDirectory));
|
||||
string? newRootDirectory = SaveUrlAndGetNewRootDirectory(container);
|
||||
for (int i = 1; i < 10; i++)
|
||||
{
|
||||
@ -1059,17 +1090,19 @@ public partial class DlibDotNet
|
||||
propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false);
|
||||
propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse);
|
||||
}
|
||||
Dictionary<int, List<(string, int)>> idToMappedFaceFilesCollection = Map.Models.Stateless.Methods.IMapLogic.DeleteEmptyDirectoriesAndGetMappedFaceFiles(_MapConfiguration, ticks, a2PeopleContentDirectory, eDistanceContentDirectory, personContainers);
|
||||
containers = Shared.Models.Stateless.Methods.IContainer.SortContainers(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, _ArgZeroIsConfigurationRootDirectory, argZero, containers);
|
||||
FullDoWork(argZero, propertyRoot, ticks, propertyLogic, t, containers, eDistanceContentDirectory, idToMappedFaceFilesCollection);
|
||||
MapLogicSupport mapLogicSupport = new(_Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeFaceAreaPermilleTolerance, _Configuration.SortingMaximumPerFaceShouldBeHigh);
|
||||
MapLogic? mapLogic = _Configuration.DistanceMoveUnableToMatch ? null : new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, personContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory, mapLogicSupport);
|
||||
FullDoWork(argZero, propertyRoot, ticks, personContainers, propertyLogic, t, containers, a2PeopleContentDirectory, eDistanceContentDirectory);
|
||||
_Distance.Clear();
|
||||
mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, personContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory, mapLogicSupport);
|
||||
foreach (string outputResolution in _Configuration.OutputResolutions)
|
||||
{
|
||||
if (_PropertyRootExistedBefore || container is not null)
|
||||
break;
|
||||
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
|
||||
if (_Configuration.SaveResizedImagesByPersonKeyFormattedForOutputResolutions.Contains(outputResolution))
|
||||
MapLogicSupport.BeforeSaveResizedImagesByPersonKeyFormatted(_Configuration.JLinks, a2PeopleSingletonDirectory);
|
||||
if (_Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution))
|
||||
MapLogicSupport.BeforeSaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, a2PeopleSingletonDirectory);
|
||||
if (_ArgZeroIsConfigurationRootDirectory
|
||||
&& _Configuration.SaveResizedSubfiles
|
||||
&& outputResolution == _Configuration.OutputResolutions[0]
|
||||
@ -1080,7 +1113,7 @@ public partial class DlibDotNet
|
||||
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any())
|
||||
break;
|
||||
SetMapping(argZero, containers);
|
||||
MapLogic(argZero, ticks, personContainers, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eDistanceContentDirectory, outputResolution);
|
||||
MapLogic(argZero, ticks, personContainers, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, mapLogicSupport, mapLogic, outputResolution);
|
||||
if (_IsEnvironment.Development)
|
||||
continue;
|
||||
List<MappingFromItem> mappingFromItemCollection = GetMappingFromItemCollection(containers);
|
||||
|
||||
Reference in New Issue
Block a user