phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Testing

Browse Source
This commit is contained in:
Mike Phares
2025-06-30 16:33:14 -07:00
parent c7ded16e50
commit 5d11335eda
38 changed files with 901 additions and 960 deletions
Download Patch File Download Diff File Expand all files Collapse all files

374
Instance/DlibDotNet.cs
View File

@ -29,9 +29,10 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
string FilesCollectionRootDirectory,
bool FilesCollectionCountIsOne,
ReadOnlyCollection<ReadOnlyCollection<FilePath>> FilePathsCollection,
ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>>? IdToFilePaths,
ReadOnlyDictionary<int, Identifier>? SplatNineIdentifiers);
public long Ticks { get; init; }
private readonly D_Face _Faces;
private ProgressBar? _ProgressBar;
private readonly C_Resize _Resize;
@ -67,11 +68,12 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
_Exceptions = [];
_Console = console;
_AppSettings = appSettings;
IDlibDotNet dlibDotNet = this;
_IsEnvironment = isEnvironment;
long ticks = DateTime.Now.Ticks;
Ticks = DateTime.Now.Ticks;
_JLinkResolvedDirectories = [];
if (ticks.ToString().Last() == '0')
ticks += 1;
if (Ticks.ToString().Last() == '0')
Ticks += 1;
ReadOnlyCollection<PersonContainer> personContainers;
_ProgressBarOptions = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot);
@ -126,10 +128,10 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
personContainers = new([]);
else
{
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - dlibDotNet.Ticks).TotalSeconds);
message = $") Building People Collection - {totalSeconds} total second(s)";
using ProgressBar progressBar = new(1, message, _ProgressBarOptions);
progressBar.Tick();
dlibDotNet.ConstructProgressBar(1, message);
dlibDotNet.Tick();
string peopleRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(A2_People));
string? rootResultsDirectory = Path.GetDirectoryName(Path.GetDirectoryName(peopleRootDirectory)) ?? throw new Exception();
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(peopleRootDirectory, propertyConfiguration.ResultSingleton));
@ -158,12 +160,12 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
filenameExtension);
}
if (!configuration.SkipSearch)
Search(ticks, personContainers, argZero, propertyRoot);
Search(personContainers, argZero, propertyRoot);
if (!_PropertyRootExistedBefore && !_IsEnvironment.Development && _Exceptions.Count == 0 && _ArgZeroIsConfigurationRootDirectory)
{
string d2FacePartsRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(D2_FaceParts));
_Logger?.LogInformation(string.Concat("Cleaning <", d2FacePartsRootDirectory, ">"));
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(d2FacePartsRootDirectory, ticks);
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(d2FacePartsRootDirectory, dlibDotNet.Ticks);
}
message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}";
_Logger?.LogInformation(message);
@ -196,7 +198,9 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
string[] changesFrom = [nameof(A_Property)];
List<Tuple<string, DateTime>> subFileTuples = [];
FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(cResultsFullGroupDirectory, item, outputResolutionHasNumber);
if (item.ExifDirectory is null || item.ExifDirectory.FilePath.Id is null || !item.SourceDirectoryFileHolder.Exists || item.SourceDirectoryFileHolder.CreationTime is null || item.SourceDirectoryFileHolder.LastWriteTime is null || item.Any())
if (item.ExifDirectory is null || item.ExifDirectory.FilePath?.Id is null)
throw new Exception();
if (!item.SourceDirectoryFileHolder.Exists || item.SourceDirectoryFileHolder.CreationTime is null || item.SourceDirectoryFileHolder.LastWriteTime is null || item.Any())
throw new Exception();
if (_Configuration.PropertyConfiguration.ForcePropertyLastWriteTimeToCreationTime && item.SourceDirectoryFileHolder.LastWriteTime.Value != item.SourceDirectoryFileHolder.CreationTime.Value)
{
@ -220,7 +224,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
else
{
ReadOnlyCollection<string> keywords = IMetaBase.GetKeywords(exifDirectory.ExifBaseDirectories);
ReadOnlyCollection<string> keywords = IMetaBase.GetKeywords(exifDirectory);
shouldIgnore = _Configuration.PropertyConfiguration.IgnoreRulesKeyWords.Any(keywords.Contains);
if (shouldIgnore.Value)
{
@ -261,47 +265,54 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
distance += 1;
}
}
Dictionary<string, int[]> outputResolutionToResize = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, item.ExifDirectory, mappingFromItem);
if (_Configuration.SaveResizedSubfiles)
{
if (shouldIgnore is not null && item.FilePath.HasIgnoreKeyword is not null && item.FilePath.HasIgnoreKeyword.Value != shouldIgnore.Value)
faces = [];
else
_Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.ExifDirectory, mappingFromItem, outputResolutionToResize);
}
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
faces = [];
else if (!mappingFromItem.ResizedFileHolder.Exists && !File.Exists(mappingFromItem.ResizedFileHolder.FullName))
int? orientation = IMetaBase.GetOrientation(exifDirectory);
if (exifDirectory is null || orientation is null || exifDirectory.Width is null || exifDirectory.Height is null)
faces = [];
else
{
List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection;
if (!fileNameToCollection.TryGetValue(mappingFromItem.Id, out mappingFromPhotoPrismCollection))
mappingFromPhotoPrismCollection = null;
bool move = _Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution);
faces = _Faces.GetFaces(outputResolution, cResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, item.ExifDirectory, mappingFromItem, outputResolutionToResize, mappingFromPhotoPrismCollection);
result = GetNotMappedCountAndUpdateMappingFromPersonThenSetMapping(mapLogic, item, isFocusRelativePath, mappingFromItem, mappingFromPhotoPrismCollection, faces);
List<(Shared.Models.Face, FileHolder?, string, bool Saved)> faceCollection = _Faces.SaveFaces(item.FilePath, subFileTuples, parseExceptions, mappingFromItem, exifDirectory, faces);
if (move && faceCollection.All(l => !l.Saved))
Dictionary<string, int[]> outputResolutionToResize = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, item.ExifDirectory, mappingFromItem);
(int outputResolutionWidth, int outputResolutionHeight, int outputResolutionOrientation) = Resize.Models.Stateless.Methods.IResize.Get(outputResolution, outputResolutionToResize);
if (_Configuration.SaveResizedSubfiles)
{
ReadOnlyCollection<LocationContainer> locationContainers = mapLogic.GetLocationContainers(item);
if (locationContainers.Count > 0)
if (shouldIgnore is not null && item.FilePath.HasIgnoreKeyword is not null && item.FilePath.HasIgnoreKeyword.Value != shouldIgnore.Value)
faces = [];
else
_Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.ExifDirectory, mappingFromItem, outputResolutionToResize);
}
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
faces = [];
else if (!mappingFromItem.ResizedFileHolder.Exists && !File.Exists(mappingFromItem.ResizedFileHolder.FullName))
faces = [];
else
{
List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection;
if (!fileNameToCollection.TryGetValue(mappingFromItem.Id, out mappingFromPhotoPrismCollection))
mappingFromPhotoPrismCollection = null;
bool move = _Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution);
faces = _Faces.GetFaces(cResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, item.ExifDirectory, mappingFromItem, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation, mappingFromPhotoPrismCollection);
result = GetNotMappedCountAndUpdateMappingFromPersonThenSetMapping(mapLogic, item, isFocusRelativePath, mappingFromItem, mappingFromPhotoPrismCollection, faces);
List<(Shared.Models.Face, FileHolder?, string, bool Saved)> faceCollection = _Faces.SaveFaces(item.FilePath, subFileTuples, parseExceptions, mappingFromItem, exifDirectory, faces);
if (move && faceCollection.All(l => !l.Saved))
{
Map.Models.Stateless.Methods.IMapLogic.SetCreationTime(mappingFromItem, locationContainers);
if (_Configuration.LocationContainerDistanceTolerance is null && _Configuration.MoveToDecade)
Map.Models.Stateless.Methods.IMapLogic.MoveToDecade(_Configuration.PropertyConfiguration, mappingFromItem, locationContainers);
_Distance.LookForMatchFacesAndPossiblyRename(_Configuration.OverrideForFaceImages, _DistanceLimits, _Faces, item.FilePath, mappingFromItem, exifDirectory, faces, locationContainers);
ReadOnlyCollection<LocationContainer> locationContainers = mapLogic.GetLocationContainers(item);
if (locationContainers.Count > 0)
{
Map.Models.Stateless.Methods.IMapLogic.SetCreationTime(mappingFromItem, locationContainers);
if (_Configuration.LocationContainerDistanceTolerance is null && _Configuration.MoveToDecade)
Map.Models.Stateless.Methods.IMapLogic.MoveToDecade(_Configuration.PropertyConfiguration, mappingFromItem, locationContainers);
_Distance.LookForMatchFacesAndPossiblyRename(_Configuration.OverrideForFaceImages, _DistanceLimits, _Faces, item.FilePath, mappingFromItem, exifDirectory, faces, locationContainers);
}
}
(bool review, int[] eyesCollection) = Shared.Models.Stateless.Methods.IFace.GetEyeCollection(_Configuration.EyeThreshold, faces);
if (review || _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
{
bool saveRotated = false;
string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(_Configuration.PropertyConfiguration, container.SourceDirectory);
_FaceParts.SaveFaceLandmarkImages(_Configuration.PropertyConfiguration, d2ResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, mappingFromItem, exifDirectory, faces, saveRotated);
}
if (_Configuration.SaveFaceLandmarkForOutputResolutionsV2.Contains(outputResolution))
_FaceParts.SaveFaceLandmarkImages(d2ResultsFullGroupDirectory, mappingFromItem, exifDirectory, faces);
}
(bool review, int[] eyesCollection) = Shared.Models.Stateless.Methods.IFace.GetEyeCollection(_Configuration.EyeThreshold, faces);
if (review || _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
{
bool saveRotated = false;
string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(_Configuration.PropertyConfiguration, container.SourceDirectory);
_FaceParts.SaveFaceLandmarkImages(_Configuration.PropertyConfiguration, d2ResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, mappingFromItem, exifDirectory, faces, saveRotated);
}
if (_Configuration.SaveFaceLandmarkForOutputResolutionsV2.Contains(outputResolution))
_FaceParts.SaveFaceLandmarkImages(d2ResultsFullGroupDirectory, mappingFromItem, exifDirectory, faces);
}
lock (sourceDirectoryChanges)
{
@ -327,14 +338,18 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
{
int result = 0;
int exceptionsCount = 0;
IDlibDotNet dlibDotNet = this;
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
DateTime[] containerDateTimes = Container.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems);
string focusRelativePath = Path.GetFullPath(string.Concat(_Configuration.PropertyConfiguration.RootDirectory, _Configuration.FocusDirectory));
bool? isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath);
bool anyPropertiesChangedForX = _Configuration.PropertyConfiguration.PropertiesChangedForProperty || _Configuration.PropertiesChangedForDistance || _Configuration.PropertiesChangedForFaces || _Configuration.PropertiesChangedForIndex || _Configuration.PropertiesChangedForMetadata || _Configuration.PropertiesChangedForResize;
using ProgressBar progressBar = new(filteredItems.Count, message, _ProgressBarOptions);
dlibDotNet.ConstructProgressBar(filteredItems.Count, message);
_ = Parallel.For(0, filteredItems.Count, parallelOptions, (i, state) =>
{
Item item = filteredItems[i];
if (!item.SourceDirectoryFileHolder.Exists || item.SourceDirectoryFileHolder.CreationTime is null || item.SourceDirectoryFileHolder.LastWriteTime is null || item.Any())
return;
try
{
result += FullParallelForWork(metadata,
@ -347,11 +362,11 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
fileNameToCollection,
record,
container,
filteredItems[i],
item,
containerDateTimes,
isFocusRelativePath);
if (!anyPropertiesChangedForX && (i == 0 || sourceDirectoryChanges.Count > 0))
progressBar.Tick();
dlibDotNet.Tick();
}
catch (Exception)
{
@ -365,7 +380,6 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
private void FullDoWork(string argZero,
string propertyRoot,
long ticks,
string fPhotoPrismSingletonDirectory,
int count,
B_Metadata metadata,
@ -413,7 +427,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
continue;
sourceDirectoryChanges.Clear();
anyNullOrNoIsUniqueFileName = filteredItems.Any(l => !l.IsUniqueFileName);
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - dlibDotNet.Ticks).TotalSeconds);
message = $"{totalSeconds} total second(s) - {outputResolution} - {i + 1:000} / {readOnlyContainers.Count:000} - {total} / {count} total - <{container.SourceDirectory}> [{filteredItems.Count:000}] - total not mapped {totalNotMapped:000000}";
if (outputResolutionHasNumber)
_Resize.SetAngleBracketCollection(cResultsFullGroupDirectory, container.SourceDirectory);
@ -448,10 +462,10 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
total += container.Items.Count;
}
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - dlibDotNet.Ticks).TotalSeconds);
message = $"{totalSeconds} total second(s) - {outputResolution} - ### [###] / {readOnlyContainers.Count:000} - {total} / {count} total - <> - total not mapped {totalNotMapped:000000}";
using ProgressBar progressBar = new(1, message, _ProgressBarOptions);
progressBar.Tick();
dlibDotNet.ConstructProgressBar(1, message);
dlibDotNet.Tick();
}
}
@ -607,14 +621,36 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
{
ExifDirectory? result;
if (filePair.Match is null)
result = null;
{
try
{ result = Metadata.Models.Stateless.Methods.IMetadata.GetExifDirectory(filePair.FilePath); }
catch (Exception) { result = null; }
}
else
{
string json = File.ReadAllText(filePair.Match.FullName);
if (string.IsNullOrEmpty(json))
result = null;
else
{
result = JsonSerializer.Deserialize(json, ExifDirectorySourceGenerationContext.Default.ExifDirectory);
if (result?.FilePath?.Id is null)
{
try
{
result = Metadata.Models.Stateless.Methods.IMetadata.GetExifDirectory(filePair.FilePath);
json = JsonSerializer.Serialize(result, ExifDirectorySourceGenerationContext.Default.ExifDirectory);
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(filePair.Match.FullName, json, updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null);
}
catch (Exception) { result = null; }
}
}
}
if (result is not null && result.FilePath?.Id is null)
{
try
{ result = Metadata.Models.Stateless.Methods.IMetadata.GetExifDirectory(filePair.FilePath); }
catch (Exception) { result = null; }
}
return result;
}
@ -715,7 +751,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
List<(string, string, string)> collection = [];
foreach (Item item in distinctValidImageItems)
{
if (item.ExifDirectory?.FilePath.Id is null)
if (item.ExifDirectory?.FilePath?.Id is null)
continue;
if (item.IsNotUniqueAndNeedsReview is null || !item.IsNotUniqueAndNeedsReview.Value)
continue;
@ -728,7 +764,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
foreach (Item item in distinctValidImageItems)
{
if (item.ExifDirectory?.FilePath.Id is null)
if (item.ExifDirectory?.FilePath?.Id is null)
continue;
dateTime = IDate.GetDateTimeOriginal(item.ExifDirectory);
if (dateTime is null)
@ -809,7 +845,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
return result;
}
private static ReadOnlyCollection<FilePath> HideSplatNineAndGetMovedDuplicatesWithSameSplatNine(Property.Models.Configuration propertyConfiguration, ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>> keyValuePairs)
private static ReadOnlyCollection<FilePath> HideSplatNineAndGetMovedDuplicatesWithSameSplatNine(Property.Models.Configuration propertyConfiguration, ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>> idToFilePaths)
{
List<FilePath> results = [];
string checkFile;
@ -817,7 +853,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
FilePath? filePath;
string checkDirectory;
List<FilePath> collection = [];
foreach (KeyValuePair<int, ReadOnlyCollection<FilePath>> keyValuePair in keyValuePairs)
foreach (KeyValuePair<int, ReadOnlyCollection<FilePath>> keyValuePair in idToFilePaths)
{
collection.Clear();
for (int i = 0; i < keyValuePair.Value.Count; i++)
@ -851,7 +887,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
return results.AsReadOnly();
}
private static ReadOnlyDictionary<int, Identifier> GetSplatNineIdentifiers(Property.Models.Configuration propertyConfiguration, string bResultsFullGroupDirectory, ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>> keyValuePairs)
private static ReadOnlyDictionary<int, Identifier> GetSplatNineIdentifiers(Property.Models.Configuration propertyConfiguration, string bResultsFullGroupDirectory, ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>> idToFilePaths)
{
Dictionary<int, Identifier> results = [];
string json;
@ -862,7 +898,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
string bMetadataCollectionDirectory = Path.Combine(bResultsFullGroupDirectory, propertyConfiguration.ResultCollection);
if (!Directory.Exists(bMetadataCollectionDirectory))
_ = Directory.CreateDirectory(bMetadataCollectionDirectory);
foreach (KeyValuePair<int, ReadOnlyCollection<FilePath>> keyValuePair in keyValuePairs)
foreach (KeyValuePair<int, ReadOnlyCollection<FilePath>> keyValuePair in idToFilePaths)
{
filePath = null;
for (int i = 0; i < keyValuePair.Value.Count; i++)
@ -894,17 +930,18 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
return results.AsReadOnly();
}
private static ReadOnlyDictionary<int, Identifier> GetSplatNineIdentifiersAndHideSplatNine(Property.Models.Configuration propertyConfiguration, string bResultsFullGroupDirectory, ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>> keyValuePairs)
private static ReadOnlyDictionary<int, Identifier> GetSplatNineIdentifiersAndHideSplatNine(Property.Models.Configuration propertyConfiguration, string bResultsFullGroupDirectory, ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection)
{
ReadOnlyDictionary<int, Identifier> results;
if (keyValuePairs.Count == 0)
if (filePathsCollection.Count == 0)
results = new(new Dictionary<int, Identifier>());
else
{
ReadOnlyCollection<FilePath> moved = HideSplatNineAndGetMovedDuplicatesWithSameSplatNine(propertyConfiguration, keyValuePairs);
ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>> idToFilePaths = FilePath.GetKeyValuePairs(filePathsCollection);
ReadOnlyCollection<FilePath> moved = HideSplatNineAndGetMovedDuplicatesWithSameSplatNine(propertyConfiguration, idToFilePaths);
if (moved.Count > 0)
throw new Exception($"House cleaning needed!{Environment.NewLine}{string.Join(Environment.NewLine, moved.Select(l => l.Id))}");
results = GetSplatNineIdentifiers(propertyConfiguration, bResultsFullGroupDirectory, keyValuePairs);
results = GetSplatNineIdentifiers(propertyConfiguration, bResultsFullGroupDirectory, idToFilePaths);
}
return results;
}
@ -983,13 +1020,13 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
return result;
}
private ReadOnlyCollection<FilePath> GetFilePath(long ticks, string dFacesContentDirectory)
private ReadOnlyCollection<FilePath> GetFilePath(string dFacesContentDirectory)
{
List<FilePath> results = [];
FilePath filePath;
FileHolder fileHolder;
string[] files = Directory.GetFiles(dFacesContentDirectory, $"*{_Faces.FileNameExtension}", SearchOption.AllDirectories);
long? skipOlderThan = _Configuration.SkipOlderThanDays is null ? null : new DateTime(ticks).AddDays(-_Configuration.SkipOlderThanDays.Value).Ticks;
long? skipOlderThan = _Configuration.SkipOlderThanDays is null ? null : new DateTime(Ticks).AddDays(-_Configuration.SkipOlderThanDays.Value).Ticks;
foreach (string file in files)
{
fileHolder = Shared.Models.Stateless.Methods.IFileHolder.Get(file);
@ -1003,19 +1040,22 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
return results.AsReadOnly();
}
private void ParallelFor(IDlibDotNet dlibDotNet, FilePair filePair, Dictionary<int, ExifDirectory> results)
private void ParallelFor(FilePair filePair, Dictionary<int, ExifDirectory> results, Action? tick)
{
dlibDotNet?.Tick();
tick?.Invoke();
if (filePair.FilePath.Id is null || results.ContainsKey(filePair.FilePath.Id.Value))
return;
ExifDirectory? exifDirectory = GetExifDirectory(filePair);
if (exifDirectory is null)
return;
lock (results)
results.Add(filePair.FilePath.Id.Value, exifDirectory);
{
if (!results.ContainsKey(filePair.FilePath.Id.Value))
results.Add(filePair.FilePath.Id.Value, exifDirectory);
}
}
private bool GetRunToDoCollectionFirst(Models.Configuration configuration, long ticks, string[] checkDirectories)
private bool GetRunToDoCollectionFirst(Models.Configuration configuration, string[] checkDirectories)
{
bool result = configuration.SaveSortingWithoutPerson;
if (!result)
@ -1030,7 +1070,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
{
string seasonDirectory;
DirectoryInfo directoryInfo;
DateTime dateTime = new(ticks);
DateTime dateTime = new(Ticks);
string rootDirectory = _Configuration.PropertyConfiguration.RootDirectory;
string eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), _Configuration.PropertyConfiguration.ResultContent);
(int season, string seasonName) = Shared.Models.Stateless.Methods.IProperty.GetSeason(dateTime.DayOfYear);
@ -1067,15 +1107,14 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
return result;
}
private void Search(long ticks, ReadOnlyCollection<PersonContainer> personContainers, string argZero, string propertyRoot)
private void Search(ReadOnlyCollection<PersonContainer> personContainers, string argZero, string propertyRoot)
{
string message;
MapLogic? mapLogic;
Record? record = null;
string seasonDirectory;
A_Property propertyLogic;
IDlibDotNet dlibDotNet = this;
DateTime dateTime = new(ticks);
DateTime dateTime = new(Ticks);
string eDistanceContentDirectory;
string? a2PeopleContentDirectory;
string aResultsFullGroupDirectory;
@ -1087,25 +1126,25 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
const string directorySearchFilter = "*";
bool configurationOutputResolutionsHas = false;
ReadOnlyDictionary<long, List<int>> personKeyToIds;
(int season, string seasonName) = Shared.Models.Stateless.Methods.IProperty.GetSeason(dateTime.DayOfYear);
string[] checkDirectories =
[
Path.Combine(_Configuration.PropertyConfiguration.RootDirectory, "Ancestry"),
Path.Combine(_Configuration.PropertyConfiguration.RootDirectory, "Facebook"),
Path.Combine(_Configuration.PropertyConfiguration.RootDirectory, "LinkedIn")
];
bool runToDoCollectionFirst = GetRunToDoCollectionFirst(_Configuration, ticks, checkDirectories);
bool runToDoCollectionFirst = GetRunToDoCollectionFirst(_Configuration, checkDirectories);
(aResultsFullGroupDirectory, bResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories();
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(_Configuration.PropertyConfiguration.RootDirectory, ticks);
(int season, string seasonName) = Shared.Models.Stateless.Methods.IProperty.GetSeason(dateTime.DayOfYear);
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(_Configuration.PropertyConfiguration.RootDirectory, dlibDotNet.Ticks);
a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "([])");
eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), _Configuration.PropertyConfiguration.ResultContent);
string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), _Configuration.PropertyConfiguration.ResultSingleton);
_ = Directory.CreateDirectory(Path.Combine(eDistanceContentDirectory, ticks.ToString()));
B_Metadata metadata = new(dlibDotNet, _Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, ticks, bResultsFullGroupDirectory);
_ = Directory.CreateDirectory(Path.Combine(eDistanceContentDirectory, dlibDotNet.Ticks.ToString()));
B_Metadata metadata = new(dlibDotNet, _Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, dlibDotNet.Ticks, bResultsFullGroupDirectory);
if (runToDoCollectionFirst)
mapLogic = null;
else
mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _Distance, personContainers, ticks, a2PeopleContentDirectory, a2PeopleSingletonDirectory, eDistanceContentDirectory);
mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _Distance, personContainers, dlibDotNet.Ticks, a2PeopleContentDirectory, a2PeopleSingletonDirectory, eDistanceContentDirectory);
foreach (string outputResolution in _Configuration.OutputResolutions)
{
if (outputResolution.Any(char.IsNumber))
@ -1113,7 +1152,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
configurationOutputResolutionsHas = true;
if (!runToDoCollectionFirst)
break;
record = GetFilesCollectionThenCopyOrMove(dlibDotNet, ticks, fileSearchFilter, directorySearchFilter, bResultsFullGroupDirectory, outputResolution);
record = GetFilesCollectionThenCopyOrMove(dlibDotNet, metadata, fileSearchFilter, directorySearchFilter, bResultsFullGroupDirectory, outputResolution);
break;
}
fPhotoPrismContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), _Configuration.PropertyConfiguration.ResultContent);
@ -1145,30 +1184,21 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
{
if (outputResolution.Any(char.IsNumber))
continue;
Dictionary<int, ExifDirectory> exifDirectoriesById = [];
(cResultsFullGroupDirectory, _, _, _) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
string? filesCollectionRootDirectory = Path.Combine(cResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
ReadOnlyCollection<ReadOnlyCollection<FilePath>>? filePathsCollection = IDirectory.GetFilePathCollections(_Configuration.PropertyConfiguration, directorySearchFilter, fileSearchFilter, filesCollectionRootDirectory, useIgnoreExtensions: true, useCeilingAverage: true);
ReadOnlyDictionary<int, ExifDirectory> exifDirectoriesById = record?.ExifDirectoriesById ?? new(new Dictionary<int, ExifDirectory>());
record = new(FilesCollectionRootDirectory: filesCollectionRootDirectory,
FilesCollectionCountIsOne: false,
FilePathsCollection: filePathsCollection,
ExifDirectoriesById: new(exifDirectoriesById),
IdToFilePaths: null,
ExifDirectoriesById: exifDirectoriesById,
SplatNineIdentifiers: null);
break;
}
}
if (string.IsNullOrEmpty(record?.FilesCollectionRootDirectory) || record.FilePathsCollection.Count == 0)
throw new NullReferenceException(nameof(record.FilePathsCollection));
foreach (string checkDirectory in checkDirectories)
{
seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName} {Path.GetFileName(checkDirectory)}");
if (!Directory.Exists(seasonDirectory))
_ = Directory.CreateDirectory(seasonDirectory);
}
int count = record.FilePathsCollection.Select(l => l.Count).Sum();
message = $") Building Container(s) - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
_ProgressBar = new(count, message, _ProgressBarOptions);
ReadOnlyCollection<Container.Models.Container> readOnlyContainers =
Container.Models.Stateless.Methods.IContainer.GetContainers(dlibDotNet,
_Configuration.PropertyConfiguration,
@ -1176,24 +1206,21 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
_Faces.HiddenFileNameExtension,
eDistanceContentDirectory,
record.FilesCollectionRootDirectory,
record.IdToFilePaths,
record.SplatNineIdentifiers,
record.FilePathsCollection,
record.ExifDirectoriesById);
_ProgressBar.Dispose();
mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _Distance, personContainers, ticks, a2PeopleContentDirectory, a2PeopleSingletonDirectory, eDistanceContentDirectory);
Verify(argZero, readOnlyContainers);
foreach (string checkDirectory in checkDirectories)
{
seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName} {Path.GetFileName(checkDirectory)}");
if (!Directory.Exists(seasonDirectory))
_ = Directory.CreateDirectory(seasonDirectory);
}
mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _Distance, personContainers, dlibDotNet.Ticks, a2PeopleContentDirectory, a2PeopleSingletonDirectory, eDistanceContentDirectory);
DeleteContinueFiles(personContainers);
if (!runToDoCollectionFirst)
MapFaceFileLogic(ticks, personContainers, mapLogic, a2PeopleContentDirectory, eDistanceContentDirectory);
FullDoWork(argZero,
propertyRoot,
ticks,
fPhotoPrismSingletonDirectory,
count,
metadata,
record,
readOnlyContainers,
mapLogic);
MapFaceFileLogic(personContainers, mapLogic, a2PeopleContentDirectory, eDistanceContentDirectory);
FullDoWork(argZero, propertyRoot, fPhotoPrismSingletonDirectory, count, metadata, record, readOnlyContainers, mapLogic);
ReadOnlyCollection<Item> distinctValidImageItems = Container.Models.Stateless.Methods.IContainer.GetValidImageItems(_Configuration.PropertyConfiguration, readOnlyContainers, distinctItems: true, filterItems: true);
if (_Configuration.LookForAbandoned)
{
@ -1201,10 +1228,9 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
string d2ResultsFullGroupDirectory;
foreach (string outputResolution in _Configuration.OutputResolutions)
{
_ProgressBar = new(5, nameof(mapLogic.LookForAbandoned), _ProgressBarOptions);
dlibDotNet.ConstructProgressBar(5, nameof(mapLogic.LookForAbandoned));
(cResultsFullGroupDirectory, _, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
mapLogic.LookForAbandoned(dlibDotNet, _Configuration.PropertyConfiguration, bResultsFullGroupDirectory, readOnlyContainers, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory);
_ProgressBar.Dispose();
mapLogic.LookForAbandoned(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, readOnlyContainers, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, dlibDotNet.Tick);
}
}
_Distance.Clear();
@ -1215,7 +1241,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
if (!Directory.Exists(eDistanceContentDirectory))
_ = Directory.CreateDirectory(eDistanceContentDirectory);
string json = JsonSerializer.Serialize(distinctValidImageMappingCollection);
File.WriteAllText(Path.Combine(eDistanceContentDirectory, $"{ticks}.json"), json);
File.WriteAllText(Path.Combine(eDistanceContentDirectory, $"{Ticks}.json"), json);
}
foreach (string outputResolution in _Configuration.OutputResolutions)
{
@ -1233,7 +1259,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
&& outputResolution == _Configuration.OutputResolutions[0]
&& _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
&& _Exceptions.Count == 0)
MapLogic(ticks, readOnlyContainers, fPhotoPrismContentDirectory, mapLogic, outputResolution, new(personKeyToIds), distinctValidImageFaces, distinctValidImageMappingCollection);
MapLogic(readOnlyContainers, fPhotoPrismContentDirectory, mapLogic, outputResolution, personKeyToIds.AsReadOnly(), distinctValidImageFaces, distinctValidImageMappingCollection);
if (runToDoCollectionFirst && _Configuration.SaveRandomForOutputResolutions.Contains(outputResolution) && personKeyToIds.Count > 0 && record?.SplatNineIdentifiers is not null && distinctValidImageMappingCollection.Count > 0)
_Random.Random(_Configuration.PropertyConfiguration, _Configuration.ImmichAssetsFile, _Configuration.ImmichOwnerId, _Configuration.ImmichRoot, _Configuration.RadomUseBirthdayMinimum, _Configuration.ValidKeyWordsToIgnoreInRandom, personKeyToIds, record.SplatNineIdentifiers, distinctValidImageMappingCollection);
if (_IsEnvironment.Development)
@ -1253,59 +1279,96 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
}
private Record GetFilesCollectionThenCopyOrMove(IDlibDotNet dlibDotNet, long ticks, string fileSearchFilter, string directorySearchFilter, string bResultsFullGroupDirectory, string outputResolution)
private void Verify(string argZero, ReadOnlyCollection<Container.Models.Container> readOnlyContainers)
{
int count = 0;
List<Item> items = [];
Container.Models.Container container;
ReadOnlyCollection<Item> filteredItems;
foreach (string outputResolution in _Configuration.OutputResolutions)
{
for (int i = 0; i < readOnlyContainers.Count; i++)
{
container = readOnlyContainers[i];
if (container.Items.Count == 0)
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = Container.Models.Stateless.Methods.IContainer.GetValidImageItems(_Configuration.PropertyConfiguration, container);
if (filteredItems.Count == 0)
continue;
foreach (Item item in filteredItems)
{
count++;
if (item.ExifDirectory is null || item.ExifDirectory.FilePath?.Id is null)
{
items.Add(item);
continue;
}
if (!item.SourceDirectoryFileHolder.Exists || item.SourceDirectoryFileHolder.CreationTime is null || item.SourceDirectoryFileHolder.LastWriteTime is null || item.Any())
{
items.Add(item);
continue;
}
}
}
}
if (items.Count > 0)
throw new Exception($"{items.Count} item(s) of {count} item(s) are not setup!");
}
private Record GetFilesCollectionThenCopyOrMove(IDlibDotNet dlibDotNet, B_Metadata metadata, string fileSearchFilter, string directorySearchFilter, string bResultsFullGroupDirectory, string outputResolution)
{
Record result;
int count;
string message;
ProgressBar progressBar;
const string extension = ".json";
ReadOnlyCollection<FilePair> filePairs;
Dictionary<int, ExifDirectory> results = [];
int maxDegreeOfParallelism = Environment.ProcessorCount;
Dictionary<int, ExifDirectory> exifDirectoriesById = [];
string filesCollectionRootDirectory = _Configuration.PropertyConfiguration.RootDirectory;
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
(string cResultsFullGroupDirectory, _, _, _) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
string jsonGroupDirectory = Path.Combine(bResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton);
if (!Directory.Exists(jsonGroupDirectory))
_ = Directory.CreateDirectory(jsonGroupDirectory);
IReadOnlyDictionary<string, ReadOnlyDictionary<byte, ReadOnlyCollection<string>>> keyValuePairs =
ReadOnlyDictionary<string, ReadOnlyDictionary<byte, ReadOnlyCollection<string>>> keyValuePairs =
Shared.Models.Stateless.Methods.IPath.GetKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, [_Configuration.PropertyConfiguration.ResultContent]);
ReadOnlyDictionary<byte, ReadOnlyCollection<string>> fileGroups = keyValuePairs[_Configuration.PropertyConfiguration.ResultContent];
ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection = IDirectory.GetFilePathCollections(_Configuration.PropertyConfiguration, directorySearchFilter, fileSearchFilter, filesCollectionRootDirectory, useIgnoreExtensions: true, useCeilingAverage: false);
foreach (KeyValuePair<int, ExifDirectory> keyValuePair in metadata.GetKeyValuePairsAndClear())
results.Add(keyValuePair.Key, keyValuePair.Value);
count = filePathsCollection.Select(l => l.Count).Sum();
filePairs = IFilePair.GetFilePairs(_Configuration.PropertyConfiguration, directorySearchFilter, extension, jsonGroupDirectory, filePathsCollection);
message = $") Preloading ExifDirectory Dictionary - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
progressBar = new(count, message, _ProgressBarOptions);
_ = Parallel.For(0, filePairs.Count, parallelOptions, (i, state) => ParallelFor(dlibDotNet, filePairs[i], exifDirectoriesById));
progressBar.Dispose();
message = $") {nameof(DlibDotNet)} - Preloading ExifDirectory Dictionary - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - dlibDotNet.Ticks).TotalSeconds)} total second(s)";
dlibDotNet.ConstructProgressBar(count, message);
_ = Parallel.For(0, filePairs.Count, parallelOptions, (i, state) => ParallelFor(filePairs[i], results, dlibDotNet.Tick));
if (results.Count == 0)
throw new Exception("No exif directories were found!");
count = filePathsCollection.Select(l => l.Count).Sum();
ReadOnlyDictionary<int, ExifDirectory> exifDirectoriesById = results.AsReadOnly();
bool filesCollectionCountIsOne = GetFilesCollectionCountIsOne(filePathsCollection);
message = $") Selecting for ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
progressBar = new(count, message, _ProgressBarOptions);
(string[] distinctDirectories, List<(FilePath, string)> toDoCollection) = IDirectory.GetToDoCollection(_Configuration.PropertyConfiguration, filePathsCollection, fileGroups, exifDirectoriesById, () => progressBar.Tick());
progressBar.Dispose();
message = $") Selecting for ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - dlibDotNet.Ticks).TotalSeconds)} total second(s)";
dlibDotNet.ConstructProgressBar(count, message);
(string[] distinctDirectories, List<(FilePath, string)> toDoCollection) = IDirectory.GetToDoCollection(_Configuration.PropertyConfiguration, filePathsCollection, fileGroups, exifDirectoriesById, dlibDotNet.Tick);
foreach (string distinctDirectory in distinctDirectories)
{
if (!Directory.Exists(distinctDirectory))
_ = Directory.CreateDirectory(distinctDirectory);
}
message = $") Copying to ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
progressBar = new(count, message, _ProgressBarOptions);
_ = IDirectory.CopyOrMove(toDoCollection, move: false, moveBack: false, () => progressBar.Tick());
progressBar.Dispose();
ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>> idToFilePaths = FilePath.GetKeyValuePairs(filePathsCollection);
ReadOnlyDictionary<int, Identifier> splatNineIdentifiers = GetSplatNineIdentifiersAndHideSplatNine(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, idToFilePaths);
result = new(ExifDirectoriesById: new(exifDirectoriesById),
message = $") Copying to ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - dlibDotNet.Ticks).TotalSeconds)} total second(s)";
dlibDotNet.ConstructProgressBar(count, message);
_ = IDirectory.CopyOrMove(toDoCollection, move: false, moveBack: false, dlibDotNet.Tick);
ReadOnlyDictionary<int, Identifier> splatNineIdentifiers = GetSplatNineIdentifiersAndHideSplatNine(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, filePathsCollection);
result = new(ExifDirectoriesById: exifDirectoriesById,
FilesCollectionRootDirectory: filesCollectionRootDirectory,
FilesCollectionCountIsOne: filesCollectionCountIsOne,
FilePathsCollection: filePathsCollection,
IdToFilePaths: idToFilePaths,
SplatNineIdentifiers: splatNineIdentifiers);
return result;
}
private void MapFaceFileLogic(long ticks, ReadOnlyCollection<PersonContainer> personContainers, MapLogic mapLogic, string? a2PeopleContentDirectory, string eDistanceContentDirectory)
private void MapFaceFileLogic(ReadOnlyCollection<PersonContainer> personContainers, MapLogic mapLogic, string? a2PeopleContentDirectory, string eDistanceContentDirectory)
{
foreach (string outputResolution in _Configuration.OutputResolutions)
{
@ -1323,7 +1386,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
continue;
if (!_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
continue;
List<SaveContainer> saveContainers = GetSaveContainers(ticks, personContainers, a2PeopleContentDirectory, eDistanceContentDirectory, mapLogic, outputResolution);
List<SaveContainer> saveContainers = GetSaveContainers(personContainers, a2PeopleContentDirectory, eDistanceContentDirectory, mapLogic, outputResolution);
if (saveContainers.Count > 0)
{
int updated = 0;
@ -1333,24 +1396,24 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
}
private List<SaveContainer> GetSaveContainers(long ticks, ReadOnlyCollection<PersonContainer> personContainers, string a2PeopleSingletonDirectory, string eDistanceContentDirectory, MapLogic mapLogic, string outputResolution)
private List<SaveContainer> GetSaveContainers(ReadOnlyCollection<PersonContainer> personContainers, string a2PeopleSingletonDirectory, string eDistanceContentDirectory, MapLogic mapLogic, string outputResolution)
{
List<SaveContainer> results;
IDlibDotNet dlibDotNet = this;
long[] jLinkResolvedPersonKeys = _JLinkResolvedDirectories.Select(l => l.PersonKey).ToArray();
(string cResultsFullGroupDirectory, string _, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
ReadOnlyDictionary<int, ReadOnlyDictionary<int, LocationContainer>> mapped = Map.Models.Stateless.Methods.IMapLogic.GetMapped(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, personContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory);
ReadOnlyDictionary<int, ReadOnlyDictionary<int, LocationContainer>> mapped = Map.Models.Stateless.Methods.IMapLogic.GetMapped(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, dlibDotNet.Ticks, personContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory);
if (mapped.Count == 0 && !_Configuration.SaveSortingWithoutPerson)
throw new NotSupportedException($"Switch {nameof(_Configuration.SaveSortingWithoutPerson)}!");
ReadOnlyCollection<FilePath> filePaths = GetFilePath(ticks, dFacesContentDirectory);
List<LocationContainer> available = Map.Models.Stateless.Methods.IMapLogic.GetAvailable(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, _Faces, ticks, filePaths);
ReadOnlyCollection<FilePath> filePaths = GetFilePath(dFacesContentDirectory);
List<LocationContainer> available = Map.Models.Stateless.Methods.IMapLogic.GetAvailable(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, _Faces, dlibDotNet.Ticks, filePaths);
if (!string.IsNullOrEmpty(_Configuration.FocusDirectory) && _Configuration.FocusDirectory.Length != 2)
throw new NotSupportedException($"{nameof(_Configuration.FocusDirectory)} currently only works with output directory! Example 00.");
ReadOnlyDictionary<int, ReadOnlyDictionary<int, LocationContainer>> mappedWithEncoding = E_Distance.GetMappedWithEncoding(mapped);
if (mappedWithEncoding.Count == 0 && !_Configuration.SaveSortingWithoutPerson)
throw new NotSupportedException($"Switch {nameof(_Configuration.SaveSortingWithoutPerson)}!");
List<LocationContainer> preFiltered = E_Distance.GetPreFilterLocationContainer(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, _Configuration.FocusDirectory, _Configuration.FocusModel, _Configuration.SkipPersonWithMoreThen, ticks, mapLogic, jLinkResolvedPersonKeys, mapped, available);
List<LocationContainer> preFiltered = E_Distance.GetPreFilterLocationContainer(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, _Configuration.FocusDirectory, _Configuration.FocusModel, _Configuration.SkipPersonWithMoreThen, dlibDotNet.Ticks, mapLogic, jLinkResolvedPersonKeys, mapped, available);
if (preFiltered.Count == 0)
results = [];
else
@ -1361,10 +1424,9 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
results = [];
else
{
string message = $") Building Matrix - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
_ProgressBar = new(postFiltered.Count, message, _ProgressBarOptions);
ReadOnlyCollection<LocationContainer> matrix = E_Distance.GetMatrixLocationContainers(dlibDotNet, _MapConfiguration, ticks, mapLogic, mappedWithEncoding, preFiltered, distanceLimits, postFiltered);
_ProgressBar.Dispose();
string message = $") Building Matrix - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - dlibDotNet.Ticks).TotalSeconds)} total second(s)";
dlibDotNet.ConstructProgressBar(postFiltered.Count, message);
ReadOnlyCollection<LocationContainer> matrix = E_Distance.GetMatrixLocationContainers(_MapConfiguration, dlibDotNet.Ticks, mapLogic, mappedWithEncoding, preFiltered, distanceLimits, postFiltered, dlibDotNet.Tick);
ReadOnlyDictionary<string, LocationContainer> onlyOne = GetOnlyOne(distanceLimits, matrix);
if (onlyOne.Count == 0)
results = [];
@ -1403,7 +1465,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath);
foreach (Item item in filteredItems)
{
if (item.ExifDirectory?.FilePath.Id is null || item.ResizedFileHolder is null)
if (item.ExifDirectory?.FilePath?.Id is null || item.ResizedFileHolder is null)
continue;
mappingFromItem = IMappingFromItem.GetMappingFromItem(containerDateTimes, item, item.ResizedFileHolder);
if (distinctItems)
@ -1458,7 +1520,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
return (result, notMapped);
}
private void MapLogic(long ticks, ReadOnlyCollection<Container.Models.Container> containers, string fPhotoPrismContentDirectory, MapLogic mapLogic, string outputResolution, ReadOnlyDictionary<long, List<int>> personKeyToIds, ReadOnlyCollection<Shared.Models.Face> distinctValidImageFaces, ReadOnlyCollection<Mapping> distinctValidImageMappingCollection)
private void MapLogic(ReadOnlyCollection<Container.Models.Container> containers, string fPhotoPrismContentDirectory, MapLogic mapLogic, string outputResolution, ReadOnlyDictionary<long, List<int>> personKeyToIds, ReadOnlyCollection<Shared.Models.Face> distinctValidImageFaces, ReadOnlyCollection<Mapping> distinctValidImageMappingCollection)
{
IDlibDotNet dlibDotNet = this;
(_, _, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
@ -1467,24 +1529,25 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
string d2FacePartsContentCollectionDirectory = Path.Combine(d2ResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContentCollection);
if (distinctValidImageMappingCollection.Count > 0)
{
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(d2FacePartsContentDirectory, ticks);
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(d2FacePartsContentDirectory, dlibDotNet.Ticks);
if (Directory.Exists(d2FacePartsContentCollectionDirectory))
Shared.Models.Stateless.Methods.IPath.MakeHiddenIfAllItemsAreHidden(d2FacePartsContentCollectionDirectory);
}
if (Directory.Exists(fPhotoPrismContentDirectory))
F_PhotoPrism.WriteMatches(fPhotoPrismContentDirectory, _Configuration.PersonBirthdayFormat, _Configuration.RectangleIntersectMinimums, ticks, distinctValidImageFaces, mapLogic);
F_PhotoPrism.WriteMatches(fPhotoPrismContentDirectory, _Configuration.PersonBirthdayFormat, _Configuration.RectangleIntersectMinimums, dlibDotNet.Ticks, distinctValidImageFaces, mapLogic);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
mapLogic.SaveShortcutsForOutputResolutionsDuringMapLogic(containers, personKeyToIds, dFacesContentDirectory, distinctValidImageMappingCollection);
ReadOnlyDictionary<int, ReadOnlyDictionary<int, Mapping>> idToWholePercentagesToMapping = Map.Models.Stateless.Methods.IMapLogic.GetIdToWholePercentagesToFace(distinctValidImageMappingCollection);
if (_Configuration.SaveMappedForOutputResolutions.Contains(outputResolution))
mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, personKeyToIds, distinctValidImageMappingCollection, idToWholePercentagesToMapping);
if (_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution))
SaveFaceDistances(ticks, mapLogic, distinctValidImageFaces, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping);
SaveFaceDistances(mapLogic, distinctValidImageFaces, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping);
}
private void SaveFaceDistances(long ticks, MapLogic mapLogic, ReadOnlyCollection<Shared.Models.Face> distinctValidImageFaces, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, ReadOnlyDictionary<int, ReadOnlyDictionary<int, Mapping>> idToWholePercentagesToMapping)
private void SaveFaceDistances(MapLogic mapLogic, ReadOnlyCollection<Shared.Models.Face> distinctValidImageFaces, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, ReadOnlyDictionary<int, ReadOnlyDictionary<int, Mapping>> idToWholePercentagesToMapping)
{
E_Distance.PreFilterSetFaceDistances(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, distinctValidImageFaces);
IDlibDotNet dlibDotNet = this;
E_Distance.PreFilterSetFaceDistances(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, dlibDotNet.Ticks, distinctValidImageFaces);
ReadOnlyCollection<FaceDistanceContainer> faceDistanceContainers = E_Distance.GetFaceDistanceContainers(distinctValidImageFaces);
if (faceDistanceContainers.Count > 0)
{
@ -1495,24 +1558,25 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
continue;
faceDistanceEncodings.Add(faceDistanceContainer.FaceDistance);
}
SaveFaceDistances(ticks, mapLogic, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, new(faceDistanceEncodings), faceDistanceContainers);
SaveFaceDistances(mapLogic, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, faceDistanceEncodings.AsReadOnly(), faceDistanceContainers);
}
}
private void SaveFaceDistances(long ticks, MapLogic mapLogic, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, ReadOnlyDictionary<int, ReadOnlyDictionary<int, Mapping>> idToWholePercentagesToMapping, ReadOnlyCollection<FaceDistance> faceDistanceEncodings, ReadOnlyCollection<FaceDistanceContainer> faceDistanceContainers)
private void SaveFaceDistances(MapLogic mapLogic, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, ReadOnlyDictionary<int, ReadOnlyDictionary<int, Mapping>> idToWholePercentagesToMapping, ReadOnlyCollection<FaceDistance> faceDistanceEncodings, ReadOnlyCollection<FaceDistanceContainer> faceDistanceContainers)
{
int? useFiltersCounter = null;
IDlibDotNet dlibDotNet = this;
DistanceLimits distanceLimits;
ReadOnlyCollection<SortingContainer> sortingContainers;
FaceDistanceContainer[] filteredFaceDistanceContainers;
long? skipOlderThan = _Configuration.SkipOlderThanDays is null ? null : new DateTime(ticks).AddDays(-_Configuration.SkipOlderThanDays.Value).Ticks;
long? skipOlderThan = _Configuration.SkipOlderThanDays is null ? null : new DateTime(Ticks).AddDays(-_Configuration.SkipOlderThanDays.Value).Ticks;
distanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh);
filteredFaceDistanceContainers = E_Distance.FilteredPostLoadFaceDistanceContainers(mapLogic, faceDistanceContainers, skipOlderThan, distanceLimits);
if (filteredFaceDistanceContainers.Length == 0)
_Logger?.LogInformation("All images have been filtered!");
else
{
sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortedSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distanceLimits, faceDistanceEncodings, filteredFaceDistanceContainers);
sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortedSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, dlibDotNet.Ticks, mapLogic, distanceLimits, faceDistanceEncodings, filteredFaceDistanceContainers);
if (sortingContainers.Count == 0)
{
for (useFiltersCounter = 1; useFiltersCounter < _Configuration.UseFilterTries; useFiltersCounter++)
@ -1523,7 +1587,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
_Logger?.LogInformation("All images have been filtered!");
else
{
sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortedSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distanceLimits, faceDistanceEncodings, filteredFaceDistanceContainers);
sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortedSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, dlibDotNet.Ticks, mapLogic, distanceLimits, faceDistanceEncodings, filteredFaceDistanceContainers);
if (sortingContainers.Count == 0)
break;
}
@ -1599,11 +1663,11 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
MappingFromFilterPre mappingFromFilterPre;
MappingFromFilterPost mappingFromFilterPost;
bool? isFocusModel = GetIsFocusModel(item.ExifDirectory);
ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>? wholePercentagesToPersonContainers = mapLogic.GetWholePercentagesToPersonContainers(item.ExifDirectory?.FilePath.Id);
ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>? wholePercentagesToPersonContainers = mapLogic.GetWholePercentagesToPersonContainers(item.ExifDirectory?.FilePath?.Id);
long[] jLinkResolvedPersonKeys = _JLinkResolvedDirectories.Select(l => l.PersonKey).ToArray();
foreach (Shared.Models.Face face in faces)
{
if (item.ExifDirectory?.FilePath.Id is null || face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
if (item.ExifDirectory?.FilePath?.Id is null || face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
{
canReMap = null;
isFocusPerson = null;