Switch to ExifDirectory from Property

This commit is contained in:
2025-04-06 18:23:57 -07:00
parent 3f7affceef
commit c7ded16e50
50 changed files with 2647 additions and 1846 deletions

View File

@ -25,6 +25,13 @@ namespace View_by_Distance.Instance;
public partial class DlibDotNet : IDlibDotNet, IDisposable
{
public record Record(ReadOnlyDictionary<int, ExifDirectory> ExifDirectoriesById,
string FilesCollectionRootDirectory,
bool FilesCollectionCountIsOne,
ReadOnlyCollection<ReadOnlyCollection<FilePath>> FilePathsCollection,
ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>>? IdToFilePaths,
ReadOnlyDictionary<int, Identifier>? SplatNineIdentifiers);
private readonly D_Face _Faces;
private ProgressBar? _ProgressBar;
private readonly C_Resize _Resize;
@ -40,6 +47,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
private readonly DistanceLimits _DistanceLimits;
private readonly bool _PropertyRootExistedBefore;
private readonly Models.Configuration _Configuration;
private readonly ProgressBarOptions _ProgressBarOptions;
private readonly bool _ArgZeroIsConfigurationRootDirectory;
private readonly Map.Models.Configuration _MapConfiguration;
private readonly List<(string Directory, long PersonKey)> _JLinkResolvedDirectories;
@ -56,15 +64,16 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
{
string message;
_Logger = logger;
_Exceptions = [];
_Console = console;
_AppSettings = appSettings;
_IsEnvironment = isEnvironment;
long ticks = DateTime.Now.Ticks;
_Exceptions = [];
_JLinkResolvedDirectories = [];
if (ticks.ToString().Last() == '0')
ticks += 1;
ReadOnlyCollection<PersonContainer> personContainers;
_ProgressBarOptions = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot);
Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration);
_Logger?.LogInformation(propertyConfiguration.RootDirectory);
@ -119,8 +128,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
{
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
message = $") Building People Collection - {totalSeconds} total second(s)";
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
using ProgressBar progressBar = new(1, message, options);
using ProgressBar progressBar = new(1, message, _ProgressBarOptions);
progressBar.Tick();
string peopleRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(A2_People));
string? rootResultsDirectory = Path.GetDirectoryName(Path.GetDirectoryName(peopleRootDirectory)) ?? throw new Exception();
@ -165,8 +173,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
_Logger?.LogInformation("First run completed. Run again if wanted");
}
private int FullParallelForWork(A_Property propertyLogic,
B_Metadata metadata,
private int FullParallelForWork(B_Metadata metadata,
MapLogic mapLogic,
string outputResolution,
bool outputResolutionHasNumber,
@ -174,59 +181,47 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
string d2ResultsFullGroupDirectory,
List<Tuple<string, DateTime>> sourceDirectoryChanges,
Dictionary<int, List<MappingFromPhotoPrism>> fileNameToCollection,
Record record,
Container.Models.Container container,
Item item,
DateTime[] containerDateTimes,
bool? isFocusRelativePath)
{
int result = 0;
bool? shouldIgnore;
List<Shared.Models.Face> faces;
long ticks = DateTime.Now.Ticks;
DateTime dateTime = DateTime.Now;
Shared.Models.Property? property;
List<string> parseExceptions = [];
string[] changesFrom = [nameof(A_Property)];
List<Tuple<string, DateTime>> subFileTuples = [];
FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(cResultsFullGroupDirectory, item, outputResolutionHasNumber);
if (item.Property is null || item.Property.Id is null || !item.SourceDirectoryFileHolder.Exists || item.SourceDirectoryFileHolder.CreationTime is null || item.SourceDirectoryFileHolder.LastWriteTime is null || item.Any())
if (item.ExifDirectory is null || item.ExifDirectory.FilePath.Id is null || !item.SourceDirectoryFileHolder.Exists || item.SourceDirectoryFileHolder.CreationTime is null || item.SourceDirectoryFileHolder.LastWriteTime is null || item.Any())
throw new Exception();
if (_Configuration.PropertyConfiguration.ForcePropertyLastWriteTimeToCreationTime && item.SourceDirectoryFileHolder.LastWriteTime.Value != item.SourceDirectoryFileHolder.CreationTime.Value)
{
LogItemPropertyIsNull(item);
int? propertyHashCode = item.Property?.GetHashCode();
property = propertyLogic.GetProperty(metadata, item, subFileTuples, parseExceptions);
item.Update(property);
if (propertyHashCode is null)
{
lock (sourceDirectoryChanges)
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), DateTime.Now));
}
else if (propertyHashCode.Value != property.GetHashCode())
{
lock (sourceDirectoryChanges)
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), DateTime.Now));
}
File.SetLastWriteTime(item.SourceDirectoryFileHolder.FullName, item.SourceDirectoryFileHolder.CreationTime.Value);
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), item.SourceDirectoryFileHolder.CreationTime.Value));
}
else if (item.SourceDirectoryFileHolder.LastWriteTime is not null)
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), item.SourceDirectoryFileHolder.LastWriteTime.Value));
else
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), new FileInfo(item.SourceDirectoryFileHolder.FullName).LastWriteTime));
if (resizedFileHolder.Exists && item.ExifDirectory.Width is not null && item.ExifDirectory.Width.Value > 4 && _Configuration.SaveBlurHashForOutputResolutions.Contains(outputResolution))
{
string? file = _BlurHasher.GetFile(item.FilePath);
if (file is not null && !File.Exists(file))
_ = _BlurHasher.EncodeAndSave(item.FilePath, resizedFileHolder);
}
if (item.FilePath.Id is null || !record.ExifDirectoriesById.TryGetValue(item.FilePath.Id.Value, out ExifDirectory? exifDirectory))
{
shouldIgnore = null;
exifDirectory = null;
}
else
{
property = item.Property;
if (_Configuration.PropertyConfiguration.ForcePropertyLastWriteTimeToCreationTime && item.SourceDirectoryFileHolder.LastWriteTime.Value != item.SourceDirectoryFileHolder.CreationTime.Value)
{
File.SetLastWriteTime(item.SourceDirectoryFileHolder.FullName, item.SourceDirectoryFileHolder.CreationTime.Value);
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), item.SourceDirectoryFileHolder.CreationTime.Value));
}
else if (item.SourceDirectoryFileHolder.LastWriteTime is not null)
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), item.SourceDirectoryFileHolder.LastWriteTime.Value));
else
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), new FileInfo(item.SourceDirectoryFileHolder.FullName).LastWriteTime));
if (resizedFileHolder.Exists && item.Property.Width is not null && item.Property.Width.Value > 4 && _Configuration.SaveBlurHashForOutputResolutions.Contains(outputResolution))
{
string? file = _BlurHasher.GetFile(item.FilePath);
if (file is not null && !File.Exists(file))
_ = _BlurHasher.EncodeAndSave(item.FilePath, resizedFileHolder);
}
}
bool? shouldIgnore = property is null || property.Keywords is null ? null : _Configuration.PropertyConfiguration.IgnoreRulesKeyWords.Any(l => property.Keywords.Contains(l));
if (shouldIgnore is not null)
{
ReadOnlyCollection<string> keywords = IMetaBase.GetKeywords(exifDirectory.ExifBaseDirectories);
shouldIgnore = _Configuration.PropertyConfiguration.IgnoreRulesKeyWords.Any(keywords.Contains);
if (shouldIgnore.Value)
{
FileInfo fileInfo = new(resizedFileHolder.FullName);
@ -241,11 +236,9 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
throw new NotSupportedException($"Rename File! <{item.FilePath.FileNameFirstSegment}>");
}
}
if (property is null || item.Property is null)
throw new NullReferenceException(nameof(property));
item.SetResizedFileHolder(_Resize.FileNameExtension, resizedFileHolder);
MappingFromItem mappingFromItem = IMappingFromItem.GetMappingFromItem(containerDateTimes, item, resizedFileHolder);
ExifDirectory exifDirectory = metadata.GetMetadataCollection(item.FilePath, subFileTuples, parseExceptions, changesFrom, mappingFromItem);
exifDirectory ??= metadata.GetMetadataCollection(item.FilePath, subFileTuples, parseExceptions, changesFrom, mappingFromItem);
if (_AppSettings.Places.Count > 0)
{
float latitude;
@ -268,13 +261,13 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
distance += 1;
}
}
Dictionary<string, int[]> outputResolutionToResize = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, item.Property, mappingFromItem);
Dictionary<string, int[]> outputResolutionToResize = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, item.ExifDirectory, mappingFromItem);
if (_Configuration.SaveResizedSubfiles)
{
if (shouldIgnore is not null && item.FilePath.HasIgnoreKeyword is not null && item.FilePath.HasIgnoreKeyword.Value != shouldIgnore.Value)
faces = [];
else
_Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.Property, mappingFromItem, outputResolutionToResize);
_Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.ExifDirectory, mappingFromItem, outputResolutionToResize);
}
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
faces = [];
@ -286,7 +279,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
if (!fileNameToCollection.TryGetValue(mappingFromItem.Id, out mappingFromPhotoPrismCollection))
mappingFromPhotoPrismCollection = null;
bool move = _Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution);
faces = _Faces.GetFaces(outputResolution, cResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionToResize, mappingFromPhotoPrismCollection);
faces = _Faces.GetFaces(outputResolution, cResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, item.ExifDirectory, mappingFromItem, outputResolutionToResize, mappingFromPhotoPrismCollection);
result = GetNotMappedCountAndUpdateMappingFromPersonThenSetMapping(mapLogic, item, isFocusRelativePath, mappingFromItem, mappingFromPhotoPrismCollection, faces);
List<(Shared.Models.Face, FileHolder?, string, bool Saved)> faceCollection = _Faces.SaveFaces(item.FilePath, subFileTuples, parseExceptions, mappingFromItem, exifDirectory, faces);
if (move && faceCollection.All(l => !l.Saved))
@ -319,7 +312,6 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
private (int, bool) FullParallelWork(int maxDegreeOfParallelism,
A_Property propertyLogic,
B_Metadata metadata,
MapLogic mapLogic,
string outputResolution,
@ -328,6 +320,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
string d2ResultsFullGroupDirectory,
List<Tuple<string, DateTime>> sourceDirectoryChanges,
Dictionary<int, List<MappingFromPhotoPrism>> fileNameToCollection,
Record record,
Container.Models.Container container,
ReadOnlyCollection<Item> filteredItems,
string message)
@ -336,17 +329,15 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
int exceptionsCount = 0;
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
DateTime[] containerDateTimes = Container.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems);
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
string focusRelativePath = Path.GetFullPath(string.Concat(_Configuration.PropertyConfiguration.RootDirectory, _Configuration.FocusDirectory));
bool? isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath);
bool anyPropertiesChangedForX = _Configuration.PropertyConfiguration.PropertiesChangedForProperty || _Configuration.PropertiesChangedForDistance || _Configuration.PropertiesChangedForFaces || _Configuration.PropertiesChangedForIndex || _Configuration.PropertiesChangedForMetadata || _Configuration.PropertiesChangedForResize;
using ProgressBar progressBar = new(filteredItems.Count, message, options);
using ProgressBar progressBar = new(filteredItems.Count, message, _ProgressBarOptions);
_ = Parallel.For(0, filteredItems.Count, parallelOptions, (i, state) =>
{
try
{
result += FullParallelForWork(propertyLogic,
metadata,
result += FullParallelForWork(metadata,
mapLogic,
outputResolution,
outputResolutionHasNumber,
@ -354,6 +345,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
d2ResultsFullGroupDirectory,
sourceDirectoryChanges,
fileNameToCollection,
record,
container,
filteredItems[i],
containerDateTimes,
@ -371,6 +363,98 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
return (result, exceptionsCount > 0);
}
private void FullDoWork(string argZero,
string propertyRoot,
long ticks,
string fPhotoPrismSingletonDirectory,
int count,
B_Metadata metadata,
Record record,
ReadOnlyCollection<Container.Models.Container> readOnlyContainers,
MapLogic mapLogic)
{
int total;
int notMapped;
string message;
bool exceptions;
int totalSeconds;
int totalNotMapped = 0;
IDlibDotNet dlibDotNet = this;
bool outputResolutionHasNumber;
bool anyNullOrNoIsUniqueFileName;
string cResultsFullGroupDirectory;
string dResultsFullGroupDirectory;
string c2ResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
Container.Models.Container container;
ReadOnlyCollection<Item> filteredItems;
List<Tuple<string, DateTime>> sourceDirectoryChanges = [];
int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism;
Dictionary<int, List<MappingFromPhotoPrism>> fileNameToCollection = !Directory.Exists(fPhotoPrismSingletonDirectory) ? fileNameToCollection = [] : F_PhotoPrism.GetFileNameToCollection(fPhotoPrismSingletonDirectory);
string dResultsDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(D_Face));
foreach (string outputResolution in _Configuration.OutputResolutions)
{
total = 0;
outputResolutionHasNumber = outputResolution.Any(char.IsNumber);
(cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
_Faces.Update(dResultsFullGroupDirectory);
_Resize.Update(cResultsFullGroupDirectory);
_FaceParts.Update(d2ResultsFullGroupDirectory);
_BlurHasher.Update(c2ResultsFullGroupDirectory);
for (int i = 0; i < readOnlyContainers.Count; i++)
{
container = readOnlyContainers[i];
if (container.Items.Count == 0)
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = Container.Models.Stateless.Methods.IContainer.GetValidImageItems(_Configuration.PropertyConfiguration, container);
if (filteredItems.Count == 0)
continue;
sourceDirectoryChanges.Clear();
anyNullOrNoIsUniqueFileName = filteredItems.Any(l => !l.IsUniqueFileName);
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
message = $"{totalSeconds} total second(s) - {outputResolution} - {i + 1:000} / {readOnlyContainers.Count:000} - {total} / {count} total - <{container.SourceDirectory}> [{filteredItems.Count:000}] - total not mapped {totalNotMapped:000000}";
if (outputResolutionHasNumber)
_Resize.SetAngleBracketCollection(cResultsFullGroupDirectory, container.SourceDirectory);
(notMapped, exceptions) = FullParallelWork(maxDegreeOfParallelism,
metadata,
mapLogic,
outputResolution,
outputResolutionHasNumber,
cResultsFullGroupDirectory,
d2ResultsFullGroupDirectory,
sourceDirectoryChanges,
fileNameToCollection,
record,
container,
filteredItems,
message);
totalNotMapped += notMapped;
if (exceptions)
{
_Exceptions.Add(container.SourceDirectory);
continue;
}
if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Length > 0)
{
for (int y = 0; y < int.MaxValue; y++)
{
_Logger?.LogInformation("Press \"Y\" key when ready to continue or close console");
if (_Console.ReadKey() == ConsoleKey.Y)
break;
}
_Logger?.LogInformation(". . .");
}
total += container.Items.Count;
}
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
message = $"{totalSeconds} total second(s) - {outputResolution} - ### [###] / {readOnlyContainers.Count:000} - {total} / {count} total - <> - total not mapped {totalNotMapped:000000}";
using ProgressBar progressBar = new(1, message, _ProgressBarOptions);
progressBar.Tick();
}
}
void IDlibDotNet.Tick() =>
_ProgressBar?.Tick();
@ -399,6 +483,12 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory);
}
void IDlibDotNet.ConstructProgressBar(int maxTicks, string message)
{
_ProgressBar?.Dispose();
_ProgressBar = new(maxTicks, message, _ProgressBarOptions);
}
(string, string, string, string) IDlibDotNet.GetResultsFullGroupDirectories(string outputResolution)
{
string cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
@ -466,7 +556,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
if (filePath.Id is null)
throw new Exception();
string[] directoryNames = keyValuePair.Value.Select(l => l.DirectoryFullPath.Replace('\\', '/')).ToArray();
string paddedId = IId.GetPaddedId(propertyConfiguration, filePath.Id.Value, filePath.HasIgnoreKeyword, filePath.HasDateTimeOriginal, index: null);
string paddedId = IId.GetPaddedId(propertyConfiguration, filePath.Id.Value, filePath.ExtensionLowered, filePath.HasIgnoreKeyword, filePath.HasDateTimeOriginal, index: null);
result = new(directoryNames, filePath.ExtensionLowered, filePath.HasDateTimeOriginal, filePath.Id.Value, filePath.Length, paddedId, filePath.LastWriteTicks);
return result;
}
@ -513,6 +603,22 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
throw new NullReferenceException(nameof(configuration.ModelDirectory));
}
private static ExifDirectory? GetExifDirectory(FilePair filePair)
{
ExifDirectory? result;
if (filePair.Match is null)
result = null;
else
{
string json = File.ReadAllText(filePair.Match.FullName);
if (string.IsNullOrEmpty(json))
result = null;
else
result = JsonSerializer.Deserialize(json, ExifDirectorySourceGenerationContext.Default.ExifDirectory);
}
return result;
}
private static void DeleteContinueFiles(ReadOnlyCollection<PersonContainer> personContainers)
{
foreach (PersonContainer personContainer in personContainers)
@ -554,7 +660,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
}
private static bool IsFilesCollectionCountIsOne(ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection)
private static bool GetFilesCollectionCountIsOne(ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection)
{
bool result = true;
int count = 0;
@ -582,11 +688,11 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
foreach (Item item in distinctFilteredItems)
{
found = false;
if (item.Property?.Id is null)
if (item.ExifDirectory?.FilePath.Id is null)
continue;
foreach (Mapping mapping in distinctFilteredMappingCollection)
{
if (mapping.MappingFromItem.Id != item.Property.Id.Value)
if (mapping.MappingFromItem.Id != item.ExifDirectory.FilePath.Id.Value)
continue;
found = true;
break;
@ -601,13 +707,15 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
string fileName;
string directory;
bool? isWrongYear;
List<DateTime> dateTimes;
DateTime? dateTime;
string? exifDirectoryModel;
List<string> distinct = [];
WindowsShortcut windowsShortcut;
ReadOnlyCollection<DateTime> dateTimes;
List<(string, string, string)> collection = [];
foreach (Item item in distinctValidImageItems)
{
if (item.Property?.Id is null)
if (item.ExifDirectory?.FilePath.Id is null)
continue;
if (item.IsNotUniqueAndNeedsReview is null || !item.IsNotUniqueAndNeedsReview.Value)
continue;
@ -620,15 +728,19 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
foreach (Item item in distinctValidImageItems)
{
if (item.Property?.Id is null || item.Property.DateTimeOriginal is null)
if (item.ExifDirectory?.FilePath.Id is null)
continue;
dateTimes = item.Property.GetDateTimes();
(isWrongYear, _) = Shared.Models.Stateless.Methods.IProperty.IsWrongYear(item.FilePath, item.Property.DateTimeOriginal, dateTimes);
dateTime = IDate.GetDateTimeOriginal(item.ExifDirectory);
if (dateTime is null)
continue;
dateTimes = IDate.GetDateTimes(item.ExifDirectory);
(isWrongYear, _) = Shared.Models.Stateless.Methods.IProperty.IsWrongYear(item.FilePath, dateTime, dateTimes.ToList());
if (isWrongYear is null || !isWrongYear.Value)
continue;
// Remove-Item -LiteralPath "\\?\D:\Tmp\a\EX-Z70 "
model = string.IsNullOrEmpty(item.Property.Model) ? "Unknown" : CameraRegex().Replace(item.Property.Model.Trim(), "_");
directory = Path.Combine($"{eDistanceContentDirectory[..^1]}{nameof(Item)})", item.Property.DateTimeOriginal.Value.Year.ToString(), model);
exifDirectoryModel = IMetaBase.GetModel(item.ExifDirectory?.ExifBaseDirectories);
model = string.IsNullOrEmpty(exifDirectoryModel) ? "Unknown" : CameraRegex().Replace(exifDirectoryModel.Trim(), "_");
directory = Path.Combine($"{eDistanceContentDirectory[..^1]}{nameof(Item)})", dateTime.Value.Year.ToString(), model);
fileName = item.IsNotUniqueAndNeedsReview is not null && item.IsNotUniqueAndNeedsReview.Value ? Path.Combine(directory, $"{item.FilePath.Name} {item.FilePath.Length}.lnk") : Path.Combine(directory, $"{item.FilePath.Name}.lnk");
collection.Add((item.FilePath.FullName, directory, fileName));
if (distinct.Contains(directory))
@ -797,15 +909,19 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
return results;
}
private bool? GetIsFocusModel(Shared.Models.Property? property)
private bool? GetIsFocusModel(ExifDirectory? exifDirectory)
{
bool? result;
if (string.IsNullOrEmpty(_Configuration.FocusModel))
result = null;
else if (property is null || string.IsNullOrEmpty(property.Model))
result = null;
else
result = property.Model.Contains(_Configuration.FocusModel);
{
string? model = IMetaBase.GetModel(exifDirectory?.ExifBaseDirectories);
if (exifDirectory is null || string.IsNullOrEmpty(model))
result = null;
else
result = model.Contains(_Configuration.FocusModel);
}
return result;
}
@ -867,7 +983,39 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
return result;
}
private bool GetRunToDoCollectionFirst(Models.Configuration configuration, long ticks)
private ReadOnlyCollection<FilePath> GetFilePath(long ticks, string dFacesContentDirectory)
{
List<FilePath> results = [];
FilePath filePath;
FileHolder fileHolder;
string[] files = Directory.GetFiles(dFacesContentDirectory, $"*{_Faces.FileNameExtension}", SearchOption.AllDirectories);
long? skipOlderThan = _Configuration.SkipOlderThanDays is null ? null : new DateTime(ticks).AddDays(-_Configuration.SkipOlderThanDays.Value).Ticks;
foreach (string file in files)
{
fileHolder = Shared.Models.Stateless.Methods.IFileHolder.Get(file);
filePath = FilePath.Get(_Configuration.PropertyConfiguration, fileHolder, index: null);
if (filePath.Id is null)
continue;
if (skipOlderThan is not null && (fileHolder.LastWriteTime is null || fileHolder.LastWriteTime.Value.Ticks < skipOlderThan.Value))
continue;
results.Add(filePath);
}
return results.AsReadOnly();
}
private void ParallelFor(IDlibDotNet dlibDotNet, FilePair filePair, Dictionary<int, ExifDirectory> results)
{
dlibDotNet?.Tick();
if (filePair.FilePath.Id is null || results.ContainsKey(filePair.FilePath.Id.Value))
return;
ExifDirectory? exifDirectory = GetExifDirectory(filePair);
if (exifDirectory is null)
return;
lock (results)
results.Add(filePair.FilePath.Id.Value, exifDirectory);
}
private bool GetRunToDoCollectionFirst(Models.Configuration configuration, long ticks, string[] checkDirectories)
{
bool result = configuration.SaveSortingWithoutPerson;
if (!result)
@ -887,20 +1035,16 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
string eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), _Configuration.PropertyConfiguration.ResultContent);
(int season, string seasonName) = Shared.Models.Stateless.Methods.IProperty.GetSeason(dateTime.DayOfYear);
FileSystemInfo fileSystemInfo = new DirectoryInfo(eDistanceContentDirectory);
string[] checkDirectories =
[
Path.Combine(rootDirectory, "Ancestry"),
Path.Combine(rootDirectory, "Facebook"),
Path.Combine(rootDirectory, "LinkedIn")
];
foreach (string checkDirectory in checkDirectories)
{
if (!Directory.Exists(checkDirectory))
result = true;
if (checkDirectory == rootDirectory)
seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName}");
else
seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName} {Path.GetFileName(checkDirectory)}");
if (!Directory.Exists(seasonDirectory))
_ = Directory.CreateDirectory(seasonDirectory);
result = true;
if (result)
continue;
directories = Directory.GetDirectories(checkDirectory, "*", SearchOption.TopDirectoryOnly);
@ -923,32 +1067,15 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
return result;
}
private ReadOnlyCollection<FilePath> GetFilePath(long ticks, string dFacesContentDirectory)
{
List<FilePath> results = [];
FilePath filePath;
FileHolder fileHolder;
string[] files = Directory.GetFiles(dFacesContentDirectory, $"*{_Faces.FileNameExtension}", SearchOption.AllDirectories);
long? skipOlderThan = _Configuration.SkipOlderThanDays is null ? null : new DateTime(ticks).AddDays(-_Configuration.SkipOlderThanDays.Value).Ticks;
foreach (string file in files)
{
fileHolder = Shared.Models.Stateless.Methods.IFileHolder.Get(file);
filePath = FilePath.Get(_Configuration.PropertyConfiguration, fileHolder, index: null);
if (filePath.Id is null)
continue;
if (skipOlderThan is not null && (fileHolder.LastWriteTime is null || fileHolder.LastWriteTime.Value.Ticks < skipOlderThan.Value))
continue;
results.Add(filePath);
}
return results.AsReadOnly();
}
private void Search(long ticks, ReadOnlyCollection<PersonContainer> personContainers, string argZero, string propertyRoot)
{
string message;
MapLogic? mapLogic;
Record? record = null;
string seasonDirectory;
A_Property propertyLogic;
IDlibDotNet dlibDotNet = this;
DateTime dateTime = new(ticks);
string eDistanceContentDirectory;
string? a2PeopleContentDirectory;
string aResultsFullGroupDirectory;
@ -957,23 +1084,24 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
string fPhotoPrismContentDirectory;
const string fileSearchFilter = "*";
string fPhotoPrismSingletonDirectory;
bool filesCollectionCountIsOne = false;
const string directorySearchFilter = "*";
string? filesCollectionRootDirectory = null;
bool configurationOutputResolutionsHas = false;
ReadOnlyDictionary<long, List<int>> personKeyToIds;
ReadOnlyDictionary<int, Identifier>? splatNineIdentifiers = null;
ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>>? keyValuePairs = null;
ReadOnlyCollection<ReadOnlyCollection<FilePath>>? filePathsCollection = null;
bool runToDoCollectionFirst = GetRunToDoCollectionFirst(_Configuration, ticks);
(int season, string seasonName) = Shared.Models.Stateless.Methods.IProperty.GetSeason(dateTime.DayOfYear);
string[] checkDirectories =
[
Path.Combine(_Configuration.PropertyConfiguration.RootDirectory, "Ancestry"),
Path.Combine(_Configuration.PropertyConfiguration.RootDirectory, "Facebook"),
Path.Combine(_Configuration.PropertyConfiguration.RootDirectory, "LinkedIn")
];
bool runToDoCollectionFirst = GetRunToDoCollectionFirst(_Configuration, ticks, checkDirectories);
(aResultsFullGroupDirectory, bResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories();
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(_Configuration.PropertyConfiguration.RootDirectory, ticks);
a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "([])");
B_Metadata metadata = new(_Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, bResultsFullGroupDirectory);
eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), _Configuration.PropertyConfiguration.ResultContent);
string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), _Configuration.PropertyConfiguration.ResultSingleton);
_ = Directory.CreateDirectory(Path.Combine(eDistanceContentDirectory, ticks.ToString()));
B_Metadata metadata = new(dlibDotNet, _Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, ticks, bResultsFullGroupDirectory);
if (runToDoCollectionFirst)
mapLogic = null;
else
@ -985,18 +1113,16 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
configurationOutputResolutionsHas = true;
if (!runToDoCollectionFirst)
break;
(filesCollectionRootDirectory, filePathsCollection, filesCollectionCountIsOne) = GetFilesCollectionThenCopyOrMove(ticks, fileSearchFilter, directorySearchFilter, options, outputResolution);
keyValuePairs = FilePath.GetKeyValuePairs(filePathsCollection);
splatNineIdentifiers = GetSplatNineIdentifiersAndHideSplatNine(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, keyValuePairs);
record = GetFilesCollectionThenCopyOrMove(dlibDotNet, ticks, fileSearchFilter, directorySearchFilter, bResultsFullGroupDirectory, outputResolution);
break;
}
fPhotoPrismContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), _Configuration.PropertyConfiguration.ResultContent);
fPhotoPrismSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), _Configuration.PropertyConfiguration.ResultSingleton);
propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory);
if (filesCollectionCountIsOne)
if (record is not null && record.FilesCollectionCountIsOne)
{
if (filePathsCollection is null)
throw new NullReferenceException(nameof(filePathsCollection));
if (record.FilePathsCollection is null)
throw new NullReferenceException(nameof(record.FilePathsCollection));
string resultsGroupDirectory;
a2PeopleContentDirectory = null;
eDistanceContentDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", _Configuration.PropertyConfiguration.ResultContent);
@ -1007,7 +1133,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
resultsGroupDirectory = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, string.Empty, create: true);
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(resultsGroupDirectory);
}
argZero = SaveUrlAndGetNewRootDirectory(filePathsCollection.First());
argZero = SaveUrlAndGetNewRootDirectory(record.FilePathsCollection.First());
_Configuration.PropertyConfiguration.ChangeRootDirectory(argZero);
(aResultsFullGroupDirectory, bResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories();
propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false);
@ -1019,24 +1145,55 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
{
if (outputResolution.Any(char.IsNumber))
continue;
Dictionary<int, ExifDirectory> exifDirectoriesById = [];
(cResultsFullGroupDirectory, _, _, _) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
filesCollectionRootDirectory = Path.Combine(cResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
filePathsCollection = IDirectory.GetFilePathCollections(_Configuration.PropertyConfiguration, directorySearchFilter, fileSearchFilter, filesCollectionRootDirectory, useCeilingAverage: true);
string? filesCollectionRootDirectory = Path.Combine(cResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
ReadOnlyCollection<ReadOnlyCollection<FilePath>>? filePathsCollection = IDirectory.GetFilePathCollections(_Configuration.PropertyConfiguration, directorySearchFilter, fileSearchFilter, filesCollectionRootDirectory, useIgnoreExtensions: true, useCeilingAverage: true);
record = new(FilesCollectionRootDirectory: filesCollectionRootDirectory,
FilesCollectionCountIsOne: false,
FilePathsCollection: filePathsCollection,
ExifDirectoriesById: new(exifDirectoriesById),
IdToFilePaths: null,
SplatNineIdentifiers: null);
break;
}
}
if (filesCollectionRootDirectory is null || filePathsCollection is null)
throw new NullReferenceException(nameof(filePathsCollection));
int count = filePathsCollection.Select(l => l.Count).Sum();
if (string.IsNullOrEmpty(record?.FilesCollectionRootDirectory) || record.FilePathsCollection.Count == 0)
throw new NullReferenceException(nameof(record.FilePathsCollection));
foreach (string checkDirectory in checkDirectories)
{
seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName} {Path.GetFileName(checkDirectory)}");
if (!Directory.Exists(seasonDirectory))
_ = Directory.CreateDirectory(seasonDirectory);
}
int count = record.FilePathsCollection.Select(l => l.Count).Sum();
message = $") Building Container(s) - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
_ProgressBar = new(count, message, options);
ReadOnlyCollection<Container.Models.Container> readOnlyContainers = Container.Models.Stateless.Methods.IContainer.GetContainers(this, _Configuration.PropertyConfiguration, _Faces.FileNameExtension, _Faces.HiddenFileNameExtension, eDistanceContentDirectory, filesCollectionRootDirectory, keyValuePairs, splatNineIdentifiers, filePathsCollection);
_ProgressBar = new(count, message, _ProgressBarOptions);
ReadOnlyCollection<Container.Models.Container> readOnlyContainers =
Container.Models.Stateless.Methods.IContainer.GetContainers(dlibDotNet,
_Configuration.PropertyConfiguration,
_Faces.FileNameExtension,
_Faces.HiddenFileNameExtension,
eDistanceContentDirectory,
record.FilesCollectionRootDirectory,
record.IdToFilePaths,
record.SplatNineIdentifiers,
record.FilePathsCollection,
record.ExifDirectoriesById);
_ProgressBar.Dispose();
mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _Distance, personContainers, ticks, a2PeopleContentDirectory, a2PeopleSingletonDirectory, eDistanceContentDirectory);
DeleteContinueFiles(personContainers);
if (!runToDoCollectionFirst)
MapFaceFileLogic(ticks, personContainers, mapLogic, a2PeopleContentDirectory, eDistanceContentDirectory, options);
FullDoWork(argZero, propertyRoot, ticks, aResultsFullGroupDirectory, fPhotoPrismSingletonDirectory, count, metadata, readOnlyContainers, propertyLogic, mapLogic);
MapFaceFileLogic(ticks, personContainers, mapLogic, a2PeopleContentDirectory, eDistanceContentDirectory);
FullDoWork(argZero,
propertyRoot,
ticks,
fPhotoPrismSingletonDirectory,
count,
metadata,
record,
readOnlyContainers,
mapLogic);
ReadOnlyCollection<Item> distinctValidImageItems = Container.Models.Stateless.Methods.IContainer.GetValidImageItems(_Configuration.PropertyConfiguration, readOnlyContainers, distinctItems: true, filterItems: true);
if (_Configuration.LookForAbandoned)
{
@ -1044,9 +1201,9 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
string d2ResultsFullGroupDirectory;
foreach (string outputResolution in _Configuration.OutputResolutions)
{
_ProgressBar = new(5, nameof(mapLogic.LookForAbandoned), options);
_ProgressBar = new(5, nameof(mapLogic.LookForAbandoned), _ProgressBarOptions);
(cResultsFullGroupDirectory, _, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
mapLogic.LookForAbandoned(this, _Configuration.PropertyConfiguration, bResultsFullGroupDirectory, readOnlyContainers, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory);
mapLogic.LookForAbandoned(dlibDotNet, _Configuration.PropertyConfiguration, bResultsFullGroupDirectory, readOnlyContainers, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory);
_ProgressBar.Dispose();
}
}
@ -1077,8 +1234,8 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
&& _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
&& _Exceptions.Count == 0)
MapLogic(ticks, readOnlyContainers, fPhotoPrismContentDirectory, mapLogic, outputResolution, new(personKeyToIds), distinctValidImageFaces, distinctValidImageMappingCollection);
if (runToDoCollectionFirst && _Configuration.SaveRandomForOutputResolutions.Contains(outputResolution) && personKeyToIds.Count > 0 && splatNineIdentifiers is not null && distinctValidImageMappingCollection.Count > 0)
_Random.Random(_Configuration.PropertyConfiguration, _Configuration.ImmichAssetsFile, _Configuration.ImmichOwnerId, _Configuration.ImmichRoot, _Configuration.RadomUseBirthdayMinimum, _Configuration.ValidKeyWordsToIgnoreInRandom, personKeyToIds, splatNineIdentifiers, distinctValidImageMappingCollection);
if (runToDoCollectionFirst && _Configuration.SaveRandomForOutputResolutions.Contains(outputResolution) && personKeyToIds.Count > 0 && record?.SplatNineIdentifiers is not null && distinctValidImageMappingCollection.Count > 0)
_Random.Random(_Configuration.PropertyConfiguration, _Configuration.ImmichAssetsFile, _Configuration.ImmichOwnerId, _Configuration.ImmichRoot, _Configuration.RadomUseBirthdayMinimum, _Configuration.ValidKeyWordsToIgnoreInRandom, personKeyToIds, record.SplatNineIdentifiers, distinctValidImageMappingCollection);
if (_IsEnvironment.Development)
continue;
if (!_IsEnvironment.Development)
@ -1096,19 +1253,37 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
}
private (string, ReadOnlyCollection<ReadOnlyCollection<FilePath>>, bool) GetFilesCollectionThenCopyOrMove(long ticks, string fileSearchFilter, string directorySearchFilter, ProgressBarOptions options, string outputResolution)
private Record GetFilesCollectionThenCopyOrMove(IDlibDotNet dlibDotNet, long ticks, string fileSearchFilter, string directorySearchFilter, string bResultsFullGroupDirectory, string outputResolution)
{
Record result;
int count;
string message;
ProgressBar progressBar;
IDlibDotNet dlibDotNet = this;
const string extension = ".json";
ReadOnlyCollection<FilePair> filePairs;
int maxDegreeOfParallelism = Environment.ProcessorCount;
Dictionary<int, ExifDirectory> exifDirectoriesById = [];
string filesCollectionRootDirectory = _Configuration.PropertyConfiguration.RootDirectory;
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
(string cResultsFullGroupDirectory, _, _, _) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
IReadOnlyDictionary<string, ReadOnlyDictionary<byte, ReadOnlyCollection<string>>> fileGroups = Shared.Models.Stateless.Methods.IPath.GetKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, [_Configuration.PropertyConfiguration.ResultContent]);
ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection = IDirectory.GetFilePathCollections(_Configuration.PropertyConfiguration, directorySearchFilter, fileSearchFilter, filesCollectionRootDirectory, useCeilingAverage: false);
int count = filePathsCollection.Select(l => l.Count).Sum();
bool filesCollectionCountIsOne = IsFilesCollectionCountIsOne(filePathsCollection);
string message = $") Selecting for ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
progressBar = new(count, message, options);
(string[] distinctDirectories, List<(FilePath, string)> toDoCollection) = IDirectory.GetToDoCollection(_Configuration.PropertyConfiguration, filePathsCollection, fileGroups, () => progressBar.Tick());
string jsonGroupDirectory = Path.Combine(bResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton);
if (!Directory.Exists(jsonGroupDirectory))
_ = Directory.CreateDirectory(jsonGroupDirectory);
IReadOnlyDictionary<string, ReadOnlyDictionary<byte, ReadOnlyCollection<string>>> keyValuePairs =
Shared.Models.Stateless.Methods.IPath.GetKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, [_Configuration.PropertyConfiguration.ResultContent]);
ReadOnlyDictionary<byte, ReadOnlyCollection<string>> fileGroups = keyValuePairs[_Configuration.PropertyConfiguration.ResultContent];
ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection = IDirectory.GetFilePathCollections(_Configuration.PropertyConfiguration, directorySearchFilter, fileSearchFilter, filesCollectionRootDirectory, useIgnoreExtensions: true, useCeilingAverage: false);
count = filePathsCollection.Select(l => l.Count).Sum();
filePairs = IFilePair.GetFilePairs(_Configuration.PropertyConfiguration, directorySearchFilter, extension, jsonGroupDirectory, filePathsCollection);
message = $") Preloading ExifDirectory Dictionary - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
progressBar = new(count, message, _ProgressBarOptions);
_ = Parallel.For(0, filePairs.Count, parallelOptions, (i, state) => ParallelFor(dlibDotNet, filePairs[i], exifDirectoriesById));
progressBar.Dispose();
count = filePathsCollection.Select(l => l.Count).Sum();
bool filesCollectionCountIsOne = GetFilesCollectionCountIsOne(filePathsCollection);
message = $") Selecting for ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
progressBar = new(count, message, _ProgressBarOptions);
(string[] distinctDirectories, List<(FilePath, string)> toDoCollection) = IDirectory.GetToDoCollection(_Configuration.PropertyConfiguration, filePathsCollection, fileGroups, exifDirectoriesById, () => progressBar.Tick());
progressBar.Dispose();
foreach (string distinctDirectory in distinctDirectories)
{
@ -1116,13 +1291,21 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
_ = Directory.CreateDirectory(distinctDirectory);
}
message = $") Copying to ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
progressBar = new(count, message, options);
progressBar = new(count, message, _ProgressBarOptions);
_ = IDirectory.CopyOrMove(toDoCollection, move: false, moveBack: false, () => progressBar.Tick());
progressBar.Dispose();
return (filesCollectionRootDirectory, filePathsCollection, filesCollectionCountIsOne);
ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>> idToFilePaths = FilePath.GetKeyValuePairs(filePathsCollection);
ReadOnlyDictionary<int, Identifier> splatNineIdentifiers = GetSplatNineIdentifiersAndHideSplatNine(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, idToFilePaths);
result = new(ExifDirectoriesById: new(exifDirectoriesById),
FilesCollectionRootDirectory: filesCollectionRootDirectory,
FilesCollectionCountIsOne: filesCollectionCountIsOne,
FilePathsCollection: filePathsCollection,
IdToFilePaths: idToFilePaths,
SplatNineIdentifiers: splatNineIdentifiers);
return result;
}
private void MapFaceFileLogic(long ticks, ReadOnlyCollection<PersonContainer> personContainers, MapLogic mapLogic, string? a2PeopleContentDirectory, string eDistanceContentDirectory, ProgressBarOptions options)
private void MapFaceFileLogic(long ticks, ReadOnlyCollection<PersonContainer> personContainers, MapLogic mapLogic, string? a2PeopleContentDirectory, string eDistanceContentDirectory)
{
foreach (string outputResolution in _Configuration.OutputResolutions)
{
@ -1140,7 +1323,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
continue;
if (!_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
continue;
List<SaveContainer> saveContainers = GetSaveContainers(ticks, personContainers, a2PeopleContentDirectory, eDistanceContentDirectory, options, mapLogic, outputResolution);
List<SaveContainer> saveContainers = GetSaveContainers(ticks, personContainers, a2PeopleContentDirectory, eDistanceContentDirectory, mapLogic, outputResolution);
if (saveContainers.Count > 0)
{
int updated = 0;
@ -1150,7 +1333,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
}
private List<SaveContainer> GetSaveContainers(long ticks, ReadOnlyCollection<PersonContainer> personContainers, string a2PeopleSingletonDirectory, string eDistanceContentDirectory, ProgressBarOptions options, MapLogic mapLogic, string outputResolution)
private List<SaveContainer> GetSaveContainers(long ticks, ReadOnlyCollection<PersonContainer> personContainers, string a2PeopleSingletonDirectory, string eDistanceContentDirectory, MapLogic mapLogic, string outputResolution)
{
List<SaveContainer> results;
IDlibDotNet dlibDotNet = this;
@ -1179,8 +1362,8 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
else
{
string message = $") Building Matrix - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
_ProgressBar = new(postFiltered.Count, message, options);
ReadOnlyCollection<LocationContainer> matrix = E_Distance.GetMatrixLocationContainers(this, _MapConfiguration, ticks, mapLogic, mappedWithEncoding, preFiltered, distanceLimits, postFiltered);
_ProgressBar = new(postFiltered.Count, message, _ProgressBarOptions);
ReadOnlyCollection<LocationContainer> matrix = E_Distance.GetMatrixLocationContainers(dlibDotNet, _MapConfiguration, ticks, mapLogic, mappedWithEncoding, preFiltered, distanceLimits, postFiltered);
_ProgressBar.Dispose();
ReadOnlyDictionary<string, LocationContainer> onlyOne = GetOnlyOne(distanceLimits, matrix);
if (onlyOne.Count == 0)
@ -1195,92 +1378,6 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
return results;
}
private void FullDoWork(string argZero, string propertyRoot, long ticks, string aResultsFullGroupDirectory, string fPhotoPrismSingletonDirectory, int count, B_Metadata metadata, ReadOnlyCollection<Container.Models.Container> readOnlyContainers, A_Property propertyLogic, MapLogic mapLogic)
{
int total;
int notMapped;
string message;
bool exceptions;
int totalSeconds;
int totalNotMapped = 0;
IDlibDotNet dlibDotNet = this;
bool outputResolutionHasNumber;
bool anyNullOrNoIsUniqueFileName;
string cResultsFullGroupDirectory;
string dResultsFullGroupDirectory;
string c2ResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
Container.Models.Container container;
ReadOnlyCollection<Item> filteredItems;
List<Tuple<string, DateTime>> sourceDirectoryChanges = [];
int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism;
Dictionary<int, List<MappingFromPhotoPrism>> fileNameToCollection = !Directory.Exists(fPhotoPrismSingletonDirectory) ? fileNameToCollection = [] : F_PhotoPrism.GetFileNameToCollection(fPhotoPrismSingletonDirectory);
string dResultsDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(D_Face));
foreach (string outputResolution in _Configuration.OutputResolutions)
{
total = 0;
outputResolutionHasNumber = outputResolution.Any(char.IsNumber);
(cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
_Faces.Update(dResultsFullGroupDirectory);
_Resize.Update(cResultsFullGroupDirectory);
_FaceParts.Update(d2ResultsFullGroupDirectory);
_BlurHasher.Update(c2ResultsFullGroupDirectory);
for (int i = 0; i < readOnlyContainers.Count; i++)
{
container = readOnlyContainers[i];
if (container.Items.Count == 0)
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = Container.Models.Stateless.Methods.IContainer.GetValidImageItems(_Configuration.PropertyConfiguration, container);
if (filteredItems.Count == 0)
continue;
sourceDirectoryChanges.Clear();
anyNullOrNoIsUniqueFileName = filteredItems.Any(l => !l.IsUniqueFileName);
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
message = $"{totalSeconds} total second(s) - {outputResolution} - {i + 1:000} / {readOnlyContainers.Count:000} - {total} / {count} total - <{container.SourceDirectory}> [{filteredItems.Count:000}] - total not mapped {totalNotMapped:000000}";
propertyLogic.SetAngleBracketCollection(aResultsFullGroupDirectory, container.SourceDirectory, anyNullOrNoIsUniqueFileName);
if (outputResolutionHasNumber)
_Resize.SetAngleBracketCollection(cResultsFullGroupDirectory, container.SourceDirectory);
(notMapped, exceptions) = FullParallelWork(maxDegreeOfParallelism,
propertyLogic,
metadata,
mapLogic,
outputResolution,
outputResolutionHasNumber,
cResultsFullGroupDirectory,
d2ResultsFullGroupDirectory,
sourceDirectoryChanges,
fileNameToCollection,
container,
filteredItems,
message);
totalNotMapped += notMapped;
if (exceptions)
{
_Exceptions.Add(container.SourceDirectory);
continue;
}
if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Length > 0)
{
for (int y = 0; y < int.MaxValue; y++)
{
_Logger?.LogInformation("Press \"Y\" key when ready to continue or close console");
if (_Console.ReadKey() == ConsoleKey.Y)
break;
}
_Logger?.LogInformation(". . .");
}
total += container.Items.Count;
}
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
message = $"{totalSeconds} total second(s) - {outputResolution} - ### [###] / {readOnlyContainers.Count:000} - {total} / {count} total - <> - total not mapped {totalNotMapped:000000}";
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
using ProgressBar progressBar = new(1, message, options);
progressBar.Tick();
}
}
private ReadOnlyCollection<Mapping> GetMappings(Property.Models.Configuration propertyConfiguration, string eDistanceContentDirectory, ReadOnlyCollection<Container.Models.Container> readOnlyContainers, MapLogic mapLogic, bool distinctItems)
{
List<Mapping> results = [];
@ -1306,14 +1403,14 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath);
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null || item.ResizedFileHolder is null)
if (item.ExifDirectory?.FilePath.Id is null || item.ResizedFileHolder is null)
continue;
mappingFromItem = IMappingFromItem.GetMappingFromItem(containerDateTimes, item, item.ResizedFileHolder);
if (distinctItems)
{
if (distinct.Contains(item.Property.Id.Value))
if (distinct.Contains(item.ExifDirectory.FilePath.Id.Value))
continue;
distinct.Add(item.Property.Id.Value);
distinct.Add(item.ExifDirectory.FilePath.Id.Value);
}
count++;
anyValidFaces = false;
@ -1345,7 +1442,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
bool? eyeReview = null;
int confidencePercent = 0;
int faceAreaPermyriad = 0;
bool? isFocusModel = GetIsFocusModel(item.Property);
bool? isFocusModel = GetIsFocusModel(item.ExifDirectory);
long[] jLinkResolvedPersonKeys = _JLinkResolvedDirectories.Select(l => l.PersonKey).ToArray();
int wholePercentRectangle = Shared.Models.Stateless.Methods.ILocation.GetWholePercentages(Shared.Models.Stateless.ILocation.Digits);
string deterministicHashCodeKey = IMapping.GetDeterministicHashCodeKey(item.FilePath, Shared.Models.Stateless.ILocation.Digits);
@ -1501,12 +1598,12 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
MappingFromLocation? mappingFromLocation;
MappingFromFilterPre mappingFromFilterPre;
MappingFromFilterPost mappingFromFilterPost;
bool? isFocusModel = GetIsFocusModel(item.Property);
ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>? wholePercentagesToPersonContainers = mapLogic.GetWholePercentagesToPersonContainers(item.Property?.Id);
bool? isFocusModel = GetIsFocusModel(item.ExifDirectory);
ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>? wholePercentagesToPersonContainers = mapLogic.GetWholePercentagesToPersonContainers(item.ExifDirectory?.FilePath.Id);
long[] jLinkResolvedPersonKeys = _JLinkResolvedDirectories.Select(l => l.PersonKey).ToArray();
foreach (Shared.Models.Face face in faces)
{
if (item.Property?.Id is null || face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
if (item.ExifDirectory?.FilePath.Id is null || face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
{
canReMap = null;
isFocusPerson = null;
@ -1529,7 +1626,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
wholePercentRectangle = Shared.Models.Stateless.Methods.ILocation.GetWholePercentages(face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution);
deterministicHashCodeKey = IMapping.GetDeterministicHashCodeKey(item.FilePath, face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution);
mappingFromLocation = new(faceAreaPermyriad, confidencePercent, deterministicHashCodeKey, eyeα, eyeReview, wholePercentRectangle);
inSkipCollection = mapLogic.InSkipCollection(item.Property.Id.Value, mappingFromLocation);
inSkipCollection = mapLogic.InSkipCollection(item.ExifDirectory.FilePath.Id.Value, mappingFromLocation);
mappingFromFilterPre = new(inSkipCollection, isFocusModel, isFocusRelativePath);
canReMap = Map.Models.Stateless.Methods.IMapLogic.CanReMap(jLinkResolvedPersonKeys, wholePercentagesToPersonContainers, mappingFromLocation);
isFocusPerson = mapLogic.IsFocusPerson(_Configuration.SkipPersonWithMoreThen, jLinkResolvedPersonKeys, wholePercentagesToPersonContainers, mappingFromLocation);