phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Runs but broken

Browse Source
This commit is contained in:
Mike Phares
2024-08-31 08:32:06 -07:00
parent f458af776a
commit 326e579d5c
43 changed files with 1763 additions and 654 deletions
Download Patch File Download Diff File Expand all files Collapse all files

572
Instance/DlibDotNet.cs
View File

@ -22,23 +22,25 @@ using WindowsShortcutFactory;
namespace View_by_Distance.Instance;
public partial class DlibDotNet
public partial class DlibDotNet : IDlibDotNet, IDisposable
{
[GeneratedRegex(@"[\\,\/,\:,\*,\?,\"",\<,\>,\|]")]
private static partial Regex CameraRegex();
private readonly D_Face _Faces;
private ProgressBar? _ProgressBar;
private readonly C_Resize _Resize;
private readonly F_Random _Random;
private readonly IConsole _Console;
private readonly E_Distance _Distance;
private readonly IBlurHasher _BlurHasher;
private readonly D2_FaceParts _FaceParts;
private readonly IBlurHasher _BlurHasher;
private readonly AppSettings _AppSettings;
private readonly List<string> _Exceptions;
private readonly ILogger<Program>? _Logger;
private readonly IsEnvironment _IsEnvironment;
private readonly DistanceLimits _DistanceLimits;
private readonly bool _PropertyRootExistedBefore;
private readonly Models.Configuration _Configuration;
private readonly bool _ArgZeroIsConfigurationRootDirectory;
@ -111,7 +113,8 @@ public partial class DlibDotNet
(ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetGifLowQuality();
_FaceParts = new D2_FaceParts(_Configuration.PropertyConfiguration, imageCodecInfo, encoderParameters, filenameExtension, configuration.CheckDFaceAndUpWriteDates, configuration.OverrideForFaceLandmarkImages);
}
_MapConfiguration = Get(configuration, _Faces.FileNameExtension, _Faces.HiddenFileNameExtension, _FaceParts.FileNameExtension);
_DistanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaPermyriadTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh);
_MapConfiguration = Get(configuration, _DistanceLimits, _Faces.FileNameExtension, _Faces.HiddenFileNameExtension, _FaceParts.FileNameExtension);
_Distance = new(configuration.DistanceMoveUnableToMatch, configuration.DistanceRenameToMatch, configuration.FaceConfidencePercent, configuration.RangeDistanceTolerance, configuration.RectangleIntersectMinimums);
if (_PropertyRootExistedBefore || !_ArgZeroIsConfigurationRootDirectory)
personContainers = new(new List<PersonContainer>());
@ -165,6 +168,15 @@ public partial class DlibDotNet
_Logger?.LogInformation("First run completed. Run again if wanted");
}
void IDlibDotNet.Tick() =>
_ProgressBar?.Tick();
void IDisposable.Dispose()
{
_ProgressBar?.Dispose();
GC.SuppressFinalize(this);
}
private static void Verify(Models.Configuration configuration)
{
if (configuration.RangeDaysDeltaTolerance.Length != 3)
@ -232,35 +244,84 @@ public partial class DlibDotNet
throw new Exception("Configuration has to match interface!");
if (configuration.LocationFactor != Shared.Models.Stateless.ILocation.Factor)
throw new Exception("Configuration has to match interface!");
if (configuration.SaveSortingWithoutPerson && configuration.JLinks.Length > 0)
throw new Exception("Configuration has SaveSortingWithoutPerson and JLinks!");
if (configuration.SaveSortingWithoutPerson && !string.IsNullOrEmpty(configuration.FocusModel))
throw new Exception("Configuration has SaveSortingWithoutPerson and FocusModel!");
if (configuration.SaveSortingWithoutPerson && !string.IsNullOrEmpty(configuration.FocusDirectory))
throw new Exception("Configuration has SaveSortingWithoutPerson and FocusDirectory!");
}
private ReadOnlyCollection<int> GetNotNineCollection(ReadOnlyCollection<string[]> filesCollection)
private ReadOnlyCollection<int> GetNotNineCollection(ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection)
{
List<int> results = [];
FilePath filePath;
FileInfo fileInfo;
FileHolder fileHolder;
foreach (string[] files in filesCollection)
FilePath checkFilePath;
foreach (ReadOnlyCollection<FilePath> filePaths in filePathsCollection)
{
foreach (string file in files)
foreach (FilePath filePath in filePaths)
{
if (!file.Contains(" !9"))
if (!filePath.FullName.Contains(" !9"))
continue;
fileHolder = Shared.Models.Stateless.Methods.IFileHolder.Get(file);
filePath = FilePath.Get(_Configuration.PropertyConfiguration, fileHolder, index: null);
if (filePath.Id is null)
fileInfo = new(filePath.FullName);
fileHolder = Shared.Models.Stateless.Methods.IFileHolder.Get(fileInfo);
if (!fileInfo.Attributes.HasFlag(FileAttributes.Hidden))
File.SetAttributes(fileHolder.FullName, FileAttributes.Hidden);
checkFilePath = FilePath.Get(_Configuration.PropertyConfiguration, fileHolder, index: null);
if (checkFilePath.Id is null)
continue;
results.Add(filePath.Id.Value);
results.Add(checkFilePath.Id.Value);
}
}
return new(results);
}
private static void DeleteContinueFiles(ReadOnlyCollection<PersonContainer> personContainers)
{
foreach (PersonContainer personContainer in personContainers)
{
foreach (FilePath filePath in personContainer.DisplayDirectoryAllFilePaths)
{
if (filePath.ExtensionLowered != ".continue")
continue;
File.Delete(filePath.FullName);
}
}
}
private void MapFaceFileLogic(long ticks, ReadOnlyCollection<PersonContainer> personContainers, MapLogic mapLogic, string? a2PeopleContentDirectory, string eDistanceContentDirectory, ProgressBarOptions options)
{
foreach (string outputResolution in _Configuration.OutputResolutions)
{
if (_Exceptions.Count != 0)
break;
if (!_Configuration.SaveResizedSubfiles)
continue;
if (!_ArgZeroIsConfigurationRootDirectory)
continue;
if (outputResolution != _Configuration.OutputResolutions[0])
continue;
if (_PropertyRootExistedBefore || a2PeopleContentDirectory is null)
break;
if (!_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution))
continue;
if (!_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
continue;
List<SaveContainer> saveContainers = GetSaveContainers(ticks, personContainers, a2PeopleContentDirectory, eDistanceContentDirectory, options, mapLogic, outputResolution);
if (saveContainers.Count > 0)
{
int updated = 0;
mapLogic.SaveContainers(updated, saveContainers);
throw new NotSupportedException("Done Restart! :)");
}
}
}
private void Search(long ticks, ReadOnlyCollection<PersonContainer> personContainers, string argZero, string propertyRoot)
{
int t;
string message;
MapLogic? mapLogic;
Container[] containers;
A_Property propertyLogic;
string eDistanceContentDirectory;
string? a2PeopleContentDirectory;
@ -274,10 +335,11 @@ public partial class DlibDotNet
const string directorySearchFilter = "*";
string? filesCollectionRootDirectory = null;
bool configurationOutputResolutionsHas = false;
ReadOnlyCollection<Container> readOnlyContainers;
ReadOnlyCollection<int>? notNineCollection = null;
ReadOnlyDictionary<long, List<int>> personKeyToIds;
ReadOnlyCollection<string[]>? filesCollection = null;
bool runToDoCollectionFirst = GetRunToDoCollectionFirst(ticks);
ReadOnlyCollection<ReadOnlyCollection<FilePath>>? filePathsCollection = null;
bool runToDoCollectionFirst = GetRunToDoCollectionFirst(_Configuration, ticks);
(aResultsFullGroupDirectory, bResultsFullGroupDirectory) = GetResultsFullGroupDirectories();
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(_Configuration.PropertyConfiguration.RootDirectory, ticks);
@ -296,8 +358,8 @@ public partial class DlibDotNet
configurationOutputResolutionsHas = true;
if (!runToDoCollectionFirst)
break;
(filesCollectionRootDirectory, filesCollection, filesCollectionCountIsOne) = GetFilesCollectionThenCopyOrMove(ticks, fileSearchFilter, directorySearchFilter, options, outputResolution);
notNineCollection = GetNotNineCollection(filesCollection);
(filesCollectionRootDirectory, filePathsCollection, filesCollectionCountIsOne) = GetFilesCollectionThenCopyOrMove(ticks, fileSearchFilter, directorySearchFilter, options, outputResolution);
notNineCollection = GetNotNineCollection(filePathsCollection);
break;
}
fPhotoPrismContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), _Configuration.PropertyConfiguration.ResultContent);
@ -305,8 +367,8 @@ public partial class DlibDotNet
propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory);
if (filesCollectionCountIsOne)
{
if (filesCollection is null)
throw new NullReferenceException(nameof(filesCollection));
if (filePathsCollection is null)
throw new NullReferenceException(nameof(filePathsCollection));
string resultsGroupDirectory;
a2PeopleContentDirectory = null;
eDistanceContentDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", _Configuration.PropertyConfiguration.ResultContent);
@ -317,7 +379,7 @@ public partial class DlibDotNet
resultsGroupDirectory = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, string.Empty, create: true);
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(resultsGroupDirectory);
}
argZero = SaveUrlAndGetNewRootDirectory(filesCollection.First());
argZero = SaveUrlAndGetNewRootDirectory(filePathsCollection.First());
_Configuration.PropertyConfiguration.ChangeRootDirectory(argZero);
(aResultsFullGroupDirectory, bResultsFullGroupDirectory) = GetResultsFullGroupDirectories();
propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false);
@ -325,53 +387,53 @@ public partial class DlibDotNet
}
if (configurationOutputResolutionsHas)
{
int count;
foreach (string outputResolution in _Configuration.OutputResolutions)
{
if (outputResolution.Any(char.IsNumber))
continue;
(cResultsFullGroupDirectory, _, _, _) = GetResultsFullGroupDirectories(outputResolution);
filesCollectionRootDirectory = Path.Combine(cResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
filesCollection = IDirectory.GetFilesCollection(filesCollectionRootDirectory, directorySearchFilter, fileSearchFilter, useCeilingAverage: true);
count = filesCollection.Select(l => l.Length).Sum();
filePathsCollection = IDirectory.GetFilePathCollections(_Configuration.PropertyConfiguration, filesCollectionRootDirectory, directorySearchFilter, fileSearchFilter, useCeilingAverage: true);
break;
}
}
if (filesCollectionRootDirectory is null || filesCollection is null)
throw new NullReferenceException(nameof(filesCollection));
if (filesCollectionRootDirectory is null || filePathsCollection is null)
throw new NullReferenceException(nameof(filePathsCollection));
string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton);
if (!Directory.Exists(aPropertySingletonDirectory))
_ = Directory.CreateDirectory(aPropertySingletonDirectory);
int count = filePathsCollection.Select(l => l.Count).Sum();
SaveDistinctIds(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, filePathsCollection);
message = $") Building Container(s) - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
using (ProgressBar progressBar = new(2, message, options))
{
progressBar.Tick();
string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton);
if (!Directory.Exists(aPropertySingletonDirectory))
_ = Directory.CreateDirectory(aPropertySingletonDirectory);
(t, containers) = Shared.Models.Stateless.Methods.IContainer.GetContainers(_Configuration.PropertyConfiguration, aPropertySingletonDirectory, filesCollectionRootDirectory, filesCollection);
progressBar.Tick();
}
ReadOnlyCollection<Container> readOnlyContainers = new(containers);
SaveDistinctIds(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, readOnlyContainers);
_ProgressBar = new(count, message, options);
readOnlyContainers = Shared.Models.Stateless.Methods.IContainer.GetContainers(this, _Configuration.PropertyConfiguration, aPropertySingletonDirectory, filesCollectionRootDirectory, filePathsCollection);
_ProgressBar.Dispose();
mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _Distance, personContainers, ticks, a2PeopleContentDirectory, a2PeopleSingletonDirectory, eDistanceContentDirectory);
FullDoWork(argZero, propertyRoot, ticks, aResultsFullGroupDirectory, bResultsFullGroupDirectory, fPhotoPrismSingletonDirectory, t, readOnlyContainers, propertyLogic, mapLogic);
ReadOnlyCollection<Item> distinctFilteredItems = Shared.Models.Stateless.Methods.IContainer.GetItems(_Configuration.PropertyConfiguration, readOnlyContainers, distinctItems: true, filterItems: true);
DeleteContinueFiles(personContainers);
if (!runToDoCollectionFirst)
MapFaceFileLogic(ticks, personContainers, mapLogic, a2PeopleContentDirectory, eDistanceContentDirectory, options);
FullDoWork(argZero, propertyRoot, ticks, aResultsFullGroupDirectory, bResultsFullGroupDirectory, fPhotoPrismSingletonDirectory, count, readOnlyContainers, propertyLogic, mapLogic);
ReadOnlyCollection<Item> distinctValidImageItems = Shared.Models.Stateless.Methods.IContainer.GetValidImageItems(_Configuration.PropertyConfiguration, readOnlyContainers, distinctItems: true, filterItems: true);
if (_Configuration.LookForAbandoned)
{
string dResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
foreach (string outputResolution in _Configuration.OutputResolutions)
{
_ProgressBar = new(5, nameof(mapLogic.LookForAbandoned), options);
(cResultsFullGroupDirectory, _, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
mapLogic.LookForAbandoned(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, readOnlyContainers, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory);
mapLogic.LookForAbandoned(this, _Configuration.PropertyConfiguration, bResultsFullGroupDirectory, readOnlyContainers, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory);
_ProgressBar.Dispose();
}
}
_Distance.Clear();
ReadOnlyCollection<Shared.Models.Face> distinctFilteredFaces = Map.Models.Stateless.Methods.IMapLogic.GetFaces(distinctFilteredItems);
ReadOnlyCollection<Mapping> distinctFilteredMappingCollection = GetMappings(_Configuration.PropertyConfiguration, eDistanceContentDirectory, readOnlyContainers, mapLogic, distinctItems: true);
ReadOnlyCollection<Shared.Models.Face> distinctValidImageFaces = Map.Models.Stateless.Methods.IMapLogic.GetFaces(distinctValidImageItems);
ReadOnlyCollection<Mapping> distinctValidImageMappingCollection = GetMappings(_Configuration.PropertyConfiguration, eDistanceContentDirectory, readOnlyContainers, mapLogic, distinctItems: true);
if (runToDoCollectionFirst)
{
if (!Directory.Exists(eDistanceContentDirectory))
_ = Directory.CreateDirectory(eDistanceContentDirectory);
string json = JsonSerializer.Serialize(distinctFilteredMappingCollection);
string json = JsonSerializer.Serialize(distinctValidImageMappingCollection);
File.WriteAllText(Path.Combine(eDistanceContentDirectory, $"{ticks}.json"), json);
}
foreach (string outputResolution in _Configuration.OutputResolutions)
@ -380,19 +442,19 @@ public partial class DlibDotNet
break;
personKeyToIds = mapLogic.GetPersonKeyToIds();
if (_Configuration.SavePropertyShortcutsForOutputResolutions.Contains(outputResolution))
SavePropertyShortcutsForOutputResolutions(eDistanceContentDirectory, distinctFilteredItems);
SavePropertyShortcutsForOutputResolutions(eDistanceContentDirectory, distinctValidImageItems);
if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
mapLogic.SaveShortcutsForOutputResolutionsPreMapLogic(eDistanceContentDirectory, personKeyToIds, distinctFilteredMappingCollection);
mapLogic.SaveShortcutsForOutputResolutionsPreMapLogic(eDistanceContentDirectory, personKeyToIds, distinctValidImageMappingCollection);
if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.JLinks.Where(l => !string.IsNullOrEmpty(l)).Any() && _Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution))
mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, personContainers, a2PeopleContentDirectory, personKeyToIds, distinctFilteredMappingCollection);
mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, personContainers, a2PeopleContentDirectory, personKeyToIds, distinctValidImageMappingCollection);
if (_ArgZeroIsConfigurationRootDirectory
&& _Configuration.SaveResizedSubfiles
&& outputResolution == _Configuration.OutputResolutions[0]
&& _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
&& _Exceptions.Count == 0)
MapLogic(ticks, readOnlyContainers, fPhotoPrismContentDirectory, mapLogic, outputResolution, new(personKeyToIds), distinctFilteredFaces, distinctFilteredMappingCollection);
if (runToDoCollectionFirst && _Configuration.SaveRandomForOutputResolutions.Contains(outputResolution) && personKeyToIds.Count > 0 && distinctFilteredMappingCollection.Count > 0)
_Random.Random(_Configuration.PropertyConfiguration, _Configuration.RadomUseBirthdayMinimum, _Configuration.ValidKeyWordsToIgnoreInRandom, personKeyToIds, notNineCollection, distinctFilteredMappingCollection);
MapLogic(ticks, readOnlyContainers, fPhotoPrismContentDirectory, mapLogic, outputResolution, new(personKeyToIds), distinctValidImageFaces, distinctValidImageMappingCollection);
if (runToDoCollectionFirst && _Configuration.SaveRandomForOutputResolutions.Contains(outputResolution) && personKeyToIds.Count > 0 && distinctValidImageMappingCollection.Count > 0)
_Random.Random(_Configuration.PropertyConfiguration, _Configuration.ImmichAssetsFile, _Configuration.RadomUseBirthdayMinimum, _Configuration.ValidKeyWordsToIgnoreInRandom, personKeyToIds, notNineCollection, distinctValidImageMappingCollection);
if (_IsEnvironment.Development)
continue;
if (!_IsEnvironment.Development)
@ -410,47 +472,52 @@ public partial class DlibDotNet
}
}
private bool GetRunToDoCollectionFirst(long ticks)
private bool GetRunToDoCollectionFirst(Models.Configuration configuration, long ticks)
{
bool result = false;
string[] directories;
directories = Directory.GetDirectories(_Configuration.PropertyConfiguration.RootDirectory, "*", SearchOption.TopDirectoryOnly);
if (directories.Length == 0)
result = true;
else
bool result = configuration.SaveSortingWithoutPerson;
if (!result)
result = !IId.IsOffsetDeterministicHashCode(configuration.PropertyConfiguration);
if (!result)
{
string seasonDirectory;
DirectoryInfo directoryInfo;
DateTime dateTime = new(ticks);
string rootDirectory = _Configuration.PropertyConfiguration.RootDirectory;
string eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), _Configuration.PropertyConfiguration.ResultContent);
(int season, string seasonName) = Shared.Models.Stateless.Methods.IProperty.GetSeason(dateTime.DayOfYear);
FileSystemInfo fileSystemInfo = new DirectoryInfo(eDistanceContentDirectory);
string[] checkDirectories =
[
Path.Combine(rootDirectory, "Ancestry"),
Path.Combine(rootDirectory, "Facebook"),
Path.Combine(rootDirectory, "LinkedIn"),
rootDirectory,
];
foreach (string checkDirectory in checkDirectories)
string[] directories;
directories = Directory.GetDirectories(_Configuration.PropertyConfiguration.RootDirectory, "*", SearchOption.TopDirectoryOnly);
if (directories.Length == 0)
result = true;
else
{
if (checkDirectory == rootDirectory)
seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName}");
else
seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName} {Path.GetFileName(checkDirectory)}");
if (!Directory.Exists(seasonDirectory))
_ = Directory.CreateDirectory(seasonDirectory);
if (result)
continue;
directories = Directory.GetDirectories(checkDirectory, "*", SearchOption.TopDirectoryOnly);
foreach (string directory in directories)
string seasonDirectory;
DirectoryInfo directoryInfo;
DateTime dateTime = new(ticks);
string rootDirectory = _Configuration.PropertyConfiguration.RootDirectory;
string eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), _Configuration.PropertyConfiguration.ResultContent);
(int season, string seasonName) = Shared.Models.Stateless.Methods.IProperty.GetSeason(dateTime.DayOfYear);
FileSystemInfo fileSystemInfo = new DirectoryInfo(eDistanceContentDirectory);
string[] checkDirectories =
[
Path.Combine(rootDirectory, "Ancestry"),
Path.Combine(rootDirectory, "Facebook"),
Path.Combine(rootDirectory, "LinkedIn"),
rootDirectory,
];
foreach (string checkDirectory in checkDirectories)
{
directoryInfo = new(directory);
if (directoryInfo.LastWriteTime > fileSystemInfo.LastWriteTime)
if (checkDirectory == rootDirectory)
seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName}");
else
seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName} {Path.GetFileName(checkDirectory)}");
if (!Directory.Exists(seasonDirectory))
_ = Directory.CreateDirectory(seasonDirectory);
if (result)
continue;
directories = Directory.GetDirectories(checkDirectory, "*", SearchOption.TopDirectoryOnly);
foreach (string directory in directories)
{
result = true;
break;
directoryInfo = new(directory);
if (directoryInfo.LastWriteTime > fileSystemInfo.LastWriteTime)
{
result = true;
break;
}
}
}
}
@ -462,12 +529,12 @@ public partial class DlibDotNet
return result;
}
private string SaveUrlAndGetNewRootDirectory(string[] files)
private string SaveUrlAndGetNewRootDirectory(ReadOnlyCollection<FilePath> filePaths)
{
string result;
if (files.Length == 0)
if (filePaths.Count == 0)
throw new NotSupportedException();
string? sourceDirectory = Path.GetDirectoryName(files.First());
string? sourceDirectory = filePaths[0].DirectoryName;
if (string.IsNullOrEmpty(sourceDirectory))
throw new NotSupportedException();
Uri uri;
@ -508,7 +575,7 @@ public partial class DlibDotNet
return result;
}
private void FullDoWork(string argZero, string propertyRoot, long ticks, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string fPhotoPrismSingletonDirectory, int t, ReadOnlyCollection<Container> readOnlyContainers, A_Property propertyLogic, MapLogic mapLogic)
private void FullDoWork(string argZero, string propertyRoot, long ticks, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string fPhotoPrismSingletonDirectory, int count, ReadOnlyCollection<Container> readOnlyContainers, A_Property propertyLogic, MapLogic mapLogic)
{
int total;
int notMapped;
@ -516,7 +583,6 @@ public partial class DlibDotNet
bool exceptions;
int totalSeconds;
Container container;
Item[] filteredItems;
int totalNotMapped = 0;
bool outputResolutionHasNumber;
bool anyNullOrNoIsUniqueFileName;
@ -524,6 +590,7 @@ public partial class DlibDotNet
string dResultsFullGroupDirectory;
string c2ResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
ReadOnlyCollection<Item> filteredItems;
int containersLength = readOnlyContainers.Count;
List<Tuple<string, DateTime>> sourceDirectoryChanges = [];
int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism;
@ -546,13 +613,13 @@ public partial class DlibDotNet
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container);
if (filteredItems.Length == 0)
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetValidImageItems(_Configuration.PropertyConfiguration, container);
if (filteredItems.Count == 0)
continue;
sourceDirectoryChanges.Clear();
anyNullOrNoIsUniqueFileName = filteredItems.Any(l => !l.IsUniqueFileName);
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
message = $"{i + 1:000} [{filteredItems.Length:000}] / {containersLength:000} - {total} / {t} total - {totalSeconds} total second(s) - {outputResolution} - <{container.SourceDirectory}> - total not mapped {totalNotMapped:000000}";
message = $"{i + 1:000} [{filteredItems.Count:000}] / {containersLength:000} - {total} / {count} total - {totalSeconds} total second(s) - {outputResolution} - <{container.SourceDirectory}> - total not mapped {totalNotMapped:000000}";
propertyLogic.SetAngleBracketCollection(aResultsFullGroupDirectory, container.SourceDirectory, anyNullOrNoIsUniqueFileName);
if (outputResolutionHasNumber)
_Resize.SetAngleBracketCollection(cResultsFullGroupDirectory, container.SourceDirectory);
@ -589,14 +656,14 @@ public partial class DlibDotNet
total += container.Items.Count;
}
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
message = $"### [###] / {containersLength:000} - {total} / {t} total - {totalSeconds} total second(s) - {outputResolution} - <> - total not mapped {totalNotMapped:000000}";
message = $"### [###] / {containersLength:000} - {total} / {count} total - {totalSeconds} total second(s) - {outputResolution} - <> - total not mapped {totalNotMapped:000000}";
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
using ProgressBar progressBar = new(1, message, options);
progressBar.Tick();
}
}
private static void SaveDistinctIds(Property.Models.Configuration propertyConfiguration, string bResultsFullGroupDirectory, ReadOnlyCollection<Container> readOnlyContainers)
private static void SaveDistinctIds(Property.Models.Configuration propertyConfiguration, string bResultsFullGroupDirectory, ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection)
{
string paddedId;
List<int> distinct = [];
@ -604,21 +671,19 @@ public partial class DlibDotNet
string bMetadataCollectionDirectory = Path.Combine(bResultsFullGroupDirectory, propertyConfiguration.ResultCollection);
if (!Directory.Exists(bMetadataCollectionDirectory))
_ = Directory.CreateDirectory(bMetadataCollectionDirectory);
foreach (Container container in readOnlyContainers)
foreach (ReadOnlyCollection<FilePath> filePaths in filePathsCollection)
{
if (container.Items.Count == 0)
if (filePaths.Count == 0)
continue;
foreach (Item item in container.Items)
foreach (FilePath filePath in filePaths)
{
if (item.Property?.Id is null)
if (filePath.Id is null)
continue;
if (item.Property.Id != item.FilePath.Id)
throw new NotSupportedException();
if (distinct.Contains(item.Property.Id.Value))
if (distinct.Contains(filePath.Id.Value))
continue;
distinct.Add(item.Property.Id.Value);
paddedId = IId.GetPaddedId(propertyConfiguration, item.Property.Id.Value, item.FilePath.IsIgnore, index: null);
identifiers.Add(new(item.Property.Id.Value, paddedId));
distinct.Add(filePath.Id.Value);
paddedId = IId.GetPaddedId(propertyConfiguration, filePath.Id.Value, filePath.HasIgnoreKeyword, filePath.HasDateTimeOriginal, index: null);
identifiers.Add(new(filePath.Id.Value, paddedId));
}
}
string json = JsonSerializer.Serialize(identifiers.OrderBy(l => l.PaddedId).ToArray(), IdentifierCollectionSourceGenerationContext.Default.IdentifierArray);
@ -627,7 +692,7 @@ public partial class DlibDotNet
private ReadOnlyCollection<Mapping> GetMappings(Property.Models.Configuration propertyConfiguration, string eDistanceContentDirectory, ReadOnlyCollection<Container> readOnlyContainers, MapLogic mapLogic, bool distinctItems)
{
ReadOnlyCollection<Mapping> results;
List<Mapping> results = [];
int count = 0;
int notMapped;
Mapping mapping;
@ -636,15 +701,14 @@ public partial class DlibDotNet
string focusRelativePath;
bool? isFocusRelativePath;
DateTime[] containerDateTimes;
IEnumerable<Item> filteredItems;
MappingFromItem mappingFromItem;
List<Mapping> mappingCollection = [];
ReadOnlyCollection<Item> filteredItems;
foreach (Container container in readOnlyContainers)
{
if (container.Items.Count == 0)
continue;
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(propertyConfiguration, container);
if (!filteredItems.Any())
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetValidImageItems(propertyConfiguration, container);
if (filteredItems.Count == 0)
continue;
containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems);
focusRelativePath = Path.GetFullPath(string.Concat(_Configuration.PropertyConfiguration.RootDirectory, _Configuration.FocusDirectory));
@ -665,24 +729,25 @@ public partial class DlibDotNet
foreach (Shared.Models.Face face in item.Faces)
{
if (face.Mapping is null)
throw new NotSupportedException();
if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
continue;
anyValidFaces = true;
mappingCollection.Add(face.Mapping);
results.Add(face.Mapping);
}
if (!anyValidFaces)
{
(mapping, notMapped) = GetMappingAndUpdateMappingFromPerson(mapLogic, item, isFocusRelativePath, mappingFromItem);
mappingCollection.Add(mapping);
results.Add(mapping);
}
}
}
if (_Configuration.MoveToDecade && _Configuration.LocationContainerDistanceTolerance is null)
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(eDistanceContentDirectory);
results = new((from l in mappingCollection orderby l.MappingFromItem.Id select l).ToArray());
return results;
return new(results);
}
private static void SavePropertyShortcutsForOutputResolutions(string eDistanceContentDirectory, ReadOnlyCollection<Item> distinctFilteredItems)
private static void SavePropertyShortcutsForOutputResolutions(string eDistanceContentDirectory, ReadOnlyCollection<Item> distinctValidImageItems)
{
#if VerifyItem
bool found;
@ -713,7 +778,7 @@ public partial class DlibDotNet
List<string> distinct = [];
WindowsShortcut windowsShortcut;
List<(string, string, string)> collection = [];
foreach (Item item in distinctFilteredItems)
foreach (Item item in distinctValidImageItems)
{
if (item.Property?.Id is null)
continue;
@ -726,7 +791,7 @@ public partial class DlibDotNet
continue;
distinct.Add(directory);
}
foreach (Item item in distinctFilteredItems)
foreach (Item item in distinctValidImageItems)
{
if (item.Property?.Id is null || item.Property.DateTimeOriginal is null)
continue;
@ -771,27 +836,119 @@ public partial class DlibDotNet
}
}
private void MapLogic(long ticks, ReadOnlyCollection<Container> containers, string fPhotoPrismContentDirectory, MapLogic mapLogic, string outputResolution, ReadOnlyDictionary<long, List<int>> personKeyToIds, ReadOnlyCollection<Shared.Models.Face> distinctFilteredFaces, ReadOnlyCollection<Mapping> distinctFilteredMappingCollection)
private ReadOnlyCollection<FilePath> GetFilePath(long ticks, string dFacesContentDirectory)
{
List<FilePath> results = [];
FilePath filePath;
FileHolder fileHolder;
string[] files = Directory.GetFiles(dFacesContentDirectory, $"*{_Faces.FileNameExtension}", SearchOption.AllDirectories);
long? skipOlderThan = _Configuration.SkipOlderThanDays is null ? null : new DateTime(ticks).AddDays(-_Configuration.SkipOlderThanDays.Value).Ticks;
foreach (string file in files)
{
fileHolder = Shared.Models.Stateless.Methods.IFileHolder.Get(file);
filePath = FilePath.Get(_Configuration.PropertyConfiguration, fileHolder, index: null);
if (filePath.Id is null)
continue;
if (skipOlderThan is not null && (fileHolder.LastWriteTime is null || fileHolder.LastWriteTime.Value.Ticks < skipOlderThan.Value))
continue;
results.Add(filePath);
}
return new(results);
}
public ReadOnlyDictionary<string, LocationContainer> GetOnlyOne(IDistanceLimits distanceLimits, ReadOnlyCollection<LocationContainer> matrix)
{
Dictionary<string, LocationContainer> results = [];
List<string> added = [];
LocationContainer? tryGetValue;
foreach (LocationContainer locationContainer in matrix)
{
if (_Configuration.SaveIndividually)
break;
if (locationContainer.LengthSource is null)
continue;
if (_Configuration.UseExtraPersonKeyCheck)
{
if (results.TryGetValue(locationContainer.LengthSource.Name, out tryGetValue))
{
if (locationContainer.PersonKey is not null && tryGetValue.PersonKey is not null && locationContainer.PersonKey.Value != tryGetValue.PersonKey)
_ = results.Remove(locationContainer.LengthSource.Name);
continue;
}
}
if (added.Contains(locationContainer.LengthSource.Name))
continue;
added.Add(locationContainer.LengthSource.Name);
results.Add(locationContainer.LengthSource.Name, locationContainer);
}
return new(results);
}
private List<SaveContainer> GetSaveContainers(long ticks, ReadOnlyCollection<PersonContainer> personContainers, string a2PeopleSingletonDirectory, string eDistanceContentDirectory, ProgressBarOptions options, MapLogic mapLogic, string outputResolution)
{
List<SaveContainer> results;
long[] jLinkResolvedPersonKeys = _JLinkResolvedDirectories.Select(l => l.PersonKey).ToArray();
(string cResultsFullGroupDirectory, string _, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
ReadOnlyDictionary<int, ReadOnlyDictionary<int, LocationContainer>> mapped = Map.Models.Stateless.Methods.IMapLogic.GetMapped(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, personContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory);
if (mapped.Count == 0 && !_Configuration.SaveSortingWithoutPerson)
throw new NotSupportedException($"Switch {nameof(_Configuration.SaveSortingWithoutPerson)}!");
ReadOnlyCollection<FilePath> filePaths = GetFilePath(ticks, dFacesContentDirectory);
List<LocationContainer> available = Map.Models.Stateless.Methods.IMapLogic.GetAvailable(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, _Faces, ticks, filePaths);
if (!string.IsNullOrEmpty(_Configuration.FocusDirectory) && _Configuration.FocusDirectory.Length != 2)
throw new NotSupportedException($"{nameof(_Configuration.FocusDirectory)} currently only works with output directory! Example 00.");
ReadOnlyDictionary<int, ReadOnlyDictionary<int, LocationContainer>> mappedWithEncoding = E_Distance.GetMappedWithEncoding(mapped);
if (mappedWithEncoding.Count == 0 && !_Configuration.SaveSortingWithoutPerson)
throw new NotSupportedException($"Switch {nameof(_Configuration.SaveSortingWithoutPerson)}!");
List<LocationContainer> preFiltered = E_Distance.GetPreFilterLocationContainer(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, _Configuration.FocusDirectory, _Configuration.FocusModel, _Configuration.SkipPersonWithMoreThen, ticks, mapLogic, jLinkResolvedPersonKeys, mapped, available);
if (preFiltered.Count == 0)
results = [];
else
{
DistanceLimits distanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaPermyriadTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh);
List<LocationContainer> postFiltered = E_Distance.GetPostFilterLocationContainer(mapLogic, preFiltered, distanceLimits);
if (postFiltered.Count == 0)
results = [];
else
{
string message = $") Building Matrix - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
_ProgressBar = new(postFiltered.Count, message, options);
ReadOnlyCollection<LocationContainer> matrix = E_Distance.GetMatrixLocationContainers(this, _MapConfiguration, ticks, mapLogic, mappedWithEncoding, preFiltered, distanceLimits, postFiltered);
_ProgressBar.Dispose();
ReadOnlyDictionary<string, LocationContainer> onlyOne = GetOnlyOne(distanceLimits, matrix);
if (onlyOne.Count == 0)
results = [];
else
{
ReadOnlyDictionary<long, PersonContainer> personKeyToPersonContainer = mapLogic.GetPersonKeyToPersonContainer();
results = mapLogic.GetSaveContainers(cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, distanceLimits, onlyOne, personKeyToPersonContainer);
}
}
}
return results;
}
private void MapLogic(long ticks, ReadOnlyCollection<Container> containers, string fPhotoPrismContentDirectory, MapLogic mapLogic, string outputResolution, ReadOnlyDictionary<long, List<int>> personKeyToIds, ReadOnlyCollection<Shared.Models.Face> distinctValidImageFaces, ReadOnlyCollection<Mapping> distinctValidImageMappingCollection)
{
(_, _, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
string d2FacePartsContentDirectory = Path.Combine(d2ResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
string d2FacePartsContentCollectionDirectory = Path.Combine(d2ResultsFullGroupDirectory, "[()]");
if (distinctFilteredMappingCollection.Count > 0)
if (distinctValidImageMappingCollection.Count > 0)
{
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(d2FacePartsContentDirectory, ticks);
if (Directory.Exists(d2FacePartsContentCollectionDirectory))
Shared.Models.Stateless.Methods.IPath.MakeHiddenIfAllItemsAreHidden(d2FacePartsContentCollectionDirectory);
}
if (Directory.Exists(fPhotoPrismContentDirectory))
F_PhotoPrism.WriteMatches(fPhotoPrismContentDirectory, _Configuration.PersonBirthdayFormat, _Configuration.RectangleIntersectMinimums, ticks, distinctFilteredFaces, mapLogic);
F_PhotoPrism.WriteMatches(fPhotoPrismContentDirectory, _Configuration.PersonBirthdayFormat, _Configuration.RectangleIntersectMinimums, ticks, distinctValidImageFaces, mapLogic);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
mapLogic.SaveShortcutsForOutputResolutionsDuringMapLogic(containers, personKeyToIds, dFacesContentDirectory, distinctFilteredMappingCollection);
ReadOnlyDictionary<int, ReadOnlyDictionary<int, Mapping>> idToWholePercentagesToMapping = Map.Models.Stateless.Methods.IMapLogic.GetIdToWholePercentagesToFace(distinctFilteredMappingCollection);
mapLogic.SaveShortcutsForOutputResolutionsDuringMapLogic(containers, personKeyToIds, dFacesContentDirectory, distinctValidImageMappingCollection);
ReadOnlyDictionary<int, ReadOnlyDictionary<int, Mapping>> idToWholePercentagesToMapping = Map.Models.Stateless.Methods.IMapLogic.GetIdToWholePercentagesToFace(distinctValidImageMappingCollection);
if (_Configuration.SaveMappedForOutputResolutions.Contains(outputResolution))
mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, personKeyToIds, distinctFilteredMappingCollection, idToWholePercentagesToMapping);
mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, personKeyToIds, distinctValidImageMappingCollection, idToWholePercentagesToMapping);
if (_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution))
SaveFaceDistances(ticks, mapLogic, distinctFilteredFaces, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping);
SaveFaceDistances(ticks, mapLogic, distinctValidImageFaces, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping);
}
private bool? GetIsFocusModel(Shared.Models.Property? property)
@ -818,7 +975,7 @@ public partial class DlibDotNet
_Logger?.LogInformation(string.Concat("Moved <", item.FilePath.FullName, '>'));
}
private int GetNotMappedCountAndUpdateMappingFromPersonThenSetMapping(MapLogic mapLogic, Item item, bool? isFocusRelativePath, ReadOnlyCollection<LocationContainer> locationContainers, MappingFromItem mappingFromItem, List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection, List<Shared.Models.Face> faces)
private int GetNotMappedCountAndUpdateMappingFromPersonThenSetMapping(MapLogic mapLogic, Item item, bool? isFocusRelativePath, MappingFromItem mappingFromItem, List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection, List<Shared.Models.Face> faces)
{
int result;
double? α;
@ -839,7 +996,6 @@ public partial class DlibDotNet
bool? isFocusModel = GetIsFocusModel(item.Property);
long[] jLinkResolvedPersonKeys = _JLinkResolvedDirectories.Select(l => l.PersonKey).ToArray();
ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>? wholePercentagesToPersonContainers;
ReadOnlyCollection<string> locationContainersFiles = new((from l in locationContainers select l.FilePath.FullName).ToArray());
foreach (Shared.Models.Face face in faces)
{
wholePercentagesToPersonContainers = mapLogic.GetWholePercentagesToPersonContainers(item.Property?.Id);
@ -880,36 +1036,37 @@ public partial class DlibDotNet
return result;
}
private static Map.Models.Configuration Get(Models.Configuration configuration, string facesFileNameExtension, string facesHiddenFileNameExtension, string facePartsFileNameExtension)
private static Map.Models.Configuration Get(Models.Configuration configuration, IDistanceLimits distanceLimits, string facesFileNameExtension, string facesHiddenFileNameExtension, string facePartsFileNameExtension)
{
Map.Models.Configuration result = new(
configuration.DeletePossibleDuplicates,
configuration.DistanceMoveUnableToMatch,
configuration.DistanceRenameToMatch,
configuration.FaceConfidencePercent,
configuration.FaceDistancePermyriad,
configuration.LinkedAlpha,
configuration.LocationContainerDebugDirectory,
configuration.LocationContainerDirectoryPattern,
configuration.LocationContainerDistanceGroupMinimum,
configuration.LocationContainerDistanceTake,
configuration.LocationContainerDistanceTolerance,
configuration.LocationDigits,
configuration.MappingDefaultName,
configuration.PersonBirthdayFirstYear,
configuration.PersonBirthdayFormat,
configuration.PersonCharacters.ToArray(),
configuration.RangeDaysDeltaTolerance,
configuration.RangeDistanceTolerance,
configuration.ReMap,
configuration.SaveIndividually,
configuration.SaveSortingWithoutPerson,
configuration.SkipNotSkipDirectories,
configuration.SortingMaximumPerKey,
configuration.SortingMinimumToUseSigma,
facesFileNameExtension,
facesHiddenFileNameExtension,
facePartsFileNameExtension);
Map.Models.Configuration result = new(distanceLimits,
configuration.PropertyConfiguration,
configuration.DeletePossibleDuplicates,
configuration.DistanceMoveUnableToMatch,
configuration.DistanceRenameToMatch,
configuration.FaceConfidencePercent,
configuration.FaceDistancePermyriad,
configuration.LinkedAlpha,
configuration.LocationContainerDebugDirectory,
configuration.LocationContainerDirectoryPattern,
configuration.LocationContainerDistanceGroupMinimum,
configuration.LocationContainerDistanceTake,
configuration.LocationContainerDistanceTolerance,
configuration.LocationDigits,
configuration.MappingDefaultName,
configuration.PersonBirthdayFirstYear,
configuration.PersonBirthdayFormat,
configuration.PersonCharacters.ToArray(),
configuration.RangeDaysDeltaTolerance,
configuration.RangeDistanceTolerance,
configuration.ReMap,
configuration.SaveIndividually,
configuration.SaveSortingWithoutPerson,
configuration.SkipNotSkipDirectories,
configuration.SortingMaximumPerKey,
configuration.SortingMinimumToUseSigma,
facesFileNameExtension,
facesHiddenFileNameExtension,
facePartsFileNameExtension);
return result;
}
@ -962,7 +1119,6 @@ public partial class DlibDotNet
string[] changesFrom = [nameof(A_Property)];
List<Tuple<string, DateTime>> subFileTuples = [];
FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(cResultsFullGroupDirectory, item, outputResolutionHasNumber);
ReadOnlyCollection<LocationContainer> locationContainers = mapLogic.GetLocationContainers(item);
if (item.Property is null || item.Property.Id is null || !item.SourceDirectoryFileHolder.Exists || item.SourceDirectoryFileHolder.CreationTime is null || item.SourceDirectoryFileHolder.LastWriteTime is null || item.Any())
{
LogItemPropertyIsNull(item);
@ -999,12 +1155,30 @@ public partial class DlibDotNet
_ = _BlurHasher.EncodeAndSave(item.FilePath, resizedFileHolder);
}
}
bool? shouldIgnore = property is null || property.Keywords is null ? null : _Configuration.PropertyConfiguration.IgnoreRulesKeyWords.Any(l => property.Keywords.Contains(l));
if (shouldIgnore is not null)
{
if (shouldIgnore.Value)
{
FileInfo fileInfo = new(resizedFileHolder.FullName);
if (!fileInfo.Attributes.HasFlag(FileAttributes.Hidden))
File.SetAttributes(resizedFileHolder.FullName, FileAttributes.Hidden);
}
if (resizedFileHolder.Exists && item.FilePath.HasIgnoreKeyword is not null && item.FilePath.HasIgnoreKeyword.Value != shouldIgnore.Value)
{
if (!item.FilePath.DirectoryName.Contains("Results") || !item.FilePath.DirectoryName.Contains("Resize"))
throw new NotSupportedException($"Rename File! <{item.FilePath.FileNameFirstSegment}>");
else
{
File.Delete(resizedFileHolder.FullName);
}
}
}
if (property is null || item.Property is null)
throw new NullReferenceException(nameof(property));
item.SetResizedFileHolder(_Resize.FileNameExtension, resizedFileHolder);
MappingFromItem mappingFromItem = IMappingFromItem.GetMappingFromItem(containerDateTimes, item, resizedFileHolder);
ExifDirectory exifDirectory = metadata.GetMetadataCollection(item.FilePath, subFileTuples, parseExceptions, changesFrom, mappingFromItem);
Map.Models.Stateless.Methods.IMapLogic.SetCreationTimeMaybeMoveToDecade(_Configuration.PropertyConfiguration, _Configuration.MoveToDecade && _Configuration.LocationContainerDistanceTolerance is null, mappingFromItem, locationContainers);
if (_AppSettings.Places.Count > 0)
{
float latitude;
@ -1029,7 +1203,12 @@ public partial class DlibDotNet
}
Dictionary<string, int[]> outputResolutionToResize = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, item.Property, mappingFromItem);
if (_Configuration.SaveResizedSubfiles)
_Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.Property, mappingFromItem, outputResolutionToResize);
{
if (shouldIgnore is not null && item.FilePath.HasIgnoreKeyword is not null && item.FilePath.HasIgnoreKeyword.Value != shouldIgnore.Value)
faces = [];
else
_Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.Property, mappingFromItem, outputResolutionToResize);
}
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
faces = [];
else if (!mappingFromItem.ResizedFileHolder.Exists && !File.Exists(mappingFromItem.ResizedFileHolder.FullName))
@ -1039,15 +1218,23 @@ public partial class DlibDotNet
List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection;
if (!fileNameToCollection.TryGetValue(mappingFromItem.Id, out mappingFromPhotoPrismCollection))
mappingFromPhotoPrismCollection = null;
faces = _Faces.GetFaces(outputResolution, dResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionToResize, locationContainers, mappingFromPhotoPrismCollection);
result = GetNotMappedCountAndUpdateMappingFromPersonThenSetMapping(mapLogic, item, isFocusRelativePath, locationContainers, mappingFromItem, mappingFromPhotoPrismCollection, faces);
List<(Shared.Models.Face, FileInfo?, string, bool Saved)> faceCollection = _Faces.SaveFaces(_FaceParts.FileNameExtension, dResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, mappingFromItem, faces);
bool move = _Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution);
faces = _Faces.GetFaces(outputResolution, cResultsFullGroupDirectory, dResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionToResize, mappingFromPhotoPrismCollection);
result = GetNotMappedCountAndUpdateMappingFromPersonThenSetMapping(mapLogic, item, isFocusRelativePath, mappingFromItem, mappingFromPhotoPrismCollection, faces);
List<(Shared.Models.Face, FileHolder?, string, bool Saved)> faceCollection = _Faces.SaveFaces(_FaceParts.FileNameExtension, dResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, mappingFromItem, exifDirectory, faces);
if (_Configuration.CopyFacesAndSaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
_FaceParts.CopyFacesAndSaveFaceLandmarkImage(facePartsCollectionDirectory, mappingFromItem, faceCollection);
if ((_Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch)
&& _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
&& locationContainers is not null && faceCollection.All(l => !l.Saved))
_Distance.LookForMatchFacesAndPossiblyRename(_Faces.FileNameExtension, item.FilePath, mappingFromItem, faces, locationContainers);
if (move && faceCollection.All(l => !l.Saved))
{
ReadOnlyCollection<LocationContainer> locationContainers = mapLogic.GetLocationContainers(item);
if (locationContainers.Count > 0)
{
Map.Models.Stateless.Methods.IMapLogic.SetCreationTime(mappingFromItem, locationContainers);
if (_Configuration.LocationContainerDistanceTolerance is null && _Configuration.MoveToDecade)
Map.Models.Stateless.Methods.IMapLogic.MoveToDecade(_Configuration.PropertyConfiguration, mappingFromItem, locationContainers);
_Distance.LookForMatchFacesAndPossiblyRename(_Configuration.OverrideForFaceImages, _DistanceLimits, _Faces, item.FilePath, mappingFromItem, exifDirectory, faces, locationContainers);
}
}
(bool review, int[] eyesCollection) = Shared.Models.Stateless.Methods.IFace.GetEyeCollection(_Configuration.EyeThreshold, faces);
if (review || _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
{
@ -1076,7 +1263,7 @@ public partial class DlibDotNet
List<Tuple<string, DateTime>> sourceDirectoryChanges,
Dictionary<int, List<MappingFromPhotoPrism>> fileNameToCollection,
Container container,
Item[] filteredItems,
ReadOnlyCollection<Item> filteredItems,
string message)
{
int result = 0;
@ -1088,8 +1275,8 @@ public partial class DlibDotNet
bool? isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath);
string facePartsCollectionDirectory = _Configuration.CopyFacesAndSaveFaceLandmarkForOutputResolutions.Contains(outputResolution) ? _FaceParts.GetFacePartsDirectory(_Configuration.PropertyConfiguration, d2ResultsFullGroupDirectory, item: filteredItems.First(), includeNameWithoutExtension: false) : string.Empty;
bool anyPropertiesChangedForX = _Configuration.PropertyConfiguration.PropertiesChangedForProperty || _Configuration.PropertiesChangedForDistance || _Configuration.PropertiesChangedForFaces || _Configuration.PropertiesChangedForIndex || _Configuration.PropertiesChangedForMetadata || _Configuration.PropertiesChangedForResize;
using ProgressBar progressBar = new(filteredItems.Length, message, options);
_ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) =>
using ProgressBar progressBar = new(filteredItems.Count, message, options);
_ = Parallel.For(0, filteredItems.Count, parallelOptions, (i, state) =>
{
try
{
@ -1115,7 +1302,7 @@ public partial class DlibDotNet
catch (Exception)
{
exceptionsCount++;
if (exceptionsCount == filteredItems.Length)
if (exceptionsCount == filteredItems.Count)
throw new Exception(string.Concat("All in [", container.SourceDirectory, "] failed!"));
}
});
@ -1210,18 +1397,17 @@ public partial class DlibDotNet
if (filteredFaceDistanceContainers.Length > 0)
{
int updated = sortingContainers.Count == 0 ? 0 : mapLogic.UpdateFromSortingContainers(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, sortingContainers);
List<SaveContainer> saveContainers;
saveContainers = mapLogic.GetSaveContainers(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, distanceLimits, useFiltersCounter, sortingContainers);
List<SaveContainer> saveContainers = mapLogic.GetSaveContainers(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, distanceLimits, useFiltersCounter, sortingContainers);
if (saveContainers.Count > 0)
mapLogic.SaveContainers(updated, saveContainers);
}
}
}
private void SaveFaceDistances(long ticks, MapLogic mapLogic, ReadOnlyCollection<Shared.Models.Face> distinctFilteredFaces, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, ReadOnlyDictionary<int, ReadOnlyDictionary<int, Mapping>> idToWholePercentagesToMapping)
private void SaveFaceDistances(long ticks, MapLogic mapLogic, ReadOnlyCollection<Shared.Models.Face> distinctValidImageFaces, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, ReadOnlyDictionary<int, ReadOnlyDictionary<int, Mapping>> idToWholePercentagesToMapping)
{
E_Distance.PreFilterSetFaceDistances(_AppSettings.MaxDegreeOfParallelism, ticks, distinctFilteredFaces);
ReadOnlyCollection<FaceDistanceContainer> faceDistanceContainers = E_Distance.GetFaceDistanceContainers(distinctFilteredFaces);
E_Distance.PreFilterSetFaceDistances(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, distinctValidImageFaces);
ReadOnlyCollection<FaceDistanceContainer> faceDistanceContainers = E_Distance.GetFaceDistanceContainers(distinctValidImageFaces);
if (faceDistanceContainers.Count > 0)
{
List<FaceDistance> faceDistanceEncodings = [];
@ -1235,41 +1421,41 @@ public partial class DlibDotNet
}
}
private static void CheckForAllWindowsLinks(ReadOnlyCollection<string[]> filesCollection)
private static void CheckForAllWindowsLinks(ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection)
{
string fileFullPath;
WindowsShortcut windowsShortcut;
foreach (string[] files in filesCollection)
foreach (ReadOnlyCollection<FilePath> filePaths in filePathsCollection)
{
if (files.Length == 0)
if (filePaths.Count == 0)
continue;
if (files.All(l => l.EndsWith(".lnk")))
if (filePaths.All(l => l.ExtensionLowered == ".lnk"))
{
foreach (string file in files)
foreach (FilePath filePath in filePaths)
{
windowsShortcut = WindowsShortcut.Load(file);
windowsShortcut = WindowsShortcut.Load(filePath.FullName);
if (windowsShortcut.Path is null)
{
File.Delete(file);
File.Delete(filePath.FullName);
continue;
}
fileFullPath = windowsShortcut.Path;
windowsShortcut.Dispose();
File.WriteAllText(Path.ChangeExtension(file, ".url"), fileFullPath);
File.Delete(file);
File.WriteAllText(Path.ChangeExtension(filePath.FullName, ".url"), fileFullPath);
File.Delete(filePath.FullName);
}
throw new NotSupportedException("All are Windows *.lnk files!");
}
}
}
private static bool IsFilesCollectionCountIsOne(ReadOnlyCollection<string[]> filesCollection)
private static bool IsFilesCollectionCountIsOne(ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection)
{
bool result = true;
int count = 0;
foreach (string[] files in filesCollection)
foreach (ReadOnlyCollection<FilePath> filePaths in filePathsCollection)
{
if (files.Length == 0)
if (filePaths.Count == 0)
continue;
count += 1;
if (count > 1)
@ -1279,22 +1465,22 @@ public partial class DlibDotNet
}
}
if (result)
CheckForAllWindowsLinks(filesCollection);
CheckForAllWindowsLinks(filePathsCollection);
return result;
}
private (string, ReadOnlyCollection<string[]>, bool) GetFilesCollectionThenCopyOrMove(long ticks, string fileSearchFilter, string directorySearchFilter, ProgressBarOptions options, string outputResolution)
private (string, ReadOnlyCollection<ReadOnlyCollection<FilePath>>, bool) GetFilesCollectionThenCopyOrMove(long ticks, string fileSearchFilter, string directorySearchFilter, ProgressBarOptions options, string outputResolution)
{
ProgressBar progressBar;
string filesCollectionRootDirectory = _Configuration.PropertyConfiguration.RootDirectory;
(string cResultsFullGroupDirectory, _, _, _) = GetResultsFullGroupDirectories(outputResolution);
IReadOnlyDictionary<string, string[]> fileGroups = Shared.Models.Stateless.Methods.IPath.GetKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, [_Configuration.PropertyConfiguration.ResultContent]);
ReadOnlyCollection<string[]> filesCollection = IDirectory.GetFilesCollection(filesCollectionRootDirectory, directorySearchFilter, fileSearchFilter, useCeilingAverage: false);
int count = filesCollection.Select(l => l.Length).Sum();
bool filesCollectionCountIsOne = IsFilesCollectionCountIsOne(filesCollection);
ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection = IDirectory.GetFilePathCollections(_Configuration.PropertyConfiguration, filesCollectionRootDirectory, directorySearchFilter, fileSearchFilter, useCeilingAverage: false);
int count = filePathsCollection.Select(l => l.Count).Sum();
bool filesCollectionCountIsOne = IsFilesCollectionCountIsOne(filePathsCollection);
string message = $") Selecting for ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
progressBar = new(count, message, options);
(string[] distinctDirectories, List<(FilePath, string)> toDoCollection) = IDirectory.GetToDoCollection(_Configuration.PropertyConfiguration, filesCollection, fileGroups[_Configuration.PropertyConfiguration.ResultContent], () => progressBar.Tick());
(string[] distinctDirectories, List<(FilePath, string)> toDoCollection) = IDirectory.GetToDoCollection(_Configuration.PropertyConfiguration, filePathsCollection, fileGroups[_Configuration.PropertyConfiguration.ResultContent], () => progressBar.Tick());
progressBar.Dispose();
foreach (string distinctDirectory in distinctDirectories)
{
@ -1305,7 +1491,7 @@ public partial class DlibDotNet
progressBar = new(count, message, options);
_ = IDirectory.CopyOrMove(toDoCollection, move: false, moveBack: false, () => progressBar.Tick());
progressBar.Dispose();
return (filesCollectionRootDirectory, new(filesCollection), filesCollectionCountIsOne);
return (filesCollectionRootDirectory, filePathsCollection, filesCollectionCountIsOne);
}
}