phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reuse count over t

Browse Source 
FilePath over String
Support first run better
Bump
This commit is contained in:
Mike Phares
2024-07-20 10:48:26 -07:00
parent 5b9e748ba4
commit 9ebfdc8af2
10 changed files with 216 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

155
Instance/DlibDotNet.cs
View File

@ -252,26 +252,26 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
throw new Exception("Configuration has SaveSortingWithoutPerson and FocusDirectory!");
}
private ReadOnlyCollection<int> GetNotNineCollection(ReadOnlyCollection<string[]> filesCollection)
private ReadOnlyCollection<int> GetNotNineCollection(ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection)
{
List<int> results = [];
FileInfo fileInfo;
FilePath filePath;
FileHolder fileHolder;
foreach (string[] files in filesCollection)
FilePath checkFilePath;
foreach (ReadOnlyCollection<FilePath> filePaths in filePathsCollection)
{
foreach (string file in files)
foreach (FilePath filePath in filePaths)
{
if (!file.Contains(" !9"))
if (!filePath.FullName.Contains(" !9"))
continue;
fileInfo = new(file);
fileInfo = new(filePath.FullName);
fileHolder = Shared.Models.Stateless.Methods.IFileHolder.Get(fileInfo);
if (!fileInfo.Attributes.HasFlag(FileAttributes.Hidden))
File.SetAttributes(fileHolder.FullName, FileAttributes.Hidden);
filePath = FilePath.Get(_Configuration.PropertyConfiguration, fileHolder, index: null);
if (filePath.Id is null)
checkFilePath = FilePath.Get(_Configuration.PropertyConfiguration, fileHolder, index: null);
if (checkFilePath.Id is null)
continue;
results.Add(filePath.Id.Value);
results.Add(checkFilePath.Id.Value);
}
}
return new(results);
@ -320,10 +320,8 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
private void Search(long ticks, ReadOnlyCollection<PersonContainer> personContainers, string argZero, string propertyRoot)
{
int t;
string message;
MapLogic? mapLogic;
Container[] containers;
A_Property propertyLogic;
string eDistanceContentDirectory;
string? a2PeopleContentDirectory;
@ -337,11 +335,12 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
const string directorySearchFilter = "*";
string? filesCollectionRootDirectory = null;
bool configurationOutputResolutionsHas = false;
ReadOnlyCollection<Container> readOnlyContainers;
ReadOnlyCollection<int>? notNineCollection = null;
ReadOnlyDictionary<long, List<int>> personKeyToIds;
ReadOnlyCollection<string[]>? filesCollection = null;
ReadOnlyCollection<ReadOnlyCollection<FilePath>>? filePathsCollection = null;
bool runToDoCollectionFirst = GetRunToDoCollectionFirst(_Configuration, ticks);
(aResultsFullGroupDirectory, bResultsFullGroupDirectory) = GetResultsFullGroupDirectories();
bool runToDoCollectionFirst = GetRunToDoCollectionFirst(_Configuration.PropertyConfiguration, ticks);
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(_Configuration.PropertyConfiguration.RootDirectory, ticks);
a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "([])");
@ -359,8 +358,8 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
configurationOutputResolutionsHas = true;
if (!runToDoCollectionFirst)
break;
(filesCollectionRootDirectory, filesCollection, filesCollectionCountIsOne) = GetFilesCollectionThenCopyOrMove(ticks, fileSearchFilter, directorySearchFilter, options, outputResolution);
notNineCollection = GetNotNineCollection(filesCollection);
(filesCollectionRootDirectory, filePathsCollection, filesCollectionCountIsOne) = GetFilesCollectionThenCopyOrMove(ticks, fileSearchFilter, directorySearchFilter, options, outputResolution);
notNineCollection = GetNotNineCollection(filePathsCollection);
break;
}
fPhotoPrismContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), _Configuration.PropertyConfiguration.ResultContent);
@ -368,8 +367,8 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory);
if (filesCollectionCountIsOne)
{
if (filesCollection is null)
throw new NullReferenceException(nameof(filesCollection));
if (filePathsCollection is null)
throw new NullReferenceException(nameof(filePathsCollection));
string resultsGroupDirectory;
a2PeopleContentDirectory = null;
eDistanceContentDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", _Configuration.PropertyConfiguration.ResultContent);
@ -380,7 +379,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
resultsGroupDirectory = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, string.Empty, create: true);
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(resultsGroupDirectory);
}
argZero = SaveUrlAndGetNewRootDirectory(filesCollection.First());
argZero = SaveUrlAndGetNewRootDirectory(filePathsCollection.First());
_Configuration.PropertyConfiguration.ChangeRootDirectory(argZero);
(aResultsFullGroupDirectory, bResultsFullGroupDirectory) = GetResultsFullGroupDirectories();
propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false);
@ -388,34 +387,32 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
if (configurationOutputResolutionsHas)
{
int count;
foreach (string outputResolution in _Configuration.OutputResolutions)
{
if (outputResolution.Any(char.IsNumber))
continue;
(cResultsFullGroupDirectory, _, _, _) = GetResultsFullGroupDirectories(outputResolution);
filesCollectionRootDirectory = Path.Combine(cResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
filesCollection = IDirectory.GetFilesCollection(filesCollectionRootDirectory, directorySearchFilter, fileSearchFilter, useCeilingAverage: true);
count = filesCollection.Select(l => l.Length).Sum();
filePathsCollection = IDirectory.GetFilePathCollections(_Configuration.PropertyConfiguration, filesCollectionRootDirectory, directorySearchFilter, fileSearchFilter, useCeilingAverage: true);
break;
}
}
if (filesCollectionRootDirectory is null || filesCollection is null)
throw new NullReferenceException(nameof(filesCollection));
message = $") Building Container(s) - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
if (filesCollectionRootDirectory is null || filePathsCollection is null)
throw new NullReferenceException(nameof(filePathsCollection));
string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton);
if (!Directory.Exists(aPropertySingletonDirectory))
_ = Directory.CreateDirectory(aPropertySingletonDirectory);
_ProgressBar = new(short.MaxValue, message, options);
(t, containers) = Shared.Models.Stateless.Methods.IContainer.GetContainers(this, _Configuration.PropertyConfiguration, aPropertySingletonDirectory, filesCollectionRootDirectory, filesCollection);
int count = filePathsCollection.Select(l => l.Count).Sum();
SaveDistinctIds(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, filePathsCollection);
message = $") Building Container(s) - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
_ProgressBar = new(count, message, options);
readOnlyContainers = Shared.Models.Stateless.Methods.IContainer.GetContainers(this, _Configuration.PropertyConfiguration, aPropertySingletonDirectory, filesCollectionRootDirectory, filePathsCollection);
_ProgressBar.Dispose();
ReadOnlyCollection<Container> readOnlyContainers = new(containers);
SaveDistinctIds(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, readOnlyContainers);
mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _Distance, personContainers, ticks, a2PeopleContentDirectory, a2PeopleSingletonDirectory, eDistanceContentDirectory);
DeleteContinueFiles(personContainers);
if (!runToDoCollectionFirst)
MapFaceFileLogic(ticks, personContainers, mapLogic, a2PeopleContentDirectory, eDistanceContentDirectory, options);
FullDoWork(argZero, propertyRoot, ticks, aResultsFullGroupDirectory, bResultsFullGroupDirectory, fPhotoPrismSingletonDirectory, t, readOnlyContainers, propertyLogic, mapLogic);
FullDoWork(argZero, propertyRoot, ticks, aResultsFullGroupDirectory, bResultsFullGroupDirectory, fPhotoPrismSingletonDirectory, count, readOnlyContainers, propertyLogic, mapLogic);
ReadOnlyCollection<Item> distinctValidImageItems = Shared.Models.Stateless.Methods.IContainer.GetValidImageItems(_Configuration.PropertyConfiguration, readOnlyContainers, distinctItems: true, filterItems: true);
if (_Configuration.LookForAbandoned)
{
@ -475,9 +472,11 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
}
private bool GetRunToDoCollectionFirst(Property.Models.Configuration propertyConfiguration, long ticks)
private bool GetRunToDoCollectionFirst(Models.Configuration configuration, long ticks)
{
bool result = !IId.IsOffsetDeterministicHashCode(propertyConfiguration);
bool result = configuration.SaveSortingWithoutPerson;
if (!result)
result = !IId.IsOffsetDeterministicHashCode(configuration.PropertyConfiguration);
if (!result)
{
string[] directories;
@ -530,12 +529,12 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
return result;
}
private string SaveUrlAndGetNewRootDirectory(string[] files)
private string SaveUrlAndGetNewRootDirectory(ReadOnlyCollection<FilePath> filePaths)
{
string result;
if (files.Length == 0)
if (filePaths.Count == 0)
throw new NotSupportedException();
string? sourceDirectory = Path.GetDirectoryName(files.First());
string? sourceDirectory = filePaths[0].DirectoryName;
if (string.IsNullOrEmpty(sourceDirectory))
throw new NotSupportedException();
Uri uri;
@ -576,7 +575,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
return result;
}
private void FullDoWork(string argZero, string propertyRoot, long ticks, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string fPhotoPrismSingletonDirectory, int t, ReadOnlyCollection<Container> readOnlyContainers, A_Property propertyLogic, MapLogic mapLogic)
private void FullDoWork(string argZero, string propertyRoot, long ticks, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string fPhotoPrismSingletonDirectory, int count, ReadOnlyCollection<Container> readOnlyContainers, A_Property propertyLogic, MapLogic mapLogic)
{
int total;
int notMapped;
@ -620,7 +619,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
sourceDirectoryChanges.Clear();
anyNullOrNoIsUniqueFileName = filteredItems.Any(l => !l.IsUniqueFileName);
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
message = $"{i + 1:000} [{filteredItems.Count:000}] / {containersLength:000} - {total} / {t} total - {totalSeconds} total second(s) - {outputResolution} - <{container.SourceDirectory}> - total not mapped {totalNotMapped:000000}";
message = $"{i + 1:000} [{filteredItems.Count:000}] / {containersLength:000} - {total} / {count} total - {totalSeconds} total second(s) - {outputResolution} - <{container.SourceDirectory}> - total not mapped {totalNotMapped:000000}";
propertyLogic.SetAngleBracketCollection(aResultsFullGroupDirectory, container.SourceDirectory, anyNullOrNoIsUniqueFileName);
if (outputResolutionHasNumber)
_Resize.SetAngleBracketCollection(cResultsFullGroupDirectory, container.SourceDirectory);
@ -657,14 +656,14 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
total += container.Items.Count;
}
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
message = $"### [###] / {containersLength:000} - {total} / {t} total - {totalSeconds} total second(s) - {outputResolution} - <> - total not mapped {totalNotMapped:000000}";
message = $"### [###] / {containersLength:000} - {total} / {count} total - {totalSeconds} total second(s) - {outputResolution} - <> - total not mapped {totalNotMapped:000000}";
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
using ProgressBar progressBar = new(1, message, options);
progressBar.Tick();
}
}
private static void SaveDistinctIds(Property.Models.Configuration propertyConfiguration, string bResultsFullGroupDirectory, ReadOnlyCollection<Container> readOnlyContainers)
private static void SaveDistinctIds(Property.Models.Configuration propertyConfiguration, string bResultsFullGroupDirectory, ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection)
{
string paddedId;
List<int> distinct = [];
@ -672,21 +671,19 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
string bMetadataCollectionDirectory = Path.Combine(bResultsFullGroupDirectory, propertyConfiguration.ResultCollection);
if (!Directory.Exists(bMetadataCollectionDirectory))
_ = Directory.CreateDirectory(bMetadataCollectionDirectory);
foreach (Container container in readOnlyContainers)
foreach (ReadOnlyCollection<FilePath> filePaths in filePathsCollection)
{
if (container.Items.Count == 0)
if (filePaths.Count == 0)
continue;
foreach (Item item in container.Items)
foreach (FilePath filePath in filePaths)
{
if (item.Property?.Id is null)
if (filePath.Id is null)
continue;
if (item.Property.Id != item.FilePath.Id)
throw new NotSupportedException();
if (distinct.Contains(item.Property.Id.Value))
if (distinct.Contains(filePath.Id.Value))
continue;
distinct.Add(item.Property.Id.Value);
paddedId = IId.GetPaddedId(propertyConfiguration, item.Property.Id.Value, item.FilePath.HasIgnoreKeyword, item.FilePath.HasDateTimeOriginal, index: null);
identifiers.Add(new(item.Property.Id.Value, paddedId));
distinct.Add(filePath.Id.Value);
paddedId = IId.GetPaddedId(propertyConfiguration, filePath.Id.Value, filePath.HasIgnoreKeyword, filePath.HasDateTimeOriginal, index: null);
identifiers.Add(new(filePath.Id.Value, paddedId));
}
}
string json = JsonSerializer.Serialize(identifiers.OrderBy(l => l.PaddedId).ToArray(), IdentifierCollectionSourceGenerationContext.Default.IdentifierArray);
@ -1158,6 +1155,23 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
_ = _BlurHasher.EncodeAndSave(item.FilePath, resizedFileHolder);
}
}
bool? shouldIgnore = property is null || property.Keywords is null ? null : _Configuration.PropertyConfiguration.IgnoreRulesKeyWords.Any(l => property.Keywords.Contains(l));
if (shouldIgnore is not null)
{
if (shouldIgnore.Value)
{
FileInfo fileInfo = new(resizedFileHolder.FullName);
if (!fileInfo.Attributes.HasFlag(FileAttributes.Hidden))
File.SetAttributes(resizedFileHolder.FullName, FileAttributes.Hidden);
}
if (resizedFileHolder.Exists && item.FilePath.HasIgnoreKeyword is not null && item.FilePath.HasIgnoreKeyword.Value != shouldIgnore.Value)
{
if (item.FilePath.DirectoryName.Contains("Results") && item.FilePath.DirectoryName.Contains("Resize"))
File.Delete(resizedFileHolder.FullName);
else
throw new NotSupportedException($"Rename File! <{item.FilePath.FileNameFirstSegment}>");
}
}
if (property is null || item.Property is null)
throw new NullReferenceException(nameof(property));
item.SetResizedFileHolder(_Resize.FileNameExtension, resizedFileHolder);
@ -1187,7 +1201,12 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
Dictionary<string, int[]> outputResolutionToResize = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, item.Property, mappingFromItem);
if (_Configuration.SaveResizedSubfiles)
_Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.Property, mappingFromItem, outputResolutionToResize);
{
if (shouldIgnore is not null && item.FilePath.HasIgnoreKeyword is not null && item.FilePath.HasIgnoreKeyword.Value != shouldIgnore.Value)
faces = [];
else
_Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.Property, mappingFromItem, outputResolutionToResize);
}
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
faces = [];
else if (!mappingFromItem.ResizedFileHolder.Exists && !File.Exists(mappingFromItem.ResizedFileHolder.FullName))
@ -1400,41 +1419,41 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
}
private static void CheckForAllWindowsLinks(ReadOnlyCollection<string[]> filesCollection)
private static void CheckForAllWindowsLinks(ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection)
{
string fileFullPath;
WindowsShortcut windowsShortcut;
foreach (string[] files in filesCollection)
foreach (ReadOnlyCollection<FilePath> filePaths in filePathsCollection)
{
if (files.Length == 0)
if (filePaths.Count == 0)
continue;
if (files.All(l => l.EndsWith(".lnk")))
if (filePaths.All(l => l.ExtensionLowered == ".lnk"))
{
foreach (string file in files)
foreach (FilePath filePath in filePaths)
{
windowsShortcut = WindowsShortcut.Load(file);
windowsShortcut = WindowsShortcut.Load(filePath.FullName);
if (windowsShortcut.Path is null)
{
File.Delete(file);
File.Delete(filePath.FullName);
continue;
}
fileFullPath = windowsShortcut.Path;
windowsShortcut.Dispose();
File.WriteAllText(Path.ChangeExtension(file, ".url"), fileFullPath);
File.Delete(file);
File.WriteAllText(Path.ChangeExtension(filePath.FullName, ".url"), fileFullPath);
File.Delete(filePath.FullName);
}
throw new NotSupportedException("All are Windows *.lnk files!");
}
}
}
private static bool IsFilesCollectionCountIsOne(ReadOnlyCollection<string[]> filesCollection)
private static bool IsFilesCollectionCountIsOne(ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection)
{
bool result = true;
int count = 0;
foreach (string[] files in filesCollection)
foreach (ReadOnlyCollection<FilePath> filePaths in filePathsCollection)
{
if (files.Length == 0)
if (filePaths.Count == 0)
continue;
count += 1;
if (count > 1)
@ -1444,22 +1463,22 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
}
if (result)
CheckForAllWindowsLinks(filesCollection);
CheckForAllWindowsLinks(filePathsCollection);
return result;
}
private (string, ReadOnlyCollection<string[]>, bool) GetFilesCollectionThenCopyOrMove(long ticks, string fileSearchFilter, string directorySearchFilter, ProgressBarOptions options, string outputResolution)
private (string, ReadOnlyCollection<ReadOnlyCollection<FilePath>>, bool) GetFilesCollectionThenCopyOrMove(long ticks, string fileSearchFilter, string directorySearchFilter, ProgressBarOptions options, string outputResolution)
{
ProgressBar progressBar;
string filesCollectionRootDirectory = _Configuration.PropertyConfiguration.RootDirectory;
(string cResultsFullGroupDirectory, _, _, _) = GetResultsFullGroupDirectories(outputResolution);
IReadOnlyDictionary<string, string[]> fileGroups = Shared.Models.Stateless.Methods.IPath.GetKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, [_Configuration.PropertyConfiguration.ResultContent]);
ReadOnlyCollection<string[]> filesCollection = IDirectory.GetFilesCollection(filesCollectionRootDirectory, directorySearchFilter, fileSearchFilter, useCeilingAverage: false);
int count = filesCollection.Select(l => l.Length).Sum();
bool filesCollectionCountIsOne = IsFilesCollectionCountIsOne(filesCollection);
ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection = IDirectory.GetFilePathCollections(_Configuration.PropertyConfiguration, filesCollectionRootDirectory, directorySearchFilter, fileSearchFilter, useCeilingAverage: false);
int count = filePathsCollection.Select(l => l.Count).Sum();
bool filesCollectionCountIsOne = IsFilesCollectionCountIsOne(filePathsCollection);
string message = $") Selecting for ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
progressBar = new(count, message, options);
(string[] distinctDirectories, List<(FilePath, string)> toDoCollection) = IDirectory.GetToDoCollection(_Configuration.PropertyConfiguration, filesCollection, fileGroups[_Configuration.PropertyConfiguration.ResultContent], () => progressBar.Tick());
(string[] distinctDirectories, List<(FilePath, string)> toDoCollection) = IDirectory.GetToDoCollection(_Configuration.PropertyConfiguration, filePathsCollection, fileGroups[_Configuration.PropertyConfiguration.ResultContent], () => progressBar.Tick());
progressBar.Dispose();
foreach (string distinctDirectory in distinctDirectories)
{
@ -1470,7 +1489,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
progressBar = new(count, message, options);
_ = IDirectory.CopyOrMove(toDoCollection, move: false, moveBack: false, () => progressBar.Tick());
progressBar.Dispose();
return (filesCollectionRootDirectory, new(filesCollection), filesCollectionCountIsOne);
return (filesCollectionRootDirectory, filePathsCollection, filesCollectionCountIsOne);
}
}