phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

logger for !9

Browse Source 
xmp and json sidecar support

Alignment with Phares 8.0.118.14905 for Shared and Metadata

A_Metadata

Parameter constructors

Removed force-property-last-write-time-to-creation-time

House Cleaning
This commit is contained in:
Mike Phares
2025-07-27 10:57:26 -07:00
parent 30d8a270f9
commit 08164a880d
47 changed files with 1038 additions and 677 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
Instance/DlibDotNet.cs
View File

@ -31,6 +31,8 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
ReadOnlyCollection<ReadOnlyCollection<FilePath>> FilePathsCollection,
ReadOnlyDictionary<int, Identifier>? SplatNineIdentifiers);
public int? CurrentTick =>
_ProgressBar?.CurrentTick;
public long Ticks { get; init; }
private readonly D_Face _Faces;
@ -87,7 +89,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
if (configuration.IgnoreExtensions is null)
throw new NullReferenceException(nameof(configuration.IgnoreExtensions));
_BlurHasher = new BlurHash.Models.C2_BlurHasher(configuration.PropertyConfiguration);
string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(propertyConfiguration, nameof(A_Property), create: false);
string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(propertyConfiguration, nameof(A_Metadata), create: false);
_PropertyRootExistedBefore = !Directory.Exists(propertyRoot);
string argZero = args.Count > 0 ? Path.GetFullPath(args[0]) : Path.GetFullPath(propertyConfiguration.RootDirectory);
_ArgZeroIsConfigurationRootDirectory = propertyConfiguration.RootDirectory == argZero;
@ -175,7 +177,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
_Logger?.LogInformation("First run completed. Run again if wanted");
}
private int FullParallelForWork(B_Metadata metadata,
private int FullParallelForWork(A_Metadata metadata,
MapLogic mapLogic,
string outputResolution,
bool outputResolutionHasNumber,
@ -195,22 +197,22 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
long ticks = DateTime.Now.Ticks;
DateTime dateTime = DateTime.Now;
List<string> parseExceptions = [];
string[] changesFrom = [nameof(A_Property)];
string[] changesFrom = [nameof(A_Metadata)];
List<Tuple<string, DateTime>> subFileTuples = [];
FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(cResultsFullGroupDirectory, item, outputResolutionHasNumber);
if (item.ExifDirectory is null || item.ExifDirectory.FilePath?.Id is null)
throw new Exception();
if (!item.SourceDirectoryFileHolder.Exists || item.SourceDirectoryFileHolder.CreationTime is null || item.SourceDirectoryFileHolder.LastWriteTime is null || item.Any())
throw new Exception();
if (_Configuration.PropertyConfiguration.ForcePropertyLastWriteTimeToCreationTime && item.SourceDirectoryFileHolder.LastWriteTime.Value != item.SourceDirectoryFileHolder.CreationTime.Value)
if (item.SourceDirectoryFileHolder.LastWriteTime.Value != item.SourceDirectoryFileHolder.CreationTime.Value)
{
File.SetLastWriteTime(item.SourceDirectoryFileHolder.FullName, item.SourceDirectoryFileHolder.CreationTime.Value);
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), item.SourceDirectoryFileHolder.CreationTime.Value));
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Metadata), item.SourceDirectoryFileHolder.CreationTime.Value));
}
else if (item.SourceDirectoryFileHolder.LastWriteTime is not null)
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), item.SourceDirectoryFileHolder.LastWriteTime.Value));
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Metadata), item.SourceDirectoryFileHolder.LastWriteTime.Value));
else
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), new FileInfo(item.SourceDirectoryFileHolder.FullName).LastWriteTime));
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Metadata), new FileInfo(item.SourceDirectoryFileHolder.FullName).LastWriteTime));
if (resizedFileHolder.Exists && item.ExifDirectory.Width is not null && item.ExifDirectory.Width.Value > 4 && _Configuration.SaveBlurHashForOutputResolutions.Contains(outputResolution))
{
string? file = _BlurHasher.GetFile(item.FilePath);
@ -323,7 +325,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
private (int, bool) FullParallelWork(int maxDegreeOfParallelism,
B_Metadata metadata,
A_Metadata metadata,
MapLogic mapLogic,
string outputResolution,
bool outputResolutionHasNumber,
@ -343,7 +345,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
DateTime[] containerDateTimes = Container.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems);
string focusRelativePath = Path.GetFullPath(string.Concat(_Configuration.PropertyConfiguration.RootDirectory, _Configuration.FocusDirectory));
bool? isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath);
bool anyPropertiesChangedForX = _Configuration.PropertyConfiguration.PropertiesChangedForProperty || _Configuration.PropertiesChangedForDistance || _Configuration.PropertiesChangedForFaces || _Configuration.PropertiesChangedForIndex || _Configuration.PropertiesChangedForMetadata || _Configuration.PropertiesChangedForResize;
bool anyPropertiesChangedForX = _Configuration.PropertiesChangedForDistance || _Configuration.PropertiesChangedForFaces || _Configuration.PropertiesChangedForIndex || _Configuration.PropertiesChangedForMetadata || _Configuration.PropertiesChangedForResize;
dlibDotNet.ConstructProgressBar(filteredItems.Count, message);
_ = Parallel.For(0, filteredItems.Count, parallelOptions, (i, state) =>
{
@ -382,7 +384,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
string propertyRoot,
string fPhotoPrismSingletonDirectory,
int count,
B_Metadata metadata,
A_Metadata metadata,
Record record,
ReadOnlyCollection<Container.Models.Container> readOnlyContainers,
MapLogic mapLogic)
@ -478,23 +480,16 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
GC.SuppressFinalize(this);
}
(string, string) IDlibDotNet.GetResultsFullGroupDirectories()
string IDlibDotNet.GetResultsFullGroupDirectories()
{
string aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
string result = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
_Configuration.PropertyConfiguration,
nameof(A_Property),
nameof(A_Metadata),
string.Empty,
includeResizeGroup: false,
includeModel: false,
includePredictorModel: false);
string bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
_Configuration.PropertyConfiguration,
nameof(B_Metadata),
string.Empty,
includeResizeGroup: false,
includeModel: false,
includePredictorModel: false);
return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory);
return new(result);
}
void IDlibDotNet.ConstructProgressBar(int maxTicks, string message)
@ -570,7 +565,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
if (filePath.Id is null)
throw new Exception();
string[] directoryNames = keyValuePair.Value.Select(l => l.DirectoryFullPath.Replace('\\', '/')).ToArray();
string paddedId = IId.GetPaddedId(propertyConfiguration, filePath.Id.Value, filePath.ExtensionLowered, filePath.HasIgnoreKeyword, filePath.HasDateTimeOriginal, index: null);
string paddedId = IId.GetPaddedId(propertyConfiguration, filePath, index: null);
result = new(directoryNames, filePath.ExtensionLowered, filePath.HasDateTimeOriginal, filePath.Id.Value, filePath.Length, paddedId, filePath.LastWriteTicks);
return result;
}
@ -887,7 +882,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
return results.AsReadOnly();
}
private static ReadOnlyDictionary<int, Identifier> GetSplatNineIdentifiers(Property.Models.Configuration propertyConfiguration, string bResultsFullGroupDirectory, ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>> idToFilePaths)
private static ReadOnlyDictionary<int, Identifier> GetSplatNineIdentifiers(ILogger<Program>? logger, Property.Models.Configuration propertyConfiguration, string aResultsFullGroupDirectory, ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>> idToFilePaths)
{
Dictionary<int, Identifier> results = [];
string json;
@ -895,7 +890,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
Identifier identifier;
List<Identifier> identifiers = [];
string rootDirectory = propertyConfiguration.RootDirectory.Replace('\\', '/');
string bMetadataCollectionDirectory = Path.Combine(bResultsFullGroupDirectory, propertyConfiguration.ResultCollection);
string bMetadataCollectionDirectory = Path.Combine(aResultsFullGroupDirectory, propertyConfiguration.ResultCollection);
if (!Directory.Exists(bMetadataCollectionDirectory))
_ = Directory.CreateDirectory(bMetadataCollectionDirectory);
foreach (KeyValuePair<int, ReadOnlyCollection<FilePath>> keyValuePair in idToFilePaths)
@ -924,13 +919,19 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
results.Add(keyValuePair.Key, identifier);
}
json = JsonSerializer.Serialize(results.Values.ToArray(), IdentifierCollectionSourceGenerationContext.Default.IdentifierArray);
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(Path.Combine(bMetadataCollectionDirectory, "!9.json"), json, updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null);
if (Shared.Models.Stateless.Methods.IPath.WriteAllText(Path.Combine(bMetadataCollectionDirectory, "!9.json"), json, updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null))
logger?.LogWarning("!9.json has been updated!");
else
logger?.LogInformation("!9.json matches");
json = JsonSerializer.Serialize((from l in identifiers orderby l.PaddedId select l).ToArray(), IdentifierCollectionSourceGenerationContext.Default.IdentifierArray);
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(Path.Combine(bMetadataCollectionDirectory, ".json"), json.Replace(rootDirectory, string.Empty), updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null);
if (Shared.Models.Stateless.Methods.IPath.WriteAllText(Path.Combine(bMetadataCollectionDirectory, ".json"), json.Replace(rootDirectory, string.Empty), updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null))
logger?.LogWarning(".json has been updated!");
else
logger?.LogInformation(".json matches");
return results.AsReadOnly();
}
private static ReadOnlyDictionary<int, Identifier> GetSplatNineIdentifiersAndHideSplatNine(Property.Models.Configuration propertyConfiguration, string bResultsFullGroupDirectory, ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection)
private static ReadOnlyDictionary<int, Identifier> GetSplatNineIdentifiersAndHideSplatNine(ILogger<Program>? logger, Property.Models.Configuration propertyConfiguration, string aResultsFullGroupDirectory, ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection)
{
ReadOnlyDictionary<int, Identifier> results;
if (filePathsCollection.Count == 0)
@ -941,7 +942,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
ReadOnlyCollection<FilePath> moved = HideSplatNineAndGetMovedDuplicatesWithSameSplatNine(propertyConfiguration, idToFilePaths);
if (moved.Count > 0)
throw new Exception($"House cleaning needed!{Environment.NewLine}{string.Join(Environment.NewLine, moved.Select(l => l.Id))}");
results = GetSplatNineIdentifiers(propertyConfiguration, bResultsFullGroupDirectory, idToFilePaths);
results = GetSplatNineIdentifiers(logger, propertyConfiguration, aResultsFullGroupDirectory, idToFilePaths);
}
return results;
}
@ -1112,13 +1113,11 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
MapLogic? mapLogic;
Record? record = null;
string seasonDirectory;
A_Property propertyLogic;
IDlibDotNet dlibDotNet = this;
DateTime dateTime = new(Ticks);
string eDistanceContentDirectory;
string? a2PeopleContentDirectory;
string aResultsFullGroupDirectory;
string bResultsFullGroupDirectory;
string cResultsFullGroupDirectory;
string fPhotoPrismContentDirectory;
const string fileSearchFilter = "*";
@ -1132,15 +1131,15 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
Path.Combine(_Configuration.PropertyConfiguration.RootDirectory, "Facebook"),
Path.Combine(_Configuration.PropertyConfiguration.RootDirectory, "LinkedIn")
];
aResultsFullGroupDirectory = dlibDotNet.GetResultsFullGroupDirectories();
bool runToDoCollectionFirst = GetRunToDoCollectionFirst(_Configuration, checkDirectories);
(aResultsFullGroupDirectory, bResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories();
(int season, string seasonName) = Shared.Models.Stateless.Methods.IProperty.GetSeason(dateTime.DayOfYear);
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(_Configuration.PropertyConfiguration.RootDirectory, dlibDotNet.Ticks);
a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "([])");
eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), _Configuration.PropertyConfiguration.ResultContent);
string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), _Configuration.PropertyConfiguration.ResultSingleton);
_ = Directory.CreateDirectory(Path.Combine(eDistanceContentDirectory, dlibDotNet.Ticks.ToString()));
B_Metadata metadata = new(dlibDotNet, _Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, dlibDotNet.Ticks, bResultsFullGroupDirectory);
A_Metadata metadata = new(dlibDotNet, _Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, dlibDotNet.Ticks, aResultsFullGroupDirectory);
if (runToDoCollectionFirst)
mapLogic = null;
else
@ -1152,12 +1151,11 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
configurationOutputResolutionsHas = true;
if (!runToDoCollectionFirst)
break;
record = GetFilesCollectionThenCopyOrMove(dlibDotNet, metadata, fileSearchFilter, directorySearchFilter, bResultsFullGroupDirectory, outputResolution);
record = GetFilesCollectionThenCopyOrMove(dlibDotNet, metadata, fileSearchFilter, directorySearchFilter, aResultsFullGroupDirectory, outputResolution);
break;
}
fPhotoPrismContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), _Configuration.PropertyConfiguration.ResultContent);
fPhotoPrismSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), _Configuration.PropertyConfiguration.ResultSingleton);
propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory);
if (record is not null && record.FilesCollectionCountIsOne)
{
if (record.FilePathsCollection is null)
@ -1174,9 +1172,8 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
argZero = SaveUrlAndGetNewRootDirectory(record.FilePathsCollection.First());
_Configuration.PropertyConfiguration.ChangeRootDirectory(argZero);
(aResultsFullGroupDirectory, bResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories();
aResultsFullGroupDirectory = dlibDotNet.GetResultsFullGroupDirectories();
propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false);
propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory);
}
if (configurationOutputResolutionsHas)
{
@ -1230,7 +1227,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
{
dlibDotNet.ConstructProgressBar(5, nameof(mapLogic.LookForAbandoned));
(cResultsFullGroupDirectory, _, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
mapLogic.LookForAbandoned(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, readOnlyContainers, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, dlibDotNet.Tick);
mapLogic.LookForAbandoned(_Configuration.PropertyConfiguration, aResultsFullGroupDirectory, readOnlyContainers, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, dlibDotNet.Tick);
}
}
_Distance.Clear();
@ -1271,7 +1268,6 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
string d2ResultsFullGroupDirectory;
(cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(aResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton));
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(bResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton));
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(cResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton));
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultCollection));
@ -1314,10 +1310,10 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
}
}
if (items.Count > 0)
throw new Exception($"{items.Count} item(s) of {count} item(s) are not setup!");
throw new Exception($"{items.Count} item(s) of {count} item(s) are not setup!{Environment.NewLine}{string.Join(Environment.NewLine, items.Select(l => l.FilePath.FileNameFirstSegment))}");
}
private Record GetFilesCollectionThenCopyOrMove(IDlibDotNet dlibDotNet, B_Metadata metadata, string fileSearchFilter, string directorySearchFilter, string bResultsFullGroupDirectory, string outputResolution)
private Record GetFilesCollectionThenCopyOrMove(IDlibDotNet dlibDotNet, A_Metadata metadata, string fileSearchFilter, string directorySearchFilter, string aResultsFullGroupDirectory, string outputResolution)
{
Record result;
int count;
@ -1329,7 +1325,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
string filesCollectionRootDirectory = _Configuration.PropertyConfiguration.RootDirectory;
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
(string cResultsFullGroupDirectory, _, _, _) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
string jsonGroupDirectory = Path.Combine(bResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton);
string jsonGroupDirectory = Path.Combine(aResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton);
if (!Directory.Exists(jsonGroupDirectory))
_ = Directory.CreateDirectory(jsonGroupDirectory);
ReadOnlyDictionary<string, ReadOnlyDictionary<byte, ReadOnlyCollection<string>>> keyValuePairs =
@ -1359,7 +1355,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable
message = $") Copying to ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - dlibDotNet.Ticks).TotalSeconds)} total second(s)";
dlibDotNet.ConstructProgressBar(count, message);
_ = IDirectory.CopyOrMove(toDoCollection, move: false, moveBack: false, dlibDotNet.Tick);
ReadOnlyDictionary<int, Identifier> splatNineIdentifiers = GetSplatNineIdentifiersAndHideSplatNine(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, filePathsCollection);
ReadOnlyDictionary<int, Identifier> splatNineIdentifiers = GetSplatNineIdentifiersAndHideSplatNine(_Logger, _Configuration.PropertyConfiguration, aResultsFullGroupDirectory, filePathsCollection);
result = new(ExifDirectoriesById: exifDirectoriesById,
FilesCollectionRootDirectory: filesCollectionRootDirectory,
FilesCollectionCountIsOne: filesCollectionCountIsOne,