diff --git a/Container/Models/Stateless/Methods/Container.cs b/Container/Models/Stateless/Methods/Container.cs index e4e16e6..d6d70dc 100644 --- a/Container/Models/Stateless/Methods/Container.cs +++ b/Container/Models/Stateless/Methods/Container.cs @@ -48,7 +48,7 @@ internal abstract class Container continue; foreach (Item item in filteredItems) { - if (item.ExifDirectory?.FilePath.Id is null || item.ResizedFileHolder is null) + if (item.ExifDirectory?.FilePath?.Id is null || item.ResizedFileHolder is null) continue; if (results.Contains(item.ExifDirectory.FilePath.Id.Value)) continue; @@ -151,7 +151,7 @@ internal abstract class Container exifDirectory = Metadata.Models.Stateless.Methods.IMetadata.GetExifDirectory(filePair.FilePath); ReadOnlyCollection keywords = IMetaBase.GetKeywords(exifDirectory?.ExifBaseDirectories); bool? shouldIgnore = propertyConfiguration.IgnoreRulesKeyWords.Any(keywords.Contains); - bool? fileSizeChanged = exifDirectory is not null ? exifDirectory.FilePath.Length != filePair.FilePath.Length : null; + bool? fileSizeChanged = exifDirectory?.FilePath is not null ? exifDirectory.FilePath.Length != filePair.FilePath.Length : null; bool isValidImageFormatExtension = propertyConfiguration.ValidImageFormatExtensions.Contains(filePair.FilePath.ExtensionLowered); bool? isArchive = filePair.FilePath.Id is null || splatNineIdentifiers is null ? null : splatNineIdentifiers.TryGetValue(filePair.FilePath.Id.Value, out Identifier? identifier); if (exifDirectory is not null && filePair.FilePath.Id is not null && filePair.FilePath.HasIgnoreKeyword is not null && filePair.FilePath.HasDateTimeOriginal is not null) @@ -196,7 +196,7 @@ internal abstract class Container RenameFile(filePair, filePair.FilePath, change.Value, filePaths); } string relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(filePair.FilePath.FullName, rootDirectoryLength, forceExtensionToLower: true); - bool? lastWriteTimeChanged = exifDirectory is not null ? propertyConfiguration.PropertiesChangedForProperty || exifDirectory.FilePath.LastWriteTicks != filePair.FilePath.LastWriteTicks : null; + bool? lastWriteTimeChanged = exifDirectory?.FilePath is not null ? propertyConfiguration.PropertiesChangedForProperty || exifDirectory.FilePath.LastWriteTicks != filePair.FilePath.LastWriteTicks : null; if (filePair.Match is not null) sourceDirectoryFileHolder = IFileHolder.Get(filePair.Match); else if (!filePair.IsUnique) @@ -279,7 +279,7 @@ internal abstract class Container continue; foreach (Item item in filteredItems) { - if (item.ExifDirectory?.FilePath.Id is null || item.ResizedFileHolder is null) + if (item.ExifDirectory?.FilePath?.Id is null || item.ResizedFileHolder is null) continue; if (results.Contains(item.FilePath.FileNameFirstSegment)) continue; @@ -308,7 +308,7 @@ internal abstract class Container } foreach (Item item in filteredItems) { - if (item.ExifDirectory?.FilePath.Id is null || item.ResizedFileHolder is null) + if (item.ExifDirectory?.FilePath?.Id is null || item.ResizedFileHolder is null) continue; if (distinctItems) { @@ -397,7 +397,7 @@ internal abstract class Container { bool result = false; string checkFile; - FilePath filePath; + FilePath? filePath; string subDirectory; string directoryName; string checkDirectory; @@ -408,8 +408,8 @@ internal abstract class Container foreach (string directory in directories) { fileHolder = IFileHolder.Get(Path.GetFileName(directory)); - filePath = FilePath.Get(propertyConfiguration, fileHolder, index: null); - if (filePath.Id is null) + filePath = FilePath.GetNullSafe(propertyConfiguration, fileHolder, index: null); + if (filePath?.Id is null) continue; if (!fileNamesToFiles.TryGetValue(filePath.Id.Value, out collection)) throw new Exception(); diff --git a/Drag-Drop-Search/DragDropSearch.cs b/Drag-Drop-Search/DragDropSearch.cs index f8ec7ac..fde6f11 100644 --- a/Drag-Drop-Search/DragDropSearch.cs +++ b/Drag-Drop-Search/DragDropSearch.cs @@ -96,7 +96,7 @@ public partial class DragDropSearch : Form List collection = Program.GetItemCollection(_Configuration, containers); foreach (Item item in collection) { - if (item.ExifDirectory?.FilePath.Id is null) + if (item.ExifDirectory?.FilePath?.Id is null) continue; if (_IdToItem.ContainsKey(item.ExifDirectory.FilePath.Id.Value)) continue; diff --git a/Duplicate-Search/DuplicateSearch.cs b/Duplicate-Search/DuplicateSearch.cs index 210fbd8..a79726e 100644 --- a/Duplicate-Search/DuplicateSearch.cs +++ b/Duplicate-Search/DuplicateSearch.cs @@ -129,7 +129,7 @@ public class DuplicateSearch containerDateTimes = Container.Models.Stateless.Methods.IContainer.GetContainerDateTimes(validImageItems); foreach (Item item in validImageItems) { - if (item.ExifDirectory?.FilePath.Id is null) + if (item.ExifDirectory?.FilePath?.Id is null) { if (int.TryParse(item.FilePath.NameWithoutExtension, out int id)) continue; diff --git a/Instance/DlibDotNet.cs b/Instance/DlibDotNet.cs index f889b5c..c3d406f 100644 --- a/Instance/DlibDotNet.cs +++ b/Instance/DlibDotNet.cs @@ -197,7 +197,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable string[] changesFrom = [nameof(A_Property)]; List> subFileTuples = []; FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(cResultsFullGroupDirectory, item, outputResolutionHasNumber); - if (item.ExifDirectory is null || item.ExifDirectory.FilePath.Id is null) + if (item.ExifDirectory is null || item.ExifDirectory.FilePath?.Id is null) throw new Exception(); if (!item.SourceDirectoryFileHolder.Exists || item.SourceDirectoryFileHolder.CreationTime is null || item.SourceDirectoryFileHolder.LastWriteTime is null || item.Any()) throw new Exception(); @@ -339,6 +339,9 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable dlibDotNet.ConstructProgressBar(filteredItems.Count, message); _ = Parallel.For(0, filteredItems.Count, parallelOptions, (i, state) => { + Item item = filteredItems[i]; + if (!item.SourceDirectoryFileHolder.Exists || item.SourceDirectoryFileHolder.CreationTime is null || item.SourceDirectoryFileHolder.LastWriteTime is null || item.Any()) + return; try { result += FullParallelForWork(metadata, @@ -351,7 +354,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable fileNameToCollection, record, container, - filteredItems[i], + item, containerDateTimes, isFocusRelativePath); if (!anyPropertiesChangedForX && (i == 0 || sourceDirectoryChanges.Count > 0)) @@ -622,7 +625,25 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable if (string.IsNullOrEmpty(json)) result = null; else + { result = JsonSerializer.Deserialize(json, ExifDirectorySourceGenerationContext.Default.ExifDirectory); + if (result?.FilePath?.Id is null) + { + try + { + result = Metadata.Models.Stateless.Methods.IMetadata.GetExifDirectory(filePair.FilePath); + json = JsonSerializer.Serialize(result, ExifDirectorySourceGenerationContext.Default.ExifDirectory); + _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(filePair.Match.FullName, json, updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null); + } + catch (Exception) { result = null; } + } + } + } + if (result is not null && result.FilePath?.Id is null) + { + try + { result = Metadata.Models.Stateless.Methods.IMetadata.GetExifDirectory(filePair.FilePath); } + catch (Exception) { result = null; } } return result; } @@ -723,7 +744,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable List<(string, string, string)> collection = []; foreach (Item item in distinctValidImageItems) { - if (item.ExifDirectory?.FilePath.Id is null) + if (item.ExifDirectory?.FilePath?.Id is null) continue; if (item.IsNotUniqueAndNeedsReview is null || !item.IsNotUniqueAndNeedsReview.Value) continue; @@ -736,7 +757,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable } foreach (Item item in distinctValidImageItems) { - if (item.ExifDirectory?.FilePath.Id is null) + if (item.ExifDirectory?.FilePath?.Id is null) continue; dateTime = IDate.GetDateTimeOriginal(item.ExifDirectory); if (dateTime is null) @@ -1020,7 +1041,10 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable if (exifDirectory is null) return; lock (results) - results.Add(filePair.FilePath.Id.Value, exifDirectory); + { + if (!results.ContainsKey(filePair.FilePath.Id.Value)) + results.Add(filePair.FilePath.Id.Value, exifDirectory); + } } private bool GetRunToDoCollectionFirst(Models.Configuration configuration, long ticks, string[] checkDirectories) @@ -1121,7 +1145,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable configurationOutputResolutionsHas = true; if (!runToDoCollectionFirst) break; - record = GetFilesCollectionThenCopyOrMove(dlibDotNet, ticks, fileSearchFilter, directorySearchFilter, bResultsFullGroupDirectory, outputResolution); + record = GetFilesCollectionThenCopyOrMove(dlibDotNet, metadata, ticks, fileSearchFilter, directorySearchFilter, bResultsFullGroupDirectory, outputResolution); break; } fPhotoPrismContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), _Configuration.PropertyConfiguration.ResultContent); @@ -1274,7 +1298,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable foreach (Item item in filteredItems) { count++; - if (item.ExifDirectory is null || item.ExifDirectory.FilePath.Id is null) + if (item.ExifDirectory is null || item.ExifDirectory.FilePath?.Id is null) { items.Add(item); continue; @@ -1291,7 +1315,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable throw new Exception($"{items.Count} item(s) of {count} item(s) are not setup!"); } - private Record GetFilesCollectionThenCopyOrMove(IDlibDotNet dlibDotNet, long ticks, string fileSearchFilter, string directorySearchFilter, string bResultsFullGroupDirectory, string outputResolution) + private Record GetFilesCollectionThenCopyOrMove(IDlibDotNet dlibDotNet, B_Metadata metadata, long ticks, string fileSearchFilter, string directorySearchFilter, string bResultsFullGroupDirectory, string outputResolution) { Record result; int count; @@ -1300,6 +1324,8 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable ReadOnlyCollection filePairs; int maxDegreeOfParallelism = Environment.ProcessorCount; Dictionary exifDirectoriesById = []; + foreach (KeyValuePair keyValuePair in metadata.ExifDirectoriesById) + exifDirectoriesById.Add(keyValuePair.Key, keyValuePair.Value); string filesCollectionRootDirectory = _Configuration.PropertyConfiguration.RootDirectory; ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; (string cResultsFullGroupDirectory, _, _, _) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution); @@ -1312,7 +1338,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable ReadOnlyCollection> filePathsCollection = IDirectory.GetFilePathCollections(_Configuration.PropertyConfiguration, directorySearchFilter, fileSearchFilter, filesCollectionRootDirectory, useIgnoreExtensions: true, useCeilingAverage: false); count = filePathsCollection.Select(l => l.Count).Sum(); filePairs = IFilePair.GetFilePairs(_Configuration.PropertyConfiguration, directorySearchFilter, extension, jsonGroupDirectory, filePathsCollection); - message = $") Preloading ExifDirectory Dictionary - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)"; + message = $") {nameof(DlibDotNet)} - Preloading ExifDirectory Dictionary - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)"; dlibDotNet.ConstructProgressBar(count, message); _ = Parallel.For(0, filePairs.Count, parallelOptions, (i, state) => ParallelFor(filePairs[i], exifDirectoriesById, dlibDotNet.Tick)); if (exifDirectoriesById.Count == 0) @@ -1438,7 +1464,7 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath); foreach (Item item in filteredItems) { - if (item.ExifDirectory?.FilePath.Id is null || item.ResizedFileHolder is null) + if (item.ExifDirectory?.FilePath?.Id is null || item.ResizedFileHolder is null) continue; mappingFromItem = IMappingFromItem.GetMappingFromItem(containerDateTimes, item, item.ResizedFileHolder); if (distinctItems) @@ -1634,11 +1660,11 @@ public partial class DlibDotNet : IDlibDotNet, IDisposable MappingFromFilterPre mappingFromFilterPre; MappingFromFilterPost mappingFromFilterPost; bool? isFocusModel = GetIsFocusModel(item.ExifDirectory); - ReadOnlyDictionary>? wholePercentagesToPersonContainers = mapLogic.GetWholePercentagesToPersonContainers(item.ExifDirectory?.FilePath.Id); + ReadOnlyDictionary>? wholePercentagesToPersonContainers = mapLogic.GetWholePercentagesToPersonContainers(item.ExifDirectory?.FilePath?.Id); long[] jLinkResolvedPersonKeys = _JLinkResolvedDirectories.Select(l => l.PersonKey).ToArray(); foreach (Shared.Models.Face face in faces) { - if (item.ExifDirectory?.FilePath.Id is null || face.FaceEncoding is null || face.Location is null || face.OutputResolution is null) + if (item.ExifDirectory?.FilePath?.Id is null || face.FaceEncoding is null || face.Location is null || face.OutputResolution is null) { canReMap = null; isFocusPerson = null; diff --git a/Map/Models/MapLogic.cs b/Map/Models/MapLogic.cs index 1f4bf09..0f70991 100644 --- a/Map/Models/MapLogic.cs +++ b/Map/Models/MapLogic.cs @@ -320,7 +320,7 @@ public partial class MapLogic : Shared.Models.Methods.IMapLogic public ReadOnlyCollection GetLocationContainers(Item item) { LocationContainer[] results; - if (item.ExifDirectory?.FilePath.Id is null) + if (item.ExifDirectory?.FilePath?.Id is null) results = []; else { diff --git a/Map/Models/Stateless/MapLogic.cs b/Map/Models/Stateless/MapLogic.cs index 13b5880..8d1fc30 100644 --- a/Map/Models/Stateless/MapLogic.cs +++ b/Map/Models/Stateless/MapLogic.cs @@ -110,7 +110,7 @@ internal abstract class MapLogic List results = []; foreach (Item item in items) { - if (item.ExifDirectory?.FilePath.Id is null || item.ResizedFileHolder is null) + if (item.ExifDirectory?.FilePath?.Id is null || item.ResizedFileHolder is null) continue; foreach (Face face in item.Faces) { diff --git a/Metadata/Models/B_Metadata.cs b/Metadata/Models/B_Metadata.cs index 16ff588..eeadd03 100644 --- a/Metadata/Models/B_Metadata.cs +++ b/Metadata/Models/B_Metadata.cs @@ -50,15 +50,15 @@ public class B_Metadata : IMetadata else throw new Exception(); } - List results = []; string jsonGroupDirectory; const string extension = ".json"; const string fileSearchFilter = "*"; string filesCollectionRootDirectory; const string directorySearchFilter = "*"; - int maxDegreeOfParallelism = Environment.ProcessorCount; - filesCollectionRootDirectory = propertyConfiguration.RootDirectory; Dictionary exifDirectoriesById = []; + int maxDegreeOfParallelism = Environment.ProcessorCount; + Action? tick = dlibDotNet is null ? null : dlibDotNet.Tick; + filesCollectionRootDirectory = propertyConfiguration.RootDirectory; ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; string jsonGroupSingletonDirectory = Path.Combine(bResultsFullGroupDirectory, propertyConfiguration.ResultSingleton); string jsonGroupCollectionDirectory = Path.Combine(bResultsFullGroupDirectory, propertyConfiguration.ResultCollection); @@ -67,16 +67,10 @@ public class B_Metadata : IMetadata ReadOnlyCollection> filePathsCollection = IDirectory.GetFilePathCollections(propertyConfiguration, directorySearchFilter, fileSearchFilter, filesCollectionRootDirectory, useIgnoreExtensions: true, useCeilingAverage: false); ReadOnlyDictionary> fileNamesToFiles = FilePath.GetFilesKeyValuePairs(filePathsCollection); ReadOnlyCollection filePairs = IFilePair.GetFilePairs(propertyConfiguration, directorySearchFilter, extension, jsonGroupSingletonDirectory, filePathsCollection, fileNamesToFiles); - string message = $") Preloading ExifDirectory Dictionary - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)"; + string message = $") {nameof(B_Metadata)} - Preloading ExifDirectory Dictionary - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)"; dlibDotNet?.ConstructProgressBar(filePairs.Count, message); - _ = Parallel.For(0, filePairs.Count, parallelOptions, (i, state) => ParallelFor(dlibDotNet, filePairs[i], results)); + _ = Parallel.For(0, filePairs.Count, parallelOptions, (i, state) => ParallelFor(filePairs[i], exifDirectoriesById, tick)); jsonGroupDirectory = Path.Combine(bResultsFullGroupDirectory, propertyConfiguration.ResultCollection); - foreach (ExifDirectory exifDirectory in results) - { - if (exifDirectory.FilePath.Id is null || exifDirectoriesById.ContainsKey(exifDirectory.FilePath.Id.Value)) - continue; - exifDirectoriesById.Add(exifDirectory.FilePath.Id.Value, exifDirectory); - } ExifDirectoriesById = new(exifDirectoriesById); DateGroupDirectory = bResultsFullGroupDirectory; SingletonById = fileNamesToFiles; @@ -86,16 +80,19 @@ public class B_Metadata : IMetadata Collection = filePathsSingletonCollection[0]; } - private void ParallelFor(IDlibDotNet? dlibDotNet, FilePair filePair, List results) + private void ParallelFor(FilePair filePair, Dictionary results, Action? tick) { - dlibDotNet?.Tick(); + tick?.Invoke(); if (filePair.FilePath.Id is null) return; ExifDirectory? exifDirectory = GetExifDirectory(filePair); if (exifDirectory is null) return; lock (results) - results.Add(exifDirectory); + { + if (!results.ContainsKey(filePair.FilePath.Id.Value)) + results.Add(filePair.FilePath.Id.Value, exifDirectory); + } } private static ExifDirectory? GetExifDirectory(FilePair filePair) @@ -105,11 +102,24 @@ public class B_Metadata : IMetadata result = null; else { - string json = File.ReadAllText(filePair.Match.FullName); + string json; + json = File.ReadAllText(filePair.Match.FullName); if (string.IsNullOrEmpty(json)) result = null; else + { result = JsonSerializer.Deserialize(json, ExifDirectorySourceGenerationContext.Default.ExifDirectory); + if (result?.FilePath?.Id is null) + { + try + { + result = Stateless.Methods.IMetadata.GetExifDirectory(filePair.FilePath); + json = JsonSerializer.Serialize(result, ExifDirectorySourceGenerationContext.Default.ExifDirectory); + _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(filePair.Match.FullName, json, updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null); + } + catch (Exception) { result = null; } + } + } } return result; } diff --git a/Shared/Models/ExifDirectory.cs b/Shared/Models/ExifDirectory.cs index 70b9651..5bf0d6a 100644 --- a/Shared/Models/ExifDirectory.cs +++ b/Shared/Models/ExifDirectory.cs @@ -6,7 +6,7 @@ namespace View_by_Distance.Shared.Models; public record ExifDirectory(AviDirectory[] AviDirectories, ExifDirectoryBase[] ExifBaseDirectories, FileMetadataDirectory[] FileMetadataDirectories, - FilePath FilePath, + FilePath? FilePath, GifHeaderDirectory[] GifHeaderDirectories, GpsDirectory[] GpsDirectories, int? Height, diff --git a/Shared/Models/FilePath.cs b/Shared/Models/FilePath.cs index 7c41812..c964ca6 100644 --- a/Shared/Models/FilePath.cs +++ b/Shared/Models/FilePath.cs @@ -21,6 +21,20 @@ public record FilePath(long CreationTicks, int? SortOrder) { + public static FilePath? GetNullSafe(Properties.IPropertyConfiguration propertyConfiguration, FileHolder fileHolder, int? index) + { + FilePath? result; + if (fileHolder.CreationTime is null) + result = null; + else if (fileHolder.LastWriteTime is null) + result = null; + else if (fileHolder.Length is null) + result = null; + else + result = Get(propertyConfiguration, fileHolder, index); + return result; + } + public static FilePath Get(Properties.IPropertyConfiguration propertyConfiguration, FileHolder fileHolder, int? index) { if (fileHolder.CreationTime is null) diff --git a/Shared/Models/MappingFromItem.cs b/Shared/Models/MappingFromItem.cs index 54f97fd..6890994 100644 --- a/Shared/Models/MappingFromItem.cs +++ b/Shared/Models/MappingFromItem.cs @@ -29,7 +29,7 @@ public record MappingFromItem(DateTime[] ContainerDateTimes, internal static MappingFromItem GetMappingFromItem(DateTime[] containerDateTimes, Item item, FileHolder? resizedFileHolder) { MappingFromItem result; - if (item.ExifDirectory?.FilePath.Id is null) + if (item.ExifDirectory?.FilePath?.Id is null) throw new NotSupportedException(); if (resizedFileHolder is null) throw new NotSupportedException(); diff --git a/Shared/Models/Stateless/Methods/XDate.cs b/Shared/Models/Stateless/Methods/XDate.cs index 9bf6995..6de5bb5 100644 --- a/Shared/Models/Stateless/Methods/XDate.cs +++ b/Shared/Models/Stateless/Methods/XDate.cs @@ -40,8 +40,8 @@ internal abstract class XDate } if (results.Count == 0) { - string fileNameWithoutExtension = Path.GetFileNameWithoutExtension(exifDirectory.FilePath.Name); - DateTime? dateTime = GetDateTimeFromName(fileNameWithoutExtension); + string? fileNameWithoutExtension = exifDirectory.FilePath is null ? null : Path.GetFileNameWithoutExtension(exifDirectory.FilePath.Name); + DateTime? dateTime = fileNameWithoutExtension is null ? null : GetDateTimeFromName(fileNameWithoutExtension); if (dateTime is not null) results.Add(dateTime.Value); foreach (ExifDirectoryBase exifDirectoryBase in exifDirectory.ExifBaseDirectories)