diff --git a/.gitignore b/.gitignore index 5449c42..fa03149 100644 --- a/.gitignore +++ b/.gitignore @@ -468,3 +468,5 @@ globalStorage/ [Ll]ib/ Shared/.kanbn + +.vscode/Har-Files \ No newline at end of file diff --git a/.vscode/.yml b/.vscode/.yml new file mode 100644 index 0000000..e69de29 diff --git a/Metadata/AA.Metadata.csproj b/Metadata/AA.Metadata.csproj index fce4661..7de7496 100644 --- a/Metadata/AA.Metadata.csproj +++ b/Metadata/AA.Metadata.csproj @@ -34,8 +34,8 @@ - - + + diff --git a/Metadata/Models/A_Metadata.cs b/Metadata/Models/A_Metadata.cs index c15eb22..6199941 100644 --- a/Metadata/Models/A_Metadata.cs +++ b/Metadata/Models/A_Metadata.cs @@ -16,13 +16,13 @@ public class A_Metadata public A_Metadata(MetadataConfiguration metadataConfiguration) { _MetadataConfiguration = metadataConfiguration; - string bResultsFullGroupDirectory = IResult.GetResultsFullGroupDirectory(metadataConfiguration.ResultConfiguration, + string aResultsFullGroupDirectory = IResult.GetResultsFullGroupDirectory(metadataConfiguration.ResultConfiguration, nameof(A_Metadata), string.Empty, includeResizeGroup: false, includeModel: false, includePredictorModel: false); - _FileGroups = IPath.GetKeyValuePairs(metadataConfiguration.ResultConfiguration, bResultsFullGroupDirectory, [metadataConfiguration.ResultConfiguration.ResultSingleton]); + _FileGroups = IPath.GetKeyValuePairs(metadataConfiguration.ResultConfiguration, aResultsFullGroupDirectory, [metadataConfiguration.ResultConfiguration.ResultSingleton]); } private (int, FileInfo) GetFileInfo(ResultConfiguration resultConfiguration, FilePath filePath) @@ -132,7 +132,7 @@ public class A_Metadata return (fileInfo, result); } - public static Action SetExifDirectoryCollection(IRename rename, IRenameConfiguration renameConfiguration, A_Metadata metadata, List<(string, FileInfo, ExifDirectory)> exifDirectories) + public static Action SetExifDirectoryCollection(IRename rename, IRenameConfiguration renameConfiguration, A_Metadata metadata, List<(FilePath, FileInfo, ExifDirectory)> exifDirectories) { return file => { @@ -141,20 +141,30 @@ public class A_Metadata FilePath? ffmpegFilePath; ExifDirectory exifDirectory; FileHolder fileHolder = new(file); - ReadOnlyCollection ffmpegFiles; + ReadOnlyCollection? ffmpegFiles; DeterministicHashCode deterministicHashCode; FilePath filePath = FilePath.Get(renameConfiguration.MetadataConfiguration, fileHolder, index: null); if (!renameConfiguration.SkipIdFiles || filePath.Id is null || !filePath.IsIntelligentIdFormat && filePath.SortOrder is not null) { - (ffmpegFiles, ffmpegFilePath) = rename.ConvertAndGetFfmpegFiles(renameConfiguration, filePath); - if (ffmpegFilePath is not null) - filePath = ffmpegFilePath; - deterministicHashCode = filePath.Id is not null ? deterministicHashCode = new(null, filePath.Id, null) : deterministicHashCode = rename.GetDeterministicHashCode(filePath); + if (filePath.Id is not null) + { + ffmpegFiles = null; + deterministicHashCode = new(null, filePath.Id, null); + } + else + { + ffmpegFiles = rename.ConvertAndGetFfmpegFiles(renameConfiguration, filePath); + ffmpegFilePath = ffmpegFiles.Count == 0 ? null : FilePath.Get(renameConfiguration.MetadataConfiguration, new(ffmpegFiles[0]), index: null); + deterministicHashCode = ffmpegFilePath is null ? rename.GetDeterministicHashCode(filePath) : rename.GetDeterministicHashCode(ffmpegFilePath); + } (fileInfo, exifDirectory) = metadata.GetMetadataCollection(renameConfiguration.MetadataConfiguration, filePath, deterministicHashCode); lock (exifDirectories) - exifDirectories.Add(new(file, fileInfo, exifDirectory)); - foreach (string ffmpegFile in ffmpegFiles) - File.Delete(ffmpegFile); + exifDirectories.Add(new(filePath, fileInfo, exifDirectory)); + if (ffmpegFiles is not null) + { + foreach (string ffmpegFile in ffmpegFiles) + File.Delete(ffmpegFile); + } } }; diff --git a/Metadata/Models/Binder/MetadataConfiguration.cs b/Metadata/Models/Binder/MetadataConfiguration.cs index 97b2b48..374300a 100644 --- a/Metadata/Models/Binder/MetadataConfiguration.cs +++ b/Metadata/Models/Binder/MetadataConfiguration.cs @@ -23,17 +23,16 @@ public class MetadataConfiguration { if (configuration?.ForceMetadataLastWriteTimeToCreationTime is null) { + List paths = []; foreach (IConfigurationProvider configurationProvider in configurationRoot.Providers) { if (configurationProvider is not Microsoft.Extensions.Configuration.Json.JsonConfigurationProvider jsonConfigurationProvider) continue; if (jsonConfigurationProvider.Source.FileProvider is not Microsoft.Extensions.FileProviders.PhysicalFileProvider physicalFileProvider) continue; - if (!physicalFileProvider.Root.Contains("UserSecrets")) - continue; - throw new NotSupportedException(physicalFileProvider.Root); + paths.Add(physicalFileProvider.Root); } - throw new NotSupportedException("Not Found!"); + throw new NotSupportedException($"Not found!{Environment.NewLine}{string.Join(Environment.NewLine, paths.Distinct())}"); } } diff --git a/Metadata/Models/Binder/ResultConfiguration.cs b/Metadata/Models/Binder/ResultConfiguration.cs index a248e48..85fbe1b 100644 --- a/Metadata/Models/Binder/ResultConfiguration.cs +++ b/Metadata/Models/Binder/ResultConfiguration.cs @@ -29,17 +29,16 @@ public class ResultConfiguration { if (configuration?.DateGroup is null) { + List paths = []; foreach (IConfigurationProvider configurationProvider in configurationRoot.Providers) { if (configurationProvider is not Microsoft.Extensions.Configuration.Json.JsonConfigurationProvider jsonConfigurationProvider) continue; if (jsonConfigurationProvider.Source.FileProvider is not Microsoft.Extensions.FileProviders.PhysicalFileProvider physicalFileProvider) continue; - if (!physicalFileProvider.Root.Contains("UserSecrets")) - continue; - throw new NotSupportedException(physicalFileProvider.Root); + paths.Add(physicalFileProvider.Root); } - throw new NotSupportedException("Not Found!"); + throw new NotSupportedException($"Not found!{Environment.NewLine}{string.Join(Environment.NewLine, paths.Distinct())}"); } } diff --git a/Metadata/Models/Stateless/Base.cs b/Metadata/Models/Stateless/Base.cs index 8c56003..23e79f5 100644 --- a/Metadata/Models/Stateless/Base.cs +++ b/Metadata/Models/Stateless/Base.cs @@ -1,3 +1,4 @@ +using System.Collections.ObjectModel; using View_by_Distance.Shared.Models; namespace View_by_Distance.Metadata.Models.Stateless.Methods; @@ -5,7 +6,7 @@ namespace View_by_Distance.Metadata.Models.Stateless.Methods; internal static class Base { - internal static string GetMaker(ExifDirectoryBase[]? exifBaseDirectories) + internal static string? GetMaker(ExifDirectoryBase[]? exifBaseDirectories) { string? result = null; if (exifBaseDirectories is not null) @@ -23,12 +24,10 @@ internal static class Base } } } - if (string.IsNullOrEmpty(result)) - result = "Unknown"; return result; } - internal static string GetModel(ExifDirectoryBase[]? exifBaseDirectories) + internal static string? GetModel(ExifDirectoryBase[]? exifBaseDirectories) { string? result = null; if (exifBaseDirectories is not null) @@ -46,9 +45,24 @@ internal static class Base } } } - if (string.IsNullOrEmpty(result)) - result = "Unknown"; return result; } + internal static ReadOnlyCollection GetKeywords(ExifDirectoryBase[]? exifBaseDirectories) + { + List results = []; + if (exifBaseDirectories is not null) + { + string value; + foreach (ExifDirectoryBase exifDirectoryBase in exifBaseDirectories) + { + value = exifDirectoryBase?.WinKeywords is null ? string.Empty : exifDirectoryBase.WinKeywords.ToString().Trim(); + if (string.IsNullOrEmpty(value)) + continue; + results.Add(value); + } + } + return new(results); + } + } \ No newline at end of file diff --git a/Metadata/Models/Stateless/Methods/IMetadata.cs b/Metadata/Models/Stateless/Methods/IMetadata.cs index 1f2f513..d6e2a08 100644 --- a/Metadata/Models/Stateless/Methods/IMetadata.cs +++ b/Metadata/Models/Stateless/Methods/IMetadata.cs @@ -1,4 +1,5 @@ using MetadataExtractor; +using System.Collections.ObjectModel; using View_by_Distance.Shared.Models; namespace View_by_Distance.Metadata.Models.Stateless.Methods; @@ -19,16 +20,21 @@ public interface IMetadata static ExifDirectory GetExifDirectory(FilePath filePath, DeterministicHashCode deterministicHashCode) => Exif.GetExifDirectory(filePath, deterministicHashCode); - string TestStatic_GetMaker(ExifDirectory? exifDirectory) => + string? TestStatic_GetMaker(ExifDirectory? exifDirectory) => GetMaker(exifDirectory); - static string GetMaker(ExifDirectory? exifDirectory) => + static string? GetMaker(ExifDirectory? exifDirectory) => Base.GetMaker(exifDirectory?.ExifBaseDirectories); - string TestStatic_GetModel(ExifDirectory? exifDirectory) => + string? TestStatic_GetModel(ExifDirectory? exifDirectory) => GetModel(exifDirectory); - static string GetModel(ExifDirectory? exifDirectory) => + static string? GetModel(ExifDirectory? exifDirectory) => Base.GetModel(exifDirectory?.ExifBaseDirectories); + ReadOnlyCollection TestStatic_GetKeywords(ExifDirectory? exifDirectory) => + GetKeywords(exifDirectory); + static ReadOnlyCollection GetKeywords(ExifDirectory? exifDirectory) => + Base.GetKeywords(exifDirectory?.ExifBaseDirectories); + string? TestStatic_GetOutputResolution(ExifDirectory? exifDirectory) => GetOutputResolution(exifDirectory); static string? GetOutputResolution(ExifDirectory? exifDirectory) => diff --git a/Rename/AA.Rename.csproj b/Rename/AA.Rename.csproj index b9ce1a6..ffeab8f 100644 --- a/Rename/AA.Rename.csproj +++ b/Rename/AA.Rename.csproj @@ -1,4 +1,4 @@ - + enable enable @@ -34,11 +34,11 @@ - - - - - + + + + + diff --git a/Rename/Models/AppSettings.cs b/Rename/Models/AppSettings.cs index 0ff0b44..9d605c3 100644 --- a/Rename/Models/AppSettings.cs +++ b/Rename/Models/AppSettings.cs @@ -4,6 +4,7 @@ using System.Text.Json.Serialization; namespace View_by_Distance.Rename.Models; public record AppSettings(string Company, + string? HarFilesDirectory, int MaxDegreeOfParallelism, bool RequireRootDirectoryExists) { diff --git a/Rename/Models/Binder/AppSettings.cs b/Rename/Models/Binder/AppSettings.cs index 8639e79..bf1824d 100644 --- a/Rename/Models/Binder/AppSettings.cs +++ b/Rename/Models/Binder/AppSettings.cs @@ -8,6 +8,7 @@ public class AppSettings { public string? Company { get; set; } + public string? HarFilesDirectory { get; set; } public int? MaxDegreeOfParallelism { get; set; } public bool? RequireRootDirectoryExists { get; set; } @@ -21,17 +22,16 @@ public class AppSettings { if (appSettings?.Company is null) { + List paths = []; foreach (IConfigurationProvider configurationProvider in configurationRoot.Providers) { if (configurationProvider is not Microsoft.Extensions.Configuration.Json.JsonConfigurationProvider jsonConfigurationProvider) continue; if (jsonConfigurationProvider.Source.FileProvider is not Microsoft.Extensions.FileProviders.PhysicalFileProvider physicalFileProvider) continue; - if (!physicalFileProvider.Root.Contains("UserSecrets")) - continue; - throw new NotSupportedException(physicalFileProvider.Root); + paths.Add(physicalFileProvider.Root); } - throw new NotSupportedException("Not Found!"); + throw new NotSupportedException($"Not found!{Environment.NewLine}{string.Join(Environment.NewLine, paths.Distinct())}"); } } @@ -48,6 +48,7 @@ public class AppSettings if (appSettings.RequireRootDirectoryExists is null) throw new NullReferenceException(nameof(appSettings.RequireRootDirectoryExists)); Verify(appSettings); result = new(appSettings.Company, + appSettings.HarFilesDirectory, appSettings.MaxDegreeOfParallelism.Value, appSettings.RequireRootDirectoryExists.Value); return result; diff --git a/Rename/Models/Binder/RenameConfiguration.cs b/Rename/Models/Binder/RenameConfiguration.cs index 857a99f..3868068 100644 --- a/Rename/Models/Binder/RenameConfiguration.cs +++ b/Rename/Models/Binder/RenameConfiguration.cs @@ -7,8 +7,10 @@ namespace View_by_Distance.Rename.Models.Binder; public class RenameConfiguration { + public string? DefaultMaker { get; set; } + public bool? ForceNewId { get; set; } public string[]? IgnoreExtensions { get; set; } - public bool? MoveFilesToRoot { get; set; } + public string? RelativePropertyCollectionFile { get; set; } public bool? SkipIdFiles { get; set; } public string[]? ValidImageFormatExtensions { get; set; } public string[]? ValidVideoFormatExtensions { get; set; } @@ -21,19 +23,18 @@ public class RenameConfiguration private static void PreVerify(IConfigurationRoot configurationRoot, RenameConfiguration? configuration) { - if (configuration?.IgnoreExtensions is null) + if (configuration?.DefaultMaker is null) { + List paths = []; foreach (IConfigurationProvider configurationProvider in configurationRoot.Providers) { if (configurationProvider is not Microsoft.Extensions.Configuration.Json.JsonConfigurationProvider jsonConfigurationProvider) continue; if (jsonConfigurationProvider.Source.FileProvider is not Microsoft.Extensions.FileProviders.PhysicalFileProvider physicalFileProvider) continue; - if (!physicalFileProvider.Root.Contains("UserSecrets")) - continue; - throw new NotSupportedException(physicalFileProvider.Root); + paths.Add(physicalFileProvider.Root); } - throw new NotSupportedException("Not Found!"); + throw new NotSupportedException($"Not found!{Environment.NewLine}{string.Join(Environment.NewLine, paths.Distinct())}"); } } @@ -48,15 +49,19 @@ public class RenameConfiguration { Models.RenameConfiguration result; if (configuration is null) throw new NullReferenceException(nameof(configuration)); + if (configuration.DefaultMaker is null) throw new NullReferenceException(nameof(configuration.DefaultMaker)); + if (configuration.ForceNewId is null) throw new NullReferenceException(nameof(configuration.ForceNewId)); if (configuration.IgnoreExtensions is null) throw new NullReferenceException(nameof(configuration.IgnoreExtensions)); - if (configuration.MoveFilesToRoot is null) throw new NullReferenceException(nameof(configuration.MoveFilesToRoot)); + if (configuration.RelativePropertyCollectionFile is null) throw new NullReferenceException(nameof(configuration.RelativePropertyCollectionFile)); if (configuration.SkipIdFiles is null) throw new NullReferenceException(nameof(configuration.SkipIdFiles)); if (configuration.ValidImageFormatExtensions is null) throw new NullReferenceException(nameof(configuration.ValidImageFormatExtensions)); if (configuration.ValidVideoFormatExtensions is null) throw new NullReferenceException(nameof(configuration.ValidVideoFormatExtensions)); Verify(configuration); result = new(metadataConfiguration, + configuration.DefaultMaker, + configuration.ForceNewId.Value, configuration.IgnoreExtensions, - configuration.MoveFilesToRoot.Value, + configuration.RelativePropertyCollectionFile, configuration.SkipIdFiles.Value, configuration.ValidImageFormatExtensions, configuration.ValidVideoFormatExtensions); diff --git a/Rename/Models/Identifier.cs b/Rename/Models/Identifier.cs new file mode 100644 index 0000000..a044e42 --- /dev/null +++ b/Rename/Models/Identifier.cs @@ -0,0 +1,27 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace View_by_Distance.Rename.Models; + +internal record Identifier(int Id, string PaddedId) +{ + + public override string ToString() + { + string result = JsonSerializer.Serialize(this, IdentifierSourceGenerationContext.Default.Identifier); + return result; + } + +} + +[JsonSourceGenerationOptions(WriteIndented = true)] +[JsonSerializable(typeof(Identifier))] +internal partial class IdentifierSourceGenerationContext : JsonSerializerContext +{ +} + +[JsonSourceGenerationOptions(WriteIndented = true)] +[JsonSerializable(typeof(Identifier[]))] +internal partial class IdentifierCollectionSourceGenerationContext : JsonSerializerContext +{ +} \ No newline at end of file diff --git a/Rename/Models/RenameConfiguration.cs b/Rename/Models/RenameConfiguration.cs index 4737ca0..16d15b7 100644 --- a/Rename/Models/RenameConfiguration.cs +++ b/Rename/Models/RenameConfiguration.cs @@ -5,8 +5,10 @@ using System.Text.Json.Serialization; namespace View_by_Distance.Rename.Models; public record RenameConfiguration(Shared.Models.MetadataConfiguration MetadataConfiguration, + string DefaultMaker, + bool ForceNewId, string[] IgnoreExtensions, - bool MoveFilesToRoot, + string RelativePropertyCollectionFile, bool SkipIdFiles, string[] ValidImageFormatExtensions, string[] ValidVideoFormatExtensions) : Shared.Models.Properties.IRenameConfiguration diff --git a/Rename/Rename.cs b/Rename/Rename.cs index f26a22f..a72a605 100644 --- a/Rename/Rename.cs +++ b/Rename/Rename.cs @@ -3,9 +3,11 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; using ShellProgressBar; using System.Collections.ObjectModel; +using System.Data; using System.Drawing; using System.Drawing.Imaging; using System.Runtime.InteropServices; +using System.Text.Json; using View_by_Distance.Metadata.Models; using View_by_Distance.Metadata.Models.Stateless.Methods; using View_by_Distance.Rename.Models; @@ -15,11 +17,11 @@ using View_by_Distance.Shared.Models.Stateless.Methods; namespace View_by_Distance.Rename; -public class Rename : IRename +public partial class Rename : IRename { - private record ToDo(string? Directory, FileHolder FileHolder, string File, bool JsonFile); - private record Record(DateTime DateTime, ExifDirectory ExifDirectory, string File, string JsonFile); + private record ToDo(string? Directory, FilePath FilePath, string File, bool JsonFile); + private record Record(DateTime DateTime, ExifDirectory ExifDirectory, FilePath FilePath, bool HasDateTimeOriginal, bool HasIgnoreKeyword, string JsonFile); private ProgressBar? _ProgressBar; @@ -42,35 +44,37 @@ public class Rename : IRename void IRename.Tick() => _ProgressBar?.Tick(); - (ReadOnlyCollection, FilePath?) IRename.ConvertAndGetFfmpegFiles(IRenameConfiguration renameConfiguration, FilePath filePath) + ReadOnlyCollection IRename.ConvertAndGetFfmpegFiles(IRenameConfiguration renameConfiguration, FilePath filePath) { List results = []; - FilePath? result; bool isValidVideoFormatExtensions = renameConfiguration.ValidVideoFormatExtensions.Contains(filePath.ExtensionLowered); - if (!isValidVideoFormatExtensions) - result = null; - else + if (isValidVideoFormatExtensions) { - CommandTask commandTask = Cli.Wrap("ffmpeg.exe") - .WithArguments(new[] { "-i", filePath.FullName, "-vf", "select=eq(n\\,0)", "-q:v", "1", $"{filePath.Name}-%4d.jpg" }) - .WithWorkingDirectory(filePath.DirectoryName) - .ExecuteAsync(); - commandTask.Task.Wait(); - results.AddRange(Directory.GetFiles(filePath.DirectoryName, $"{filePath.Name}-*.jpg", SearchOption.TopDirectoryOnly)); - if (results.Count == 0) - throw new Exception(); - FileHolder fileHolder = new(results[0]); - result = FilePath.Get(renameConfiguration.MetadataConfiguration, fileHolder, index: null); - if (!result.Name.EndsWith("-0001.jpg")) - throw new Exception(); - bool isValidImageFormatExtension = renameConfiguration.ValidImageFormatExtensions.Contains(result.ExtensionLowered); - bool isIgnoreExtension = isValidImageFormatExtension && renameConfiguration.IgnoreExtensions.Contains(result.ExtensionLowered); - if (isIgnoreExtension || !isValidImageFormatExtension) - throw new Exception(); - if (result.DirectoryName is null) - throw new NullReferenceException(nameof(result.DirectoryName)); + bool check; + try + { + CommandTask commandTask = Cli.Wrap("ffmpeg.exe") + .WithArguments(new[] { "-i", filePath.FullName, "-vf", "select=eq(n\\,0)", "-q:v", "1", $"{filePath.Name}-%4d.jpg" }) + .WithWorkingDirectory(filePath.DirectoryName) + .ExecuteAsync(); + commandTask.Task.Wait(); + check = true; + } + catch (Exception) + { + check = false; + } + if (check) + { + results.AddRange(Directory.GetFiles(filePath.DirectoryName, $"{filePath.Name}-*.jpg", SearchOption.TopDirectoryOnly)); + if (results.Count == 0) + throw new Exception(); + File.SetCreationTime(results[0], new(filePath.CreationTicks)); + File.SetLastWriteTime(results[0], new(filePath.LastWriteTicks)); + Thread.Sleep(100); + } } - return new(new(results), result); + return new(results); } #pragma warning disable CA1416 @@ -126,21 +130,24 @@ public class Rename : IRename } } - private static void GetExifDirectoryCollection(IRename rename, RenameConfiguration renameConfiguration, List<(string, FileInfo, ExifDirectory)> exifDirectories, IEnumerable files, A_Metadata metadata) + private static List<(FilePath, FileInfo, ExifDirectory)> GetExifDirectoryCollection(IRename rename, RenameConfiguration renameConfiguration, IEnumerable files, A_Metadata metadata) { + List<(FilePath, FileInfo, ExifDirectory)> results = []; int index = -1; FileInfo fileInfo; FilePath filePath; FileHolder fileHolder; FilePath? ffmpegFilePath; ExifDirectory exifDirectory; - ReadOnlyCollection ffmpegFiles; + ReadOnlyCollection? ffmpegFiles; DeterministicHashCode deterministicHashCode; foreach (string file in files) { index += 1; rename.Tick(); fileHolder = new(file); + if (renameConfiguration.IgnoreExtensions.Contains(fileHolder.ExtensionLowered)) + continue; filePath = FilePath.Get(renameConfiguration.MetadataConfiguration, fileHolder, index); if (filePath.ExtensionLowered == ".url" && filePath.Id is not null) { @@ -149,29 +156,43 @@ public class Rename : IRename } if (renameConfiguration.SkipIdFiles && filePath.Id is not null && (filePath.IsIntelligentIdFormat || filePath.SortOrder is not null)) continue; - (ffmpegFiles, ffmpegFilePath) = rename.ConvertAndGetFfmpegFiles(renameConfiguration, filePath); - if (ffmpegFilePath is not null) - filePath = ffmpegFilePath; - if (filePath.Id is not null) + if (!renameConfiguration.ForceNewId && filePath.Id is not null) + { + ffmpegFiles = null; deterministicHashCode = new(null, filePath.Id, null); + } else - deterministicHashCode = rename.GetDeterministicHashCode(filePath); + { + ffmpegFiles = rename.ConvertAndGetFfmpegFiles(renameConfiguration, filePath); + ffmpegFilePath = ffmpegFiles.Count == 0 ? null : FilePath.Get(renameConfiguration.MetadataConfiguration, new(ffmpegFiles[0]), index); + deterministicHashCode = ffmpegFilePath is null ? rename.GetDeterministicHashCode(filePath) : rename.GetDeterministicHashCode(ffmpegFilePath); + } (fileInfo, exifDirectory) = metadata.GetMetadataCollection(renameConfiguration.MetadataConfiguration, filePath, deterministicHashCode); - exifDirectories.Add(new(file, fileInfo, exifDirectory)); - foreach (string ffmpegFile in ffmpegFiles) - File.Delete(ffmpegFile); + results.Add(new(filePath, fileInfo, exifDirectory)); + if (ffmpegFiles is not null) + { + foreach (string ffmpegFile in ffmpegFiles) + File.Delete(ffmpegFile); + } } + return results; } - private static ReadOnlyCollection GetExifDirectoryCollection(List<(string, FileInfo, ExifDirectory)> exifDirectories) + private static ReadOnlyCollection GetExifDirectoryCollection(MetadataConfiguration metadataConfiguration, List<(FilePath, FileInfo, ExifDirectory)> exifDirectories) { List results = []; DateTime? dateTime; - foreach ((string file, FileInfo fileInfo, ExifDirectory exifDirectory) in exifDirectories) + bool hasIgnoreKeyword; + bool hasDateTimeOriginal; + ReadOnlyCollection keywords; + foreach ((FilePath filePath, FileInfo fileInfo, ExifDirectory exifDirectory) in exifDirectories) { dateTime = IDate.GetDateTimeOriginal(exifDirectory); + hasDateTimeOriginal = dateTime is not null; dateTime ??= IDate.GetMinimum(exifDirectory); - results.Add(new(dateTime.Value, exifDirectory, file, fileInfo.FullName)); + keywords = IMetadata.GetKeywords(exifDirectory); + hasIgnoreKeyword = metadataConfiguration.IgnoreRulesKeyWords.Any(l => keywords.Contains(l)); + results.Add(new(dateTime.Value, exifDirectory, filePath, hasDateTimeOriginal, hasIgnoreKeyword, fileInfo.FullName)); } return new(results); } @@ -179,13 +200,14 @@ public class Rename : IRename private ReadOnlyCollection GetExifDirectoryCollection(IRename rename, AppSettings appSettings, RenameConfiguration renameConfiguration, DirectoryInfo directoryInfo) { ReadOnlyCollection results; - List<(string, FileInfo, ExifDirectory)> exifDirectories = []; - int appSettingsMaxDegreeOfParallelism = appSettings.MaxDegreeOfParallelism; - IEnumerable files = Directory.EnumerateFiles(directoryInfo.FullName, "*", SearchOption.AllDirectories); + List<(FilePath, FileInfo, ExifDirectory)> exifDirectories = []; A_Metadata metadata = new(renameConfiguration.MetadataConfiguration); - _ProgressBar = new(123000, "EnumerateFiles load", new ProgressBarOptions() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }); + int appSettingsMaxDegreeOfParallelism = appSettings.MaxDegreeOfParallelism; + IEnumerable files = appSettingsMaxDegreeOfParallelism == 1 ? Directory.GetFiles(directoryInfo.FullName, "*", SearchOption.AllDirectories) : Directory.EnumerateFiles(directoryInfo.FullName, "*", SearchOption.AllDirectories); + int filesCount = appSettingsMaxDegreeOfParallelism == 1 ? files.Count() : 123000; + _ProgressBar = new(filesCount, "EnumerateFiles load", new ProgressBarOptions() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }); if (appSettingsMaxDegreeOfParallelism == 1) - GetExifDirectoryCollection(rename, renameConfiguration, exifDirectories, files, metadata); + exifDirectories.AddRange(GetExifDirectoryCollection(rename, renameConfiguration, files, metadata)); else { ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = appSettingsMaxDegreeOfParallelism }; @@ -194,13 +216,13 @@ public class Rename : IRename throw new NotSupportedException(); } _ProgressBar.Dispose(); - results = GetExifDirectoryCollection(exifDirectories); + results = GetExifDirectoryCollection(renameConfiguration.MetadataConfiguration, exifDirectories); return results; } private static void VerifyIntMinValueLength(MetadataConfiguration metadataConfiguration, ReadOnlyCollection exifDirectories) { - foreach ((DateTime _, ExifDirectory exifDirectory, string _, string _) in exifDirectories) + foreach ((DateTime _, ExifDirectory exifDirectory, FilePath _, bool _, bool _, string _) in exifDirectories) { if (exifDirectory.Id is null) continue; @@ -209,60 +231,78 @@ public class Rename : IRename } } - private static string GetCheckDirectory(RenameConfiguration renameConfiguration, Record record, FileHolder fileHolder) + private static string? GetCheckDirectory(RenameConfiguration renameConfiguration, Record record, FilePath filePath, ReadOnlyCollection ids, bool multipleDirectoriesWithFiles) { - string? checkDirectory; - if (fileHolder.DirectoryName is null) - throw new NullReferenceException(nameof(fileHolder.DirectoryName)); - string directoryName; + string? result; + if (filePath.DirectoryName is null) + throw new NullReferenceException(nameof(filePath.DirectoryName)); string year = record.DateTime.Year.ToString(); - string checkDirectoryName = Path.GetFileName(fileHolder.DirectoryName); - if (!checkDirectoryName.Contains(year)) - throw new NotImplementedException(); + string checkDirectoryName = Path.GetFileName(filePath.DirectoryName); + if (multipleDirectoriesWithFiles && !checkDirectoryName.Contains(year)) + result = null; else { - string maker = IMetadata.GetMaker(record.ExifDirectory); + string? maker = IMetadata.GetMaker(record.ExifDirectory); (int seasonValue, string seasonName) = IDate.GetSeason(record.DateTime.DayOfYear); - string splat = fileHolder.DirectoryName[^3..][1] == '!' ? fileHolder.DirectoryName[^3..] : string.Empty; - directoryName = $"{year}.{seasonValue} {seasonName} {maker.Split(' ')[0]}{splat}"; + string splat = filePath.DirectoryName[^3..][1] == '!' ? filePath.DirectoryName[^3..] : string.Empty; + string makerSplit = string.IsNullOrEmpty(maker) ? string.IsNullOrEmpty(renameConfiguration.DefaultMaker) ? string.Empty : renameConfiguration.DefaultMaker : $" {maker.Split(' ')[0]}"; + string directoryName = $"{year}.{seasonValue} {seasonName}{makerSplit}{splat}"; + result = Path.Combine(renameConfiguration.MetadataConfiguration.ResultConfiguration.RootDirectory, record.ExifDirectory.Id is null || !ids.Contains(record.ExifDirectory.Id.Value) ? "_ Destination _" : "_ Exists _", record.HasDateTimeOriginal ? "Has" : "Not", directoryName); } - string rootDirectory = renameConfiguration.MoveFilesToRoot ? renameConfiguration.MetadataConfiguration.ResultConfiguration.RootDirectory : fileHolder.DirectoryName; - checkDirectory = Path.Combine(rootDirectory, "_ Destination _", directoryName); - return checkDirectory; + return result; } - private static ReadOnlyCollection GetToDoCollection(ILogger? logger, RenameConfiguration renameConfiguration, long ticks, ReadOnlyCollection exifDirectories) + private static ReadOnlyCollection GetToDoCollection(RenameConfiguration renameConfiguration, Identifier[]? identifiers, ReadOnlyCollection records) { List results = []; Record record; string jsonFile; string paddedId; string checkFile; + FilePath filePath; string directoryName; FileHolder fileHolder; string? checkDirectory; const string jpg = ".jpg"; string checkFileExtension; + bool multipleDirectoriesWithFiles; List distinct = []; + bool? directoryCheck = null; const string jpeg = ".jpeg"; string jsonFileSubDirectory; - VerifyIntMinValueLength(renameConfiguration.MetadataConfiguration, exifDirectories); - ResultConfiguration resultConfiguration = renameConfiguration.MetadataConfiguration.ResultConfiguration; - ReadOnlyCollection records = new((from l in exifDirectories orderby l.DateTime select l).ToArray()); - for (int i = 0; i < records.Count; i++) + foreach (string directory in Directory.GetDirectories(renameConfiguration.MetadataConfiguration.ResultConfiguration.RootDirectory, "*", SearchOption.TopDirectoryOnly)) { - record = records[i]; + foreach (string _ in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories)) + { + if (directoryCheck is null) + directoryCheck = false; + else if (directoryCheck.Value) + directoryCheck = true; + break; + } + if (directoryCheck is not null && directoryCheck.Value) + break; + } + VerifyIntMinValueLength(renameConfiguration.MetadataConfiguration, records); + multipleDirectoriesWithFiles = directoryCheck is not null && directoryCheck.Value; + ReadOnlyCollection collection = new((from l in records orderby l.DateTime select l).ToArray()); + ResultConfiguration resultConfiguration = renameConfiguration.MetadataConfiguration.ResultConfiguration; + ReadOnlyCollection ids = identifiers is null ? new([]) : new((from l in identifiers select l.Id).ToArray()); + for (int i = 0; i < collection.Count; i++) + { + record = collection[i]; if (record.ExifDirectory.Id is null) continue; - fileHolder = new(record.File); - if (fileHolder.DirectoryName is null) + if (record.FilePath.DirectoryName is null) continue; - checkDirectory = GetCheckDirectory(renameConfiguration, record, fileHolder); - checkFileExtension = fileHolder.ExtensionLowered == jpeg ? jpg : fileHolder.ExtensionLowered; - paddedId = IId.GetPaddedId(renameConfiguration.MetadataConfiguration, i, record.ExifDirectory.Id.Value); + checkDirectory = GetCheckDirectory(renameConfiguration, record, record.FilePath, ids, multipleDirectoriesWithFiles); + if (string.IsNullOrEmpty(checkDirectory)) + continue; + checkFileExtension = record.FilePath.ExtensionLowered == jpeg ? jpg : record.FilePath.ExtensionLowered; + paddedId = IId.GetPaddedId(renameConfiguration.MetadataConfiguration, record.ExifDirectory.Id.Value, record.HasIgnoreKeyword, i); jsonFileSubDirectory = Path.GetDirectoryName(Path.GetDirectoryName(record.JsonFile)) ?? throw new Exception(); checkFile = Path.Combine(checkDirectory, $"{paddedId}{checkFileExtension}"); - if (checkFile == fileHolder.FullName) + if (checkFile == record.FilePath.FullName) continue; if (File.Exists(checkFile)) { @@ -273,11 +313,15 @@ public class Rename : IRename (directoryName, _) = IPath.GetDirectoryNameAndIndex(resultConfiguration, record.ExifDirectory.Id.Value); jsonFile = Path.Combine(jsonFileSubDirectory, directoryName, $"{record.ExifDirectory.Id.Value}{checkFileExtension}.json"); if (record.JsonFile != jsonFile) - results.Add(new(null, new(record.JsonFile), jsonFile, JsonFile: true)); + { + fileHolder = new(record.JsonFile); + filePath = FilePath.Get(renameConfiguration.MetadataConfiguration, fileHolder, index: null); + results.Add(new(null, filePath, jsonFile, JsonFile: true)); + } if (distinct.Contains(checkFile)) continue; distinct.Add(checkFile); - results.Add(new(checkDirectory, fileHolder, checkFile, JsonFile: false)); + results.Add(new(checkDirectory, record.FilePath, checkFile, JsonFile: false)); } return new(results); } @@ -307,7 +351,7 @@ public class Rename : IRename { if (File.Exists(toDo.File)) File.Delete(toDo.File); - File.Move(toDo.FileHolder.FullName, toDo.File); + File.Move(toDo.FilePath.FullName, toDo.File); } else if (toDo.Directory is null) throw new NotSupportedException(); @@ -316,22 +360,142 @@ public class Rename : IRename if (File.Exists(toDo.File)) File.Delete(toDo.File); try - { File.Move(toDo.FileHolder.FullName, toDo.File); } + { File.Move(toDo.FilePath.FullName, toDo.File); } catch (Exception) { continue; } - results.Add($"{toDo.FileHolder.FullName}\t{toDo.File}"); + results.Add($"{toDo.FilePath.FullName}\t{toDo.File}"); } } _ProgressBar.Dispose(); return new(results); } + private static void SaveIdentifiersToDisk(long ticks, RenameConfiguration renameConfiguration, string aMetadataCollectionDirectory, ReadOnlyCollection records) + { + string paddedId; + List identifiers = []; + foreach (Record record in records) + { + if (record.ExifDirectory.Id is null) + continue; + paddedId = IId.GetPaddedId(renameConfiguration.MetadataConfiguration, record.ExifDirectory.Id.Value, record.FilePath.IsIgnore, index: null); + identifiers.Add(new(record.ExifDirectory.Id.Value, paddedId)); + } + string json = JsonSerializer.Serialize(identifiers.OrderBy(l => l.PaddedId).ToArray(), IdentifierCollectionSourceGenerationContext.Default.IdentifierArray); + _ = IPath.WriteAllText(Path.Combine(aMetadataCollectionDirectory, $"{ticks}.json"), json, updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null); + } + + private static ReadOnlyCollection<(string, string)> GetAggregationLines(string harFile) + { + List<(string, string)> results = []; + if (!File.Exists(harFile)) + throw new Exception(); + string lastUrl = string.Empty; + string text = "\"text\": \"{"; + string[] lines = File.ReadAllLines(harFile); + foreach (string line in lines) + { + if (line.Contains("\"url\": \"")) + lastUrl = line; + if (!line.Contains(text)) + continue; + if (!line.Contains("aggregations")) + continue; + if (lastUrl.Contains("search?asset=NONE")) + continue; + results.Add(new(lastUrl, line.Trim()[(text.Length - 1)..^1].Replace("\\\"", "\""))); + lastUrl = string.Empty; + } + return new(results); + } + + private static void SaveAmazon(IReadOnlyList data, string personIdFile) + { + string json; + Dictionary keyValuePairs = []; + foreach (Datum datum in data) + _ = keyValuePairs.TryAdd(datum.Name.Split('.')[0], datum); + json = JsonSerializer.Serialize(keyValuePairs, DictionaryDatumGenerationContext.Default.DictionaryStringDatum); + File.WriteAllText(personIdFile, json); + } + + private static void SaveAmazon(AppSettings appSettings, string harFile) + { + if (string.IsNullOrEmpty(appSettings.HarFilesDirectory)) + throw new NullReferenceException(nameof(appSettings.HarFilesDirectory)); + string offset; + string personId; + RootAmazon amazon; + string? personName; + string personIdFile; + string personDirectory; + PersonAmazon personAmazon; + Dictionary keyValuePairs = []; + ReadOnlyCollection<(string Url, string AggregationLine)> aggregationLines = GetAggregationLines(harFile); + foreach ((string url, string aggregationLine) in aggregationLines) + { + if (aggregationLine.Contains(",\"category\":\"allPeople\"}")) + continue; + amazon = JsonSerializer.Deserialize(aggregationLine, RootAmazonGenerationContext.Default.RootAmazon) ?? throw new Exception(); + if (amazon.Aggregations?.People is null || amazon.Aggregations.People.Count < 1) + continue; + personAmazon = amazon.Aggregations.People[0]; + if (!url.Contains(personAmazon.Match)) + continue; + personDirectory = Path.Combine(appSettings.HarFilesDirectory, "Amazon", personAmazon.SearchData.ClusterName); + _ = Directory.CreateDirectory(personDirectory); + personIdFile = Path.Combine(personDirectory, $"000) {personAmazon.Match}.json"); + _ = keyValuePairs.TryAdd(personAmazon.Match, personAmazon.SearchData.ClusterName); + SaveAmazon(amazon.Data, personIdFile); + } + foreach ((string url, string aggregationLine) in aggregationLines) + { + if (aggregationLine.Contains(",\"category\":\"allPeople\"}")) + continue; + amazon = JsonSerializer.Deserialize(aggregationLine, RootAmazonGenerationContext.Default.RootAmazon) ?? throw new Exception(); + if (amazon.Aggregations?.People is not null && amazon.Aggregations.People.Count > 0) + continue; + if (!url.Contains("offset=")) + continue; + offset = url.Split("offset=")[1]; + if (!url.Contains("people%3A(")) + continue; + personId = url.Split("people%3A(")[1].Split(')')[0]; + if (!keyValuePairs.TryGetValue(personId, out personName)) + continue; + personDirectory = Path.Combine(appSettings.HarFilesDirectory, "Amazon", personName); + _ = Directory.CreateDirectory(personDirectory); + personIdFile = Path.Combine(personDirectory, $"{offset.Split('&')[0]}) {personId}.json"); + SaveAmazon(amazon.Data, personIdFile); + } + } + + private static void SaveAmazon(ILogger? logger, AppSettings appSettings, long ticks) + { + if (string.IsNullOrEmpty(appSettings.HarFilesDirectory)) + throw new NullReferenceException(nameof(appSettings.HarFilesDirectory)); + logger?.LogInformation("{Ticks} a", ticks); + string[] harFiles = Directory.GetFiles(appSettings.HarFilesDirectory, "*.har", SearchOption.TopDirectoryOnly); + foreach (string harFile in harFiles) + SaveAmazon(appSettings, harFile); + logger?.LogInformation("{harFiles} count", harFiles.Length); + } + private void RenameWork(ILogger? logger, AppSettings appSettings, IRename rename, long ticks, RenameConfiguration renameConfiguration) { + string aMetadataCollectionDirectory = IResult.GetResultsDateGroupDirectory(renameConfiguration.MetadataConfiguration.ResultConfiguration, nameof(A_Metadata), renameConfiguration.MetadataConfiguration.ResultConfiguration.ResultCollection); + string? propertyCollectionFile = string.IsNullOrEmpty(renameConfiguration.RelativePropertyCollectionFile) ? null : Path.GetFullPath(Path.Combine(aMetadataCollectionDirectory, renameConfiguration.RelativePropertyCollectionFile)); + string? json = !File.Exists(propertyCollectionFile) ? null : File.ReadAllText(propertyCollectionFile); + Identifier[]? identifiers = json is null ? null : JsonSerializer.Deserialize(json, IdentifierCollectionSourceGenerationContext.Default.IdentifierArray); + if (identifiers is null && !string.IsNullOrEmpty(renameConfiguration.RelativePropertyCollectionFile)) + throw new Exception($"Invalid {nameof(renameConfiguration.RelativePropertyCollectionFile)}"); DirectoryInfo directoryInfo = new(Path.GetFullPath(renameConfiguration.MetadataConfiguration.ResultConfiguration.RootDirectory)); - logger?.LogInformation("{RootDirectory}", directoryInfo.FullName); - ReadOnlyCollection exifDirectories = GetExifDirectoryCollection(rename, appSettings, renameConfiguration, directoryInfo); - ReadOnlyCollection toDoCollection = GetToDoCollection(logger, renameConfiguration, ticks, exifDirectories); + logger?.LogInformation("{Ticks} {RootDirectory}", ticks, directoryInfo.FullName); + if (!string.IsNullOrEmpty(appSettings.HarFilesDirectory) && Directory.Exists(appSettings.HarFilesDirectory)) + SaveAmazon(logger, appSettings, ticks); + ReadOnlyCollection records = GetExifDirectoryCollection(rename, appSettings, renameConfiguration, directoryInfo); + SaveIdentifiersToDisk(ticks, renameConfiguration, aMetadataCollectionDirectory, records); + ReadOnlyCollection toDoCollection = GetToDoCollection(renameConfiguration, identifiers, records); ReadOnlyCollection lines = RenameFilesInDirectories(toDoCollection); if (lines.Count != 0) { diff --git a/Shared/AA.Shared.csproj b/Shared/AA.Shared.csproj index 9a8b3b6..cbad8d6 100644 --- a/Shared/AA.Shared.csproj +++ b/Shared/AA.Shared.csproj @@ -32,7 +32,7 @@ - - + + \ No newline at end of file diff --git a/Shared/Models/Aggregations.cs b/Shared/Models/Aggregations.cs new file mode 100644 index 0000000..5182aab --- /dev/null +++ b/Shared/Models/Aggregations.cs @@ -0,0 +1,19 @@ +using System.Text.Json.Serialization; + +namespace View_by_Distance.Shared.Models; + +public record Aggregations( + [property: JsonPropertyName("allPeople")] IReadOnlyList AllPeople, + [property: JsonPropertyName("clusterId")] IReadOnlyList ClusterId, + [property: JsonPropertyName("location")] IReadOnlyList Location, + [property: JsonPropertyName("people")] IReadOnlyList People, + [property: JsonPropertyName("things")] IReadOnlyList Things, + [property: JsonPropertyName("time")] IReadOnlyList