diff --git a/Copy-Distinct/CopyDistinct.cs b/Copy-Distinct/CopyDistinct.cs index 7eae9ff..7e0ed98 100644 --- a/Copy-Distinct/CopyDistinct.cs +++ b/Copy-Distinct/CopyDistinct.cs @@ -36,15 +36,15 @@ public class CopyDistinct _Configuration = configuration; propertyConfiguration.Update(); log.Information(propertyConfiguration.RootDirectory); - (bool move, List allFiles, bool anyLenFiles, bool moveBack) = Verify(); + (bool move, List filesCollection, bool anyLenFiles, bool moveBack) = Verify(); _FileGroups = Shared.Models.Stateless.Methods.IPath.GetKeyValuePairs(propertyConfiguration, appSettings.CopyTo, new string[] { appSettings.ResultDirectoryKey }); - List lines = CopyDistinctFilesInDirectories(log, move, allFiles, anyLenFiles, moveBack); + List lines = CopyDistinctFilesInDirectories(log, move, filesCollection, anyLenFiles, moveBack); if (lines.Any()) File.WriteAllLines($"D:/Tmp/Phares/{DateTime.Now.Ticks}.tsv", lines); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(propertyConfiguration.RootDirectory); } - private (bool, List, bool, bool) Verify() + private (bool, List, bool, bool) Verify() { if (_AppSettings is null) throw new NullReferenceException(nameof(_AppSettings)); @@ -64,8 +64,7 @@ public class CopyDistinct string copyTo = Path.GetFullPath(_AppSettings.CopyTo); bool move = copyTo == _PropertyConfiguration.RootDirectory; List filesCollection = Shared.Models.Stateless.Methods.IDirectory.GetFilesCollection(_PropertyConfiguration.RootDirectory, directorySearchFilter, fileSearchFilter); - (_, List allFiles) = Get(filesCollection); - bool anyLenFiles = allFiles.Any(l => l.EndsWith("len")); + bool anyLenFiles = filesCollection.Any(l => l.Any(m => m.EndsWith("len"))); if (!move) moveBack = false; else @@ -82,198 +81,98 @@ public class CopyDistinct moveBack = true; } } - return (move, allFiles, anyLenFiles, moveBack); + return (move, filesCollection, anyLenFiles, moveBack); } - private static (List, List) Get(List filesCollection) + private static (string[], List<(FileHolder, string)>) GetMoveBackToDoCollection(List filesCollection) { - List results = new(); - string? directory; - List directories = new(); - foreach (string[] files in filesCollection) - { - if (!files.Any()) - continue; - directory = Path.GetDirectoryName(files.First()); - if (directory is null) - continue; - if (!directories.Contains(directory)) - directories.Add(directory); - results.AddRange(files); - } - return (directories, results); - } - - private List<(FileHolder, string, string)> GetToDoCollection(ProgressBar progressBar, FileHolder[] fileHolders) - { - List<(FileHolder, string, string)> results = new(); - string checkFile; - string directory; - FileInfo fileInfo; - int directoryIndex; - string directoryName; - bool wrapped = false; - List distinct = new(); - FileHolder[] sortedFileHolders = (from l in fileHolders orderby l.LastWriteTime, l.FullName.Length descending select l).ToArray(); - string key = string.IsNullOrEmpty(_AppSettings.ResultDirectoryKey) ? _PropertyConfiguration.ResultAllInOne : _AppSettings.ResultDirectoryKey; - foreach (FileHolder fileHolder in sortedFileHolders) - { - progressBar.Tick(); - if (fileHolder.Name.EndsWith("len") || fileHolder.ExtensionLowered == ".id" || fileHolder.ExtensionLowered == ".lsv" || fileHolder.DirectoryName is null) - continue; - (_, directoryIndex) = Shared.Models.Stateless.Methods.IPath.GetDirectoryNameAndIndex(_PropertyConfiguration.ResultAllInOneSubdirectoryLength, fileHolder.NameWithoutExtension); - directoryName = Path.GetFileName(fileHolder.DirectoryName); - if (directoryName.Length < _PropertyConfiguration.ResultAllInOneSubdirectoryLength + 3 || !fileHolder.Name.StartsWith(directoryName)) - { - if (wrapped) - continue; - directory = _FileGroups[key][directoryIndex]; - } - else - { - if (!wrapped) - wrapped = true; - directory = Path.Combine(_FileGroups[key][directoryIndex], directoryName); - } - checkFile = Path.Combine(directory, $"{fileHolder.NameWithoutExtension}{fileHolder.ExtensionLowered}"); - if (distinct.Contains(checkFile)) - { - if (!_AppSettings.CopyDuplicates) - continue; - for (int i = 1; i < int.MaxValue; i++) - { - fileInfo = new(checkFile); - if (!fileInfo.Exists || fileHolder.Length == fileInfo.Length && fileHolder.LastWriteTime == fileInfo.LastWriteTime) - checkFile = Path.Combine(directory, $"{fileHolder.NameWithoutExtension}.{i}dup{fileHolder.ExtensionLowered}"); - else - checkFile = Path.Combine(directory, $"{fileHolder.NameWithoutExtension}.{i}why{fileHolder.ExtensionLowered}"); - if (distinct.Contains(checkFile)) - continue; - distinct.Add(checkFile); - results.Add(new(fileHolder, directory, checkFile)); - break; - } - continue; - } - distinct.Add(checkFile); - results.Add(new(fileHolder, directory, checkFile)); - } - return results; - } - - private static List<(FileHolder, string, string)> GetMoveBackToDoCollection(List files) - { - List<(FileHolder, string, string)> results = new(); + List<(FileHolder, string)> results = new(); string key; string? value; string fileName; - files.Reverse(); string? directory; string destinationFile; List distinctFound = new(); List distinctNeeded = new(); + List distinctDirectories = new(); Dictionary nameToPath = new(); for (int i = 1; i < 3; i++) + { + foreach (string[] files in filesCollection) + { + foreach (string file in files.Reverse()) + { + fileName = Path.GetFileName(file); + if (fileName.EndsWith("len")) + { + key = fileName[..^3]; + destinationFile = file[..^3]; + if (nameToPath.ContainsKey(key)) + continue; + nameToPath.Add(key, destinationFile); + } + else + { + if (!distinctNeeded.Contains(file)) + distinctNeeded.Add(file); + if (!nameToPath.ContainsKey(fileName)) + continue; + if (distinctFound.Contains(file)) + continue; + distinctFound.Add(file); + } + } + if (distinctNeeded.Count != distinctFound.Count) + continue; + break; + } + } + foreach (string[] files in filesCollection) { foreach (string file in files) { + // if (distinctNeeded.Count != distinctFound.Count) + // continue; fileName = Path.GetFileName(file); if (fileName.EndsWith("len")) - { - key = fileName[..^3]; - destinationFile = file[..^3]; - if (nameToPath.ContainsKey(key)) - continue; - nameToPath.Add(key, destinationFile); - } - else - { - if (!distinctNeeded.Contains(file)) - distinctNeeded.Add(file); - if (!nameToPath.ContainsKey(fileName)) - continue; - if (distinctFound.Contains(file)) - continue; - distinctFound.Add(file); - } - } - if (distinctNeeded.Count != distinctFound.Count) - continue; - break; - } - foreach (string file in files) - { - // if (distinctNeeded.Count != distinctFound.Count) - // continue; - fileName = Path.GetFileName(file); - if (fileName.EndsWith("len")) - continue; - if (!nameToPath.TryGetValue(fileName, out value)) - continue; - directory = Path.GetDirectoryName(value); - if (string.IsNullOrEmpty(directory)) - continue; - results.Add(new(new(file), directory, value)); - } - return results; - } - - private static List CopyOrMove(ProgressBar progressBar, List<(FileHolder, string, string)> toDoCollection, bool move, bool moveBack) - { - List results = new(); - FileInfo fileInfo; - foreach ((FileHolder fileHolder, string _, string to) in toDoCollection) - { - progressBar.Tick(); - fileInfo = new(to); - if (fileInfo.Exists) - { - if (fileHolder.Length != fileInfo.Length || fileHolder.LastWriteTime != fileInfo.LastWriteTime) - fileInfo.Delete(); - else continue; + if (!nameToPath.TryGetValue(fileName, out value)) + continue; + directory = Path.GetDirectoryName(value); + if (string.IsNullOrEmpty(directory)) + continue; + results.Add(new(new(file), value)); + if (!distinctDirectories.Contains(directory)) + distinctDirectories.Add(directory); } - results.Add(fileHolder.NameWithoutExtension); - try - { - if (move || moveBack) - File.Move(fileHolder.FullName, to); - else - File.Copy(fileHolder.FullName, to); - } - catch (Exception) { } } - return results; + return (distinctDirectories.ToArray(), results); } - private List CopyDistinctFilesInDirectories(ILogger log, bool move, List allFiles, bool anyLenFiles, bool moveBack) + private List CopyDistinctFilesInDirectories(ILogger log, bool move, List filesCollection, bool anyLenFiles, bool moveBack) { List results = new(); ProgressBar progressBar; + string[] distinctDirectories; ConsoleKey? consoleKey = null; string message = nameof(CopyDistinct); - List distinctDirectories = new(); - List<(FileHolder, string, string)> toDoCollection; + List<(FileHolder, string)> toDoCollection; + int count = filesCollection.Select(l => l.Length).Sum(); ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; if (moveBack) { if (!anyLenFiles) throw new NotSupportedException(); - toDoCollection = GetMoveBackToDoCollection(allFiles); + (distinctDirectories, toDoCollection) = GetMoveBackToDoCollection(filesCollection); } else { - progressBar = new(allFiles.Count, message, options); - FileHolder[] fileHolders = (from l in allFiles select new FileHolder(l)).ToArray(); - toDoCollection = GetToDoCollection(progressBar, fileHolders); + progressBar = new(count, message, options); + string key = string.IsNullOrEmpty(_AppSettings.ResultDirectoryKey) ? _PropertyConfiguration.ResultAllInOne : _AppSettings.ResultDirectoryKey; + string[] directories = _FileGroups[key]; + (distinctDirectories, toDoCollection) = Shared.Models.Stateless.Methods.IDirectory.GetToDoCollection(_PropertyConfiguration, _AppSettings.CopyDuplicates, filesCollection, directories, () => progressBar.Tick()); progressBar.Dispose(); } - foreach ((FileHolder fileHolder, string directory, string to) in toDoCollection) - { - if (distinctDirectories.Contains(directory)) - continue; - distinctDirectories.Add(directory); - } foreach (string distinctDirectory in distinctDirectories) { if (!Directory.Exists(distinctDirectory)) @@ -307,8 +206,8 @@ public class CopyDistinct } else { - progressBar = new(allFiles.Count, message, options); - results.AddRange(CopyOrMove(progressBar, toDoCollection, move, moveBack)); + progressBar = new(count, message, options); + results.AddRange(Shared.Models.Stateless.Methods.IDirectory.CopyOrMove(toDoCollection, move, moveBack, () => progressBar.Tick())); progressBar.Dispose(); if (move || moveBack) log.Information("Done moving"); diff --git a/Instance/DlibDotNet.cs b/Instance/DlibDotNet.cs index 4af8e1e..67f8264 100644 --- a/Instance/DlibDotNet.cs +++ b/Instance/DlibDotNet.cs @@ -392,6 +392,7 @@ public partial class DlibDotNet ReadOnlyDictionary>> idToLocationContainers, MapLogic mapLogic, string outputResolution, + bool outputResolutionHasNumber, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, @@ -412,7 +413,7 @@ public partial class DlibDotNet List parseExceptions = new(); List> subFileTuples = new(); List> metadataCollection; - FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(item); + FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(cResultsFullGroupDirectory, item, outputResolutionHasNumber); if (item.Property is null || item.Property.Id is null || item.Any()) { LogItemPropertyIsNull(item); @@ -513,6 +514,7 @@ public partial class DlibDotNet ReadOnlyDictionary>> idToLocationContainers, MapLogic mapLogic, string outputResolution, + bool outputResolutionHasNumber, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, @@ -542,6 +544,7 @@ public partial class DlibDotNet idToLocationContainers, mapLogic, outputResolution, + outputResolutionHasNumber, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, @@ -665,6 +668,7 @@ public partial class DlibDotNet int exceptionCount; Container container; Item[] filteredItems; + bool outputResolutionHasNumber; bool anyNullOrNoIsUniqueFileName; string cResultsFullGroupDirectory; string dResultsFullGroupDirectory; @@ -677,6 +681,7 @@ public partial class DlibDotNet foreach (string outputResolution in _Configuration.OutputResolutions) { total = 0; + outputResolutionHasNumber = outputResolution.Any(l => char.IsNumber(l)); (cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); _Faces.Update(dResultsFullGroupDirectory); _Resize.Update(cResultsFullGroupDirectory); @@ -697,13 +702,15 @@ public partial class DlibDotNet totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); message = $"{i + 1:000} [{filteredItems.Length:000}] / {containersLength:000} - {total} / {t} total - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}"; propertyLogic.SetAngleBracketCollection(aResultsFullGroupDirectory, container.SourceDirectory, anyNullOrNoIsUniqueFileName); - _Resize.SetAngleBracketCollection(cResultsFullGroupDirectory, container.SourceDirectory); + if (outputResolutionHasNumber) + _Resize.SetAngleBracketCollection(cResultsFullGroupDirectory, container.SourceDirectory); exceptionCount = FullParallelWork(maxDegreeOfParallelism, propertyLogic, metadata, idToLocationContainers, mapLogic, outputResolution, + outputResolutionHasNumber, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, @@ -815,13 +822,17 @@ public partial class DlibDotNet SaveFaceDistances(ticks, mapLogic, distinctFilteredFaces, distinctFilteredMappingCollection, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, dFacesCollectionDirectory, idToWholePercentagesToMapping); } - private string SaveUrlAndGetNewRootDirectory(Container container) + private string SaveUrlAndGetNewRootDirectory(string[] files) { + string result; if (_Log is null) throw new NullReferenceException(nameof(_Log)); - string? result = Path.GetDirectoryName(Path.GetDirectoryName(container.SourceDirectory)) ?? throw new Exception(); + if (!files.Any()) + throw new NotSupportedException(); + string? sourceDirectory = Path.GetDirectoryName(files.First()); + if (string.IsNullOrEmpty(sourceDirectory)) + throw new NotSupportedException(); Uri uri; - Item item; string? line; string fileName; Task task; @@ -829,9 +840,10 @@ public partial class DlibDotNet FileHolder fileHolder; string extensionLowered; string sourceDirectoryFile; - int length = result.Length; HttpClient httpClient = new(); bool isValidImageFormatExtension; + result = Path.GetDirectoryName(Path.GetDirectoryName(sourceDirectory)) ?? throw new NotSupportedException(); + int length = result.Length; for (int y = 0; y < int.MaxValue; y++) { _Log.Information("Enter fileNameToCollection url for fileNameToCollection image"); @@ -843,14 +855,14 @@ public partial class DlibDotNet continue; task = httpClient.GetByteArrayAsync(uri); fileName = Path.GetFileName(uri.LocalPath); - sourceDirectoryFile = Path.Combine(container.SourceDirectory, fileName); + sourceDirectoryFile = Path.Combine(sourceDirectory, fileName); File.WriteAllBytes(sourceDirectoryFile, task.Result); extensionLowered = Path.GetExtension(uri.LocalPath); relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFile, length, forceExtensionToLower: true); isValidImageFormatExtension = _Configuration.PropertyConfiguration.ValidImageFormatExtensions.Contains(extensionLowered); fileHolder = new(sourceDirectoryFile); - item = new(fileHolder, relativePath, isValidImageFormatExtension); - container.Items.Add(item); + _ = new Item(fileHolder, relativePath, isValidImageFormatExtension); + // container.Items.Add(item); } _Log.Information(". . ."); return result; @@ -1125,6 +1137,7 @@ public partial class DlibDotNet private void Search(long ticks, string argZero, string propertyRoot) { int t; + string message; Container[] containers; A_Property propertyLogic; string eDistanceContentDirectory; @@ -1136,31 +1149,22 @@ public partial class DlibDotNet string c2ResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; string fPhotoPrismContentDirectory; + const string fileSearchFilter = "*"; string fPhotoPrismSingletonDirectory; + const string directorySearchFilter = "*"; Dictionary> personKeyToIds; Dictionary> fileNameToCollection; (aResultsFullGroupDirectory, bResultsFullGroupDirectory) = GetResultsFullGroupDirectories(); + ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "{}"); a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "([])"); eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), "()"); fPhotoPrismContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), "()"); fPhotoPrismSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), "{}"); propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory); - MapLogic mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, new(_PersonContainers), ticks, a2PeopleSingletonDirectory, eDistanceContentDirectory); - _PersonContainers.AddRange(Shared.Models.Stateless.Methods.IPersonContainer.GetNonSpecificPeopleCollection(_Configuration.MappingDefaultName, _Configuration.PersonBirthdayFirstYear, _Configuration.PersonCharacters.ToArray(), _PersonContainers, ticks)); - int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); - string message = $") Building Container(s) - {totalSeconds} total second(s)"; - ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; - using (ProgressBar progressBar = new(2, message, options)) - { - progressBar.Tick(); - string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, "{}"); - if (!Directory.Exists(aPropertySingletonDirectory)) - _ = Directory.CreateDirectory(aPropertySingletonDirectory); - (t, containers) = Shared.Models.Stateless.Methods.IContainer.GetContainers(_Configuration.PropertyConfiguration, aPropertySingletonDirectory); - progressBar.Tick(); - } - if (containers.Length == 1) + List filesCollection = Shared.Models.Stateless.Methods.IDirectory.GetFilesCollection(_Configuration.PropertyConfiguration.RootDirectory, directorySearchFilter, fileSearchFilter); + int count = filesCollection.Select(l => l.Length).Sum(); + if (filesCollection.Count == 1) { string resultsGroupDirectory; a2PeopleContentDirectory = null; @@ -1172,14 +1176,46 @@ public partial class DlibDotNet resultsGroupDirectory = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, string.Empty, create: true); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(resultsGroupDirectory); } - argZero = SaveUrlAndGetNewRootDirectory(containers[0]); + argZero = SaveUrlAndGetNewRootDirectory(filesCollection.First()); _Configuration.PropertyConfiguration.ChangeRootDirectory(argZero); (aResultsFullGroupDirectory, bResultsFullGroupDirectory) = GetResultsFullGroupDirectories(); propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false); propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory); } + TimeSpan eLastWriteTimeTimeSpan = new(ticks - new DirectoryInfo(eDistanceContentDirectory).LastWriteTime.Ticks); + foreach (string outputResolution in _Configuration.OutputResolutions) + { + if (outputResolution.Any(l => char.IsNumber(l))) + continue; + if (eLastWriteTimeTimeSpan.TotalDays < 1) + break; + ProgressBar progressBar; + (cResultsFullGroupDirectory, _, _, _) = GetResultsFullGroupDirectories(outputResolution); + IReadOnlyDictionary fileGroups = Shared.Models.Stateless.Methods.IPath.GetKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, new string[] { _Configuration.PropertyConfiguration.ResultContent }); + message = $") Selecting for ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)"; + progressBar = new(count, message, options); + (string[] distinctDirectories, List<(FileHolder, string)> toDoCollection) = Shared.Models.Stateless.Methods.IDirectory.GetToDoCollection(_Configuration.PropertyConfiguration, copyDuplicates: false, filesCollection, fileGroups[_Configuration.PropertyConfiguration.ResultContent], () => progressBar.Tick()); + progressBar.Dispose(); + message = $") Copying to ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)"; + progressBar = new(count, message, options); + _ = Shared.Models.Stateless.Methods.IDirectory.CopyOrMove(toDoCollection, move: false, moveBack: false, () => progressBar.Tick()); + progressBar.Dispose(); + break; + } + message = $") Building Container(s) - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)"; + using (ProgressBar progressBar = new(2, message, options)) + { + progressBar.Tick(); + string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, "{}"); + if (!Directory.Exists(aPropertySingletonDirectory)) + _ = Directory.CreateDirectory(aPropertySingletonDirectory); + (t, containers) = Shared.Models.Stateless.Methods.IContainer.GetContainers(_Configuration.PropertyConfiguration, aPropertySingletonDirectory, filesCollection); + progressBar.Tick(); + } fileNameToCollection = !Directory.Exists(fPhotoPrismSingletonDirectory) ? fileNameToCollection = new() : F_PhotoPrism.GetFileNameToCollection(fPhotoPrismSingletonDirectory); B_Metadata metadata = new(_Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, bResultsFullGroupDirectory); + MapLogic mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, new(_PersonContainers), ticks, a2PeopleSingletonDirectory, eDistanceContentDirectory); + _PersonContainers.AddRange(Shared.Models.Stateless.Methods.IPersonContainer.GetNonSpecificPeopleCollection(_Configuration.MappingDefaultName, _Configuration.PersonBirthdayFirstYear, _Configuration.PersonCharacters.ToArray(), _PersonContainers, ticks)); containers = Shared.Models.Stateless.Methods.IContainer.SortContainers(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, _ArgZeroIsConfigurationRootDirectory, argZero, containers); personKeyToIds = mapLogic.GetPersonKeyToIds(); if (!string.IsNullOrEmpty(_Configuration.GenealogicalDataCommunicationFile) && !string.IsNullOrEmpty(a2PeopleContentDirectory) && _GenealogicalDataCommunicationHeaderLines is not null && _GenealogicalDataCommunicationFooterLines is not null && _GenealogicalDataCommunicationHeaderLines.Any() && _GenealogicalDataCommunicationFooterLines.Any()) diff --git a/Resize/Models/_C_Resize.cs b/Resize/Models/_C_Resize.cs index 3d20c92..e43dfba 100644 --- a/Resize/Models/_C_Resize.cs +++ b/Resize/Models/_C_Resize.cs @@ -419,9 +419,16 @@ public class C_Resize return results; } - public FileHolder GetResizedFileHolder(Item item) + public FileHolder GetResizedFileHolder(string cResultsFullGroupDirectory, Item item, bool outputResolutionHasNumber) { - FileHolder result = new(Path.Combine(AngleBracketCollection[0].Replace("<>", _PropertyConfiguration.ResultContent), Path.GetFileName(item.ImageFileHolder.FullName))); + FileHolder result; + if (outputResolutionHasNumber) + result = new(Path.Combine(AngleBracketCollection[0].Replace("<>", _PropertyConfiguration.ResultContent), Path.GetFileName(item.ImageFileHolder.FullName))); + else + { + (string directoryName, _) = Shared.Models.Stateless.Methods.IPath.GetDirectoryNameAndIndex(_PropertyConfiguration.ResultAllInOneSubdirectoryLength, item.ImageFileHolder.Name); + result = new(Path.Combine(cResultsFullGroupDirectory, _PropertyConfiguration.ResultContent, _PropertyConfiguration.ResultAllInOne, directoryName, Path.GetFileName(item.ImageFileHolder.FullName))); + } return result; } diff --git a/Set-Created-Date/SetCreatedDate.cs b/Set-Created-Date/SetCreatedDate.cs index 24b8726..9a5c580 100644 --- a/Set-Created-Date/SetCreatedDate.cs +++ b/Set-Created-Date/SetCreatedDate.cs @@ -61,53 +61,39 @@ public class SetCreatedDate { } } - private static (List, List) Get(List filesCollection) - { - List results = new(); - string? directory; - List directories = new(); - foreach (string[] files in filesCollection) - { - if (!files.Any()) - continue; - directory = Path.GetDirectoryName(files.First()); - if (directory is null) - continue; - if (!directories.Contains(directory)) - directories.Add(directory); - results.AddRange(files); - } - return (directories, results); - } - - private List<(FileHolder, DateTime)> GetToDoCollection(ProgressBar progressBar, FileHolder[] fileHolders) + private List<(FileHolder, DateTime)> GetToDoCollection(ProgressBar progressBar, List filesCollection) { List<(FileHolder, DateTime)> results = new(); int? id; string? message; DateTime? dateTime; + FileHolder fileHolder; DateTime?[] dateTimes; bool isIgnoreExtension; DateTime? dateTimeOriginal; bool isValidImageFormatExtension; - foreach (FileHolder fileHolder in fileHolders) + foreach (string[] files in filesCollection) { - progressBar.Tick(); - if (fileHolder.ExtensionLowered == ".id" || fileHolder.ExtensionLowered == ".lsv" || fileHolder.DirectoryName is null) - continue; - if (_PropertyConfiguration.IgnoreExtensions.Contains(fileHolder.ExtensionLowered)) - continue; - isValidImageFormatExtension = _PropertyConfiguration.ValidImageFormatExtensions.Contains(fileHolder.ExtensionLowered); - isIgnoreExtension = isValidImageFormatExtension && _PropertyConfiguration.IgnoreExtensions.Contains(fileHolder.ExtensionLowered); - if (isIgnoreExtension || !isValidImageFormatExtension) - continue; - (dateTimeOriginal, dateTimes, id, message) = Shared.Models.Stateless.Methods.IProperty.Get(fileHolder, isIgnoreExtension, isValidImageFormatExtension, _PropertyConfiguration.PopulatePropertyId); - dateTime = dateTimeOriginal is not null ? dateTimeOriginal : dateTime = dateTimes.Min(); - if (dateTime is null) - continue; - if (dateTime.Value == fileHolder.CreationTime) - continue; - results.Add((fileHolder, dateTime.Value)); + foreach (string file in files) + { + progressBar.Tick(); + fileHolder = new(file); + if (fileHolder.ExtensionLowered == ".id" || fileHolder.ExtensionLowered == ".lsv" || fileHolder.DirectoryName is null) + continue; + if (_PropertyConfiguration.IgnoreExtensions.Contains(fileHolder.ExtensionLowered)) + continue; + isValidImageFormatExtension = _PropertyConfiguration.ValidImageFormatExtensions.Contains(fileHolder.ExtensionLowered); + isIgnoreExtension = isValidImageFormatExtension && _PropertyConfiguration.IgnoreExtensions.Contains(fileHolder.ExtensionLowered); + if (isIgnoreExtension || !isValidImageFormatExtension) + continue; + (dateTimeOriginal, dateTimes, id, message) = Shared.Models.Stateless.Methods.IProperty.Get(fileHolder, isIgnoreExtension, isValidImageFormatExtension, _PropertyConfiguration.PopulatePropertyId); + dateTime = dateTimeOriginal is not null ? dateTimeOriginal : dateTime = dateTimes.Min(); + if (dateTime is null) + continue; + if (dateTime.Value == fileHolder.CreationTime) + continue; + results.Add((fileHolder, dateTime.Value)); + } } return results; } @@ -135,11 +121,10 @@ public class SetCreatedDate string message = nameof(SetCreatedDate); const string directorySearchFilter = "*"; List filesCollection = Shared.Models.Stateless.Methods.IDirectory.GetFilesCollection(_PropertyConfiguration.RootDirectory, directorySearchFilter, fileSearchFilter); - (_, List allFiles) = Get(filesCollection); + int count = filesCollection.Select(l => l.Length).Sum(); ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; - progressBar = new(allFiles.Count, message, options); - FileHolder[] fileHolders = (from l in allFiles select new FileHolder(l)).ToArray(); - List<(FileHolder, DateTime)> toDoCollection = GetToDoCollection(progressBar, fileHolders); + progressBar = new(count, message, options); + List<(FileHolder, DateTime)> toDoCollection = GetToDoCollection(progressBar, filesCollection); progressBar.Dispose(); log.Information($"Ready to set created date {toDoCollection.Count} file(s)?"); for (int y = 0; y < int.MaxValue; y++) @@ -154,7 +139,7 @@ public class SetCreatedDate log.Information("Nothing set!"); else { - progressBar = new(allFiles.Count, message, options); + progressBar = new(count, message, options); results.AddRange(SetCreatedDateForeach(progressBar, toDoCollection)); progressBar.Dispose(); log.Information("Done setting created date"); diff --git a/Shared/Models/Stateless/Methods/Container.cs b/Shared/Models/Stateless/Methods/Container.cs index f588dae..525cb07 100644 --- a/Shared/Models/Stateless/Methods/Container.cs +++ b/Shared/Models/Stateless/Methods/Container.cs @@ -158,20 +158,7 @@ internal abstract class Container return results; } - private static void CreateShell(Properties.IPropertyConfiguration propertyConfiguration, List directories) - { - string checkDirectory; - int startIndex = propertyConfiguration.RootDirectory.Length; - foreach (string directory in directories) - { - checkDirectory = directory.Insert(startIndex, "-Shell"); - if (Directory.Exists(checkDirectory)) - continue; - _ = Directory.CreateDirectory(checkDirectory); - } - } - - internal static (int, Models.Container[]) GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory) + private static (int, Models.Container[]) GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory, string directorySearchFilter, List filesCollection) { List results = new(); string? directory; @@ -179,10 +166,7 @@ internal abstract class Container Models.Container container; const string extension = ".json"; List directories = new(); - const string fileSearchFilter = "*"; - const string directorySearchFilter = "*"; Dictionary> directoryToItems = new(); - List filesCollection = IDirectory.GetFilesCollection(propertyConfiguration.RootDirectory, directorySearchFilter, fileSearchFilter); foreach (string[] files in filesCollection) { if (!files.Any()) @@ -199,7 +183,6 @@ internal abstract class Container throw new Exception(); } } - CreateShell(propertyConfiguration, directories); List filePairs = GetFilePairs(propertyConfiguration, directorySearchFilter, extension, aPropertySingletonDirectory, filesCollection); List collection = GetFilePairs(propertyConfiguration, aPropertySingletonDirectory, extension, filePairs); foreach (FilePair filePair in collection) @@ -224,6 +207,26 @@ internal abstract class Container return (collection.Count, results.ToArray()); } + internal static (int, Models.Container[]) GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory, List filesCollection) + { + int count; + Models.Container[] results; + const string directorySearchFilter = "*"; + (count, results) = GetContainers(propertyConfiguration, aPropertySingletonDirectory, directorySearchFilter, filesCollection); + return (count, results); + } + + internal static (int, Models.Container[]) GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory) + { + int count; + Models.Container[] results; + const string fileSearchFilter = "*"; + const string directorySearchFilter = "*"; + List filesCollection = IDirectory.GetFilesCollection(propertyConfiguration.RootDirectory, directorySearchFilter, fileSearchFilter); + (count, results) = GetContainers(propertyConfiguration, aPropertySingletonDirectory, directorySearchFilter, filesCollection); + return (count, results); + } + internal static List GetFilteredDistinctIds(Properties.IPropertyConfiguration propertyConfiguration, Models.Container[] containers) { List results = new(); diff --git a/Shared/Models/Stateless/Methods/IContainer.cs b/Shared/Models/Stateless/Methods/IContainer.cs index df4a141..1e5fa79 100644 --- a/Shared/Models/Stateless/Methods/IContainer.cs +++ b/Shared/Models/Stateless/Methods/IContainer.cs @@ -33,6 +33,11 @@ public interface IContainer static (int, Models.Container[]) GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory) => Container.GetContainers(propertyConfiguration, aPropertySingletonDirectory); + (int, Models.Container[]) TestStatic_GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory, List filesCollection) => + GetContainers(propertyConfiguration, aPropertySingletonDirectory, filesCollection); + static (int, Models.Container[]) GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory, List filesCollection) => + Container.GetContainers(propertyConfiguration, aPropertySingletonDirectory, filesCollection); + List TestStatic_GetFilteredDistinctIds(Properties.IPropertyConfiguration propertyConfiguration, Models.Container[] containers) => GetFilteredDistinctIds(propertyConfiguration, containers); static List GetFilteredDistinctIds(Properties.IPropertyConfiguration propertyConfiguration, Models.Container[] containers) => diff --git a/Shared/Models/Stateless/Methods/IDirectory.cs b/Shared/Models/Stateless/Methods/IDirectory.cs index 2e03ac1..3eadc5b 100644 --- a/Shared/Models/Stateless/Methods/IDirectory.cs +++ b/Shared/Models/Stateless/Methods/IDirectory.cs @@ -43,4 +43,14 @@ public interface IDirectory static void MoveFiles(List files, string find, string replace) => XDirectory.MoveFiles(files, find, replace); + (string[], List<(Models.FileHolder, string)>) TestStatic_GetToDoCollection(Properties.IPropertyConfiguration propertyConfiguration, bool copyDuplicates, List filesCollection, string[] directories, Action? tick) => + GetToDoCollection(propertyConfiguration, copyDuplicates, filesCollection, directories, tick); + static (string[], List<(Models.FileHolder, string)>) GetToDoCollection(Properties.IPropertyConfiguration propertyConfiguration, bool copyDuplicates, List filesCollection, string[] directories, Action? tick) => + XDirectory.GetToDoCollection(propertyConfiguration, copyDuplicates, filesCollection, directories, tick); + + void TestStatic_CopyOrMove(List<(Models.FileHolder, string)> toDoCollection, bool move, bool moveBack, Action? tick) => + CopyOrMove(toDoCollection, move, moveBack, tick); + static List CopyOrMove(List<(Models.FileHolder, string)> toDoCollection, bool move, bool moveBack, Action? tick) => + XDirectory.CopyOrMove(toDoCollection, move, moveBack, tick); + } \ No newline at end of file diff --git a/Shared/Models/Stateless/Methods/XDirectory.cs b/Shared/Models/Stateless/Methods/XDirectory.cs index f4acc4c..507132d 100644 --- a/Shared/Models/Stateless/Methods/XDirectory.cs +++ b/Shared/Models/Stateless/Methods/XDirectory.cs @@ -278,4 +278,102 @@ internal abstract partial class XDirectory } } + private static List GetFileHolders(List filesCollection) + { + List results = new(); + foreach (string[] files in filesCollection) + results.AddRange(files.Select(l => new Models.FileHolder(l))); + return results; + } + + internal static (string[], List<(Models.FileHolder, string)>) GetToDoCollection(Properties.IPropertyConfiguration propertyConfiguration, bool copyDuplicates, List filesCollection, string[] directories, Action? tick) + { + List<(Models.FileHolder, string)> results = new(); + string checkFile; + string directory; + FileInfo fileInfo; + int directoryIndex; + string directoryName; + bool wrapped = false; + List distinct = new(); + List distinctDirectories = new(); + List fileHolders = GetFileHolders(filesCollection); + Models.FileHolder[] sortedFileHolders = (from l in fileHolders orderby l.LastWriteTime, l.FullName.Length descending select l).ToArray(); + foreach (Models.FileHolder fileHolder in sortedFileHolders) + { + tick?.Invoke(); + if (fileHolder.Name.EndsWith("len") || fileHolder.ExtensionLowered == ".id" || fileHolder.ExtensionLowered == ".lsv" || fileHolder.DirectoryName is null) + continue; + (_, directoryIndex) = IPath.GetDirectoryNameAndIndex(propertyConfiguration.ResultAllInOneSubdirectoryLength, fileHolder.NameWithoutExtension); + directoryName = Path.GetFileName(fileHolder.DirectoryName); + if (directoryName.Length < propertyConfiguration.ResultAllInOneSubdirectoryLength + 3 || !fileHolder.Name.StartsWith(directoryName)) + { + if (wrapped) + continue; + directory = directories[directoryIndex]; + } + else + { + if (!wrapped) + wrapped = true; + directory = Path.Combine(directories[directoryIndex], directoryName); + } + checkFile = Path.Combine(directory, $"{fileHolder.NameWithoutExtension}{fileHolder.ExtensionLowered}"); + if (distinct.Contains(checkFile)) + { + if (!copyDuplicates) + continue; + for (int i = 1; i < int.MaxValue; i++) + { + fileInfo = new(checkFile); + if (!fileInfo.Exists || fileHolder.Length == fileInfo.Length && fileHolder.LastWriteTime == fileInfo.LastWriteTime) + checkFile = Path.Combine(directory, $"{fileHolder.NameWithoutExtension}.{i}dup{fileHolder.ExtensionLowered}"); + else + checkFile = Path.Combine(directory, $"{fileHolder.NameWithoutExtension}.{i}why{fileHolder.ExtensionLowered}"); + if (distinct.Contains(checkFile)) + continue; + distinct.Add(checkFile); + results.Add(new(fileHolder, checkFile)); + if (!distinctDirectories.Contains(directory)) + distinctDirectories.Add(directory); + break; + } + continue; + } + distinct.Add(checkFile); + results.Add(new(fileHolder, checkFile)); + if (!distinctDirectories.Contains(directory)) + distinctDirectories.Add(directory); + } + return (distinctDirectories.ToArray(), results); + } + + internal static List CopyOrMove(List<(Models.FileHolder, string)> toDoCollection, bool move, bool moveBack, Action? tick) + { + List results = new(); + FileInfo fileInfo; + foreach ((Models.FileHolder fileHolder, string to) in toDoCollection) + { + tick?.Invoke(); + fileInfo = new(to); + if (fileInfo.Exists) + { + if (fileHolder.Length != fileInfo.Length || fileHolder.LastWriteTime != fileInfo.LastWriteTime) + fileInfo.Delete(); + else + continue; + } + results.Add(fileHolder.NameWithoutExtension); + try + { + if (move || moveBack) + File.Move(fileHolder.FullName, to); + else + File.Copy(fileHolder.FullName, to); + } + catch (Exception) { } + } + return results; + } + } \ No newline at end of file diff --git a/Tests/UnitTestResize.cs b/Tests/UnitTestResize.cs index 161038d..4f31c36 100644 --- a/Tests/UnitTestResize.cs +++ b/Tests/UnitTestResize.cs @@ -144,6 +144,7 @@ public class UnitTestResize List> metadataCollection; int length = _PropertyConfiguration.RootDirectory.Length; string outputResolution = _Configuration.OutputResolutions[0]; + bool outputResolutionHasNumber = outputResolution.Any(l => char.IsNumber(l)); (string cResultsFullGroupDirectory, _, _) = GetResultsFullGroupDirectories(outputResolution); (string aResultsFullGroupDirectory, string bResultsFullGroupDirectory) = GetResultsFullGroupDirectories(); _Logger.Information(_Configuration.ModelDirectory); @@ -174,7 +175,7 @@ public class UnitTestResize } if (property is null || item.Property is null) throw new NullReferenceException(nameof(property)); - resizedFileHolder = resize.GetResizedFileHolder(item); + resizedFileHolder = resize.GetResizedFileHolder(cResultsFullGroupDirectory, item, outputResolutionHasNumber); Shared.Models.Methods.IBlurHasher blurHasher = new BlurHash.Models.C2_BlurHasher(_Configuration.PropertyConfiguration, resultsFullGroupDirectory: null); _ = blurHasher.Encode(resizedFileHolder); item.SetResizedFileHolder(resize.FileNameExtension, resizedFileHolder); diff --git a/TestsWithFaceRecognitionDotNet/UnitTestFace.cs b/TestsWithFaceRecognitionDotNet/UnitTestFace.cs index 014c2b7..9129a6b 100644 --- a/TestsWithFaceRecognitionDotNet/UnitTestFace.cs +++ b/TestsWithFaceRecognitionDotNet/UnitTestFace.cs @@ -218,6 +218,7 @@ public class UnitTestFace List> metadataCollection; int length = _PropertyConfiguration.RootDirectory.Length; string outputResolution = _Configuration.OutputResolutions[0]; + bool outputResolutionHasNumber = outputResolution.Any(l => char.IsNumber(l)); (string cResultsFullGroupDirectory, _, _) = GetResultsFullGroupDirectories(outputResolution); (string aResultsFullGroupDirectory, string bResultsFullGroupDirectory) = GetResultsFullGroupDirectories(); _Logger.Information(_Configuration.ModelDirectory); @@ -248,7 +249,7 @@ public class UnitTestFace } if (property is null || item.Property is null) throw new NullReferenceException(nameof(property)); - resizedFileHolder = resize.GetResizedFileHolder(item); + resizedFileHolder = resize.GetResizedFileHolder(cResultsFullGroupDirectory, item, outputResolutionHasNumber); item.SetResizedFileHolder(resize.FileNameExtension, resizedFileHolder); MappingFromItem mappingFromItem = IMappingFromItem.GetMappingFromItem(item); (int _, metadataCollection) = metadata.GetMetadataCollection(subFileTuples, parseExceptions, mappingFromItem);