diff --git a/Compare/Compare.cs b/Compare/Compare.cs index f48b5dc..9fe82cd 100644 --- a/Compare/Compare.cs +++ b/Compare/Compare.cs @@ -162,7 +162,7 @@ public class Compare } if (_IsEnvironment.Development && propertyConfiguration.PopulatePropertyId) throw new Exception("Copy keyValuePairs-####.json file"); - (int j, int f, int t, Shared.Models.Container[] containers) = Property.Models.Stateless.Container.GetContainers(propertyConfiguration, propertyLogic); + (int j, int f, int t, Shared.Models.Container[] containers) = Shared.Models.Stateless.Methods.IContainer.GetContainers(propertyConfiguration, propertyLogic); if (propertyLogic.ExceptionsDirectories.Any()) throw new Exception(); if (propertyConfiguration.PopulatePropertyId && Shared.Models.Stateless.Methods.IProperty.Any(containers)) diff --git a/Date-Group/DateGroup.cs b/Date-Group/DateGroup.cs index ed62a33..0ada996 100644 --- a/Date-Group/DateGroup.cs +++ b/Date-Group/DateGroup.cs @@ -42,11 +42,9 @@ public class DateGroup Verify(configuration); bool reverse = false; _Configuration = configuration; - string outputExtension = ".jpg"; if (!_IsEnvironment.Development) throw new Exception("This program only allows development environments!"); long ticks = DateTime.Now.Ticks; - A_Property propertyLogic = GetPropertyLogic(reverse, outputExtension); string[] dbFiles = Directory.GetFiles(propertyConfiguration.RootDirectory, "*.db", SearchOption.AllDirectories); foreach (string dbFile in dbFiles) File.Delete(dbFile); @@ -116,7 +114,10 @@ public class DateGroup // continue; // File.Move(moveBackFileName, checkFile); // } - (int j, int f, int t, Container[] containers) = Property.Models.Stateless.Container.GetContainers(propertyConfiguration, propertyLogic); + string aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property)); + string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, "{}"); + (int f, Container[] containers) = Shared.Models.Stateless.Methods.IContainer.GetContainers(propertyConfiguration, aPropertySingletonDirectory); + A_Property propertyLogic = GetPropertyLogic(reverse, aResultsFullGroupDirectory, aResultsFullGroupDirectory); if (propertyLogic.ExceptionsDirectories.Any()) throw new Exception(); if (propertyConfiguration.PopulatePropertyId && (configuration.ByCreateDateShortcut || configuration.ByHash) && Shared.Models.Stateless.Methods.IProperty.Any(containers)) @@ -359,12 +360,12 @@ public class DateGroup return results; } - private A_Property GetPropertyLogic(bool reverse, string outputExtension) + private A_Property GetPropertyLogic(bool reverse, string outputExtension, string aResultsFullGroupDirectory) { A_Property result; if (_Configuration?.PropertyConfiguration is null) throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration)); - result = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, outputExtension, reverse); + result = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, outputExtension, reverse, aResultsFullGroupDirectory); return result; } @@ -373,8 +374,7 @@ public class DateGroup List results = new(); foreach (Item item in container.Items) { - if (item.ImageFileHolder is not null - && (item.Abandoned is null || !item.Abandoned.Value)) + if (item.ImageFileHolder is not null) results.Add(item); } return results.ToArray(); diff --git a/Drag-Drop-Search/DragDropSearch.cs b/Drag-Drop-Search/DragDropSearch.cs index e980d01..d83a031 100644 --- a/Drag-Drop-Search/DragDropSearch.cs +++ b/Drag-Drop-Search/DragDropSearch.cs @@ -4,6 +4,7 @@ using Serilog; using System.Diagnostics; using System.Reflection; using View_by_Distance.Drag_Drop.Models; +using View_by_Distance.Property.Models; using View_by_Distance.Resize.Models; using View_by_Distance.Shared.Models; using View_by_Distance.Shared.Models.Stateless.Methods; @@ -18,10 +19,10 @@ public partial class DragDropSearch : Form private readonly AppSettings _AppSettings; private readonly ProgressBar _ProgressBar; private readonly string _WorkingDirectory; - private readonly Configuration _Configuration; private readonly IsEnvironment _IsEnvironment; private readonly Dictionary _IdToItem; private readonly string _ResizeFileNameExtension; + private readonly Models.Configuration _Configuration; private readonly IConfigurationRoot _ConfigurationRoot; private readonly Property.Models.Configuration _PropertyConfiguration; @@ -29,11 +30,11 @@ public partial class DragDropSearch : Form { InitializeComponent(); ILogger logger; + _IdToItem = new(); AppSettings appSettings; string workingDirectory; - Configuration configuration; IsEnvironment isEnvironment; - _IdToItem = new(); + Models.Configuration configuration; IConfigurationRoot configurationRoot; LoggerConfiguration loggerConfiguration = new(); Property.Models.Configuration propertyConfiguration; @@ -116,8 +117,8 @@ public partial class DragDropSearch : Form void LoadData() { Container[] containers; - Property.Models.A_Property propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _ResizeFileNameExtension, _Configuration.Reverse); - (_, _, _, containers) = Property.Models.Stateless.Container.GetContainers(_Configuration.PropertyConfiguration, propertyLogic); + string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), "{}"); + (_, containers) = IContainer.GetContainers(_Configuration.PropertyConfiguration, aPropertySingletonDirectory); List collection = Program.GetItemCollection(_Configuration, containers); foreach (Item item in collection) { diff --git a/Drag-Drop-Search/Program.cs b/Drag-Drop-Search/Program.cs index 881af4a..fd9ef43 100644 --- a/Drag-Drop-Search/Program.cs +++ b/Drag-Drop-Search/Program.cs @@ -20,8 +20,7 @@ static class Program foreach (Item item in container.Items) { if (item.ImageFileHolder is not null - && (item.Abandoned is null || !item.Abandoned.Value) - && item.ValidImageFormatExtension + && item.IsValidImageFormatExtension && !configuration.IgnoreExtensions.Contains(item.ImageFileHolder.ExtensionLowered)) results.Add(item); } diff --git a/Duplicate-Search/DuplicateSearch.cs b/Duplicate-Search/DuplicateSearch.cs index ed75619..f1a4be0 100644 --- a/Duplicate-Search/DuplicateSearch.cs +++ b/Duplicate-Search/DuplicateSearch.cs @@ -128,18 +128,16 @@ public class DuplicateSearch private static Container[] GetContainers(AppSettings appSettings, long ticks, string argZero, Configuration configuration, bool argZeroIsConfigurationRootDirectory) { - int j; int f; - int t; Container[] containers; int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); string message = $") Building Container(s) - {totalSeconds} total second(s)"; - A_Property propertyLogic = new(appSettings.MaxDegreeOfParallelism, configuration, appSettings.OutputExtension, appSettings.Reverse); ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; using (ProgressBar progressBar = new(1, message, options)) { progressBar.Tick(); - (j, f, t, containers) = Property.Models.Stateless.Container.GetContainers(configuration, propertyLogic); + string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(A_Property), "{}"); + (f, containers) = Shared.Models.Stateless.Methods.IContainer.GetContainers(configuration, aPropertySingletonDirectory); } if (appSettings.SortContainers) containers = Shared.Models.Stateless.Methods.IContainer.SortContainers(configuration, appSettings.IgnoreRelativePaths, argZeroIsConfigurationRootDirectory, argZero, containers); diff --git a/Instance/DlibDotNet.cs b/Instance/DlibDotNet.cs index 694a78c..7ee95bd 100644 --- a/Instance/DlibDotNet.cs +++ b/Instance/DlibDotNet.cs @@ -4,7 +4,6 @@ using ShellProgressBar; using System.Diagnostics; using System.Drawing; using System.Drawing.Imaging; -using System.Text.Json; using View_by_Distance.Distance.Models; using View_by_Distance.Face.Models; using View_by_Distance.FaceParts.Models; @@ -29,7 +28,6 @@ public partial class DlibDotNet private readonly E3_Rename _Rename; private readonly IConsole _Console; private readonly E_Distance _Distance; - private readonly B_Metadata _Metadata; private readonly Serilog.ILogger? _Log; private readonly D2_FaceParts _FaceParts; private readonly AppSettings _AppSettings; @@ -104,7 +102,6 @@ public partial class DlibDotNet (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetGifLowQuality(); _FaceParts = new D2_FaceParts(imageCodecInfo, encoderParameters, filenameExtension, configuration.CheckDFaceAndUpWriteDates, configuration.OverrideForFaceLandmarkImages); } - _Metadata = new(configuration.ForceMetadataLastWriteTimeToCreationTime, configuration.PropertiesChangedForMetadata); _MapConfiguration = Get(configuration, _Faces.FileNameExtension, _Faces.HiddenFileNameExtension, _FaceParts.FileNameExtension); _Distance = new(configuration.DistanceMoveUnableToMatch, configuration.DistanceRenameToMatch, _Configuration.FaceConfidencePercent, configuration.RangeDistanceTolerance, configuration.RangeFaceConfidence); if (_PropertyRootExistedBefore || !_ArgZeroIsConfigurationRootDirectory) @@ -285,26 +282,20 @@ public partial class DlibDotNet return result; } - private void FullParallelForWork( - A_Property propertyLogic, - Dictionary>> idToLocationContainers, - string outputResolution, - string bResultsFullGroupDirectory, - string cResultsFullGroupDirectory, - string dResultsDateGroupDirectory, - string dResultsFullGroupDirectory, - string eDistanceContentDirectory, - List> sourceDirectoryChanges, - List propertyFileHolderCollection, - List propertyCollection, - List>> metadataCollections, - List> resizeKeyValuePairs, - Dictionary> fileNameToCollection, - List> imageFaceCollections, - Container container, - int index, - Item item, - DateTime[] containerDateTimes) + private void FullParallelForWork(A_Property propertyLogic, + B_Metadata metadata, + Dictionary>> idToLocationContainers, + string outputResolution, + string cResultsFullGroupDirectory, + string dResultsDateGroupDirectory, + string dResultsFullGroupDirectory, + string eDistanceContentDirectory, + List> sourceDirectoryChanges, + Dictionary> fileNameToCollection, + Container container, + int index, + Item item, + DateTime[] containerDateTimes) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); @@ -329,8 +320,8 @@ public partial class DlibDotNet else { int? propertyHashCode = item.Property?.GetHashCode(); - if (item.Abandoned.HasValue && item.Abandoned.Value) - _Log.Information(string.Concat("Abandoned <", item.ImageFileHolder.FullName, '>')); + if (!item.SourceDirectoryFileHolder.Exists) + _Log.Information(string.Concat("NoJson <", item.ImageFileHolder.FullName, '>')); else if (item.FileSizeChanged.HasValue && item.FileSizeChanged.Value) _Log.Information(string.Concat("FileSizeChanged <", item.ImageFileHolder.FullName, '>')); else if (item.LastWriteTimeChanged.HasValue && item.LastWriteTimeChanged.Value) @@ -357,7 +348,7 @@ public partial class DlibDotNet string facesDirectory = _Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution) ? _Faces.GetFacesDirectory(dResultsFullGroupDirectory, item) : string.Empty; string facePartsDirectory = _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution) ? _FaceParts.GetFacePartsDirectory(_Configuration.PropertyConfiguration, dResultsFullGroupDirectory, item) : string.Empty; MappingFromItem mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item, resizedFileHolder); - (int metadataGroups, metadataCollection) = _Metadata.GetMetadataCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem); + (int metadataGroups, metadataCollection) = metadata.GetMetadataCollection(subFileTuples, parseExceptions, mappingFromItem); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection)); Dictionary outputResolutionToResize = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, subFileTuples, parseExceptions, metadataCollection, item.Property, mappingFromItem); @@ -404,31 +395,22 @@ public partial class DlibDotNet } lock (sourceDirectoryChanges) { - imageFaceCollections[index] = faces; - propertyCollection[index] = property; - metadataCollections[index] = metadataCollection; - resizeKeyValuePairs[index] = outputResolutionToResize; - propertyFileHolderCollection[index] = item.ImageFileHolder; + item.Faces.AddRange(faces); sourceDirectoryChanges.AddRange(from l in subFileTuples where l.Item2 > dateTime select l); } } private int FullParallelWork(int maxDegreeOfParallelism, A_Property propertyLogic, + B_Metadata metadata, Dictionary>> idToLocationContainers, string outputResolution, - string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsDateGroupDirectory, string dResultsFullGroupDirectory, string eDistanceContentDirectory, List> sourceDirectoryChanges, - List propertyFileHolderCollection, - List propertyCollection, - List>> metadataCollection, - List> resizeKeyValuePairs, Dictionary> fileNameToCollection, - List> imageFaceCollections, Container container, Item[] filteredItems, string message) @@ -438,20 +420,6 @@ public partial class DlibDotNet int result = 0; ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; - if (imageFaceCollections.Count != filteredItems.Length - || metadataCollection.Count != filteredItems.Length - || resizeKeyValuePairs.Count != filteredItems.Length - || propertyCollection.Count != filteredItems.Length) - { - for (int f = 0; f < filteredItems.Length; f++) - { - propertyCollection.Add(null); - metadataCollection.Add(new()); - imageFaceCollections.Add(new()); - resizeKeyValuePairs.Add(new()); - propertyFileHolderCollection.Add(null); - } - } DateTime[] containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems); using ProgressBar progressBar = new(filteredItems.Length, message, options); _ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) => @@ -459,20 +427,15 @@ public partial class DlibDotNet try { FullParallelForWork(propertyLogic, + metadata, idToLocationContainers, outputResolution, - bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsDateGroupDirectory, dResultsFullGroupDirectory, eDistanceContentDirectory, sourceDirectoryChanges, - propertyFileHolderCollection, - propertyCollection, - metadataCollection, - resizeKeyValuePairs, fileNameToCollection, - imageFaceCollections, container, index: i, filteredItems[i], @@ -526,95 +489,27 @@ public partial class DlibDotNet } } - private void WriteGroup(A_Property propertyLogic, Shared.Models.Property[] propertyCollection, List>> metadataCollection, List> resizeKeyValuePairs, List> imageFaceCollections, string outputResolution, Container container, Item[] filteredItems) - { - Item item; - string key; - string json; - string checkFile; - int sourceDirectoryLength = container.SourceDirectory.Length; - JsonSerializerOptions writeIndentedJsonSerializerOptions = new() { WriteIndented = false }; - if (!(from l in propertyCollection where l?.Width is null select true).Any()) - { - string checkDirectory; - List> propertyCollectionKeyValuePairs = new(); - List>> resizeKeyValuePairsCollections = new(); - List>> imageFaceCollectionsKeyValuePairs = new(); - List>>> metadataCollectionKeyValuePairs = new(); - (int level, List directories) = Shared.Models.Stateless.Methods.IPath.Get( - _Configuration.PropertyConfiguration.RootDirectory, - container.SourceDirectory); - string fileName = string.Concat(string.Join(_Configuration.PropertyConfiguration.FileNameDirectorySeparator, directories), ".json"); - for (int i = 0; i < filteredItems.Length; i++) - { - item = filteredItems[i]; - if (item.Property is null) - continue; - key = Shared.Models.Stateless.Methods.IPath.GetRelativePath(item.ImageFileHolder.FullName, sourceDirectoryLength); - propertyCollectionKeyValuePairs.Add(new KeyValuePair(key, propertyCollection[i])); - resizeKeyValuePairsCollections.Add(new KeyValuePair>(key, resizeKeyValuePairs[i])); - imageFaceCollectionsKeyValuePairs.Add(new KeyValuePair>(key, imageFaceCollections[i])); - metadataCollectionKeyValuePairs.Add(new KeyValuePair>>(key, metadataCollection[i])); - } - if (propertyLogic.AngleBracketCollection.Any()) - { - checkDirectory = Shared.Models.Stateless.Methods.IPath.GetDirectory(propertyLogic.AngleBracketCollection[0], level, "[{}]"); - checkFile = Path.Combine(checkDirectory, fileName); - if (File.Exists(checkFile)) - File.Move(checkFile, Path.Combine(checkDirectory, fileName)); - checkFile = Path.Combine(checkDirectory, fileName); - json = JsonSerializer.Serialize(propertyCollectionKeyValuePairs, writeIndentedJsonSerializerOptions); - _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true); - } - if (_Metadata.AngleBracketCollection.Any()) - { - checkDirectory = Shared.Models.Stateless.Methods.IPath.GetDirectory(_Metadata.AngleBracketCollection[0], level, "[{}]"); - checkFile = Path.Combine(checkDirectory, fileName); - if (File.Exists(checkFile)) - File.Move(checkFile, Path.Combine(checkDirectory, fileName)); - checkFile = Path.Combine(checkDirectory, fileName); - json = JsonSerializer.Serialize(metadataCollectionKeyValuePairs, writeIndentedJsonSerializerOptions); - _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true); - } - if (_Resize.AngleBracketCollection.Any()) - { - checkDirectory = Shared.Models.Stateless.Methods.IPath.GetDirectory(_Resize.AngleBracketCollection[0], level, "[{}]"); - checkFile = Path.Combine(checkDirectory, fileName); - if (File.Exists(checkFile)) - File.Move(checkFile, Path.Combine(checkDirectory, fileName)); - checkFile = Path.Combine(checkDirectory, fileName); - json = JsonSerializer.Serialize(resizeKeyValuePairsCollections, writeIndentedJsonSerializerOptions); - _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true); - } - if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution) && _Faces.AngleBracketCollection.Any()) - { - checkDirectory = Shared.Models.Stateless.Methods.IPath.GetDirectory(_Faces.AngleBracketCollection[0], level, "[{}]"); - checkFile = Path.Combine(checkDirectory, fileName); - if (File.Exists(checkFile)) - File.Move(checkFile, Path.Combine(checkDirectory, fileName)); - checkFile = Path.Combine(checkDirectory, fileName); - json = JsonSerializer.Serialize(imageFaceCollectionsKeyValuePairs, writeIndentedJsonSerializerOptions); - _ = Shared.Models.Stateless.Methods.IPath.WriteAllText(checkFile, json, updateDateWhenMatches: false, compareBeforeWrite: true); - } - } - } - - private (string, string, string, string, string) GetResultsFullGroupDirectories(string outputResolution) + private (string, string) GetResultsFullGroupDirectories() { string aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(A_Property), - outputResolution, + string.Empty, includeResizeGroup: false, includeModel: false, includePredictorModel: false); string bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(B_Metadata), - outputResolution, + string.Empty, includeResizeGroup: false, includeModel: false, includePredictorModel: false); + return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory); + } + + private (string, string, string) GetResultsFullGroupDirectories(string outputResolution) + { string cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( _Configuration.PropertyConfiguration, nameof(C_Resize), @@ -636,17 +531,10 @@ public partial class DlibDotNet includeResizeGroup: true, includeModel: true, includePredictorModel: true); - return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory); + return new(cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory); } - private void SetAngleBracketCollections(A_Property propertyLogic, Container container, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory) - { - propertyLogic.SetAngleBracketCollection(aResultsFullGroupDirectory, container.SourceDirectory); - _Resize.SetAngleBracketCollection(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, container.SourceDirectory); - _Metadata.SetAngleBracketCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, container.SourceDirectory); - } - - private void FullDoWork(string argZero, string propertyRoot, long ticks, A_Property propertyLogic, int t, Container[] containers, string eDistanceContentDirectory, Dictionary> fileNameToCollection, Dictionary>> idToLocationContainers) + private void FullDoWork(string argZero, string propertyRoot, long ticks, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, int t, Container[] containers, A_Property propertyLogic, B_Metadata metadata, string eDistanceContentDirectory, Dictionary> fileNameToCollection, Dictionary>> idToLocationContainers) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); @@ -656,25 +544,18 @@ public partial class DlibDotNet int exceptionCount; Container container; Item[] filteredItems; - string aResultsFullGroupDirectory; - string bResultsFullGroupDirectory; + bool anyNullOrNoIsUniqueFileName; string cResultsFullGroupDirectory; string dResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; int containersLength = containers.Length; - Shared.Models.Property[] propertyCollection; - List propertyFileHolderCollection = new(); - List> resizeKeyValuePairs = new(); - List> imageFaceCollections = new(); List> sourceDirectoryChanges = new(); int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism; - List nullablePropertyCollection = new(); - List>> metadataCollection = new(); string dResultsDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(D_Face)); foreach (string outputResolution in _Configuration.OutputResolutions) { total = 0; - (aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); + (cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); // _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(dResultsFullGroupDirectory); for (int i = 0; i < containers.Length; i++) { @@ -686,51 +567,35 @@ public partial class DlibDotNet filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container); if (!filteredItems.Any()) continue; - metadataCollection.Clear(); - resizeKeyValuePairs.Clear(); - imageFaceCollections.Clear(); sourceDirectoryChanges.Clear(); - nullablePropertyCollection.Clear(); - propertyFileHolderCollection.Clear(); totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); - message = $"{i + 1:000}.{container.G} [{filteredItems.Length:000} collection] / {containersLength:000} - {total} / {t} total collection - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}"; + anyNullOrNoIsUniqueFileName = filteredItems.Any(l => l.IsUniqueFileName is null || !l.IsUniqueFileName.Value); + message = $"{i + 1:000} [{filteredItems.Length:000} collection] / {containersLength:000} - {total} / {t} total collection - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}"; if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution)) _Faces.SetAngleBracketCollection(dResultsFullGroupDirectory, container.SourceDirectory); if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution)) _FaceParts.SetAngleBracketCollection(_Configuration.PropertyConfiguration, d2ResultsFullGroupDirectory, container.SourceDirectory); - SetAngleBracketCollections(propertyLogic, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory); - exceptionCount = FullParallelWork( - maxDegreeOfParallelism, - propertyLogic, - idToLocationContainers, - outputResolution, - bResultsFullGroupDirectory, - cResultsFullGroupDirectory, - dResultsDateGroupDirectory, - dResultsFullGroupDirectory, - eDistanceContentDirectory, - sourceDirectoryChanges, - propertyFileHolderCollection, - nullablePropertyCollection, - metadataCollection, - resizeKeyValuePairs, - fileNameToCollection, - imageFaceCollections, - container, - filteredItems, - message); - if (metadataCollection.Count != filteredItems.Length || nullablePropertyCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || imageFaceCollections.Count != filteredItems.Length) - throw new Exception("Counts don't match!"); + propertyLogic.SetAngleBracketCollection(aResultsFullGroupDirectory, container.SourceDirectory, anyNullOrNoIsUniqueFileName); + _Resize.SetAngleBracketCollection(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, container.SourceDirectory); + exceptionCount = FullParallelWork(maxDegreeOfParallelism, + propertyLogic, + metadata, + idToLocationContainers, + outputResolution, + cResultsFullGroupDirectory, + dResultsDateGroupDirectory, + dResultsFullGroupDirectory, + eDistanceContentDirectory, + sourceDirectoryChanges, + fileNameToCollection, + container, + filteredItems, + message); if (exceptionCount != 0) { _Exceptions.Add(container.SourceDirectory); continue; } - for (int f = 0; f < imageFaceCollections.Count; f++) - filteredItems[f].Faces.AddRange(from l in imageFaceCollections[f] select l); - propertyCollection = (from l in nullablePropertyCollection where l is not null select l).ToArray(); - if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0) - WriteGroup(propertyLogic, propertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, outputResolution, container, filteredItems); if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any()) { for (int y = 0; y < int.MaxValue; y++) @@ -803,68 +668,6 @@ public partial class DlibDotNet } } - private void RenameAbandoned(string argZero, Container[] containers) - { - List renameCollection = new(); - foreach (Container container in containers) - { - if (!container.Items.Any()) - continue; - if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero)) - continue; - foreach (Item item in container.Items) - { - if (item.Abandoned is not null && item.Abandoned.Value) - renameCollection.Add(item.SourceDirectoryFileHolder.FullName); - } - } - if (renameCollection.Any()) - { - string checkFile; - foreach (string file in renameCollection) - { - if (!File.Exists(file)) - continue; - checkFile = $"{file}.abd"; - if (File.Exists(checkFile)) - continue; - File.Move(file, checkFile); - } - throw new Exception($"Renamed {renameCollection.Count}(s) files!"); - } - } - - private void RenameMoved(string argZero, Container[] containers) - { - List renameCollection = new(); - foreach (Container container in containers) - { - if (!container.Items.Any()) - continue; - if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero)) - continue; - foreach (Item item in container.Items) - { - if (item.Moved is not null && item.Moved.Value) - renameCollection.Add(item.SourceDirectoryFileHolder.FullName); - } - } - if (renameCollection.Any()) - { - string checkFile; - foreach (string file in renameCollection) - { - if (!File.Exists(file)) - continue; - checkFile = $"{file}.mvd"; - if (File.Exists(checkFile)) - continue; - File.Move(file, checkFile); - } - throw new Exception($"Renamed {renameCollection.Count}(s) files!"); - } - } - private (List, List) GetFilteredDistinct(string argZero, Container[] containers) { List resultIds = new(); @@ -1152,7 +955,7 @@ public partial class DlibDotNet for (int i = 0; i < collection.Count; i++) { file = collection[i].File; - if (!file.EndsWith(".dup") && !file.EndsWith(".unk")) + if (!file.EndsWith(".dup") && !file.EndsWith(".unk") && !file.EndsWith(".abd")) continue; if (!File.Exists(file)) continue; @@ -1221,14 +1024,10 @@ public partial class DlibDotNet private void Search(long ticks, string argZero, string propertyRoot) { - int f; - int j; int t; Container[] containers; string eDistanceContentDirectory; string? a2PeopleContentDirectory; - string aResultsFullGroupDirectory; - string bResultsFullGroupDirectory; string cResultsFullGroupDirectory; string dResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; @@ -1236,19 +1035,21 @@ public partial class DlibDotNet string fPhotoPrismSingletonDirectory; Dictionary> personKeyToIds; Dictionary> fileNameToCollection; + (string aResultsFullGroupDirectory, string bResultsFullGroupDirectory) = GetResultsFullGroupDirectories(); string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "{}"); int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); string message = $") Building Container(s) - {totalSeconds} total second(s)"; - A_Property propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse); ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; - using (ProgressBar progressBar = new(1, message, options)) + using (ProgressBar progressBar = new(2, message, options)) { progressBar.Tick(); - (j, f, t, containers) = Property.Models.Stateless.Container.GetContainers(_Configuration.PropertyConfiguration, propertyLogic); + string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, "{}"); + (t, containers) = Shared.Models.Stateless.Methods.IContainer.GetContainers(_Configuration.PropertyConfiguration, aPropertySingletonDirectory); + progressBar.Tick(); } - RenameMoved(argZero, containers); - RenameAbandoned(argZero, containers); Container? container = AreAllSameEndsWith(argZero, containers); + A_Property propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory); + B_Metadata metadata = new(_Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, bResultsFullGroupDirectory); if (!_ArgZeroIsConfigurationRootDirectory || container is null) { a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "([])"); @@ -1272,7 +1073,7 @@ public partial class DlibDotNet argZero = newRootDirectory; _Configuration.PropertyConfiguration.ChangeRootDirectory(newRootDirectory); propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false); - propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse); + propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory); } containers = Shared.Models.Stateless.Methods.IContainer.SortContainers(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, _ArgZeroIsConfigurationRootDirectory, argZero, containers); MapLogicSupport mapLogicSupport = new(_Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeFaceAreaPermilleTolerance, _Configuration.SortingMaximumPerFaceShouldBeHigh); @@ -1280,7 +1081,7 @@ public partial class DlibDotNet personKeyToIds = mapLogic is null ? new() : mapLogic.GetPersonKeyToIds(); fileNameToCollection = !Directory.Exists(fPhotoPrismSingletonDirectory) ? fileNameToCollection = new() : F_PhotoPrism.GetFileNameToCollection(fPhotoPrismSingletonDirectory); Dictionary>> idToLocationContainers = GetDictionary(ticks, a2PeopleContentDirectory, eDistanceContentDirectory); - FullDoWork(argZero, propertyRoot, ticks, propertyLogic, t, containers, eDistanceContentDirectory, fileNameToCollection, idToLocationContainers); + FullDoWork(argZero, propertyRoot, ticks, aResultsFullGroupDirectory, bResultsFullGroupDirectory, t, containers, propertyLogic, metadata, eDistanceContentDirectory, fileNameToCollection, idToLocationContainers); _Distance.Clear(); mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, _PersonContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory, mapLogicSupport); SetMapping(fileNameToCollection, argZero, containers); @@ -1300,7 +1101,7 @@ public partial class DlibDotNet mapLogic.SaveShortcutsForOutputResolutions(a2PeopleContentDirectory, personKeyToIds, idToLocationContainers, mappingCollection, totalNotMapped); if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution)) mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, _PersonContainers, a2PeopleContentDirectory, personKeyToIds, mappingCollection, totalNotMapped); - (aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); + (cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution); if (_ArgZeroIsConfigurationRootDirectory && _Configuration.SaveResizedSubfiles && outputResolution == _Configuration.OutputResolutions[0] diff --git a/Metadata/Models/B_Metadata.cs b/Metadata/Models/B_Metadata.cs index 399dfaa..6d1c89d 100644 --- a/Metadata/Models/B_Metadata.cs +++ b/Metadata/Models/B_Metadata.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using System.Text.Json; using View_by_Distance.Metadata.Models.Stateless; +using View_by_Distance.Property.Models; using View_by_Distance.Property.Models.Stateless; using View_by_Distance.Shared.Models.Stateless; @@ -12,20 +13,32 @@ namespace View_by_Distance.Metadata.Models; public class B_Metadata { - public List AngleBracketCollection { get; } - private readonly Serilog.ILogger? _Log; private readonly bool _PropertiesChangedForMetadata; private readonly bool _ForceMetadataLastWriteTimeToCreationTime; + private readonly IReadOnlyDictionary _JsonGroups; private readonly JsonSerializerOptions _WriteIndentedJsonSerializerOptions; - public B_Metadata(bool forceMetadataLastWriteTimeToCreationTime, bool propertiesChangedForMetadata) + public B_Metadata(Configuration configuration, bool forceMetadataLastWriteTimeToCreationTime, bool propertiesChangedForMetadata, string aResultsFullGroupDirectory) { - AngleBracketCollection = new List(); _Log = Serilog.Log.ForContext(); _PropertiesChangedForMetadata = propertiesChangedForMetadata; _ForceMetadataLastWriteTimeToCreationTime = forceMetadataLastWriteTimeToCreationTime; _WriteIndentedJsonSerializerOptions = new JsonSerializerOptions { WriteIndented = true }; + string checkDirectory; + List collection = new(); + for (int i = 0; i < 11; i++) + { + if (i == 10) + checkDirectory = Path.Combine(aResultsFullGroupDirectory, "{}", configuration.ResultAllInOne, "-"); + else + checkDirectory = Path.Combine(aResultsFullGroupDirectory, "{}", configuration.ResultAllInOne, i.ToString()); + if (!Directory.Exists(checkDirectory)) + _ = Directory.CreateDirectory(checkDirectory); + collection.Add(checkDirectory); + } + Dictionary jsonGroups = new() { { "{}", collection.ToArray() } }; + _JsonGroups = jsonGroups; } public override string ToString() @@ -34,18 +47,6 @@ public class B_Metadata return result; } - public void SetAngleBracketCollection(Property.Models.Configuration configuration, string bResultsFullGroupDirectory, string sourceDirectory) - { - AngleBracketCollection.Clear(); - AngleBracketCollection.AddRange(IResult.GetDirectoryInfoCollection(configuration, - sourceDirectory, - bResultsFullGroupDirectory, - contentDescription: string.Empty, - singletonDescription: "Metadata as key value pairs", - collectionDescription: string.Empty, - converted: true)); - } - private Dictionary>> GetMetadataCollection(string subFile) { Dictionary>> results = new(); @@ -89,15 +90,16 @@ public class B_Metadata return results; } - public (int, List>) GetMetadataCollection(Property.Models.Configuration configuration, string bResultsFullGroupDirectory, List> subFileTuples, List parseExceptions, Shared.Models.MappingFromItem mappingFromItem) + public (int, List>) GetMetadataCollection(List> subFileTuples, List parseExceptions, Shared.Models.MappingFromItem mappingFromItem) { List> results = new(); string json = string.Empty; + string[] changesFrom = new string[] { nameof(A_Property) }; Dictionary>>? dictionary; - string[] changesFrom = new string[] { nameof(Property.Models.A_Property) }; + char directory = Shared.Models.Stateless.Methods.IDirectory.GetDirectory(mappingFromItem.ImageFileHolder.Name); + int directoryIndex = Shared.Models.Stateless.Methods.IDirectory.GetDirectory(directory); List dateTimes = (from l in subFileTuples where changesFrom.Contains(l.Item1) select l.Item2).ToList(); - string bMetadataSingletonFile = Path.Combine(bResultsFullGroupDirectory, "{}", configuration.ResultAllInOne, $"{mappingFromItem.Id}{mappingFromItem.ImageFileHolder.ExtensionLowered}.json"); - FileInfo fileInfo = new(bMetadataSingletonFile); + FileInfo fileInfo = new(Path.Combine(_JsonGroups["{}"][directoryIndex], $"{mappingFromItem.ImageFileHolder.NameWithoutExtension}{mappingFromItem.ImageFileHolder.ExtensionLowered}.json")); if (_ForceMetadataLastWriteTimeToCreationTime && !fileInfo.Exists && File.Exists(Path.ChangeExtension(fileInfo.FullName, ".delete"))) { File.Move(Path.ChangeExtension(fileInfo.FullName, ".delete"), fileInfo.FullName); diff --git a/Not-Copy-Copy/Not-Copy-Copy.cs b/Not-Copy-Copy/Not-Copy-Copy.cs index 04b1d9e..80a36b5 100644 --- a/Not-Copy-Copy/Not-Copy-Copy.cs +++ b/Not-Copy-Copy/Not-Copy-Copy.cs @@ -30,16 +30,14 @@ public class NotCopyCopy _Log.Information(propertyConfiguration.RootDirectory); Property.Models.Configuration.Verify(propertyConfiguration, requireExist: true); Verify(configuration); - bool reverse = false; - string outputExtension = ".jpg"; _Configuration = configuration; if (!_IsEnvironment.Development) throw new Exception("This program only allows development environments!"); - A_Property propertyLogic = GetPropertyLogic(reverse, outputExtension); propertyConfiguration.ChangeRootDirectory(configuration.CompareSource); - (_, _, _, Shared.Models.Container[] compareContainers) = Property.Models.Stateless.Container.GetContainers(propertyConfiguration, propertyLogic); + string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), "{}"); + (_, Shared.Models.Container[] compareContainers) = Shared.Models.Stateless.Methods.IContainer.GetContainers(propertyConfiguration, aPropertySingletonDirectory); propertyConfiguration.ChangeRootDirectory(configuration.SelectedSource); - (_, _, _, Shared.Models.Container[] selectedContainers) = Property.Models.Stateless.Container.GetContainers(propertyConfiguration, propertyLogic); + (_, Shared.Models.Container[] selectedContainers) = Shared.Models.Stateless.Methods.IContainer.GetContainers(propertyConfiguration, aPropertySingletonDirectory); if (compareContainers.Length == selectedContainers.Length) throw new Exception(); string directoryName; @@ -100,15 +98,6 @@ public class NotCopyCopy throw new NullReferenceException(nameof(configuration.SelectedSource)); } - private A_Property GetPropertyLogic(bool reverse, string outputExtension) - { - A_Property result; - if (_Configuration?.PropertyConfiguration is null) - throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration)); - result = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, outputExtension, reverse); - return result; - } - private List<(string Source, string[] Destination)> GetCopyCollection(List compareSourceGroupCollection, List selectedSourceGroupCollection) { List<(string Source, string[] Destination)> results = new(); diff --git a/Property/Models/A_Property.cs b/Property/Models/A_Property.cs index d8cfa1a..5539bb6 100644 --- a/Property/Models/A_Property.cs +++ b/Property/Models/A_Property.cs @@ -18,7 +18,6 @@ public class A_Property protected readonly List _ExceptionsDirectories; public bool Reverse { get; } - public List AngleBracketCollection { get; } public List ExceptionsDirectories => _ExceptionsDirectories; private readonly Serilog.ILogger? _Log; @@ -27,22 +26,38 @@ public class A_Property private readonly int _MaxDegreeOfParallelism; private readonly ASCIIEncoding _ASCIIEncoding; private readonly Configuration _Configuration; + private readonly List _AngleBracketCollection; + private readonly IReadOnlyDictionary _JsonGroups; private readonly JsonSerializerOptions _WriteIndentedJsonSerializerOptions; - public A_Property(int maxDegreeOfParallelism, Configuration configuration, string outputExtension, bool reverse) + public A_Property(int maxDegreeOfParallelism, Configuration configuration, string outputExtension, bool reverse, string aResultsFullGroupDirectory) { Reverse = reverse; _Configuration = configuration; _ExceptionsDirectories = new(); _OutputExtension = outputExtension; _ASCIIEncoding = new ASCIIEncoding(); - AngleBracketCollection = new List(); _Log = Serilog.Log.ForContext(); + _AngleBracketCollection = new List(); _MaxDegreeOfParallelism = maxDegreeOfParallelism; _WriteIndentedJsonSerializerOptions = new JsonSerializerOptions { WriteIndented = true }; if (configuration.VerifyToSeason is null || !configuration.VerifyToSeason.Any()) throw new Exception(); _VerifyToSeason = configuration.VerifyToSeason.Select(l => Path.Combine(configuration.RootDirectory, l)).ToArray(); + string checkDirectory; + List collection = new(); + for (int i = 0; i < 11; i++) + { + if (i == 10) + checkDirectory = Path.Combine(aResultsFullGroupDirectory, "{}", configuration.ResultAllInOne, "-"); + else + checkDirectory = Path.Combine(aResultsFullGroupDirectory, "{}", configuration.ResultAllInOne, i.ToString()); + if (!Directory.Exists(checkDirectory)) + _ = Directory.CreateDirectory(checkDirectory); + collection.Add(checkDirectory); + } + Dictionary jsonGroups = new() { { "{}", collection.ToArray() } }; + _JsonGroups = jsonGroups; } public override string ToString() @@ -128,7 +143,7 @@ public class A_Property if (populateId && id is null) { using Bitmap bitmap = new(image); - string angleBracket = AngleBracketCollection[0]; + string angleBracket = _AngleBracketCollection[0]; Rectangle rectangle = new(0, 0, image.Width, image.Height); BitmapData bitmapData = bitmap.LockBits(rectangle, ImageLockMode.ReadOnly, bitmap.PixelFormat); IntPtr intPtr = bitmapData.Scan0; @@ -266,17 +281,17 @@ public class A_Property Shared.Models.Property? result; string json; int? id = null; + FileInfo fileInfo; bool hasWrongYearProperty = false; string[] changesFrom = Array.Empty(); - string angleBracket = AngleBracketCollection[0]; + string angleBracket = _AngleBracketCollection[0]; bool populateId = _Configuration.PopulatePropertyId; - string without = Path.Combine(angleBracket.Replace("<>", "{}"), $"{item.ImageFileHolder.NameWithoutExtension}.json"); - FileInfo fileInfo = new(Path.Combine(angleBracket.Replace("<>", "{}"), $"{item.ImageFileHolder.NameWithoutExtension}{item.ImageFileHolder.ExtensionLowered}.json")); - if (item.ValidImageFormatExtension && File.Exists(without)) - { - File.Move(without, fileInfo.FullName); - fileInfo.Refresh(); - } + char directory = Shared.Models.Stateless.Methods.IDirectory.GetDirectory(item.ImageFileHolder.Name); + int directoryIndex = Shared.Models.Stateless.Methods.IDirectory.GetDirectory(directory); + if (item.IsUniqueFileName is null || !item.IsUniqueFileName.Value) + fileInfo = new(Path.Combine(angleBracket.Replace("<>", "{}"), $"{item.ImageFileHolder.NameWithoutExtension}{item.ImageFileHolder.ExtensionLowered}.json")); + else + fileInfo = new(Path.Combine(_JsonGroups["{}"][directoryIndex], $"{item.ImageFileHolder.NameWithoutExtension}{item.ImageFileHolder.ExtensionLowered}.json")); List dateTimes = (from l in sourceDirectoryFileTuples where changesFrom.Contains(l.Item1) select l.Item2).ToList(); if (_Configuration.ForcePropertyLastWriteTimeToCreationTime && !fileInfo.Exists && File.Exists(Path.ChangeExtension(fileInfo.FullName, ".delete"))) { @@ -311,29 +326,24 @@ public class A_Property hasWrongYearProperty = true; result = null; } - if (!isIgnoreExtension && item.ValidImageFormatExtension && ((populateId && result?.Id is null) || result?.Width is null || result.Height is null)) + if (!isIgnoreExtension && item.IsValidImageFormatExtension && ((populateId && result?.Id is null) || result?.Width is null || result.Height is null)) { id = result?.Id; result = null; } - if (!isIgnoreExtension && item.ValidImageFormatExtension && populateId && result is not null) - { - id = result.Id; - result = null; - } - if (!isIgnoreExtension && item.ValidImageFormatExtension && populateId && result is not null && result.LastWriteTime != item.ImageFileHolder.LastWriteTime) + if (!isIgnoreExtension && item.IsValidImageFormatExtension && populateId && result is not null && result.LastWriteTime != item.ImageFileHolder.LastWriteTime) { id = null; result = null; } - if (!isIgnoreExtension && item.ValidImageFormatExtension && result?.Width is not null && result.Height is not null && result.Width.Value == result.Height.Value && item.ImageFileHolder.Exists) + if (!isIgnoreExtension && item.IsValidImageFormatExtension && result?.Width is not null && result.Height is not null && result.Width.Value == result.Height.Value && item.ImageFileHolder.Exists) { id = result.Id; result = null; if (result?.Width is not null && result.Height is not null && result.Width.Value != result.Height.Value) throw new Exception("Was square!"); } - if (!isIgnoreExtension && item.ValidImageFormatExtension && result is not null && result.FileSize != fileInfo.Length) + if (!isIgnoreExtension && item.IsValidImageFormatExtension && result is not null && result.FileSize != item.ImageFileHolder.Length) { id = result.Id; result = null; @@ -344,7 +354,14 @@ public class A_Property // property = ; // } if (result is not null) + { sourceDirectoryFileTuples.Add(new Tuple(nameof(A_Property), fileInfo.LastWriteTime)); + if (fileInfo.CreationTime != result.LastWriteTime) + { + File.SetCreationTime(fileInfo.FullName, result.LastWriteTime); + File.SetLastWriteTime(fileInfo.FullName, fileInfo.LastWriteTime); + } + } } catch (Exception) { @@ -355,10 +372,11 @@ public class A_Property if (result is null) { id ??= item.ImageFileHolder.Id; - result = GetImageProperty(item.ImageFileHolder, result, populateId, isIgnoreExtension, item.ValidImageFormatExtension, isValidMetadataExtensions, id); + result = GetImageProperty(item.ImageFileHolder, result, populateId, isIgnoreExtension, item.IsValidImageFormatExtension, isValidMetadataExtensions, id); json = JsonSerializer.Serialize(result, _WriteIndentedJsonSerializerOptions); if (populateId && Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, json, updateDateWhenMatches: true, compareBeforeWrite: true)) { + File.SetCreationTime(fileInfo.FullName, result.LastWriteTime); if (!_Configuration.ForcePropertyLastWriteTimeToCreationTime) sourceDirectoryFileTuples.Add(new Tuple(nameof(A_Property), DateTime.Now)); else @@ -374,6 +392,7 @@ public class A_Property json = JsonSerializer.Serialize(result, _WriteIndentedJsonSerializerOptions); if (Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, json, updateDateWhenMatches: true, compareBeforeWrite: true)) { + File.SetCreationTime(fileInfo.FullName, result.LastWriteTime); File.SetLastWriteTime(fileInfo.FullName, fileInfo.CreationTime); fileInfo.Refresh(); sourceDirectoryFileTuples.Add(new Tuple(nameof(A_Property), fileInfo.CreationTime)); @@ -400,7 +419,7 @@ public class A_Property DateTime directoryMaximumOfMinimumDateTime = DateTime.MinValue; foreach (Item item in items) { - if (!item.ValidImageFormatExtension || item.Property is null || !item.ImageFileHolder.Exists) + if (!item.IsValidImageFormatExtension || item.Property is null || !item.ImageFileHolder.Exists) continue; minimumDateTime = Shared.Models.Stateless.Methods.IProperty.GetMinimumDateTime(item.Property); if (minimumDateTime > directoryMaximumOfMinimumDateTime) @@ -487,9 +506,9 @@ public class A_Property Shared.Models.Property property; List parseExceptions = new(); bool isValidMetadataExtensions = _Configuration.ValidMetadataExtensions.Contains(item.ImageFileHolder.ExtensionLowered); - bool isIgnoreExtension = item.ValidImageFormatExtension && _Configuration.IgnoreExtensions.Contains(item.ImageFileHolder.ExtensionLowered); + bool isIgnoreExtension = item.IsValidImageFormatExtension && _Configuration.IgnoreExtensions.Contains(item.ImageFileHolder.ExtensionLowered); string filteredSourceDirectoryFileExtensionLowered = Path.Combine(sourceDirectory, $"{item.ImageFileHolder.NameWithoutExtension}{item.ImageFileHolder.ExtensionLowered}"); - if (item.ValidImageFormatExtension && item.ImageFileHolder.FullName.Length == filteredSourceDirectoryFileExtensionLowered.Length && item.ImageFileHolder.FullName != filteredSourceDirectoryFileExtensionLowered) + if (item.IsValidImageFormatExtension && item.ImageFileHolder.FullName.Length == filteredSourceDirectoryFileExtensionLowered.Length && item.ImageFileHolder.FullName != filteredSourceDirectoryFileExtensionLowered) File.Move(item.ImageFileHolder.FullName, filteredSourceDirectoryFileExtensionLowered); if (item.FileSizeChanged is null || item.FileSizeChanged.Value || item.LastWriteTimeChanged is null || item.LastWriteTimeChanged.Value || item.Property is null) { @@ -501,7 +520,7 @@ public class A_Property } } - private void SavePropertyParallelWork(List exceptions, List> sourceDirectoryChanges, Shared.Models.Container container, List items, string message) + private void SavePropertyParallelWork(List exceptions, List> sourceDirectoryChanges, Container container, List items, string message) { List> sourceDirectoryFileTuples = new(); ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _MaxDegreeOfParallelism }; @@ -530,38 +549,42 @@ public class A_Property }); } - public void SetAngleBracketCollection(string aResultsFullGroupDirectory, string sourceDirectory) + public void SetAngleBracketCollection(string aResultsFullGroupDirectory, string sourceDirectory, bool anyNullOrNoIsUniqueFileName = true) { - AngleBracketCollection.Clear(); - AngleBracketCollection.AddRange(IResult.GetDirectoryInfoCollection(_Configuration, - sourceDirectory, - aResultsFullGroupDirectory, - contentDescription: string.Empty, - singletonDescription: "Properties for each image", - collectionDescription: string.Empty, - converted: false)); + _AngleBracketCollection.Clear(); + if (!anyNullOrNoIsUniqueFileName) + _AngleBracketCollection.AddRange(new[] { Path.Combine(aResultsFullGroupDirectory, "<>") }); + else + _AngleBracketCollection.AddRange(IResult.GetDirectoryInfoCollection(_Configuration, + sourceDirectory, + aResultsFullGroupDirectory, + contentDescription: string.Empty, + singletonDescription: "Properties for each image", + collectionDescription: string.Empty, + converted: false)); } - public void SetAngleBracketCollection(string sourceDirectory) + private void SetAngleBracketCollection(string sourceDirectory, bool anyNullOrNoIsUniqueFileName) { - AngleBracketCollection.Clear(); + _AngleBracketCollection.Clear(); string aResultsFullGroupDirectory = IResult.GetResultsFullGroupDirectory(_Configuration, nameof(A_Property), string.Empty, includeResizeGroup: false, includeModel: false, includePredictorModel: false); - SetAngleBracketCollection(aResultsFullGroupDirectory, sourceDirectory); + SetAngleBracketCollection(aResultsFullGroupDirectory, sourceDirectory, anyNullOrNoIsUniqueFileName); } - public void SavePropertyParallelWork(long ticks, Shared.Models.Container[] containers) + public void SavePropertyParallelWork(long ticks, Container[] containers) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); string message; int totalSeconds; bool? anyFilesMoved; - Shared.Models.Container container; + Container container; + bool anyNullOrNoIsUniqueFileName; List exceptions = new(); int containersCount = containers.Length; List> sourceDirectoryChanges = new(); @@ -574,9 +597,10 @@ public class A_Property sourceDirectoryChanges.Clear(); if (!container.Items.Any()) continue; - SetAngleBracketCollection(container.SourceDirectory); + anyNullOrNoIsUniqueFileName = container.Items.Any(l => l.IsUniqueFileName is null || !l.IsUniqueFileName.Value); + SetAngleBracketCollection(container.SourceDirectory, anyNullOrNoIsUniqueFileName); totalSeconds = (int)Math.Truncate(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); - message = $"{i + 1:000}.{container.G} / {containersCount:000}) {container.Items.Count:000} file(s) - {totalSeconds} total second(s) - {container.SourceDirectory}"; + message = $"{i + 1:000} / {containersCount:000}) {container.Items.Count:000} file(s) - {totalSeconds} total second(s) - {container.SourceDirectory}"; SavePropertyParallelWork(exceptions, sourceDirectoryChanges, container, container.Items, message); foreach (Exception exception in exceptions) _Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, exception.Message, Environment.NewLine, exception.StackTrace), exception); @@ -604,54 +628,20 @@ public class A_Property public Shared.Models.Property GetProperty(Item item, List> sourceDirectoryFileTuples, List parseExceptions) { Shared.Models.Property result; - bool angleBracketCollectionAny = AngleBracketCollection.Any(); + bool angleBracketCollectionAny = _AngleBracketCollection.Any(); if (!angleBracketCollectionAny) { if (item.ImageFileHolder.DirectoryName is null) throw new NullReferenceException(nameof(item.ImageFileHolder.DirectoryName)); - SetAngleBracketCollection(item.ImageFileHolder.DirectoryName); + bool anyNullOrNoIsUniqueFileName = item.IsUniqueFileName is null || !item.IsUniqueFileName.Value; + SetAngleBracketCollection(item.ImageFileHolder.DirectoryName, anyNullOrNoIsUniqueFileName); } bool isValidMetadataExtensions = _Configuration.ValidMetadataExtensions.Contains(item.ImageFileHolder.ExtensionLowered); - bool isIgnoreExtension = item.ValidImageFormatExtension && _Configuration.IgnoreExtensions.Contains(item.ImageFileHolder.ExtensionLowered); + bool isIgnoreExtension = item.IsValidImageFormatExtension && _Configuration.IgnoreExtensions.Contains(item.ImageFileHolder.ExtensionLowered); result = GetPropertyOfPrivate(item, sourceDirectoryFileTuples, parseExceptions, isIgnoreExtension, isValidMetadataExtensions); if (!angleBracketCollectionAny) - AngleBracketCollection.Clear(); + _AngleBracketCollection.Clear(); return result; } - public (long Ticks, string FilteredSourceDirectoryFile, string PropertyDirectory, int PropertyId)[] GetPropertyIds(List groupCollection, bool saveToCollection) - { - List<(long Ticks, string FilteredSourceDirectoryFile, string PropertyDirectory, int PropertyId)> results = new(); - int level; - string checkDirectory; - List directories; - string propertyDirectory; - Shared.Models.Property? property; - string angleBracket = AngleBracketCollection[0]; - foreach (DirectoryInfo group in groupCollection) - { - SetAngleBracketCollection(group.SourceDirectory); - if (string.IsNullOrEmpty(group.SourceDirectory)) - throw new Exception(); - if (!saveToCollection) - propertyDirectory = angleBracket.Replace("<>", "()"); - else - { - (level, directories) = Shared.Models.Stateless.Methods.IPath.Get(_Configuration.RootDirectory, group.SourceDirectory); - checkDirectory = Shared.Models.Stateless.Methods.IPath.GetDirectory(angleBracket, level, "[()]"); - propertyDirectory = Path.Combine(checkDirectory, string.Join(_Configuration.FileNameDirectorySeparator, directories)); - } - if (!Directory.Exists(propertyDirectory)) - _ = Directory.CreateDirectory(propertyDirectory); - for (int i = 0; i < group.SourceDirectoryFileHolderCollection.Length; i++) - { - property = group.PropertyCollection[i]; - if (property?.Id is null) - continue; - results.Add(new(property.GetDateTimes().Min().Ticks, group.FilteredSourceDirectoryFiles[i], propertyDirectory, property.Id.Value)); - } - } - return results.OrderBy(l => l.Ticks).ToArray(); - } - } \ No newline at end of file diff --git a/Property/Models/Stateless/Container.cs b/Property/Models/Stateless/Container.cs deleted file mode 100644 index b4e67a1..0000000 --- a/Property/Models/Stateless/Container.cs +++ /dev/null @@ -1,257 +0,0 @@ -using System.Text.Json; -using View_by_Distance.Shared.Models; - -namespace View_by_Distance.Property.Models.Stateless; - -public class Container -{ - - private static (int f, List<(int g, string sourceDirectory, FileHolder[] sourceDirectoryFiles)>) GetFileHolderGroupCollection(Configuration configuration, A_Property propertyLogic, string searchPattern, List topDirectories) - { - List<(int g, string sourceDirectory, FileHolder[] sourceDirectoryFiles)> results = new(); - int? id; - int parsedId; - string idFile; - string[] lines; - const int zero = 0; - List<(string File, int? Id)> files = new(); - List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)> collection; - (int f, collection) = Shared.Models.Stateless.Methods.IFileHolder.GetGroupCollection(configuration.RootDirectory, configuration.MaxImagesInDirectoryForTopLevelFirstPass, propertyLogic.Reverse, searchPattern, topDirectories); - foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles) in collection) - { - files.Clear(); - foreach (string sourceDirectoryFile in sourceDirectoryFiles) - { - if (sourceDirectoryFile.EndsWith(".id")) - continue; - id = null; - idFile = $"{sourceDirectoryFile}.id"; - if (sourceDirectoryFiles.Contains(idFile)) - { - lines = File.ReadAllLines(idFile); - File.Delete(idFile); - if (lines.Any() && int.TryParse(lines[zero], out parsedId)) - id = parsedId; - }; - files.Add(new(sourceDirectoryFile, id)); - } - results.Add(new(g, sourceDirectory, (from l in files select new FileHolder(l.File, l.Id)).ToArray())); - } - return new(f, results); - } - - private static void ParallelFor(List<(string, string[])> jsonCollection, int i, int length, List<(string, List<(string, Shared.Models.Property?)>)> results) - { - string key; - string json; - Shared.Models.Property? property; - (string sourceDirectory, string[] sourceDirectoryFiles) = jsonCollection[i]; - List<(string, Shared.Models.Property?)> collection = new(); - foreach (string sourceDirectoryFile in sourceDirectoryFiles) - { - json = File.ReadAllText(sourceDirectoryFile); - if (string.IsNullOrEmpty(json)) - continue; - key = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFile, length); - property = JsonSerializer.Deserialize(json); - collection.Add(new(sourceDirectoryFile, property)); - } - lock (results) - results.Add(new(sourceDirectory, collection)); - } - - private static List<(string sourceDirectory, List<(string sourceDirectoryFile, Shared.Models.Property? property)> collection)> GetCollection(string rootDirectory, List<(string sourceDirectory, string[] SourceDirectoryFiles)> jsonCollection) - { - List<(string, List<(string, Shared.Models.Property?)>)> results = new(); - int length = rootDirectory.Length; - ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = Environment.ProcessorCount }; - _ = Parallel.For(0, jsonCollection.Count, parallelOptions, (i, state) => ParallelFor(jsonCollection, i, length, results)); - return results; - } - - private static Item GetNewItem(Configuration configuration, string relativePath, FileHolder sourceDirectoryFileHolder) - { - Item result; - Shared.Models.Property? property = null; - bool isValidImageFormatExtension = configuration.ValidImageFormatExtensions.Contains(sourceDirectoryFileHolder.ExtensionLowered); - result = new(sourceDirectoryFileHolder, relativePath, sourceDirectoryFileHolder, isValidImageFormatExtension, property, null, null, null); - return result; - } - - private static Shared.Models.Container[] GetContainers(Configuration configuration, A_Property propertyLogic, string aPropertySingletonDirectory, List<(int, string, FileHolder[])> fileHolderGroupCollection, List<(string, List<(string, Shared.Models.Property?)>)> collectionFromJson) - { - Shared.Models.Container[] results; - Item item; - int length; - string inferred; - List items; - const int last = 9; - string keyWithJson; - string relativePath; - bool fileSizeChanged; - FileHolder fileHolder; - FileHolder keyFileHolder; - bool lastWriteTimeChanged; - bool isValidImageFormatExtension; - List keySourceDirectories; - Shared.Models.Container container; - Dictionary keyValuePairs = new(); - Dictionary fileHolderKeyValuePairs = new(); - length = configuration.RootDirectory.Length; - foreach ((int g, string sourceDirectory, FileHolder[] sourceDirectoryFileHolderCollection) in fileHolderGroupCollection) - { - foreach (FileHolder sourceDirectoryFileHolder in sourceDirectoryFileHolderCollection) - { - relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFileHolder.FullName, length, forceExtensionToLower: true); - if (fileHolderKeyValuePairs.ContainsKey(relativePath)) - throw new NotSupportedException(); - fileHolderKeyValuePairs.Add(relativePath, new(sourceDirectory, sourceDirectoryFileHolder)); - } - } - length = aPropertySingletonDirectory.Length; - foreach ((string _, List<(string, Shared.Models.Property?)> collection) in collectionFromJson) - { - if (!collection.Any()) - continue; - items = new(); - keySourceDirectories = new(); - foreach ((string sourceDirectoryFile, Shared.Models.Property? property) in collection) - { - fileHolder = new(sourceDirectoryFile); - keyWithJson = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFile, length, forceExtensionToLower: false); - relativePath = keyWithJson[..^5]; - if (!fileHolderKeyValuePairs.ContainsKey(relativePath)) - { - inferred = string.Concat(configuration.RootDirectory, relativePath); - keyFileHolder = new(inferred); - if (keyFileHolder.ExtensionLowered is ".json") - continue; - keySourceDirectories.Add(string.Concat(keyFileHolder.DirectoryName)); - isValidImageFormatExtension = configuration.ValidImageFormatExtensions.Contains(keyFileHolder.ExtensionLowered); - item = new(fileHolder, relativePath, keyFileHolder, isValidImageFormatExtension, property, true, null, null); - items.Add(item); - } - else - { - keyFileHolder = fileHolderKeyValuePairs[relativePath].FileHolder; - keySourceDirectories.Add(fileHolderKeyValuePairs[relativePath].SourceDirectory); - if (!fileHolderKeyValuePairs.Remove(relativePath)) - throw new Exception(); - if (keyFileHolder.ExtensionLowered is ".json") - continue; - isValidImageFormatExtension = configuration.ValidImageFormatExtensions.Contains(keyFileHolder.ExtensionLowered); - if (property?.Id is null || property?.Width is null || property?.Height is null) - item = new(fileHolder, relativePath, keyFileHolder, isValidImageFormatExtension, property, false, null, null); - else - { - fileSizeChanged = property.FileSize != keyFileHolder.Length; - lastWriteTimeChanged = configuration.PropertiesChangedForProperty || property.LastWriteTime != keyFileHolder.LastWriteTime; - item = new(fileHolder, relativePath, keyFileHolder, isValidImageFormatExtension, property, false, fileSizeChanged, lastWriteTimeChanged); - } - items.Add(item); - } - } - if (items.Any()) - { - if (keySourceDirectories.Distinct().Count() != 1) - continue; - container = new(last, items, keySourceDirectories[0]); - keyValuePairs.Add(keySourceDirectories[0], container); - } - } - length = configuration.RootDirectory.Length; - foreach ((int g, string sourceDirectory, FileHolder[] sourceDirectoryFileHolderCollection) in fileHolderGroupCollection) - { - items = new(); - propertyLogic.SetAngleBracketCollection(sourceDirectory); - foreach (FileHolder sourceDirectoryFileHolder in sourceDirectoryFileHolderCollection) - { - relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFileHolder.FullName, length, forceExtensionToLower: true); - if (!fileHolderKeyValuePairs.ContainsKey(relativePath)) - continue; - if (!fileHolderKeyValuePairs.Remove(relativePath)) - throw new Exception(); - if (sourceDirectoryFileHolder.ExtensionLowered is ".json") - continue; - item = GetNewItem(configuration, relativePath, sourceDirectoryFileHolder); - items.Add(item); - } - if (sourceDirectory == configuration.RootDirectory || items.Any()) - { - if (!keyValuePairs.ContainsKey(sourceDirectory)) - { - container = new(g, items, sourceDirectory); - keyValuePairs.Add(sourceDirectory, container); - } - else - { - container = keyValuePairs[sourceDirectory]; - items = Shared.Models.Stateless.Methods.IItem.GetMerged(container.Items, items); - container = new(container.G, items, container.SourceDirectory); - keyValuePairs[sourceDirectory] = container; - } - } - } - if (fileHolderKeyValuePairs.Any()) - throw new NotSupportedException("Unmapped left!"); - results = (from l in keyValuePairs orderby l.Value.G, l.Value.Items.Count select l.Value).ToArray(); - return results; - } - - private static void SetCollections(string aPropertySingletonDirectory, List otherCollection, List<(string, string[])> jsonCollection) - { - string searchPattern = "*"; - (string, string[])[] collection = Shared.Models.Stateless.Methods.IFileHolder.GetFiles(aPropertySingletonDirectory, searchPattern).ToArray(); - foreach ((string sourceDirectory, string[] files) in collection) - { - otherCollection.AddRange(files.Where(l => !l.EndsWith(".json"))); - jsonCollection.Add(new(sourceDirectory, files.Where(l => l.EndsWith(".json")).ToArray())); - } - } - - private static void DeleteAbandoned(Configuration configuration, string aPropertySingletonDirectory, List otherCollection, List<(int g, string sourceDirectory, FileHolder[] sourceDirectoryFiles)> fileHolderGroupCollection) - { - string relativePath; - string checkFileName; - string? checkDirectoryName; - List checkCollection = new(); - checkCollection.AddRange(otherCollection); - int length = configuration.RootDirectory.Length; - foreach ((int g, string sourceDirectory, FileHolder[] sourceDirectoryFiles) in fileHolderGroupCollection) - { - foreach (FileHolder fileHolder in sourceDirectoryFiles) - { - relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(fileHolder.FullName, length); - checkFileName = Path.GetFullPath(string.Concat(aPropertySingletonDirectory, relativePath)); - checkDirectoryName = Path.GetDirectoryName(checkFileName); - if (string.IsNullOrEmpty(checkDirectoryName) || !Directory.Exists(checkDirectoryName)) - continue; - if (!checkCollection.Remove(checkFileName)) - File.WriteAllText(checkFileName, string.Empty); - } - } - foreach (string file in checkCollection) - File.Delete(file); - } - - public static (int, int, int, Shared.Models.Container[]) GetContainers(Configuration configuration, A_Property propertyLogic) - { - Shared.Models.Container[] results; - string searchPattern = "*"; - List topDirectories = new(); - List otherCollection = new(); - List<(string, string[])> jsonCollection = new(); - List<(int g, string sourceDirectory, FileHolder[] sourceDirectoryFiles)> fileHolderGroupCollection; - string aPropertySingletonDirectory = IResult.GetResultsDateGroupDirectory(configuration, nameof(A_Property), "{}"); - SetCollections(aPropertySingletonDirectory, otherCollection, jsonCollection); - (int f, fileHolderGroupCollection) = GetFileHolderGroupCollection(configuration, propertyLogic, searchPattern, topDirectories); - DeleteAbandoned(configuration, aPropertySingletonDirectory, otherCollection, fileHolderGroupCollection); - List<(string sourceDirectory, List<(string sourceDirectoryFile, Shared.Models.Property? property)> collection)> collectionFromJson; - collectionFromJson = GetCollection(aPropertySingletonDirectory, jsonCollection); - results = GetContainers(configuration, propertyLogic, aPropertySingletonDirectory, fileHolderGroupCollection, collectionFromJson); - int t = (from l in results select l.Items.Count).Sum(); - int j = jsonCollection.Select(l => l.Item2.Length).Sum(); - return (j, f, t, results); - } - -} \ No newline at end of file diff --git a/Rename/Rename.cs b/Rename/Rename.cs index f579d0c..ac9b0c6 100644 --- a/Rename/Rename.cs +++ b/Rename/Rename.cs @@ -156,10 +156,7 @@ public class Rename if (dateTime is not null && timeSpan.TotalMinutes > 2) { checkFileExtension = fileHolder.ExtensionLowered == jpeg ? jpg : fileHolder.ExtensionLowered; - if (!isIgnoreExtension && isValidImageFormatExtension) - checkFile = Path.Combine(fileHolder.DirectoryName, $"{dateTime.Value:yyyy-MM-dd}.{dateTime.Value.Ticks}{checkFileExtension}"); - else - checkFile = Path.Combine(fileHolder.DirectoryName, $"{dateTime.Value:yyyy-MM-dd}.{dateTime.Value.Ticks}.{fileHolder.Length}{checkFileExtension}"); + checkFile = Path.Combine(fileHolder.DirectoryName, $"{dateTime.Value:yyyy-MM-dd}.{dateTime.Value.Ticks}.{fileHolder.Length}{checkFileExtension}"); if (checkFile == fileHolder.FullName) continue; if (distinctCollection.Contains(checkFile)) @@ -172,10 +169,17 @@ public class Rename continue; checkFileExtension = fileHolder.ExtensionLowered == jpeg ? jpg : fileHolder.ExtensionLowered; checkFile = Path.Combine(fileHolder.DirectoryName, $"{id.Value}{checkFileExtension}"); - if (checkFile == fileHolder.FullName || File.Exists(checkFile)) + if (checkFile == fileHolder.FullName) continue; + if (File.Exists(checkFile)) + { + checkFile = string.Concat(checkFile, ".del"); + if (File.Exists(checkFile)) + continue; + } if (distinctCollection.Contains(checkFile)) continue; + distinctCollection.Add(checkFile); results.Add(new(fileHolder, checkFile)); } return new(results, distinctCollection.Count); diff --git a/Shared/Models/Container.cs b/Shared/Models/Container.cs index 3011f73..d3c5e56 100644 --- a/Shared/Models/Container.cs +++ b/Shared/Models/Container.cs @@ -1,33 +1,10 @@ using System.Text.Json; -using System.Text.Json.Serialization; namespace View_by_Distance.Shared.Models; -public class Container : Properties.IContainer +public record Container(string SourceDirectory, List Items) { - protected readonly int _G; - protected readonly List _Items; - protected readonly string _SourceDirectory; - public int G => _G; - public List Items => _Items; - public string SourceDirectory => _SourceDirectory; - - [JsonConstructor] - public Container(int g, List items, string sourceDirectory) - { - _G = g; - _Items = items; - _SourceDirectory = sourceDirectory; - } - - public Container(int g, string sourceDirectory) - { - _G = g; - _Items = new(); - _SourceDirectory = sourceDirectory; - } - public override string ToString() { string result = JsonSerializer.Serialize(this, new JsonSerializerOptions() { WriteIndented = true }); diff --git a/Shared/Models/FilePair.cs b/Shared/Models/FilePair.cs new file mode 100644 index 0000000..9722211 --- /dev/null +++ b/Shared/Models/FilePair.cs @@ -0,0 +1,3 @@ +namespace View_by_Distance.Shared.Models; + +public record FilePair(string Path, bool IsUnique, List Collection, string? Match) { } \ No newline at end of file diff --git a/Shared/Models/Item.cs b/Shared/Models/Item.cs index 0254800..43cfba1 100644 --- a/Shared/Models/Item.cs +++ b/Shared/Models/Item.cs @@ -6,50 +6,47 @@ namespace View_by_Distance.Shared.Models; public class Item : Properties.IItem { - protected readonly bool? _Abandoned; protected List _Faces; protected readonly bool? _FileSizeChanged; protected readonly FileHolder _ImageFileHolder; + protected bool? _IsUniqueFileName; + protected bool _IsValidImageFormatExtension; protected bool? _LastWriteTimeChanged; protected bool? _Moved; - protected readonly bool? _NoJson; protected Property? _Property; protected readonly string _RelativePath; protected FileHolder? _ResizedFileHolder; protected readonly FileHolder _SourceDirectoryFileHolder; - protected bool _ValidImageFormatExtension; - public bool? Abandoned => _Abandoned; public List Faces => _Faces; public bool? FileSizeChanged => _FileSizeChanged; public FileHolder ImageFileHolder => _ImageFileHolder; + public bool? IsUniqueFileName => _IsUniqueFileName; + public bool IsValidImageFormatExtension => _IsValidImageFormatExtension; public bool? LastWriteTimeChanged => _LastWriteTimeChanged; public bool? Moved => _Moved; - public bool? NoJson => _NoJson; public Property? Property => _Property; public string RelativePath => _RelativePath; public FileHolder? ResizedFileHolder => _ResizedFileHolder; public FileHolder SourceDirectoryFileHolder => _SourceDirectoryFileHolder; - public bool ValidImageFormatExtension => _ValidImageFormatExtension; [JsonConstructor] - public Item(bool? abandoned, List faces, bool? fileSizeChanged, FileHolder imageFileHolder, bool? lastWriteTimeChanged, bool? moved, bool? noJson, Property? property, string relativePath, FileHolder? resizedFileHolder, FileHolder sourceDirectoryFileHolder, bool validImageFormatExtension) + public Item(List faces, bool? fileSizeChanged, FileHolder imageFileHolder, bool? isUniqueFileName, bool isValidImageFormatExtension, bool? lastWriteTimeChanged, bool? moved, Property? property, string relativePath, FileHolder? resizedFileHolder, FileHolder sourceDirectoryFileHolder) { _Faces = faces; - _Moved = moved; - _NoJson = noJson; - _Property = property; - _Abandoned = abandoned; - _RelativePath = relativePath; _FileSizeChanged = fileSizeChanged; _ImageFileHolder = imageFileHolder; + _IsUniqueFileName = isUniqueFileName; + _IsValidImageFormatExtension = isValidImageFormatExtension; + _LastWriteTimeChanged = lastWriteTimeChanged; + _Moved = moved; + _Property = property; + _RelativePath = relativePath; _ResizedFileHolder = resizedFileHolder; _SourceDirectoryFileHolder = sourceDirectoryFileHolder; - _LastWriteTimeChanged = lastWriteTimeChanged; - _ValidImageFormatExtension = validImageFormatExtension; } - public Item(FileHolder sourceDirectoryFileHolder, string relativePath, FileHolder imageFileInfo, bool validImageFormatExtension, Property? property, bool? abandoned, bool? fileSizeChanged, bool? lastWriteTimeChanged) : - this(abandoned, new(), fileSizeChanged, imageFileInfo, lastWriteTimeChanged, null, abandoned is null, property, relativePath, null, sourceDirectoryFileHolder, validImageFormatExtension) + public Item(FileHolder sourceDirectoryFileHolder, string relativePath, FileHolder imageFileInfo, bool? isUniqueFileName, bool isValidImageFormatExtension, Property? property, bool? abandoned, bool? fileSizeChanged, bool? lastWriteTimeChanged) : + this(new(), fileSizeChanged, imageFileInfo, isUniqueFileName, isValidImageFormatExtension, lastWriteTimeChanged, null, property, relativePath, null, sourceDirectoryFileHolder) { if (relativePath.EndsWith(".json")) throw new ArgumentException("Can not be a *.json file!"); @@ -58,7 +55,7 @@ public class Item : Properties.IItem } public Item(FileHolder sourceDirectoryFileHolder, string relativePath, bool isValidImageFormatExtension) : - this(sourceDirectoryFileHolder, relativePath, sourceDirectoryFileHolder, isValidImageFormatExtension, null, null, null, null) + this(sourceDirectoryFileHolder, relativePath, sourceDirectoryFileHolder, null, isValidImageFormatExtension, null, null, null, null) { } public override string ToString() @@ -79,7 +76,7 @@ public class Item : Properties.IItem _ResizedFileHolder = fileHolder; } - public bool Any() => (_Abandoned.HasValue && _Abandoned.Value) || (_FileSizeChanged.HasValue && _FileSizeChanged.Value) || (_LastWriteTimeChanged.HasValue && _LastWriteTimeChanged.Value) || (_Moved.HasValue && _Moved.Value) || (_NoJson.HasValue && _NoJson.Value); + public bool Any() => !_SourceDirectoryFileHolder.Exists || (_FileSizeChanged.HasValue && _FileSizeChanged.Value) || (_LastWriteTimeChanged.HasValue && _LastWriteTimeChanged.Value) || (_Moved.HasValue && _Moved.Value); public void Update(Property property) => _Property = property; diff --git a/Shared/Models/Properties/IContainer.cs b/Shared/Models/Properties/IContainer.cs deleted file mode 100644 index e433c8f..0000000 --- a/Shared/Models/Properties/IContainer.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace View_by_Distance.Shared.Models.Properties; - -public interface IContainer -{ - - public int G { get; } - public List Items { get; } - public string SourceDirectory { get; } - -} \ No newline at end of file diff --git a/Shared/Models/Properties/IItem.cs b/Shared/Models/Properties/IItem.cs index d94a3dd..a671a8c 100644 --- a/Shared/Models/Properties/IItem.cs +++ b/Shared/Models/Properties/IItem.cs @@ -3,16 +3,15 @@ namespace View_by_Distance.Shared.Models.Properties; public interface IItem { - public bool? Abandoned { get; } public bool? FileSizeChanged { get; } public List Faces { get; } public FileHolder ImageFileHolder { get; } + public bool? IsUniqueFileName { get; } + public bool IsValidImageFormatExtension { get; } public bool? Moved { get; } - public bool? NoJson { get; } public Property? Property { get; } public string RelativePath { get; } public FileHolder? ResizedFileHolder { get; } public FileHolder SourceDirectoryFileHolder { get; } - public bool ValidImageFormatExtension { get; } } \ No newline at end of file diff --git a/Shared/Models/Properties/IPropertyConfiguration.cs b/Shared/Models/Properties/IPropertyConfiguration.cs index 00b95ac..1df12fe 100644 --- a/Shared/Models/Properties/IPropertyConfiguration.cs +++ b/Shared/Models/Properties/IPropertyConfiguration.cs @@ -5,11 +5,13 @@ public interface IPropertyConfiguration public string DateGroup { init; get; } public string[] IgnoreExtensions { init; get; } + public bool PropertiesChangedForProperty { init; get; } public string[] PropertyContentCollectionFiles { init; get; } public string ResultAllInOne { init; get; } public string ResultCollection { init; get; } public string ResultContent { init; get; } public string ResultSingleton { init; get; } + public string[] ValidImageFormatExtensions { init; get; } public string[] VerifyToSeason { init; get; } public string? ModelName { get; } public int? NumberOfJitters { get; } diff --git a/Shared/Models/Stateless/Methods/Container.cs b/Shared/Models/Stateless/Methods/Container.cs index 4f7ff4c..2f3e560 100644 --- a/Shared/Models/Stateless/Methods/Container.cs +++ b/Shared/Models/Stateless/Methods/Container.cs @@ -1,8 +1,12 @@ +using System.Text.Json; + namespace View_by_Distance.Shared.Models.Stateless.Methods; internal abstract class Container { + private record FilePair(string Path, string? Directory, bool IsUnique, List Collection, Models.Item Item) { } + internal static DateTime[] GetContainerDateTimes(Models.Item[] filteredItems) { DateTime[] results; @@ -26,8 +30,7 @@ internal abstract class Container foreach (Models.Item item in container.Items) { if (item.ImageFileHolder is not null - && (item.Abandoned is null || !item.Abandoned.Value) - && item.ValidImageFormatExtension + && item.IsValidImageFormatExtension && !propertyConfiguration.IgnoreExtensions.Contains(item.ImageFileHolder.ExtensionLowered)) results.Add(item); } @@ -55,9 +58,7 @@ internal abstract class Container internal static Models.Container[] SortContainers(Properties.IPropertyConfiguration propertyConfiguration, string[] ignoreRelativePaths, bool argZeroIsConfigurationRootDirectory, string argZero, Models.Container[] containers) { List results = new(); - Models.Item[] filteredItems; bool isIgnoreRelativePath; - List collection = new(); for (int i = 1; i < 3; i++) { foreach (Models.Container container in containers) @@ -66,30 +67,145 @@ internal abstract class Container continue; if (!argZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero)) continue; - filteredItems = GetFilterItems(propertyConfiguration, container); - if (!filteredItems.Any()) - continue; isIgnoreRelativePath = ignoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && IsIgnoreRelativePath(propertyConfiguration, ignoreRelativePaths, container.SourceDirectory); if (i == 1 && isIgnoreRelativePath) continue; if (i == 2 && !isIgnoreRelativePath) continue; - foreach (Models.Item item in filteredItems) - { - if (item.Property?.Id is null) - continue; - if (collection.Contains(item.Property.Id.Value)) - { - if (i == 1) - continue; - continue; - } - collection.Add(item.Property.Id.Value); - } results.Add(container); } } return results.ToArray(); } + internal static List GetFilePairs(Properties.IPropertyConfiguration propertyConfiguration, string directorySearchFilter, string extension, string aPropertySingletonDirectory, List filesCollection) + { + int renamed; + List? filePairs = null; + List? jsonFilesCollection = null; + IReadOnlyDictionary>? compareFileNamesToFiles = null; + IReadOnlyDictionary> fileNamesToFiles = IDirectory.GetFilesKeyValuePairs(filesCollection); + for (int i = 0; i < int.MaxValue; i++) + { + renamed = 0; + jsonFilesCollection = IDirectory.GetFilesCollection(aPropertySingletonDirectory, directorySearchFilter, extension); + compareFileNamesToFiles = IDirectory.GetFilesKeyValuePairs(jsonFilesCollection); + renamed += IDirectory.LookForAbandoned(jsonFilesCollection, fileNamesToFiles, extension); + filePairs = IDirectory.GetFiles(filesCollection, fileNamesToFiles, extension, compareFileNamesToFiles); + renamed += IDirectory.MaybeMove(propertyConfiguration.RootDirectory, propertyConfiguration.ResultAllInOne, filePairs, aPropertySingletonDirectory, extension); + if (renamed == 0) + break; + } + if (filePairs is null || jsonFilesCollection is null || compareFileNamesToFiles is null) + throw new NullReferenceException(nameof(filePairs)); + return filePairs; + } + + private static Models.Property? GetProperty(Models.FilePair filePair) + { + Models.Property? property; + if (filePair.Match is null) + property = null; + else + { + string json = File.ReadAllText(filePair.Match); + if (string.IsNullOrEmpty(json)) + property = null; + else + property = JsonSerializer.Deserialize(json); + } + return property; + } + + private static void ParallelFor(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory, string extension, int length, Models.FilePair filePair, List results) + { + char directory; + string fileName; + bool abandoned = false; + Models.FileHolder sourceDirectoryFileHolder; + Models.Property? property = GetProperty(filePair); + Models.FileHolder imageFileInfo = new(filePair.Path); + bool? fileSizeChanged = property is not null ? property.FileSize != imageFileInfo.Length : null; + string relativePath = IPath.GetRelativePath(filePair.Path, length, forceExtensionToLower: true); + bool isValidImageFormatExtension = propertyConfiguration.ValidImageFormatExtensions.Contains(imageFileInfo.ExtensionLowered); + bool? lastWriteTimeChanged = property is not null ? propertyConfiguration.PropertiesChangedForProperty || property.LastWriteTime != imageFileInfo.LastWriteTime : null; + if (filePair.Match is not null) + sourceDirectoryFileHolder = new(filePair.Match); + else if (!filePair.IsUnique) + sourceDirectoryFileHolder = new(Path.GetFullPath(string.Concat(aPropertySingletonDirectory, relativePath, extension))); + else + { + fileName = Path.GetFileName(filePair.Path); + directory = IDirectory.GetDirectory(fileName); + sourceDirectoryFileHolder = new(Path.Combine(aPropertySingletonDirectory, propertyConfiguration.ResultAllInOne, directory.ToString(), $"{fileName}{extension}")); + } + if (imageFileInfo.LastWriteTime is not null && sourceDirectoryFileHolder.CreationTime is not null && sourceDirectoryFileHolder.LastWriteTime is not null && imageFileInfo.LastWriteTime.Value != sourceDirectoryFileHolder.CreationTime.Value) + { + File.SetCreationTime(sourceDirectoryFileHolder.FullName, imageFileInfo.LastWriteTime.Value); + File.SetLastWriteTime(sourceDirectoryFileHolder.FullName, sourceDirectoryFileHolder.LastWriteTime.Value); + } + Models.Item item = new(sourceDirectoryFileHolder, relativePath, imageFileInfo, filePair.IsUnique, isValidImageFormatExtension, property, abandoned, fileSizeChanged, lastWriteTimeChanged); + lock (results) + results.Add(new(filePair.Path, imageFileInfo.DirectoryName, filePair.IsUnique, filePair.Collection, item)); + } + + private static List GetFilePairs(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory, string extension, List filePairs) + { + List results = new(); + int length = propertyConfiguration.RootDirectory.Length; + int maxDegreeOfParallelism = Environment.ProcessorCount; + ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism }; + _ = Parallel.For(0, filePairs.Count, parallelOptions, (i, state) => ParallelFor(propertyConfiguration, aPropertySingletonDirectory, extension, length, filePairs[i], results)); + return results; + } + + internal static (int, Models.Container[]) GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory) + { + List results = new(); + string? directory; + List? items; + Models.Container container; + const string extension = ".json"; + const string fileSearchFilter = "*"; + const string directorySearchFilter = "*"; + Dictionary> directoryToItems = new(); + List filesCollection = IDirectory.GetFilesCollection(propertyConfiguration.RootDirectory, directorySearchFilter, fileSearchFilter); + foreach (string[] files in filesCollection) + { + if (!files.Any()) + continue; + directory = Path.GetDirectoryName(files.First()); + if (directory is null) + continue; + if (!directoryToItems.TryGetValue(directory, out items)) + { + directoryToItems.Add(directory, new()); + if (!directoryToItems.TryGetValue(directory, out items)) + throw new Exception(); + } + } + List filePairs = GetFilePairs(propertyConfiguration, directorySearchFilter, extension, aPropertySingletonDirectory, filesCollection); + List collection = GetFilePairs(propertyConfiguration, aPropertySingletonDirectory, extension, filePairs); + foreach (FilePair filePair in collection) + { + if (filePair.Directory is null) + continue; + if (!directoryToItems.TryGetValue(filePair.Directory, out items)) + { + directoryToItems.Add(filePair.Directory, new()); + if (!directoryToItems.TryGetValue(filePair.Directory, out items)) + throw new Exception(); + } + items.Add(filePair.Item); + } + foreach (KeyValuePair> keyValuePair in directoryToItems) + { + if (!keyValuePair.Value.Any()) + continue; + container = new(keyValuePair.Key, keyValuePair.Value); + results.Add(container); + } + return (collection.Count, results.ToArray()); + } + } \ No newline at end of file diff --git a/Shared/Models/Stateless/Methods/FileHolder.cs b/Shared/Models/Stateless/Methods/FileHolder.cs index b8917c3..99570a6 100644 --- a/Shared/Models/Stateless/Methods/FileHolder.cs +++ b/Shared/Models/Stateless/Methods/FileHolder.cs @@ -42,90 +42,4 @@ internal abstract class FileHolder } } - internal static (int t, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)>) GetGroupCollection(string rootDirectory, int maxImagesInDirectoryForTopLevelFirstPass, bool reverse, string searchPattern, List topDirectories) - { - int result = 0; - List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)> results = new(); - string? parentDirectory; - string[] subDirectories; - string[] sourceDirectoryFiles; - List fileCollections = new(); - if (!topDirectories.Any()) - { - if (!Directory.Exists(rootDirectory)) - _ = Directory.CreateDirectory(rootDirectory); - topDirectories.AddRange(from l in Directory.GetDirectories(rootDirectory, "*", SearchOption.TopDirectoryOnly) select Path.GetFullPath(l)); - } - for (int g = 1; g < 5; g++) - { - if (g == 4) - { - for (int i = fileCollections.Count - 1; i > -1; i--) - { - parentDirectory = Path.GetDirectoryName(fileCollections[i][0]); - if (string.IsNullOrEmpty(parentDirectory)) - continue; - results.Add(new(g, parentDirectory, fileCollections[i])); - fileCollections.RemoveAt(i); - } - } - else if (g == 2) - { - for (int i = fileCollections.Count - 1; i > -1; i--) - { - if (fileCollections[i].Length > maxImagesInDirectoryForTopLevelFirstPass * g) - break; - parentDirectory = Path.GetDirectoryName(fileCollections[i][0]); - if (string.IsNullOrEmpty(parentDirectory)) - continue; - results.Add(new(g, parentDirectory, fileCollections[i])); - fileCollections.RemoveAt(i); - } - } - else if (g == 3) - { - subDirectories = Directory.GetDirectories(rootDirectory, "*", SearchOption.AllDirectories); - if (reverse) - subDirectories = subDirectories.Reverse().ToArray(); - foreach (string subDirectory in subDirectories) - { - sourceDirectoryFiles = Directory.GetFiles(subDirectory, "*", SearchOption.TopDirectoryOnly); - result += sourceDirectoryFiles.Length; - if (!topDirectories.Contains(subDirectory)) - results.Add(new(g, subDirectory, sourceDirectoryFiles)); - } - } - else if (g == 1) - { - sourceDirectoryFiles = Directory.GetFiles(rootDirectory, searchPattern, SearchOption.TopDirectoryOnly); - result += sourceDirectoryFiles.Length; - if (sourceDirectoryFiles.Length > maxImagesInDirectoryForTopLevelFirstPass) - fileCollections.Add(sourceDirectoryFiles); - else - results.Add(new(g, rootDirectory, sourceDirectoryFiles)); - if (reverse) - topDirectories.Reverse(); - subDirectories = topDirectories.ToArray(); - foreach (string subDirectory in subDirectories) - { - sourceDirectoryFiles = Directory.GetFiles(subDirectory, searchPattern, SearchOption.TopDirectoryOnly); - result += sourceDirectoryFiles.Length; - if (sourceDirectoryFiles.Length > maxImagesInDirectoryForTopLevelFirstPass) - fileCollections.Add(sourceDirectoryFiles); - else - { - if (sourceDirectoryFiles.Any() || Directory.GetDirectories(subDirectory, "*", SearchOption.TopDirectoryOnly).Any()) - results.Add(new(g, subDirectory, sourceDirectoryFiles)); - else if (searchPattern == "*" && subDirectory != rootDirectory) - Directory.Delete(subDirectory); - } - } - fileCollections.Reverse(); - } - else - throw new Exception(); - } - return new(result, results); - } - } \ No newline at end of file diff --git a/Shared/Models/Stateless/Methods/IContainer.cs b/Shared/Models/Stateless/Methods/IContainer.cs index 05bbd5c..fcbb717 100644 --- a/Shared/Models/Stateless/Methods/IContainer.cs +++ b/Shared/Models/Stateless/Methods/IContainer.cs @@ -23,4 +23,14 @@ public interface IContainer static Models.Container[] SortContainers(Properties.IPropertyConfiguration propertyConfiguration, string[] ignoreRelativePaths, bool argZeroIsConfigurationRootDirectory, string argZero, Models.Container[] containers) => Container.SortContainers(propertyConfiguration, ignoreRelativePaths, argZeroIsConfigurationRootDirectory, argZero, containers); + List TestStatic_GetFilePairs(Properties.IPropertyConfiguration propertyConfiguration, string directorySearchFilter, string extension, string aPropertySingletonDirectory, List filesCollection) => + Container.GetFilePairs(propertyConfiguration, directorySearchFilter, extension, aPropertySingletonDirectory, filesCollection); + static List GetFilePairs(Properties.IPropertyConfiguration propertyConfiguration, string directorySearchFilter, string extension, string aPropertySingletonDirectory, List filesCollection) => + Container.GetFilePairs(propertyConfiguration, directorySearchFilter, extension, aPropertySingletonDirectory, filesCollection); + + (int, Models.Container[]) TestStatic_GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory) => + GetContainers(propertyConfiguration, aPropertySingletonDirectory); + static (int, Models.Container[]) GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory) => + Container.GetContainers(propertyConfiguration, aPropertySingletonDirectory); + } \ No newline at end of file diff --git a/Shared/Models/Stateless/Methods/IDirectory.cs b/Shared/Models/Stateless/Methods/IDirectory.cs new file mode 100644 index 0000000..a394e1f --- /dev/null +++ b/Shared/Models/Stateless/Methods/IDirectory.cs @@ -0,0 +1,39 @@ +namespace View_by_Distance.Shared.Models.Stateless.Methods; + +public interface IDirectory +{ + + char TestStatic_GetDirectory(string fileName) => + GetDirectory(fileName); + static char GetDirectory(string fileName) => fileName.Split('-').Length > 2 ? '-' : fileName.Split('.')[0][^1]; + + int TestStatic_GetDirectory(char directory) => + GetDirectory(directory); + static int GetDirectory(char directory) => directory == '-' ? 10 : int.Parse(directory.ToString()); + + List TestStatic_GetFilesCollection(string directory, string directorySearchFilter, string fileSearchFilter) => + GetFilesCollection(directory, directorySearchFilter, fileSearchFilter); + static List GetFilesCollection(string directory, string directorySearchFilter, string fileSearchFilter) => + XDirectory.GetFilesCollection(directory, directorySearchFilter, fileSearchFilter); + + IReadOnlyDictionary> TestStatic_GetFilesKeyValuePairs(List filesCollection) => + GetFilesKeyValuePairs(filesCollection); + static IReadOnlyDictionary> GetFilesKeyValuePairs(List filesCollection) => + XDirectory.GetFilesKeyValuePairs(filesCollection); + + int TestStatic_LookForAbandoned(List jsonFilesCollection, IReadOnlyDictionary> fileNamesToFiles, string extension) => + LookForAbandoned(jsonFilesCollection, fileNamesToFiles, extension); + static int LookForAbandoned(List jsonFilesCollection, IReadOnlyDictionary> fileNamesToFiles, string extension) => + XDirectory.LookForAbandoned(jsonFilesCollection, fileNamesToFiles, extension); + + int TestStatic_MaybeMove(string directory, string resultAllInOne, List filePairs, string jsonGroupDirectory, string extension) => + MaybeMove(directory, resultAllInOne, filePairs, jsonGroupDirectory, extension); + static int MaybeMove(string directory, string resultAllInOne, List filePairs, string jsonGroupDirectory, string extension) => + XDirectory.MaybeMove(directory, resultAllInOne, filePairs, jsonGroupDirectory, extension); + + List TestStatic_GetFiles(List filesCollection, IReadOnlyDictionary> fileNamesToFiles, string extension, IReadOnlyDictionary> compareFileNamesToFiles) => + GetFiles(filesCollection, fileNamesToFiles, extension, compareFileNamesToFiles); + static List GetFiles(List filesCollection, IReadOnlyDictionary> fileNamesToFiles, string extension, IReadOnlyDictionary> compareFileNamesToFiles) => + XDirectory.GetFiles(filesCollection, fileNamesToFiles, extension, compareFileNamesToFiles); + +} \ No newline at end of file diff --git a/Shared/Models/Stateless/Methods/IFileHolder.cs b/Shared/Models/Stateless/Methods/IFileHolder.cs index 580b492..7b9b085 100644 --- a/Shared/Models/Stateless/Methods/IFileHolder.cs +++ b/Shared/Models/Stateless/Methods/IFileHolder.cs @@ -23,9 +23,4 @@ public interface IFileHolder static Models.FileHolder Refresh(Models.FileHolder fileHolder) => new(fileHolder.FullName); - (int t, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)>) TestStatic_GetGroupCollection(string rootDirectory, int maxImagesInDirectoryForTopLevelFirstPass, bool reverse, string searchPattern, List topDirectories) => - GetGroupCollection(rootDirectory, maxImagesInDirectoryForTopLevelFirstPass, reverse, searchPattern, topDirectories); - static (int t, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)>) GetGroupCollection(string rootDirectory, int maxImagesInDirectoryForTopLevelFirstPass, bool reverse, string searchPattern, List topDirectories) => - FileHolder.GetGroupCollection(rootDirectory, maxImagesInDirectoryForTopLevelFirstPass, reverse, searchPattern, topDirectories); - } \ No newline at end of file diff --git a/Shared/Models/Stateless/Methods/XDirectory.cs b/Shared/Models/Stateless/Methods/XDirectory.cs new file mode 100644 index 0000000..8885c58 --- /dev/null +++ b/Shared/Models/Stateless/Methods/XDirectory.cs @@ -0,0 +1,243 @@ +namespace View_by_Distance.Shared.Models.Stateless.Methods; + +internal abstract partial class XDirectory +{ + + private static int GetCeilingAverage(List fileCollection) + { + List counts = new(); + foreach (string[] files in fileCollection) + counts.Add(files.Length); + int average = (int)Math.Ceiling(counts.Average()); + return average; + } + + private static List GetFilesCollection(List fileCollection, int ceilingAverage) + { + List results = new(); + foreach (string[] files in fileCollection) + { + if (files.Length < ceilingAverage) + results.Add(files); + } + foreach (string[] files in fileCollection) + { + if (files.Length >= ceilingAverage) + results.Add(files); + } + return results; + } + + internal static List GetFilesCollection(string directory, string directorySearchFilter, string fileSearchFilter) + { + List results = new(); + if (!fileSearchFilter.Contains('*')) + fileSearchFilter = string.Concat('*', fileSearchFilter); + if (!directorySearchFilter.Contains('*')) + directorySearchFilter = string.Concat('*', directorySearchFilter); + string[] directories = Directory.GetDirectories(directory, directorySearchFilter, SearchOption.TopDirectoryOnly); + foreach (string innerDirectory in directories) + results.Add(Directory.GetFiles(innerDirectory, fileSearchFilter, SearchOption.AllDirectories)); + int ceilingAverage = GetCeilingAverage(results); + results = GetFilesCollection(results, ceilingAverage); + return results; + } + + internal static IReadOnlyDictionary> GetFilesKeyValuePairs(List filesCollection) + { + Dictionary> results = new(); + string fileName; + List? collection; + foreach (string[] files in filesCollection) + { + foreach (string file in files) + { + fileName = Path.GetFileName(file); + if (!results.TryGetValue(fileName, out collection)) + { + results.Add(fileName, new()); + if (!results.TryGetValue(fileName, out collection)) + throw new Exception(); + } + collection.Add(file); + } + } + return results; + } + + internal static int LookForAbandoned(List jsonFilesCollection, IReadOnlyDictionary> fileNamesToFiles, string extension) + { + string fileName; + List? collection; + int length = extension.Length; + List<(string, string)> rename = new(); + foreach (string[] files in jsonFilesCollection) + { + foreach (string file in files) + { + fileName = Path.GetFileName(file); + if (fileName.Length < length || !fileName.EndsWith(extension)) + throw new Exception(); + if (!fileNamesToFiles.TryGetValue(fileName[..^length], out collection)) + rename.Add(new(file, string.Concat(file, ".del"))); + + } + } + foreach ((string from, string to) in rename) + { + if (File.Exists(to)) + continue; + File.Move(from, to); + } + return rename.Count; + } + + private static void IsNotUniqueLoop(string file, List collection) + { + TimeSpan timeSpan; + FileInfo possibleFileInfo; + FileInfo fileInfo = new(file); + foreach (string possible in collection) + { + possibleFileInfo = new(possible); + if (possibleFileInfo.LastWriteTime == fileInfo.LastWriteTime && possibleFileInfo.Length == fileInfo.Length) + continue; + timeSpan = new(possibleFileInfo.LastWriteTime.Ticks - fileInfo.LastWriteTime.Ticks); + if (timeSpan.TotalMinutes < 2 && possibleFileInfo.Length == fileInfo.Length) + { + File.SetLastWriteTime(file, new DateTime[] { possibleFileInfo.LastWriteTime, fileInfo.LastWriteTime }.Min()); + continue; + } + throw new Exception(); + } + } + + private static string? GetMatch(string file, List collection) + { + string? result = null; + FileInfo possibleFileInfo; + List lengths = new(); + List matches = new(); + FileInfo fileInfo = new(file); + List creationTimes = new(); + foreach (string possible in collection) + { + possibleFileInfo = new(possible); + lengths.Add(possibleFileInfo.Length); + creationTimes.Add(possibleFileInfo.CreationTime); + if (possibleFileInfo.CreationTime != fileInfo.LastWriteTime) + continue; + matches.Add(possible); + } + if (matches.Count == 1 || (lengths.Distinct().Count() == 1 && creationTimes.Distinct().Count() == 1)) + result = matches.First(); + return result; + } + + internal static List GetFiles(List filesCollection, IReadOnlyDictionary> fileNamesToFiles, string extension, IReadOnlyDictionary> compareFileNamesToFiles) + { + List results = new(); + string? match; + string fileName; + bool uniqueFileName; + List? collection; + foreach (string[] files in filesCollection) + { + foreach (string file in files) + { + fileName = Path.GetFileName(file); + if (!fileNamesToFiles.TryGetValue(fileName, out collection)) + throw new Exception(); + uniqueFileName = collection.Count == 1; + if (!uniqueFileName) + IsNotUniqueLoop(file, collection); + if (!compareFileNamesToFiles.TryGetValue(string.Concat(fileName, extension), out collection)) + results.Add(new(file, uniqueFileName, new(), null)); + else + { + if (!collection.Any()) + results.Add(new(file, uniqueFileName, collection, null)); + else if (uniqueFileName && collection.Count == 1) + results.Add(new(file, uniqueFileName, collection, collection.First())); + else + { + match = GetMatch(file, collection); + results.Add(new(file, uniqueFileName, collection, match)); + } + } + } + } + return results; + } + + private static void IsUniqueLoop(string resultAllInOne, string resultAllInOneDirectory, FilePair item, List<(string, string)> rename) + { + char directory; + string fileName; + foreach (string path in item.Collection) + { + if (path.Contains(resultAllInOne)) + continue; + fileName = Path.GetFileName(path); + directory = IDirectory.GetDirectory(fileName); + rename.Add(new(path, Path.Combine(resultAllInOneDirectory, directory.ToString(), fileName))); + } + } + + private static void IsNotUniqueLoop(string directory, string resultAllInOne, string jsonGroupDirectory, string extension, FilePair item, List<(string, string)> rename) + { + int length = directory.Length; + foreach (string path in item.Collection) + { + if (!path.Contains(resultAllInOne)) + continue; + if (item.Match is null || path != item.Match) + continue; + rename.Add(new(path, string.Concat(jsonGroupDirectory, item.Path[length..], extension))); + } + } + + internal static int MaybeMove(string directory, string resultAllInOne, List filePairs, string jsonGroupDirectory, string extension) + { + FileInfo? toFileInfo; + FileInfo fromFileInfo; + string checkDirectory; + List<(string, string)> rename = new(); + string resultAllInOneDirectory = Path.Combine(jsonGroupDirectory, resultAllInOne); + foreach (FilePair item in filePairs) + { + if (item.IsUnique) + IsUniqueLoop(resultAllInOne, resultAllInOneDirectory, item, rename); + else + IsNotUniqueLoop(directory, resultAllInOne, jsonGroupDirectory, extension, item, rename); + } + foreach ((string from, string to) in rename) + { + toFileInfo = null; + checkDirectory = to; + fromFileInfo = new(from); + if (!fromFileInfo.Exists) + continue; + for (int i = 0; i < int.MaxValue; i++) + { + toFileInfo = new(checkDirectory); + if (toFileInfo.Directory is null) + continue; + if (!toFileInfo.Directory.Exists) + _ = Directory.CreateDirectory(toFileInfo.Directory.FullName); + if (checkDirectory.Length > 199) + throw new Exception(); + if (!toFileInfo.Exists) + break; + else if (fromFileInfo.Length == toFileInfo.Length && fromFileInfo.LastWriteTime == toFileInfo.LastWriteTime) + checkDirectory = string.Concat(checkDirectory, ".del"); + else + checkDirectory = string.Concat(checkDirectory, ".i"); + } + File.Move(from, checkDirectory); + } + _ = IPath.DeleteEmptyDirectories(jsonGroupDirectory); + return rename.Count; + } + +} \ No newline at end of file diff --git a/Tests/UnitTestCalculations.cs b/Tests/UnitTestCalculations.cs index 6bcb568..32f0cdc 100644 --- a/Tests/UnitTestCalculations.cs +++ b/Tests/UnitTestCalculations.cs @@ -38,6 +38,7 @@ public partial class UnitTestCalculations isEnvironment = new(processesCount: null, nullASPNetCoreEnvironmentIsDevelopment: debuggerWasAttachedAtLineZero, nullASPNetCoreEnvironmentIsProduction: !debuggerWasAttachedAtLineZero); IConfigurationBuilder configurationBuilder = new ConfigurationBuilder() .AddEnvironmentVariables() + .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) .AddJsonFile(isEnvironment.AppSettingsFileName, optional: false, reloadOnChange: true); configurationRoot = configurationBuilder.Build(); appSettings = Models.Binder.AppSettings.Get(configurationRoot); @@ -368,4 +369,40 @@ public partial class UnitTestCalculations Assert.IsTrue(check == 200); } + [TestMethod] + public void TestMovingFiles() + { + char directory; + string fileName; + string checkFile; + int directoryIndex; + string checkDirectory; + List collection = new(); + string source = "C:/1234567890123456789"; + if (Directory.Exists(source)) + { + for (int i = 0; i < 11; i++) + { + if (i == 10) + checkDirectory = Path.Combine(source, "-"); + else + checkDirectory = Path.Combine(source, i.ToString()); + if (!Directory.Exists(checkDirectory)) + _ = Directory.CreateDirectory(checkDirectory); + collection.Add(checkDirectory); + } + Dictionary jsonGroups = new() { { "{}", collection.ToArray() } }; + foreach (string file in Directory.GetFiles(source)) + { + fileName = Path.GetFileName(file); + directory = IDirectory.GetDirectory(fileName); + directoryIndex = IDirectory.GetDirectory(directory); + checkFile = Path.Combine(jsonGroups["{}"][directoryIndex], fileName); + if (File.Exists(checkFile)) + continue; + File.Move(file, checkFile); + } + } + } + } \ No newline at end of file diff --git a/Tests/UnitTestResize.cs b/Tests/UnitTestResize.cs index 9db9168..52ca590 100644 --- a/Tests/UnitTestResize.cs +++ b/Tests/UnitTestResize.cs @@ -6,6 +6,7 @@ using System.Diagnostics; using System.Drawing.Imaging; using System.Reflection; using View_by_Distance.Metadata.Models; +using View_by_Distance.Property.Models; using View_by_Distance.Resize.Models; using View_by_Distance.Shared.Models; using View_by_Distance.Shared.Models.Stateless.Methods; @@ -20,8 +21,8 @@ public class UnitTestResize private readonly ILogger _Logger; private readonly AppSettings _AppSettings; private readonly string _WorkingDirectory; - private readonly Configuration _Configuration; private readonly IsEnvironment _IsEnvironment; + private readonly Models.Configuration _Configuration; private readonly IConfigurationRoot _ConfigurationRoot; private readonly Property.Models.Configuration _PropertyConfiguration; @@ -30,8 +31,8 @@ public class UnitTestResize ILogger logger; AppSettings appSettings; string workingDirectory; - Configuration configuration; IsEnvironment isEnvironment; + Models.Configuration configuration; IConfigurationRoot configurationRoot; LoggerConfiguration loggerConfiguration = new(); Property.Models.Configuration propertyConfiguration; @@ -40,6 +41,7 @@ public class UnitTestResize isEnvironment = new(processesCount: null, nullASPNetCoreEnvironmentIsDevelopment: debuggerWasAttachedAtLineZero, nullASPNetCoreEnvironmentIsProduction: !debuggerWasAttachedAtLineZero); IConfigurationBuilder configurationBuilder = new ConfigurationBuilder() .AddEnvironmentVariables() + .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) .AddJsonFile(isEnvironment.AppSettingsFileName, optional: false, reloadOnChange: true); configurationRoot = configurationBuilder.Build(); appSettings = Models.Binder.AppSettings.Get(configurationRoot); @@ -74,15 +76,48 @@ public class UnitTestResize Assert.IsFalse(_PropertyConfiguration is null); } - private Property.Models.A_Property GetPropertyLogic(bool reverse) + private A_Property GetPropertyLogic(bool reverse, string aResultsFullGroupDirectory) { - Property.Models.A_Property result; + A_Property result; if (_Configuration?.PropertyConfiguration is null) throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration)); - result = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Configuration.OutputExtension, reverse); + result = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Configuration.OutputExtension, reverse, aResultsFullGroupDirectory); return result; } + private (string, string) GetResultsFullGroupDirectories() + { + string aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + _Configuration.PropertyConfiguration, + nameof(A_Property), + string.Empty, + includeResizeGroup: false, + includeModel: false, + includePredictorModel: false); + string bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + _Configuration.PropertyConfiguration, + nameof(B_Metadata), + string.Empty, + includeResizeGroup: false, + includeModel: false, + includePredictorModel: false); + return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory); + } + + private (string, string, string) GetResultsFullGroupDirectories(string outputResolution) + { + string cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + _Configuration.PropertyConfiguration, + nameof(C_Resize), + outputResolution, + includeResizeGroup: true, + includeModel: false, + includePredictorModel: false); + string dResultsFullGroupDirectory = string.Empty; + string d2ResultsFullGroupDirectory = string.Empty; + return new(cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory); + } + [TestMethod] public void TestMethodResize() { @@ -101,9 +136,6 @@ public class UnitTestResize Item item; bool reverse = false; FileHolder resizedFileHolder; - string aResultsFullGroupDirectory; - string bResultsFullGroupDirectory; - string cResultsFullGroupDirectory; List parseExceptions = new(); bool isValidImageFormatExtension = true; Shared.Models.Property? property = null; @@ -112,29 +144,25 @@ public class UnitTestResize List> metadataCollection; int length = _PropertyConfiguration.RootDirectory.Length; string outputResolution = _Configuration.OutputResolutions[0]; - Property.Models.A_Property propertyLogic = GetPropertyLogic(reverse); + (string cResultsFullGroupDirectory, _, _) = GetResultsFullGroupDirectories(outputResolution); string sourceDirectory = Path.Combine(_PropertyConfiguration.RootDirectory, sourceDirectoryName); + (string aResultsFullGroupDirectory, string bResultsFullGroupDirectory) = GetResultsFullGroupDirectories(); _Logger.Information(_Configuration.ModelDirectory); - aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( - _PropertyConfiguration, nameof(Property.Models.A_Property), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false); - bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( - _PropertyConfiguration, nameof(B_Metadata), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false); - cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( - _PropertyConfiguration, nameof(C_Resize), outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false); - string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_PropertyConfiguration, nameof(Property.Models.A_Property), "{}"); - B_Metadata metadata = new(_Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata); - _ = metadata.ToString(); + A_Property propertyLogic = GetPropertyLogic(reverse, aResultsFullGroupDirectory); + string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, "{}"); (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetTuple(_Configuration.OutputExtension, _Configuration.OutputQuality); + B_Metadata metadata = new(_Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, bResultsFullGroupDirectory); + _ = metadata.ToString(); C_Resize resize = new(_Configuration.ForceResizeLastWriteTimeToCreationTime, _Configuration.OverrideForResizeImages, _Configuration.PropertiesChangedForResize, _Configuration.ValidResolutions, imageCodecInfo, encoderParameters, filenameExtension); _ = resize.ToString(); + bool? isUniqueFileName = null; FileHolder sourceDirectoryFileHolder = new(".json"); FileHolder fileHolder = new(Path.Combine(sourceDirectory, sourceFileName)); string relativePath = IPath.GetRelativePath(fileHolder.FullName, length); sourceDirectory = Path.Combine(aPropertySingletonDirectory, sourceDirectoryName); propertyLogic.SetAngleBracketCollection(aResultsFullGroupDirectory, sourceDirectory); resize.SetAngleBracketCollection(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, sourceDirectory); - metadata.SetAngleBracketCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, sourceDirectory); - item = new(sourceDirectoryFileHolder, relativePath, fileHolder, isValidImageFormatExtension, property, false, false, false); + item = new(sourceDirectoryFileHolder, relativePath, fileHolder, isUniqueFileName, isValidImageFormatExtension, property, false, false, false); Assert.IsNotNull(item.ImageFileHolder); if (item.Property is null) { @@ -146,7 +174,7 @@ public class UnitTestResize resizedFileHolder = resize.GetResizedFileHolder(item); item.SetResizedFileHolder(resize.FileNameExtension, resizedFileHolder); MappingFromItem mappingFromItem = IMappingFromItem.GetMappingFromItem(item); - (int _, metadataCollection) = metadata.GetMetadataCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem); + (int _, metadataCollection) = metadata.GetMetadataCollection(subFileTuples, parseExceptions, mappingFromItem); outputResolutionToResize = resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, subFileTuples, parseExceptions, metadataCollection, item.Property, mappingFromItem); Assert.IsNotNull(mappingFromItem.ResizedFileHolder); resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.Property, mappingFromItem, outputResolutionToResize); diff --git a/TestsWithFaceRecognitionDotNet/UnitTestExample.cs b/TestsWithFaceRecognitionDotNet/UnitTestExample.cs index b678aa7..c76d083 100644 --- a/TestsWithFaceRecognitionDotNet/UnitTestExample.cs +++ b/TestsWithFaceRecognitionDotNet/UnitTestExample.cs @@ -36,6 +36,7 @@ public class UnitTestExample isEnvironment = new(processesCount: null, nullASPNetCoreEnvironmentIsDevelopment: debuggerWasAttachedAtLineZero, nullASPNetCoreEnvironmentIsProduction: !debuggerWasAttachedAtLineZero); IConfigurationBuilder configurationBuilder = new ConfigurationBuilder() .AddEnvironmentVariables() + .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) .AddJsonFile(isEnvironment.AppSettingsFileName, optional: false, reloadOnChange: true); configurationRoot = configurationBuilder.Build(); appSettings = Models.Binder.AppSettings.Get(configurationRoot); diff --git a/TestsWithFaceRecognitionDotNet/UnitTestFace.cs b/TestsWithFaceRecognitionDotNet/UnitTestFace.cs index abb981c..e22c58a 100644 --- a/TestsWithFaceRecognitionDotNet/UnitTestFace.cs +++ b/TestsWithFaceRecognitionDotNet/UnitTestFace.cs @@ -7,6 +7,7 @@ using System.Drawing.Imaging; using System.Reflection; using View_by_Distance.FaceRecognitionDotNet; using View_by_Distance.Metadata.Models; +using View_by_Distance.Property.Models; using View_by_Distance.Resize.Models; using View_by_Distance.Shared.Models; using View_by_Distance.Shared.Models.Stateless; @@ -22,8 +23,8 @@ public class UnitTestFace private readonly ILogger _Logger; private readonly AppSettings _AppSettings; private readonly string _WorkingDirectory; - private readonly Configuration _Configuration; private readonly IsEnvironment _IsEnvironment; + private readonly Models.Configuration _Configuration; private readonly IConfigurationRoot _ConfigurationRoot; private readonly Property.Models.Configuration _PropertyConfiguration; @@ -32,8 +33,8 @@ public class UnitTestFace ILogger logger; AppSettings appSettings; string workingDirectory; - Configuration configuration; IsEnvironment isEnvironment; + Models.Configuration configuration; IConfigurationRoot configurationRoot; LoggerConfiguration loggerConfiguration = new(); Property.Models.Configuration propertyConfiguration; @@ -42,6 +43,7 @@ public class UnitTestFace isEnvironment = new(processesCount: null, nullASPNetCoreEnvironmentIsDevelopment: debuggerWasAttachedAtLineZero, nullASPNetCoreEnvironmentIsProduction: !debuggerWasAttachedAtLineZero); IConfigurationBuilder configurationBuilder = new ConfigurationBuilder() .AddEnvironmentVariables() + .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) .AddJsonFile(isEnvironment.AppSettingsFileName, optional: false, reloadOnChange: true); configurationRoot = configurationBuilder.Build(); appSettings = Models.Binder.AppSettings.Get(configurationRoot); @@ -85,16 +87,16 @@ public class UnitTestFace Assert.IsFalse(_PropertyConfiguration is null); } - private Property.Models.A_Property GetPropertyLogic(bool reverse) + private A_Property GetPropertyLogic(bool reverse, string aResultsFullGroupDirectory) { - Property.Models.A_Property result; + A_Property result; if (_Configuration?.PropertyConfiguration is null) throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration)); - result = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Configuration.OutputExtension, reverse); + result = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Configuration.OutputExtension, reverse, aResultsFullGroupDirectory); return result; } - private static (Model model, PredictorModel predictorModel, ModelParameter modelParameter) GetModel(Configuration configuration) + private static (Model model, PredictorModel predictorModel, ModelParameter modelParameter) GetModel(Models.Configuration configuration) { (Model, PredictorModel, ModelParameter) result; Array array; @@ -156,6 +158,39 @@ public class UnitTestFace Assert.IsTrue(checkC == ToleranceAfterFactor); } + private (string, string) GetResultsFullGroupDirectories() + { + string aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + _Configuration.PropertyConfiguration, + nameof(A_Property), + string.Empty, + includeResizeGroup: false, + includeModel: false, + includePredictorModel: false); + string bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + _Configuration.PropertyConfiguration, + nameof(B_Metadata), + string.Empty, + includeResizeGroup: false, + includeModel: false, + includePredictorModel: false); + return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory); + } + + private (string, string, string) GetResultsFullGroupDirectories(string outputResolution) + { + string cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( + _Configuration.PropertyConfiguration, + nameof(C_Resize), + outputResolution, + includeResizeGroup: true, + includeModel: false, + includePredictorModel: false); + string dResultsFullGroupDirectory = string.Empty; + string d2ResultsFullGroupDirectory = string.Empty; + return new(cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory); + } + [TestMethod] public void TestMethodFace() { @@ -176,9 +211,6 @@ public class UnitTestFace Item item; bool reverse = false; FileHolder resizedFileHolder; - string aResultsFullGroupDirectory; - string bResultsFullGroupDirectory; - string cResultsFullGroupDirectory; List parseExceptions = new(); bool isValidImageFormatExtension = true; Shared.Models.Property? property = null; @@ -187,29 +219,26 @@ public class UnitTestFace List> metadataCollection; int length = _PropertyConfiguration.RootDirectory.Length; string outputResolution = _Configuration.OutputResolutions[0]; - Property.Models.A_Property propertyLogic = GetPropertyLogic(reverse); + (string cResultsFullGroupDirectory, _, _) = GetResultsFullGroupDirectories(outputResolution); string sourceDirectory = Path.Combine(_PropertyConfiguration.RootDirectory, sourceDirectoryName); + (string aResultsFullGroupDirectory, string bResultsFullGroupDirectory) = GetResultsFullGroupDirectories(); _Logger.Information(_Configuration.ModelDirectory); - aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( - _PropertyConfiguration, nameof(Property.Models.A_Property), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false); - bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( - _PropertyConfiguration, nameof(B_Metadata), outputResolution, includeResizeGroup: false, includeModel: false, includePredictorModel: false); - cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory( - _PropertyConfiguration, nameof(C_Resize), outputResolution, includeResizeGroup: true, includeModel: false, includePredictorModel: false); - string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_PropertyConfiguration, nameof(Property.Models.A_Property), "{}"); - B_Metadata metadata = new(_Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata); - _ = metadata.ToString(); + A_Property propertyLogic = GetPropertyLogic(reverse, aResultsFullGroupDirectory); + string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, "{}"); (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetTuple(_Configuration.OutputExtension, _Configuration.OutputQuality); + B_Metadata metadata = new(_Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, bResultsFullGroupDirectory); + _ = metadata.ToString(); C_Resize resize = new(_Configuration.ForceResizeLastWriteTimeToCreationTime, _Configuration.OverrideForResizeImages, _Configuration.PropertiesChangedForResize, _Configuration.ValidResolutions, imageCodecInfo, encoderParameters, filenameExtension); _ = resize.ToString(); + bool? isUniqueFileName = null; + bool anyNullOrNoIsUniqueFileName = true; FileHolder sourceDirectoryFileHolder = new(".json"); FileHolder fileHolder = new(Path.Combine(sourceDirectory, sourceFileName)); string relativePath = IPath.GetRelativePath(fileHolder.FullName, length); sourceDirectory = Path.Combine(aPropertySingletonDirectory, sourceDirectoryName); - propertyLogic.SetAngleBracketCollection(aResultsFullGroupDirectory, sourceDirectory); + propertyLogic.SetAngleBracketCollection(aResultsFullGroupDirectory, sourceDirectory, anyNullOrNoIsUniqueFileName); resize.SetAngleBracketCollection(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, sourceDirectory); - metadata.SetAngleBracketCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, sourceDirectory); - item = new(sourceDirectoryFileHolder, relativePath, fileHolder, isValidImageFormatExtension, property, false, false, false); + item = new(sourceDirectoryFileHolder, relativePath, fileHolder, isUniqueFileName, isValidImageFormatExtension, property, false, false, false); Assert.IsNotNull(item.ImageFileHolder); if (item.Property is null) { @@ -221,7 +250,7 @@ public class UnitTestFace resizedFileHolder = resize.GetResizedFileHolder(item); item.SetResizedFileHolder(resize.FileNameExtension, resizedFileHolder); MappingFromItem mappingFromItem = IMappingFromItem.GetMappingFromItem(item); - (int _, metadataCollection) = metadata.GetMetadataCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem); + (int _, metadataCollection) = metadata.GetMetadataCollection(subFileTuples, parseExceptions, mappingFromItem); outputResolutionToResize = resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, subFileTuples, parseExceptions, metadataCollection, item.Property, mappingFromItem); Assert.IsNotNull(mappingFromItem.ResizedFileHolder); resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.Property, mappingFromItem, outputResolutionToResize);