From deff6f484cf059667097566cf2bd17ad9c67b13d Mon Sep 17 00:00:00 2001 From: Mike Phares Date: Thu, 15 Sep 2022 21:38:13 -0700 Subject: [PATCH] Removed R from Container class --- Compare/Compare.cs | 42 ++++----- Compare/appsettings.Development.json | 18 ++-- Compare/appsettings.json | 16 ++-- Date-Group/DateGroup.cs | 2 +- Date-Group/appsettings.Development.json | 4 +- Instance/DlibDotNet.cs | 53 ++++++++--- Instance/appsettings.Staging.json | 2 +- Instance/appsettings.json | 2 +- Map/Models/MapLogic.cs | 6 +- Not-Copy-Copy/Not-Copy-Copy.cs | 4 +- Not-Copy-Copy/appsettings.Development.json | 2 +- PrepareForOld/appsettings.Development.json | 2 +- .../Models/PropertyCompareLogic.cs | 6 +- Property/Models/A_Property.cs | 19 ++-- Property/Models/Stateless/Container.cs | 91 ++++++++++--------- Shared/Models/Container.cs | 8 +- Shared/Models/Properties/IContainer.cs | 1 - Shared/Models/Stateless/Methods/Mapping.cs | 8 -- 18 files changed, 154 insertions(+), 132 deletions(-) diff --git a/Compare/Compare.cs b/Compare/Compare.cs index d0b4b1d..188cc77 100644 --- a/Compare/Compare.cs +++ b/Compare/Compare.cs @@ -45,7 +45,7 @@ public class Compare string searchPattern = "*"; long ticks = DateTime.Now.Ticks; List topDirectories = new(); - List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> groupCollection; + List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)> groupCollection; Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot); Property.Models.Configuration.Verify(propertyConfiguration); Models.Configuration configuration = Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory, propertyConfiguration); @@ -106,7 +106,7 @@ public class Compare throw new Exception("Change configuration!"); _RenameCFindReplace.Add(new(renameFrom, renameTo)); } - groupCollection = Property.Models.Stateless.Container.GetGroupCollection(propertyConfiguration.RootDirectory, searchPattern, topDirectories); + (_, groupCollection) = Property.Models.Stateless.Container.GetGroupCollection(propertyConfiguration.RootDirectory, searchPattern, topDirectories); if (appSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Property.Models.Stateless.Container.GetGroupCollection)); _Log.Information($"{nameof(Property.Models.Stateless.Container.GetGroupCollection)} has finished"); @@ -118,7 +118,7 @@ public class Compare { topDirectories.Clear(); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(propertyConfiguration.RootDirectory); - groupCollection = Property.Models.Stateless.Container.GetGroupCollection(propertyConfiguration.RootDirectory, searchPattern, topDirectories); + (_, groupCollection) = Property.Models.Stateless.Container.GetGroupCollection(propertyConfiguration.RootDirectory, searchPattern, topDirectories); if (appSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Property.Models.Stateless.Container.GetGroupCollection)); } @@ -127,7 +127,7 @@ public class Compare { topDirectories.Clear(); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(propertyConfiguration.RootDirectory); - groupCollection = Property.Models.Stateless.Container.GetGroupCollection(propertyConfiguration.RootDirectory, searchPattern, topDirectories); + (_, groupCollection) = Property.Models.Stateless.Container.GetGroupCollection(propertyConfiguration.RootDirectory, searchPattern, topDirectories); if (appSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Property.Models.Stateless.Container.GetGroupCollection)); } @@ -136,7 +136,7 @@ public class Compare { topDirectories.Clear(); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(propertyConfiguration.RootDirectory); - groupCollection = Property.Models.Stateless.Container.GetGroupCollection(propertyConfiguration.RootDirectory, searchPattern, topDirectories); + (_, groupCollection) = Property.Models.Stateless.Container.GetGroupCollection(propertyConfiguration.RootDirectory, searchPattern, topDirectories); if (appSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Property.Models.Stateless.Container.GetGroupCollection)); } @@ -145,7 +145,7 @@ public class Compare { topDirectories.Clear(); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(propertyConfiguration.RootDirectory); - groupCollection = Property.Models.Stateless.Container.GetGroupCollection(propertyConfiguration.RootDirectory, searchPattern, topDirectories); + (_, groupCollection) = Property.Models.Stateless.Container.GetGroupCollection(propertyConfiguration.RootDirectory, searchPattern, topDirectories); if (appSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Property.Models.Stateless.Container.GetGroupCollection)); } @@ -158,13 +158,13 @@ public class Compare { topDirectories.Clear(); _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(propertyConfiguration.RootDirectory); - groupCollection = Property.Models.Stateless.Container.GetGroupCollection(propertyConfiguration.RootDirectory, searchPattern, topDirectories); + (_, groupCollection) = Property.Models.Stateless.Container.GetGroupCollection(propertyConfiguration.RootDirectory, searchPattern, topDirectories); if (appSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Property.Models.Stateless.Container.GetGroupCollection)); } if (_IsEnvironment.Development && propertyConfiguration.PopulatePropertyId && !mapLogic.KeyValuePairs.Any()) throw new Exception("Copy keyValuePairs-####.json file"); - Shared.Models.Container[] containers = A_Property.Get(propertyConfiguration, propertyLogic); + (int j, int f, int t, Shared.Models.Container[] containers) = A_Property.Get(propertyConfiguration, propertyLogic); if (!isSilent) { _Log.Information("First pass completed"); @@ -509,11 +509,11 @@ public class Compare string searchPattern = "*.delete"; long ticks = DateTime.Now.Ticks; List topDirectories = new(); - List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> groupCollection; - groupCollection = Property.Models.Stateless.Container.GetGroupCollection(aPropertySingletonDirectory, searchPattern, topDirectories); + List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)> groupCollection; + (_, groupCollection) = Property.Models.Stateless.Container.GetGroupCollection(aPropertySingletonDirectory, searchPattern, topDirectories); if (_AppSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Property.Models.Stateless.Container.GetGroupCollection)); - foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles, int r) in groupCollection) + foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles) in groupCollection) { if (!topDirectories.Any()) continue; @@ -522,14 +522,14 @@ public class Compare } } - private bool PossiblyRename(List topDirectories, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> groupCollection) + private bool PossiblyRename(List topDirectories, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)> groupCollection) { bool result = false; string replaceFile; string replaceDirectory; int remainingDirectories = 0; IEnumerable<(string Find, string Replace)>? found; - foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles, int r) in groupCollection) + foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles) in groupCollection) { if (!topDirectories.Any()) continue; @@ -567,13 +567,13 @@ public class Compare return result; } - private bool PossiblyRenameB(List topDirectories, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> groupCollection) + private bool PossiblyRenameB(List topDirectories, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)> groupCollection) { bool result = false; string replaceFile; string replaceDirectory; IEnumerable<(string Find, string Replace)>? found; - foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles, int r) in groupCollection) + foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles) in groupCollection) { if (!topDirectories.Any()) continue; @@ -603,13 +603,13 @@ public class Compare return result; } - private bool PossiblyRenameC(List topDirectories, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> groupCollection) + private bool PossiblyRenameC(List topDirectories, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)> groupCollection) { bool result = false; string replaceFile; string replaceDirectory; IEnumerable<(string Find, string Replace)>? found; - foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles, int r) in groupCollection) + foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles) in groupCollection) { if (!topDirectories.Any()) continue; @@ -639,7 +639,7 @@ public class Compare return result; } - private bool PossiblyCorrect(List topDirectories, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> groupCollection) + private bool PossiblyCorrect(List topDirectories, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)> groupCollection) { if (_Configuration?.PropertyConfiguration is null) throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration)); @@ -651,7 +651,7 @@ public class Compare string[] filteredSourceDirectoryFiles; (string Find, string Replace) findReplace; IEnumerable<(string Find, string Replace)>? found; - foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles, int r) in groupCollection) + foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles) in groupCollection) { if (!topDirectories.Any()) continue; @@ -687,13 +687,13 @@ public class Compare return result; } - private List GetMissingVerifyToSeasonCollection(List _, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> groupCollection) + private List GetMissingVerifyToSeasonCollection(List _, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)> groupCollection) { if (_Configuration?.PropertyConfiguration is null) throw new NullReferenceException(nameof(_Configuration.PropertyConfiguration)); List results = new(); string check; - foreach ((int _, string sourceDirectory, string[] _, int _) in groupCollection) + foreach ((int _, string sourceDirectory, string[] _) in groupCollection) { if (sourceDirectory == _Configuration.PropertyConfiguration.RootDirectory) continue; diff --git a/Compare/appsettings.Development.json b/Compare/appsettings.Development.json index b1568e1..e3b9b6d 100644 --- a/Compare/appsettings.Development.json +++ b/Compare/appsettings.Development.json @@ -79,7 +79,7 @@ "/zzz Phares Slides/Slides 2015-06-10/Magazine 01" ], "Configuration": { - "DateGroup": "2022-08-22", + "DateGroup": "2022-09-15", "DiffPropertyDirectory": "", "FileNameDirectorySeparator": ".Z.", "ForcePropertyLastWriteTimeToCreationTime": false, @@ -87,20 +87,20 @@ "Pattern": "[^ABCDEFGHIJKLMNOPQRSTUVWXYZbcdfghjklmnpqrstvwxyz0-9]", "PopulatePropertyId": true, "PropertiesChangedForProperty": false, - "RootDirectory": "C:/Tmp/Phares/Compare/Images 2022-08-22 - bc2174b - III", + "RootDirectory": "C:/Tmp/Phares/Compare/Images 2022-09-15 - 7390c13 - III", "WriteBitmapDataBytes": false, "IgnoreExtensions": [ ".gif", ".GIF" ], "PropertyContentCollectionFiles": [ - "/Images 2022-08-22 - bc2174b - III - Results/A) Property/2022-08-22/[()]/637869381676042455.json", - "/Not-Copy-Copy/Images 2019-06-08 - 34a9240ac28b52da97428d7725153a80a757ee6b - Not-Copy-Copy - Results/A) Property/2022-08-22/[()]/637869733124119330.json", - "/Not-Copy-Copy/Images 2018-12-25 - 34a9240ac28b52da97428d7725153a80a757ee6b - Not-Copy-Copy - Results/A) Property/2022-08-22/[()]/637869734240700328.json", - "/Not-Copy-Copy/Images 2018-05-12 - b01d4763d8853b6d6057a3870b2723449726da75 - Not-Copy-Copy - Results/A) Property/2022-08-22/[()]/637869734970730630.json", - "/Not-Copy-Copy/Images 2013-12-15 - d02c8791fa0b130c0bce2d39ee684e50f7ee7a97 - Not-Copy-Copy - Results/A) Property/2022-08-22/[()]/637869743752078399.json", - "/Not-Copy-Copy - Delta/Amazon Drive - Results/A) Property/2022-08-22/[()]/637869744751177715.json", - "/Not-Copy-Copy - Delta/Blackberry - Results/A) Property/2022-08-22/[()]/637869745134124462.json" + "/Images 2022-09-15 - 7390c13 - III - Results/A) Property/2022-09-15/[()]/637869381676042455.json", + "/Not-Copy-Copy/Images 2019-06-08 - 34a9240ac28b52da97428d7725153a80a757ee6b - Not-Copy-Copy - Results/A) Property/2022-09-15/[()]/637869733124119330.json", + "/Not-Copy-Copy/Images 2018-12-25 - 34a9240ac28b52da97428d7725153a80a757ee6b - Not-Copy-Copy - Results/A) Property/2022-09-15/[()]/637869734240700328.json", + "/Not-Copy-Copy/Images 2018-05-12 - b01d4763d8853b6d6057a3870b2723449726da75 - Not-Copy-Copy - Results/A) Property/2022-09-15/[()]/637869734970730630.json", + "/Not-Copy-Copy/Images 2013-12-15 - d02c8791fa0b130c0bce2d39ee684e50f7ee7a97 - Not-Copy-Copy - Results/A) Property/2022-09-15/[()]/637869743752078399.json", + "/Not-Copy-Copy - Delta/Amazon Drive - Results/A) Property/2022-09-15/[()]/637869744751177715.json", + "/Not-Copy-Copy - Delta/Blackberry - Results/A) Property/2022-09-15/[()]/637869745134124462.json" ], "ValidImageFormatExtensions": [ ".bmp", diff --git a/Compare/appsettings.json b/Compare/appsettings.json index 94360e2..f920592 100644 --- a/Compare/appsettings.json +++ b/Compare/appsettings.json @@ -50,7 +50,7 @@ "WorkingDirectoryName": "PharesApps", "Windows": { "Configuration": { - "DateGroup": "2022-08-22", + "DateGroup": "2022-09-15", "DiffPropertyDirectory": "", "FileNameDirectorySeparator": ".Z.", "ForcePropertyLastWriteTimeToCreationTime": false, @@ -94,13 +94,13 @@ ".GIF" ], "PropertyContentCollectionFiles": [ - "/Images 2022-08-22 - bc2174b - III - Results/A) Property/2022-08-22/[()]/637869381676042455.json", - "/Not-Copy-Copy/Images 2019-06-08 - 34a9240ac28b52da97428d7725153a80a757ee6b - Not-Copy-Copy - Results/A) Property/2022-08-22/[()]/637869733124119330.json", - "/Not-Copy-Copy/Images 2018-12-25 - 34a9240ac28b52da97428d7725153a80a757ee6b - Not-Copy-Copy - Results/A) Property/2022-08-22/[()]/637869734240700328.json", - "/Not-Copy-Copy/Images 2018-05-12 - b01d4763d8853b6d6057a3870b2723449726da75 - Not-Copy-Copy - Results/A) Property/2022-08-22/[()]/637869734970730630.json", - "/Not-Copy-Copy/Images 2013-12-15 - d02c8791fa0b130c0bce2d39ee684e50f7ee7a97 - Not-Copy-Copy - Results/A) Property/2022-08-22/[()]/637869743752078399.json", - "/Not-Copy-Copy - Delta/Amazon Drive - Results/A) Property/2022-08-22/[()]/637869744751177715.json", - "/Not-Copy-Copy - Delta/Blackberry - Results/A) Property/2022-08-22/[()]/637869745134124462.json" + "/Images 2022-09-15 - 7390c13 - III - Results/A) Property/2022-09-15/[()]/637869381676042455.json", + "/Not-Copy-Copy/Images 2019-06-08 - 34a9240ac28b52da97428d7725153a80a757ee6b - Not-Copy-Copy - Results/A) Property/2022-09-15/[()]/637869733124119330.json", + "/Not-Copy-Copy/Images 2018-12-25 - 34a9240ac28b52da97428d7725153a80a757ee6b - Not-Copy-Copy - Results/A) Property/2022-09-15/[()]/637869734240700328.json", + "/Not-Copy-Copy/Images 2018-05-12 - b01d4763d8853b6d6057a3870b2723449726da75 - Not-Copy-Copy - Results/A) Property/2022-09-15/[()]/637869734970730630.json", + "/Not-Copy-Copy/Images 2013-12-15 - d02c8791fa0b130c0bce2d39ee684e50f7ee7a97 - Not-Copy-Copy - Results/A) Property/2022-09-15/[()]/637869743752078399.json", + "/Not-Copy-Copy - Delta/Amazon Drive - Results/A) Property/2022-09-15/[()]/637869744751177715.json", + "/Not-Copy-Copy - Delta/Blackberry - Results/A) Property/2022-09-15/[()]/637869745134124462.json" ], "ValidImageFormatExtensions": [ ".bmp", diff --git a/Date-Group/DateGroup.cs b/Date-Group/DateGroup.cs index d58341d..824f484 100644 --- a/Date-Group/DateGroup.cs +++ b/Date-Group/DateGroup.cs @@ -61,7 +61,7 @@ public class DateGroup _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(propertyConfiguration.RootDirectory); if (true || appSettings.MaxDegreeOfParallelism < 2) ticks = LogDelta(ticks, nameof(Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories)); - Shared.Models.Container[] containers = A_Property.Get(propertyConfiguration, propertyLogic); + (int j, int f, int t, Shared.Models.Container[] containers) = A_Property.Get(propertyConfiguration, propertyLogic); if (configuration.ByCreateDateShortcut.HasValue && configuration.ByCreateDateShortcut.Value) CreateDateShortcut(propertyConfiguration, containers); else diff --git a/Date-Group/appsettings.Development.json b/Date-Group/appsettings.Development.json index 553f17b..c9d2e45 100644 --- a/Date-Group/appsettings.Development.json +++ b/Date-Group/appsettings.Development.json @@ -55,7 +55,7 @@ "ByHash": false, "BySeason": false, "ByWeek": false, - "DateGroup": "2022-08-22", + "DateGroup": "2022-09-15", "FileNameDirectorySeparator": ".Z.", "ForcePropertyLastWriteTimeToCreationTime": false, "KeepFullPath": false, @@ -63,7 +63,7 @@ "Pattern": "[^ABCDEFGHIJKLMNOPQRSTUVWXYZbcdfghjklmnpqrstvwxyz0-9]", "PopulatePropertyId": true, "PropertiesChangedForProperty": false, - "RootDirectory": "C:/Tmp/Phares/Compare/Images 2022-08-22 - bc2174b - III", + "RootDirectory": "C:/Tmp/Phares/Compare/Images 2022-09-15 - 7390c13 - III", "WriteBitmapDataBytes": false, "IgnoreExtensions": [ ".gif", diff --git a/Instance/DlibDotNet.cs b/Instance/DlibDotNet.cs index 3065081..be81169 100644 --- a/Instance/DlibDotNet.cs +++ b/Instance/DlibDotNet.cs @@ -338,7 +338,7 @@ public partial class DlibDotNet } } - private int FullParallelWork(long ticks, A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List> sourceDirectoryChanges, List propertyFileHolderCollection, List propertyCollection, List>> metadataCollection, List> resizeKeyValuePairs, List?> imageFaceCollections, int containersCount, Container container, Item[] filteredItems) + private int FullParallelWork(A_Property propertyLogic, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, List> sourceDirectoryChanges, List propertyFileHolderCollection, List propertyCollection, List>> metadataCollection, List> resizeKeyValuePairs, List?> imageFaceCollections, Container container, Item[] filteredItems, string message) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); @@ -347,7 +347,7 @@ public partial class DlibDotNet ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; if (imageFaceCollections.Count != filteredItems.Length || metadataCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || propertyCollection.Count != filteredItems.Length) { - for (int i = 0; i < filteredItems.Length; i++) + for (int f = 0; f < filteredItems.Length; f++) { propertyCollection.Add(null); metadataCollection.Add(new()); @@ -356,8 +356,6 @@ public partial class DlibDotNet propertyFileHolderCollection.Add(null); } } - int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); - string message = $"{container.R:000}.{container.G} / {containersCount:000}) {filteredItems.Length:000} file(s) - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}"; using ProgressBar progressBar = new(filteredItems.Length, message, options); _ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) => { @@ -564,11 +562,15 @@ public partial class DlibDotNet converted: true); } - private void FullDoWork(string argZero, Model? model, PredictorModel? predictorModel, string propertyRoot, long ticks, MapLogic mapLogic, A_Property propertyLogic, Container[] containers) + private void FullDoWork(string argZero, Model? model, PredictorModel? predictorModel, string propertyRoot, long ticks, A_Property propertyLogic, int t, Container[] containers) { if (_Log is null) throw new NullReferenceException(nameof(_Log)); + int total; + string message; + int totalSeconds; int exceptionCount; + Container container; Item[] filteredItems; string aResultsFullGroupDirectory; string bResultsFullGroupDirectory; @@ -576,6 +578,7 @@ public partial class DlibDotNet string dResultsFullGroupDirectory; string eResultsFullGroupDirectory; string d2ResultsFullGroupDirectory; + int containersLength = containers.Length; Shared.Models.Property[] propertyCollection; List?> imageFaceCollections = new(); List propertyFileHolderCollection = new(); @@ -586,11 +589,13 @@ public partial class DlibDotNet string aPropertySingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), "{}"); foreach (string outputResolution in _Configuration.OutputResolutions) { + total = 0; _FileKeyValuePairs.Clear(); _FilePropertiesKeyValuePairs.Clear(); (aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory) = GetResultsFullGroupDirectories(_Configuration.PropertyConfiguration, model, predictorModel, outputResolution); - foreach (Container container in containers) + for (int i = 0; i < containers.Length; i++) { + container = containers[i]; if (!container.Items.Any()) continue; if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero)) @@ -604,14 +609,16 @@ public partial class DlibDotNet sourceDirectoryChanges.Clear(); nullablePropertyCollection.Clear(); propertyFileHolderCollection.Clear(); + totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); + message = $"{i:000}.{container.G} / {containersLength:000}) {filteredItems.Length:000} file(s) - {total} / {t} total file(s) - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}"; SetAngleBracketCollections(_Configuration.PropertyConfiguration, propertyLogic, outputResolution, container, aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory); - exceptionCount = FullParallelWork(ticks, propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, nullablePropertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, containers.Length, container, filteredItems); + exceptionCount = FullParallelWork(propertyLogic, outputResolution, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, sourceDirectoryChanges, propertyFileHolderCollection, nullablePropertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, container, filteredItems, message); if (metadataCollection.Count != filteredItems.Length || nullablePropertyCollection.Count != filteredItems.Length || resizeKeyValuePairs.Count != filteredItems.Length || imageFaceCollections.Count != filteredItems.Length) throw new Exception("Counts don't match!"); if (exceptionCount != 0) _Exceptions.Add(container.SourceDirectory); - for (int i = 0; i < imageFaceCollections.Count; i++) - filteredItems[i].Faces.AddRange(from l in imageFaceCollections[i] select l); + for (int f = 0; f < imageFaceCollections.Count; f++) + filteredItems[f].Faces.AddRange(from l in imageFaceCollections[f] select l); propertyCollection = (from l in nullablePropertyCollection where l is not null select l).ToArray(); if (_ArgZeroIsConfigurationRootDirectory && exceptionCount == 0) WriteGroup(_Configuration.PropertyConfiguration, propertyLogic, propertyCollection, metadataCollection, resizeKeyValuePairs, imageFaceCollections, outputResolution, container, filteredItems); @@ -637,6 +644,7 @@ public partial class DlibDotNet // break; // if (isSilent && container.SourceDirectory.EndsWith("Texas 2015")) //46#.4 // break; + total += container.Items.Count; } } } @@ -658,9 +666,9 @@ public partial class DlibDotNet Item[] filteredItems; DateTime minimumDateTime; List distinctCheck = new(); - string deterministicHashCodeKeyDisplay; MappingFromItem mappingFromItem; MappingFromPerson mappingFromPerson; + string deterministicHashCodeKeyDisplay; MappingFromLocation mappingFromLocation; foreach (Container container in containers) { @@ -716,7 +724,12 @@ public partial class DlibDotNet private void Search(Property.Models.Configuration configuration, Model? model, PredictorModel? predictorModel, string argZero, string propertyRoot, Person[] people) { + int j; + int f; + int t; + MapLogic? mapLogic; Container[] containers; + A_Property propertyLogic; long ticks = DateTime.Now.Ticks; string aResultsFullGroupDirectory; string bResultsFullGroupDirectory; @@ -726,19 +739,27 @@ public partial class DlibDotNet string d2ResultsFullGroupDirectory; string zResultsFullGroupDirectory = GetMapLogicResultsFullGroupDirectory(configuration, model, predictorModel); string peopleDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(A2_People)); - MapLogic mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FilenameExtension, _Faces.FilenameExtension, _Faces.HiddenFilenameExtension, _FaceParts.FilenameExtension, ticks, people, peopleDateGroupDirectory, zResultsFullGroupDirectory); - A_Property propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FilenameExtension, _Configuration.Reverse, model, predictorModel, mapLogic.IndicesFromNew, mapLogic.KeyValuePairs); + if (!_FirstRun) + { + mapLogic = null; + propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FilenameExtension, _Configuration.Reverse, model, predictorModel); + } + else + { + mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FilenameExtension, _Faces.FilenameExtension, _Faces.HiddenFilenameExtension, _FaceParts.FilenameExtension, ticks, people, peopleDateGroupDirectory, zResultsFullGroupDirectory); + propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FilenameExtension, _Configuration.Reverse, model, predictorModel, mapLogic.IndicesFromNew, mapLogic.KeyValuePairs); + } int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); string message = $") Building Container(s) - {totalSeconds} total second(s)"; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; using (ProgressBar progressBar = new(1, message, options)) { if (string.IsNullOrEmpty(configuration.RootDirectory)) - containers = A_Property.Get(configuration, propertyLogic); + (j, f, t, containers) = A_Property.Get(configuration, propertyLogic); else - containers = Property.Models.Stateless.Container.GetContainers(configuration, _FirstRun, propertyLogic); + (j, f, t, containers) = Property.Models.Stateless.Container.GetContainers(configuration, _FirstRun, propertyLogic); } - FullDoWork(argZero, model, predictorModel, propertyRoot, ticks, mapLogic, propertyLogic, containers); + FullDoWork(argZero, model, predictorModel, propertyRoot, ticks, propertyLogic, t, containers); foreach (string outputResolution in _Configuration.OutputResolutions) { if (_FirstRun) @@ -754,6 +775,8 @@ public partial class DlibDotNet _ = Directory.CreateDirectory(eDistanceContentDirectory); string eDistanceContentFileName = Path.Combine(eDistanceContentDirectory, $"{Property.Models.Stateless.IResult.AllInOne}.tvs"); List distinctFilteredFaces = SetMappingThenGetDistinctFilteredFacesWithMapping(argZero, containers); + if (mapLogic is null) + mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FilenameExtension, _Faces.FilenameExtension, _Faces.HiddenFilenameExtension, _FaceParts.FilenameExtension, ticks, people, peopleDateGroupDirectory, zResultsFullGroupDirectory); SortingContainer[] sortingContainers = E_Distance.SetPersonTicksAndSetFaceDistancesAndSetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, ticks, mapLogic, outputResolution, eDistanceContentFileName, distinctFilteredFaces); mapLogic.CommonWork(_AppSettings.MaxDegreeOfParallelism, dFacesContentDirectory, d2ResultsFullGroupDirectory, distinctFilteredFaces, sortingContainers); if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution)) diff --git a/Instance/appsettings.Staging.json b/Instance/appsettings.Staging.json index c620372..0984e62 100644 --- a/Instance/appsettings.Staging.json +++ b/Instance/appsettings.Staging.json @@ -52,7 +52,7 @@ "Configuration": { "CheckJsonForDistanceResults": false, "CrossDirectoryMaxItemsInDistanceCollection": 7, - "DateGroup": "2022-08-22", + "DateGroup": "2022-09-15", "DistanceFactor": 8, "FileNameDirectorySeparator": ".Z.", "ForceFaceLastWriteTimeToCreationTime": false, diff --git a/Instance/appsettings.json b/Instance/appsettings.json index 3858bd1..4718f6b 100644 --- a/Instance/appsettings.json +++ b/Instance/appsettings.json @@ -52,7 +52,7 @@ "Configuration": { "CheckJsonForDistanceResults": false, "CrossDirectoryMaxItemsInDistanceCollection": 7, - "DateGroup": "2022-08-22", + "DateGroup": "2022-09-15", "DistanceFactor": 8, "FileNameDirectorySeparator": ".Z.", "ForceFaceLastWriteTimeToCreationTime": false, diff --git a/Map/Models/MapLogic.cs b/Map/Models/MapLogic.cs index d38ce5d..4b7e1f9 100644 --- a/Map/Models/MapLogic.cs +++ b/Map/Models/MapLogic.cs @@ -63,6 +63,9 @@ public class MapLogic string zPropertyHolderPeopleContentDirectory = Path.Combine(peopleDateGroupDirectory, "()", "(KnownPeople)"); Dictionary> incorrectIdThenNormalizedPixelPercentageKeyValuePairs = new(); string zPropertyHolderSingletonTicksDirectory = Path.Combine(zPropertyHolderSingletonDirectory, string.Concat('{', _Ticks, '}')); + for (int i = 1; i < 5; i++) + _ = IPath.DeleteEmptyDirectories(zPropertyHolderContentDirectory); + _ = IPath.DeleteEmptyDirectories(zPropertyHolderContentDirectory); if (string.IsNullOrEmpty(rootDirectoryParent)) throw new NullReferenceException(nameof(rootDirectoryParent)); if (!Directory.Exists(zPropertyHolderContentDirectory)) @@ -732,9 +735,6 @@ public class MapLogic saveContainers = GetMappingSaveContainers(dFacesContentDirectory, d2ResultsFullGroupDirectory, distinctFilteredFaces); } SaveContainers(saveContainers); - for (int i = 1; i < 5; i++) - _ = IPath.DeleteEmptyDirectories(_ZPropertyHolderContentTicksDirectory); - _ = IPath.DeleteEmptyDirectories(_ZPropertyHolderContentTicksDirectory); SaveNotMappedTicks(); } diff --git a/Not-Copy-Copy/Not-Copy-Copy.cs b/Not-Copy-Copy/Not-Copy-Copy.cs index c34ab37..c8b4169 100644 --- a/Not-Copy-Copy/Not-Copy-Copy.cs +++ b/Not-Copy-Copy/Not-Copy-Copy.cs @@ -39,9 +39,9 @@ public class NotCopyCopy throw new Exception("This program only allows development environments!"); A_Property propertyLogic = GetPropertyLogic(reverse, model, outputExtension, predictorModel); propertyConfiguration.ChangeRootDirectory(configuration.CompareSource); - Shared.Models.Container[] compareContainers = A_Property.Get(propertyConfiguration, propertyLogic); + (_, _, _, Shared.Models.Container[] compareContainers) = A_Property.Get(propertyConfiguration, propertyLogic); propertyConfiguration.ChangeRootDirectory(configuration.SelectedSource); - Shared.Models.Container[] selectedContainers = A_Property.Get(propertyConfiguration, propertyLogic); + (_, _, _, Shared.Models.Container[] selectedContainers) = A_Property.Get(propertyConfiguration, propertyLogic); if (compareContainers.Length == selectedContainers.Length) throw new Exception(); string directoryName; diff --git a/Not-Copy-Copy/appsettings.Development.json b/Not-Copy-Copy/appsettings.Development.json index 0c736de..e9aa783 100644 --- a/Not-Copy-Copy/appsettings.Development.json +++ b/Not-Copy-Copy/appsettings.Development.json @@ -50,7 +50,7 @@ "WorkingDirectoryName": "PharesApps", "Windows": { "Configuration": { - "DateGroup": "2022-08-22", + "DateGroup": "2022-09-15", "FileNameDirectorySeparator": ".Z.", "ForcePropertyLastWriteTimeToCreationTime": false, "MaxImagesInDirectoryForTopLevelFirstPass": 10, diff --git a/PrepareForOld/appsettings.Development.json b/PrepareForOld/appsettings.Development.json index 7df5e14..2156f61 100644 --- a/PrepareForOld/appsettings.Development.json +++ b/PrepareForOld/appsettings.Development.json @@ -50,7 +50,7 @@ "WorkingDirectoryName": "PharesApps", "Windows": { "Configuration": { - "DateGroup": "2022-08-22", + "DateGroup": "2022-09-15", "FileNameDirectorySeparator": ".Z.", "ForcePropertyLastWriteTimeToCreationTime": false, "KeepFullPath": false, diff --git a/Property-Compare/Models/PropertyCompareLogic.cs b/Property-Compare/Models/PropertyCompareLogic.cs index 3ed5a76..ee286b7 100644 --- a/Property-Compare/Models/PropertyCompareLogic.cs +++ b/Property-Compare/Models/PropertyCompareLogic.cs @@ -271,11 +271,11 @@ public class PropertyCompareLogic string[] filteredSourceDirectoryFiles; List collection = new(); bool isArg = aPropertySingletonDirectory.Contains(_Configuration.RootDirectory); - List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> groupCollection; + List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)> groupCollection; ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _MaxDegreeOfParallelism }; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; - groupCollection = Property.Models.Stateless.Container.GetGroupCollection(aPropertySingletonDirectory, searchPattern, topDirectories); - foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles, int r) in groupCollection) + (int r, groupCollection) = Property.Models.Stateless.Container.GetGroupCollection(aPropertySingletonDirectory, searchPattern, topDirectories); + foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles) in groupCollection) { if (!topDirectories.Any()) continue; diff --git a/Property/Models/A_Property.cs b/Property/Models/A_Property.cs index 5cae985..85b7325 100644 --- a/Property/Models/A_Property.cs +++ b/Property/Models/A_Property.cs @@ -568,12 +568,11 @@ public class A_Property } } - private void ParallelWork(bool firstPass, List exceptions, List> sourceDirectoryChanges, int containersCount, Shared.Models.Container container, Item[] filteredItems, int totalSeconds) + private void ParallelWork(bool firstPass, List exceptions, List> sourceDirectoryChanges, Shared.Models.Container container, Item[] filteredItems, string message) { List> filteredSourceDirectoryFileTuples = new(); ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _MaxDegreeOfParallelism }; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; - string message = $"{container.R:000}.{container.G} / {containersCount:000}) {filteredItems.Length:000} file(s) - {totalSeconds} total second(s) - {container.SourceDirectory}"; using ProgressBar progressBar = new(filteredItems.Length, message, options); _ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) => { @@ -619,15 +618,18 @@ public class A_Property { if (_Log is null) throw new NullReferenceException(nameof(_Log)); + string message; int totalSeconds; bool? anyFilesMoved; Item[] filteredItems; + Shared.Models.Container container; List exceptions = new(); int containersCount = containers.Length; List> sourceDirectoryChanges = new(); string propertyRoot = IResult.GetResultsGroupDirectory(_Configuration, nameof(A_Property)); - foreach (Shared.Models.Container container in containers) + for (int i = 0; i < containers.Length; i++) { + container = containers[i]; if (!container.Items.Any()) continue; sourceDirectoryChanges.Clear(); @@ -637,9 +639,10 @@ public class A_Property filteredItems = (from l in container.Items where l.ImageFileHolder is not null && !_Configuration.IgnoreExtensions.Contains(l.ImageFileHolder.ExtensionLowered) select l).ToArray(); if (!filteredItems.Any()) continue; - totalSeconds = (int)Math.Truncate(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); SetAngleBracketCollection(container.SourceDirectory); - ParallelWork(firstPass, exceptions, sourceDirectoryChanges, containersCount, container, filteredItems, totalSeconds); + totalSeconds = (int)Math.Truncate(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); + message = $"{i:000}.{container.G} / {containersCount:000}) {filteredItems.Length:000} file(s) - {totalSeconds} total second(s) - {container.SourceDirectory}"; + ParallelWork(firstPass, exceptions, sourceDirectoryChanges, container, filteredItems, message); foreach (Exception exception in exceptions) _Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, exception.Message, Environment.NewLine, exception.StackTrace), exception); if (exceptions.Count == filteredItems.Length) @@ -710,17 +713,17 @@ public class A_Property return results.OrderBy(l => l.Ticks).ToArray(); } - public static Shared.Models.Container[] Get(Configuration configuration, A_Property propertyLogic) + public static (int, int, int, Shared.Models.Container[]) Get(Configuration configuration, A_Property propertyLogic) { Shared.Models.Container[] results; bool firstRun = false; long ticks = DateTime.Now.Ticks; List exceptionsDirectories = new(); - results = Stateless.Container.GetContainers(configuration, firstRun, propertyLogic); + (int j, int f, int t, results) = Stateless.Container.GetContainers(configuration, firstRun, propertyLogic); propertyLogic.ParallelWork(ticks, results, firstPass: false); if (exceptionsDirectories.Any()) throw new Exception(); - return results; + return new(j, f, t, results); } } \ No newline at end of file diff --git a/Property/Models/Stateless/Container.cs b/Property/Models/Stateless/Container.cs index 4a2ac9d..b993fca 100644 --- a/Property/Models/Stateless/Container.cs +++ b/Property/Models/Stateless/Container.cs @@ -6,9 +6,10 @@ namespace View_by_Distance.Property.Models.Stateless; public class Container { - public static List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> GetGroupCollection(string rootDirectory, int maxImagesInDirectoryForTopLevelFirstPass, bool reverse, string searchPattern, List topDirectories) + public static (int t, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)>) GetGroupCollection(string rootDirectory, int maxImagesInDirectoryForTopLevelFirstPass, bool reverse, string searchPattern, List topDirectories) { - List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> results = new(); + int result = 0; + List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)> results = new(); string? parentDirectory; string[] subDirectories; string[] sourceDirectoryFiles; @@ -24,7 +25,7 @@ public class Container parentDirectory = Path.GetDirectoryName(fileCollections[i][0]); if (string.IsNullOrEmpty(parentDirectory)) continue; - results.Add(new(g, parentDirectory, fileCollections[i], results.Count)); + results.Add(new(g, parentDirectory, fileCollections[i])); fileCollections.RemoveAt(i); } } @@ -38,7 +39,7 @@ public class Container parentDirectory = Path.GetDirectoryName(fileCollections[i][0]); if (string.IsNullOrEmpty(parentDirectory)) continue; - results.Add(new(g, parentDirectory, fileCollections[i], results.Count)); + results.Add(new(g, parentDirectory, fileCollections[i])); fileCollections.RemoveAt(i); } } @@ -50,32 +51,36 @@ public class Container foreach (string subDirectory in subDirectories) { sourceDirectoryFiles = Directory.GetFiles(subDirectory, "*", SearchOption.TopDirectoryOnly); + result += sourceDirectoryFiles.Length; if (!topDirectories.Contains(subDirectory)) - results.Add(new(g, subDirectory, sourceDirectoryFiles, results.Count)); + results.Add(new(g, subDirectory, sourceDirectoryFiles)); } } else if (g == 1) { sourceDirectoryFiles = Directory.GetFiles(rootDirectory, searchPattern, SearchOption.TopDirectoryOnly); + result += sourceDirectoryFiles.Length; if (sourceDirectoryFiles.Length > maxImagesInDirectoryForTopLevelFirstPass) fileCollections.Add(sourceDirectoryFiles); else - results.Add(new(g, rootDirectory, sourceDirectoryFiles, results.Count)); + results.Add(new(g, rootDirectory, sourceDirectoryFiles)); if (reverse) topDirectories.Reverse(); subDirectories = topDirectories.ToArray(); foreach (string subDirectory in subDirectories) { sourceDirectoryFiles = Directory.GetFiles(subDirectory, searchPattern, SearchOption.TopDirectoryOnly); + result += sourceDirectoryFiles.Length; if (sourceDirectoryFiles.Length > maxImagesInDirectoryForTopLevelFirstPass) fileCollections.Add(sourceDirectoryFiles); else { if (sourceDirectoryFiles.Any() || Directory.GetDirectories(subDirectory, "*", SearchOption.TopDirectoryOnly).Any()) - results.Add(new(g, subDirectory, sourceDirectoryFiles, results.Count)); + results.Add(new(g, subDirectory, sourceDirectoryFiles)); else if (searchPattern == "*") { sourceDirectoryFiles = Directory.GetFiles(subDirectory, searchPattern, SearchOption.TopDirectoryOnly); + result += sourceDirectoryFiles.Length; foreach (string subFile in sourceDirectoryFiles) File.Delete(subFile); Directory.Delete(subDirectory); @@ -87,42 +92,43 @@ public class Container else throw new Exception(); } - return results; + return new(result, results); } - public static List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> GetGroupCollection(string rootDirectory, string searchPattern, List topDirectories) + public static (int t, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)>) GetGroupCollection(string rootDirectory, string searchPattern, List topDirectories) { - List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> results; + (int t, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)>) results; int maxImagesInDirectoryForTopLevelFirstPass = 50; bool reverse = false; results = GetGroupCollection(rootDirectory, maxImagesInDirectoryForTopLevelFirstPass, reverse, searchPattern, topDirectories); return results; } - private static List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> GetJsonGroupCollection(Configuration configuration, A_Property propertyLogic, string rootDirectory) + private static (int j, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)>) GetJsonGroupCollection(Configuration configuration, A_Property propertyLogic, string rootDirectory) { - List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> results; + (int j, List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)>) results; string searchPattern = "*.json"; List topDirectories = new(); results = GetGroupCollection(rootDirectory, configuration.MaxImagesInDirectoryForTopLevelFirstPass, propertyLogic.Reverse, searchPattern, topDirectories); return results; } - private static List<(int g, string sourceDirectory, FileHolder[] sourceDirectoryFiles, int r)> GetFileHolderGroupCollection(Configuration configuration, A_Property propertyLogic, string searchPattern, List topDirectories) + private static (int f, List<(int g, string sourceDirectory, FileHolder[] sourceDirectoryFiles)>) GetFileHolderGroupCollection(Configuration configuration, A_Property propertyLogic, string searchPattern, List topDirectories) { - List<(int g, string sourceDirectory, FileHolder[] sourceDirectoryFiles, int r)> results = new(); - List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)>? collection = GetGroupCollection(configuration.RootDirectory, configuration.MaxImagesInDirectoryForTopLevelFirstPass, propertyLogic.Reverse, searchPattern, topDirectories); - foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles, int r) in collection) - results.Add(new(g, sourceDirectory, (from l in sourceDirectoryFiles select new FileHolder(l)).ToArray(), r)); - return results; + List<(int g, string sourceDirectory, FileHolder[] sourceDirectoryFiles)> results = new(); + List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)> collection; + (int f, collection) = GetGroupCollection(configuration.RootDirectory, configuration.MaxImagesInDirectoryForTopLevelFirstPass, propertyLogic.Reverse, searchPattern, topDirectories); + foreach ((int g, string sourceDirectory, string[] sourceDirectoryFiles) in collection) + results.Add(new(g, sourceDirectory, (from l in sourceDirectoryFiles select new FileHolder(l)).ToArray())); + return new(f, results); } - private static void ParallelFor(List<(int, string, string[], int)> jsonCollection, int i, int length, List<(int, string, List<(string, Shared.Models.Property?)>, int)> results) + private static void ParallelFor(List<(int, string, string[])> jsonCollection, int i, int length, List<(int, string, List<(string, Shared.Models.Property?)>)> results) { string key; string json; Shared.Models.Property? property; - (int g, string sourceDirectory, string[] sourceDirectoryFiles, int r) = jsonCollection[i]; + (int g, string sourceDirectory, string[] sourceDirectoryFiles) = jsonCollection[i]; List<(string, Shared.Models.Property?)> collection = new(); foreach (string sourceDirectoryFile in sourceDirectoryFiles) { @@ -132,20 +138,21 @@ public class Container collection.Add(new(sourceDirectoryFile, property)); } lock (results) - results.Add(new(g, sourceDirectory, collection, r)); + results.Add(new(g, sourceDirectory, collection)); } - private static List<(int g, string sourceDirectory, List<(string sourceDirectoryFile, Shared.Models.Property? property)> collection, int r)> GetCollection(string rootDirectory, List<(int g, string sourceDirectory, string[] SourceDirectoryFiles, int r)> jsonCollection) + private static List<(int g, string sourceDirectory, List<(string sourceDirectoryFile, Shared.Models.Property? property)> collection)> GetCollection(string rootDirectory, List<(int g, string sourceDirectory, string[] SourceDirectoryFiles)> jsonCollection) { - List<(int, string, List<(string, Shared.Models.Property?)>, int)> results = new(); + List<(int, string, List<(string, Shared.Models.Property?)>)> results = new(); int length = rootDirectory.Length; ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = Environment.ProcessorCount }; _ = Parallel.For(0, jsonCollection.Count, parallelOptions, (i, state) => ParallelFor(jsonCollection, i, length, results)); return results; } - private static Shared.Models.Container[] GetContainers(Configuration configuration, bool firstRun, string aPropertySingletonDirectory, List<(int, string, FileHolder[], int)> fileHolderGroupCollection, List<(int, string, List<(string, Shared.Models.Property?)>, int)> collectionFromJson) + private static (int, Shared.Models.Container[]) GetContainers(Configuration configuration, bool firstRun, string aPropertySingletonDirectory, List<(int, string, FileHolder[])> fileHolderGroupCollection, List<(int, string, List<(string, Shared.Models.Property?)>)> collectionFromJson) { + int result = 0; Shared.Models.Container[] results; Item item; int length; @@ -162,7 +169,7 @@ public class Container Dictionary keyValuePairs = new(); Dictionary fileHolderKeyValuePairs = new(); length = configuration.RootDirectory.Length; - foreach ((int g, string sourceDirectory, FileHolder[] sourceDirectoryFileHolderCollection, int r) in fileHolderGroupCollection) + foreach ((int g, string sourceDirectory, FileHolder[] sourceDirectoryFileHolderCollection) in fileHolderGroupCollection) { foreach (FileHolder sourceDirectoryFileHolder in sourceDirectoryFileHolderCollection) { @@ -171,7 +178,7 @@ public class Container } } length = aPropertySingletonDirectory.Length; - foreach ((int g, string _, List<(string, Shared.Models.Property?)> collection, int r) in collectionFromJson) + foreach ((int g, string _, List<(string, Shared.Models.Property?)> collection) in collectionFromJson) { if (!collection.Any()) continue; @@ -212,14 +219,15 @@ public class Container } if (items.Any()) { + result += items.Count; if (keySourceDirectories.Distinct().Count() != 1) continue; - container = new(g, r, items, keySourceDirectories[0]); + container = new(g, items, keySourceDirectories[0]); keyValuePairs.Add(keySourceDirectories[0], container); } } length = configuration.RootDirectory.Length; - foreach ((int g, string sourceDirectory, FileHolder[] sourceDirectoryFileHolderCollection, int r) in fileHolderGroupCollection) + foreach ((int g, string sourceDirectory, FileHolder[] sourceDirectoryFileHolderCollection) in fileHolderGroupCollection) { items = new(); foreach (FileHolder sourceDirectoryFileHolder in sourceDirectoryFileHolderCollection) @@ -244,7 +252,8 @@ public class Container { if (!keyValuePairs.ContainsKey(sourceDirectory)) { - container = new(g, r, items, sourceDirectory); + result += items.Count; + container = new(g, items, sourceDirectory); keyValuePairs.Add(sourceDirectory, container); } else @@ -261,32 +270,32 @@ public class Container additional += 1; items.Add(item); } - container = new(g, r + additional, items, sourceDirectory); + result += additional; + container = new(container.G, items, container.SourceDirectory); keyValuePairs[sourceDirectory] = container; } } } if (!firstRun && fileHolderKeyValuePairs.Any()) throw new Exception(); - KeyValuePair[] sortedKeyValuePairs = (from l in keyValuePairs orderby l.Value.G, l.Value.R select l).ToArray(); - results = (from l in sortedKeyValuePairs select l.Value).ToArray(); - return results; + results = (from l in keyValuePairs orderby l.Value.G, l.Value.Items.Count select l.Value).ToArray(); + return new(result, results); } - public static Shared.Models.Container[] GetContainers(Configuration configuration, bool firstRun, A_Property propertyLogic) + public static (int, int, int, Shared.Models.Container[]) GetContainers(Configuration configuration, bool firstRun, A_Property propertyLogic) { Shared.Models.Container[] results; string searchPattern = "*"; List topDirectories = new(); - List<(int g, string sourceDirectory, string[] sourceDirectoryFiles, int r)> jsonCollection; - List<(int g, string sourceDirectory, FileHolder[] sourceDirectoryFiles, int r)> fileHolderGroupCollection; + List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)> jsonCollection; + List<(int g, string sourceDirectory, FileHolder[] sourceDirectoryFiles)> fileHolderGroupCollection; string aPropertySingletonDirectory = IResult.GetResultsDateGroupDirectory(configuration, nameof(A_Property), "{}"); - List<(int g, string sourceDirectory, List<(string sourceDirectoryFile, Shared.Models.Property? property)> collection, int r)> collectionFromJson; - jsonCollection = GetJsonGroupCollection(configuration, propertyLogic, aPropertySingletonDirectory); - fileHolderGroupCollection = GetFileHolderGroupCollection(configuration, propertyLogic, searchPattern, topDirectories); + List<(int g, string sourceDirectory, List<(string sourceDirectoryFile, Shared.Models.Property? property)> collection)> collectionFromJson; + (int j, jsonCollection) = GetJsonGroupCollection(configuration, propertyLogic, aPropertySingletonDirectory); + (int f, fileHolderGroupCollection) = GetFileHolderGroupCollection(configuration, propertyLogic, searchPattern, topDirectories); collectionFromJson = GetCollection(aPropertySingletonDirectory, jsonCollection); - results = GetContainers(configuration, firstRun, aPropertySingletonDirectory, fileHolderGroupCollection, collectionFromJson); - return results; + (int t, results) = GetContainers(configuration, firstRun, aPropertySingletonDirectory, fileHolderGroupCollection, collectionFromJson); + return (j, f, t, results); } } \ No newline at end of file diff --git a/Shared/Models/Container.cs b/Shared/Models/Container.cs index 9426626..3011f73 100644 --- a/Shared/Models/Container.cs +++ b/Shared/Models/Container.cs @@ -7,27 +7,23 @@ public class Container : Properties.IContainer { protected readonly int _G; - protected readonly int _R; protected readonly List _Items; protected readonly string _SourceDirectory; public int G => _G; public List Items => _Items; - public int R => _R; public string SourceDirectory => _SourceDirectory; [JsonConstructor] - public Container(int g, int r, List items, string sourceDirectory) + public Container(int g, List items, string sourceDirectory) { _G = g; - _R = r; _Items = items; _SourceDirectory = sourceDirectory; } - public Container(int g, int r, string sourceDirectory) + public Container(int g, string sourceDirectory) { _G = g; - _R = r; _Items = new(); _SourceDirectory = sourceDirectory; } diff --git a/Shared/Models/Properties/IContainer.cs b/Shared/Models/Properties/IContainer.cs index d9cc6f8..e433c8f 100644 --- a/Shared/Models/Properties/IContainer.cs +++ b/Shared/Models/Properties/IContainer.cs @@ -5,7 +5,6 @@ public interface IContainer public int G { get; } public List Items { get; } - public int R { get; } public string SourceDirectory { get; } } \ No newline at end of file diff --git a/Shared/Models/Stateless/Methods/Mapping.cs b/Shared/Models/Stateless/Methods/Mapping.cs index 7de94f2..3404ab2 100644 --- a/Shared/Models/Stateless/Methods/Mapping.cs +++ b/Shared/Models/Stateless/Methods/Mapping.cs @@ -5,14 +5,6 @@ namespace View_by_Distance.Shared.Models.Stateless.Methods; internal abstract class Mapping { - internal static double GetDeterministicHashCodeKey(int locationDigits, Models.Item item, Models.Face face) - { - if (item.Property?.Id is null || item.ImageFileHolder is null || face.Location?.NormalizedPixelPercentage is null) - throw new NullReferenceException(); - double result = IMapping.GetDeterministicHashCodeKeyValue(locationDigits, item.Property.Id.Value, face.Location.NormalizedPixelPercentage.Value); - return result; - } - private static void UseKeyValuePairsSaveFaceEncoding(int locationDigits, Dictionary> keyValuePairs, string file, int id, int normalizedPixelPercentageValue, double deterministicHashCodeKey, string extensionLowered) { string json;