phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GetToDoCollection

Browse Source
This commit is contained in:
Mike Phares 2023-07-04 09:04:22 -07:00
parent b331a4d015
commit 79f069445e
10 changed files with 296 additions and 251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

161
Copy-Distinct/CopyDistinct.cs
View File

@ -36,15 +36,15 @@ public class CopyDistinct
_Configuration = configuration;
propertyConfiguration.Update();
log.Information(propertyConfiguration.RootDirectory);
(bool move, List<string> allFiles, bool anyLenFiles, bool moveBack) = Verify();
(bool move, List<string[]> filesCollection, bool anyLenFiles, bool moveBack) = Verify();
_FileGroups = Shared.Models.Stateless.Methods.IPath.GetKeyValuePairs(propertyConfiguration, appSettings.CopyTo, new string[] { appSettings.ResultDirectoryKey });
List<string> lines = CopyDistinctFilesInDirectories(log, move, allFiles, anyLenFiles, moveBack);
List<string> lines = CopyDistinctFilesInDirectories(log, move, filesCollection, anyLenFiles, moveBack);
if (lines.Any())
File.WriteAllLines($"D:/Tmp/Phares/{DateTime.Now.Ticks}.tsv", lines);
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(propertyConfiguration.RootDirectory);
}
private (bool, List<string>, bool, bool) Verify()
private (bool, List<string[]>, bool, bool) Verify()
{
if (_AppSettings is null)
throw new NullReferenceException(nameof(_AppSettings));
@ -64,8 +64,7 @@ public class CopyDistinct
string copyTo = Path.GetFullPath(_AppSettings.CopyTo);
bool move = copyTo == _PropertyConfiguration.RootDirectory;
List<string[]> filesCollection = Shared.Models.Stateless.Methods.IDirectory.GetFilesCollection(_PropertyConfiguration.RootDirectory, directorySearchFilter, fileSearchFilter);
(_, List<string> allFiles) = Get(filesCollection);
bool anyLenFiles = allFiles.Any(l => l.EndsWith("len"));
bool anyLenFiles = filesCollection.Any(l => l.Any(m => m.EndsWith("len")));
if (!move)
moveBack = false;
else
@ -82,100 +81,26 @@ public class CopyDistinct
moveBack = true;
}
}
return (move, allFiles, anyLenFiles, moveBack);
return (move, filesCollection, anyLenFiles, moveBack);
}
private static (List<string>, List<string>) Get(List<string[]> filesCollection)
private static (string[], List<(FileHolder, string)>) GetMoveBackToDoCollection(List<string[]> filesCollection)
{
List<string> results = new();
string? directory;
List<string> directories = new();
foreach (string[] files in filesCollection)
{
if (!files.Any())
continue;
directory = Path.GetDirectoryName(files.First());
if (directory is null)
continue;
if (!directories.Contains(directory))
directories.Add(directory);
results.AddRange(files);
}
return (directories, results);
}
private List<(FileHolder, string, string)> GetToDoCollection(ProgressBar progressBar, FileHolder[] fileHolders)
{
List<(FileHolder, string, string)> results = new();
string checkFile;
string directory;
FileInfo fileInfo;
int directoryIndex;
string directoryName;
bool wrapped = false;
List<string> distinct = new();
FileHolder[] sortedFileHolders = (from l in fileHolders orderby l.LastWriteTime, l.FullName.Length descending select l).ToArray();
string key = string.IsNullOrEmpty(_AppSettings.ResultDirectoryKey) ? _PropertyConfiguration.ResultAllInOne : _AppSettings.ResultDirectoryKey;
foreach (FileHolder fileHolder in sortedFileHolders)
{
progressBar.Tick();
if (fileHolder.Name.EndsWith("len") || fileHolder.ExtensionLowered == ".id" || fileHolder.ExtensionLowered == ".lsv" || fileHolder.DirectoryName is null)
continue;
(_, directoryIndex) = Shared.Models.Stateless.Methods.IPath.GetDirectoryNameAndIndex(_PropertyConfiguration.ResultAllInOneSubdirectoryLength, fileHolder.NameWithoutExtension);
directoryName = Path.GetFileName(fileHolder.DirectoryName);
if (directoryName.Length < _PropertyConfiguration.ResultAllInOneSubdirectoryLength + 3 || !fileHolder.Name.StartsWith(directoryName))
{
if (wrapped)
continue;
directory = _FileGroups[key][directoryIndex];
}
else
{
if (!wrapped)
wrapped = true;
directory = Path.Combine(_FileGroups[key][directoryIndex], directoryName);
}
checkFile = Path.Combine(directory, $"{fileHolder.NameWithoutExtension}{fileHolder.ExtensionLowered}");
if (distinct.Contains(checkFile))
{
if (!_AppSettings.CopyDuplicates)
continue;
for (int i = 1; i < int.MaxValue; i++)
{
fileInfo = new(checkFile);
if (!fileInfo.Exists || fileHolder.Length == fileInfo.Length && fileHolder.LastWriteTime == fileInfo.LastWriteTime)
checkFile = Path.Combine(directory, $"{fileHolder.NameWithoutExtension}.{i}dup{fileHolder.ExtensionLowered}");
else
checkFile = Path.Combine(directory, $"{fileHolder.NameWithoutExtension}.{i}why{fileHolder.ExtensionLowered}");
if (distinct.Contains(checkFile))
continue;
distinct.Add(checkFile);
results.Add(new(fileHolder, directory, checkFile));
break;
}
continue;
}
distinct.Add(checkFile);
results.Add(new(fileHolder, directory, checkFile));
}
return results;
}
private static List<(FileHolder, string, string)> GetMoveBackToDoCollection(List<string> files)
{
List<(FileHolder, string, string)> results = new();
List<(FileHolder, string)> results = new();
string key;
string? value;
string fileName;
files.Reverse();
string? directory;
string destinationFile;
List<string> distinctFound = new();
List<string> distinctNeeded = new();
List<string> distinctDirectories = new();
Dictionary<string, string> nameToPath = new();
for (int i = 1; i < 3; i++)
{
foreach (string file in files)
foreach (string[] files in filesCollection)
{
foreach (string file in files.Reverse())
{
fileName = Path.GetFileName(file);
if (fileName.EndsWith("len"))
@ -201,6 +126,9 @@ public class CopyDistinct
continue;
break;
}
}
foreach (string[] files in filesCollection)
{
foreach (string file in files)
{
// if (distinctNeeded.Count != distinctFound.Count)
@ -213,67 +141,38 @@ public class CopyDistinct
directory = Path.GetDirectoryName(value);
if (string.IsNullOrEmpty(directory))
continue;
results.Add(new(new(file), directory, value));
results.Add(new(new(file), value));
if (!distinctDirectories.Contains(directory))
distinctDirectories.Add(directory);
}
return results;
}
return (distinctDirectories.ToArray(), results);
}
private static List<string> CopyOrMove(ProgressBar progressBar, List<(FileHolder, string, string)> toDoCollection, bool move, bool moveBack)
{
List<string> results = new();
FileInfo fileInfo;
foreach ((FileHolder fileHolder, string _, string to) in toDoCollection)
{
progressBar.Tick();
fileInfo = new(to);
if (fileInfo.Exists)
{
if (fileHolder.Length != fileInfo.Length || fileHolder.LastWriteTime != fileInfo.LastWriteTime)
fileInfo.Delete();
else
continue;
}
results.Add(fileHolder.NameWithoutExtension);
try
{
if (move || moveBack)
File.Move(fileHolder.FullName, to);
else
File.Copy(fileHolder.FullName, to);
}
catch (Exception) { }
}
return results;
}
private List<string> CopyDistinctFilesInDirectories(ILogger log, bool move, List<string> allFiles, bool anyLenFiles, bool moveBack)
private List<string> CopyDistinctFilesInDirectories(ILogger log, bool move, List<string[]> filesCollection, bool anyLenFiles, bool moveBack)
{
List<string> results = new();
ProgressBar progressBar;
string[] distinctDirectories;
ConsoleKey? consoleKey = null;
string message = nameof(CopyDistinct);
List<string> distinctDirectories = new();
List<(FileHolder, string, string)> toDoCollection;
List<(FileHolder, string)> toDoCollection;
int count = filesCollection.Select(l => l.Length).Sum();
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
if (moveBack)
{
if (!anyLenFiles)
throw new NotSupportedException();
toDoCollection = GetMoveBackToDoCollection(allFiles);
(distinctDirectories, toDoCollection) = GetMoveBackToDoCollection(filesCollection);
}
else
{
progressBar = new(allFiles.Count, message, options);
FileHolder[] fileHolders = (from l in allFiles select new FileHolder(l)).ToArray();
toDoCollection = GetToDoCollection(progressBar, fileHolders);
progressBar = new(count, message, options);
string key = string.IsNullOrEmpty(_AppSettings.ResultDirectoryKey) ? _PropertyConfiguration.ResultAllInOne : _AppSettings.ResultDirectoryKey;
string[] directories = _FileGroups[key];
(distinctDirectories, toDoCollection) = Shared.Models.Stateless.Methods.IDirectory.GetToDoCollection(_PropertyConfiguration, _AppSettings.CopyDuplicates, filesCollection, directories, () => progressBar.Tick());
progressBar.Dispose();
}
foreach ((FileHolder fileHolder, string directory, string to) in toDoCollection)
{
if (distinctDirectories.Contains(directory))
continue;
distinctDirectories.Add(directory);
}
foreach (string distinctDirectory in distinctDirectories)
{
if (!Directory.Exists(distinctDirectory))
@ -307,8 +206,8 @@ public class CopyDistinct
}
else
{
progressBar = new(allFiles.Count, message, options);
results.AddRange(CopyOrMove(progressBar, toDoCollection, move, moveBack));
progressBar = new(count, message, options);
results.AddRange(Shared.Models.Stateless.Methods.IDirectory.CopyOrMove(toDoCollection, move, moveBack, () => progressBar.Tick()));
progressBar.Dispose();
if (move || moveBack)
log.Information("Done moving");

84
Instance/DlibDotNet.cs
View File

@ -392,6 +392,7 @@ public partial class DlibDotNet
ReadOnlyDictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers,
MapLogic mapLogic,
string outputResolution,
bool outputResolutionHasNumber,
string cResultsFullGroupDirectory,
string dResultsFullGroupDirectory,
string d2ResultsFullGroupDirectory,
@ -412,7 +413,7 @@ public partial class DlibDotNet
List<string> parseExceptions = new();
List<Tuple<string, DateTime>> subFileTuples = new();
List<KeyValuePair<string, string>> metadataCollection;
FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(item);
FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(cResultsFullGroupDirectory, item, outputResolutionHasNumber);
if (item.Property is null || item.Property.Id is null || item.Any())
{
LogItemPropertyIsNull(item);
@ -513,6 +514,7 @@ public partial class DlibDotNet
ReadOnlyDictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers,
MapLogic mapLogic,
string outputResolution,
bool outputResolutionHasNumber,
string cResultsFullGroupDirectory,
string dResultsFullGroupDirectory,
string d2ResultsFullGroupDirectory,
@ -542,6 +544,7 @@ public partial class DlibDotNet
idToLocationContainers,
mapLogic,
outputResolution,
outputResolutionHasNumber,
cResultsFullGroupDirectory,
dResultsFullGroupDirectory,
d2ResultsFullGroupDirectory,
@ -665,6 +668,7 @@ public partial class DlibDotNet
int exceptionCount;
Container container;
Item[] filteredItems;
bool outputResolutionHasNumber;
bool anyNullOrNoIsUniqueFileName;
string cResultsFullGroupDirectory;
string dResultsFullGroupDirectory;
@ -677,6 +681,7 @@ public partial class DlibDotNet
foreach (string outputResolution in _Configuration.OutputResolutions)
{
total = 0;
outputResolutionHasNumber = outputResolution.Any(l => char.IsNumber(l));
(cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
_Faces.Update(dResultsFullGroupDirectory);
_Resize.Update(cResultsFullGroupDirectory);
@ -697,6 +702,7 @@ public partial class DlibDotNet
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
message = $"{i + 1:000} [{filteredItems.Length:000}] / {containersLength:000} - {total} / {t} total - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}";
propertyLogic.SetAngleBracketCollection(aResultsFullGroupDirectory, container.SourceDirectory, anyNullOrNoIsUniqueFileName);
if (outputResolutionHasNumber)
_Resize.SetAngleBracketCollection(cResultsFullGroupDirectory, container.SourceDirectory);
exceptionCount = FullParallelWork(maxDegreeOfParallelism,
propertyLogic,
@ -704,6 +710,7 @@ public partial class DlibDotNet
idToLocationContainers,
mapLogic,
outputResolution,
outputResolutionHasNumber,
cResultsFullGroupDirectory,
dResultsFullGroupDirectory,
d2ResultsFullGroupDirectory,
@ -815,13 +822,17 @@ public partial class DlibDotNet
SaveFaceDistances(ticks, mapLogic, distinctFilteredFaces, distinctFilteredMappingCollection, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, dFacesCollectionDirectory, idToWholePercentagesToMapping);
}
private string SaveUrlAndGetNewRootDirectory(Container container)
private string SaveUrlAndGetNewRootDirectory(string[] files)
{
string result;
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
string? result = Path.GetDirectoryName(Path.GetDirectoryName(container.SourceDirectory)) ?? throw new Exception();
if (!files.Any())
throw new NotSupportedException();
string? sourceDirectory = Path.GetDirectoryName(files.First());
if (string.IsNullOrEmpty(sourceDirectory))
throw new NotSupportedException();
Uri uri;
Item item;
string? line;
string fileName;
Task<byte[]> task;
@ -829,9 +840,10 @@ public partial class DlibDotNet
FileHolder fileHolder;
string extensionLowered;
string sourceDirectoryFile;
int length = result.Length;
HttpClient httpClient = new();
bool isValidImageFormatExtension;
result = Path.GetDirectoryName(Path.GetDirectoryName(sourceDirectory)) ?? throw new NotSupportedException();
int length = result.Length;
for (int y = 0; y < int.MaxValue; y++)
{
_Log.Information("Enter fileNameToCollection url for fileNameToCollection image");
@ -843,14 +855,14 @@ public partial class DlibDotNet
continue;
task = httpClient.GetByteArrayAsync(uri);
fileName = Path.GetFileName(uri.LocalPath);
sourceDirectoryFile = Path.Combine(container.SourceDirectory, fileName);
sourceDirectoryFile = Path.Combine(sourceDirectory, fileName);
File.WriteAllBytes(sourceDirectoryFile, task.Result);
extensionLowered = Path.GetExtension(uri.LocalPath);
relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFile, length, forceExtensionToLower: true);
isValidImageFormatExtension = _Configuration.PropertyConfiguration.ValidImageFormatExtensions.Contains(extensionLowered);
fileHolder = new(sourceDirectoryFile);
item = new(fileHolder, relativePath, isValidImageFormatExtension);
container.Items.Add(item);
_ = new Item(fileHolder, relativePath, isValidImageFormatExtension);
// container.Items.Add(item);
}
_Log.Information(". . .");
return result;
@ -1125,6 +1137,7 @@ public partial class DlibDotNet
private void Search(long ticks, string argZero, string propertyRoot)
{
int t;
string message;
Container[] containers;
A_Property propertyLogic;
string eDistanceContentDirectory;
@ -1136,31 +1149,22 @@ public partial class DlibDotNet
string c2ResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
string fPhotoPrismContentDirectory;
const string fileSearchFilter = "*";
string fPhotoPrismSingletonDirectory;
const string directorySearchFilter = "*";
Dictionary<long, List<int>> personKeyToIds;
Dictionary<int, List<MappingFromPhotoPrism>> fileNameToCollection;
(aResultsFullGroupDirectory, bResultsFullGroupDirectory) = GetResultsFullGroupDirectories();
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "{}");
a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "([])");
eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), "()");
fPhotoPrismContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), "()");
fPhotoPrismSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), "{}");
propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory);
MapLogic mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, new(_PersonContainers), ticks, a2PeopleSingletonDirectory, eDistanceContentDirectory);
_PersonContainers.AddRange(Shared.Models.Stateless.Methods.IPersonContainer.GetNonSpecificPeopleCollection(_Configuration.MappingDefaultName, _Configuration.PersonBirthdayFirstYear, _Configuration.PersonCharacters.ToArray(), _PersonContainers, ticks));
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string message = $") Building Container(s) - {totalSeconds} total second(s)";
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
using (ProgressBar progressBar = new(2, message, options))
{
progressBar.Tick();
string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, "{}");
if (!Directory.Exists(aPropertySingletonDirectory))
_ = Directory.CreateDirectory(aPropertySingletonDirectory);
(t, containers) = Shared.Models.Stateless.Methods.IContainer.GetContainers(_Configuration.PropertyConfiguration, aPropertySingletonDirectory);
progressBar.Tick();
}
if (containers.Length == 1)
List<string[]> filesCollection = Shared.Models.Stateless.Methods.IDirectory.GetFilesCollection(_Configuration.PropertyConfiguration.RootDirectory, directorySearchFilter, fileSearchFilter);
int count = filesCollection.Select(l => l.Length).Sum();
if (filesCollection.Count == 1)
{
string resultsGroupDirectory;
a2PeopleContentDirectory = null;
@ -1172,14 +1176,46 @@ public partial class DlibDotNet
resultsGroupDirectory = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, string.Empty, create: true);
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(resultsGroupDirectory);
}
argZero = SaveUrlAndGetNewRootDirectory(containers[0]);
argZero = SaveUrlAndGetNewRootDirectory(filesCollection.First());
_Configuration.PropertyConfiguration.ChangeRootDirectory(argZero);
(aResultsFullGroupDirectory, bResultsFullGroupDirectory) = GetResultsFullGroupDirectories();
propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false);
propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory);
}
TimeSpan eLastWriteTimeTimeSpan = new(ticks - new DirectoryInfo(eDistanceContentDirectory).LastWriteTime.Ticks);
foreach (string outputResolution in _Configuration.OutputResolutions)
{
if (outputResolution.Any(l => char.IsNumber(l)))
continue;
if (eLastWriteTimeTimeSpan.TotalDays < 1)
break;
ProgressBar progressBar;
(cResultsFullGroupDirectory, _, _, _) = GetResultsFullGroupDirectories(outputResolution);
IReadOnlyDictionary<string, string[]> fileGroups = Shared.Models.Stateless.Methods.IPath.GetKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, new string[] { _Configuration.PropertyConfiguration.ResultContent });
message = $") Selecting for ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
progressBar = new(count, message, options);
(string[] distinctDirectories, List<(FileHolder, string)> toDoCollection) = Shared.Models.Stateless.Methods.IDirectory.GetToDoCollection(_Configuration.PropertyConfiguration, copyDuplicates: false, filesCollection, fileGroups[_Configuration.PropertyConfiguration.ResultContent], () => progressBar.Tick());
progressBar.Dispose();
message = $") Copying to ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
progressBar = new(count, message, options);
_ = Shared.Models.Stateless.Methods.IDirectory.CopyOrMove(toDoCollection, move: false, moveBack: false, () => progressBar.Tick());
progressBar.Dispose();
break;
}
message = $") Building Container(s) - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
using (ProgressBar progressBar = new(2, message, options))
{
progressBar.Tick();
string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, "{}");
if (!Directory.Exists(aPropertySingletonDirectory))
_ = Directory.CreateDirectory(aPropertySingletonDirectory);
(t, containers) = Shared.Models.Stateless.Methods.IContainer.GetContainers(_Configuration.PropertyConfiguration, aPropertySingletonDirectory, filesCollection);
progressBar.Tick();
}
fileNameToCollection = !Directory.Exists(fPhotoPrismSingletonDirectory) ? fileNameToCollection = new() : F_PhotoPrism.GetFileNameToCollection(fPhotoPrismSingletonDirectory);
B_Metadata metadata = new(_Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, bResultsFullGroupDirectory);
MapLogic mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, new(_PersonContainers), ticks, a2PeopleSingletonDirectory, eDistanceContentDirectory);
_PersonContainers.AddRange(Shared.Models.Stateless.Methods.IPersonContainer.GetNonSpecificPeopleCollection(_Configuration.MappingDefaultName, _Configuration.PersonBirthdayFirstYear, _Configuration.PersonCharacters.ToArray(), _PersonContainers, ticks));
containers = Shared.Models.Stateless.Methods.IContainer.SortContainers(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, _ArgZeroIsConfigurationRootDirectory, argZero, containers);
personKeyToIds = mapLogic.GetPersonKeyToIds();
if (!string.IsNullOrEmpty(_Configuration.GenealogicalDataCommunicationFile) && !string.IsNullOrEmpty(a2PeopleContentDirectory) && _GenealogicalDataCommunicationHeaderLines is not null && _GenealogicalDataCommunicationFooterLines is not null && _GenealogicalDataCommunicationHeaderLines.Any() && _GenealogicalDataCommunicationFooterLines.Any())

11
Resize/Models/_C_Resize.cs
View File

@ -419,9 +419,16 @@ public class C_Resize
return results;
}
public FileHolder GetResizedFileHolder(Item item)
public FileHolder GetResizedFileHolder(string cResultsFullGroupDirectory, Item item, bool outputResolutionHasNumber)
{
FileHolder result = new(Path.Combine(AngleBracketCollection[0].Replace("<>", _PropertyConfiguration.ResultContent), Path.GetFileName(item.ImageFileHolder.FullName)));
FileHolder result;
if (outputResolutionHasNumber)
result = new(Path.Combine(AngleBracketCollection[0].Replace("<>", _PropertyConfiguration.ResultContent), Path.GetFileName(item.ImageFileHolder.FullName)));
else
{
(string directoryName, _) = Shared.Models.Stateless.Methods.IPath.GetDirectoryNameAndIndex(_PropertyConfiguration.ResultAllInOneSubdirectoryLength, item.ImageFileHolder.Name);
result = new(Path.Combine(cResultsFullGroupDirectory, _PropertyConfiguration.ResultContent, _PropertyConfiguration.ResultAllInOne, directoryName, Path.GetFileName(item.ImageFileHolder.FullName)));
}
return result;
}

37
Set-Created-Date/SetCreatedDate.cs
View File

@ -61,38 +61,23 @@ public class SetCreatedDate
{ }
}
private static (List<string>, List<string>) Get(List<string[]> filesCollection)
{
List<string> results = new();
string? directory;
List<string> directories = new();
foreach (string[] files in filesCollection)
{
if (!files.Any())
continue;
directory = Path.GetDirectoryName(files.First());
if (directory is null)
continue;
if (!directories.Contains(directory))
directories.Add(directory);
results.AddRange(files);
}
return (directories, results);
}
private List<(FileHolder, DateTime)> GetToDoCollection(ProgressBar progressBar, FileHolder[] fileHolders)
private List<(FileHolder, DateTime)> GetToDoCollection(ProgressBar progressBar, List<string[]> filesCollection)
{
List<(FileHolder, DateTime)> results = new();
int? id;
string? message;
DateTime? dateTime;
FileHolder fileHolder;
DateTime?[] dateTimes;
bool isIgnoreExtension;
DateTime? dateTimeOriginal;
bool isValidImageFormatExtension;
foreach (FileHolder fileHolder in fileHolders)
foreach (string[] files in filesCollection)
{
foreach (string file in files)
{
progressBar.Tick();
fileHolder = new(file);
if (fileHolder.ExtensionLowered == ".id" || fileHolder.ExtensionLowered == ".lsv" || fileHolder.DirectoryName is null)
continue;
if (_PropertyConfiguration.IgnoreExtensions.Contains(fileHolder.ExtensionLowered))
@ -109,6 +94,7 @@ public class SetCreatedDate
continue;
results.Add((fileHolder, dateTime.Value));
}
}
return results;
}
@ -135,11 +121,10 @@ public class SetCreatedDate
string message = nameof(SetCreatedDate);
const string directorySearchFilter = "*";
List<string[]> filesCollection = Shared.Models.Stateless.Methods.IDirectory.GetFilesCollection(_PropertyConfiguration.RootDirectory, directorySearchFilter, fileSearchFilter);
(_, List<string> allFiles) = Get(filesCollection);
int count = filesCollection.Select(l => l.Length).Sum();
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
progressBar = new(allFiles.Count, message, options);
FileHolder[] fileHolders = (from l in allFiles select new FileHolder(l)).ToArray();
List<(FileHolder, DateTime)> toDoCollection = GetToDoCollection(progressBar, fileHolders);
progressBar = new(count, message, options);
List<(FileHolder, DateTime)> toDoCollection = GetToDoCollection(progressBar, filesCollection);
progressBar.Dispose();
log.Information($"Ready to set created date {toDoCollection.Count} file(s)?");
for (int y = 0; y < int.MaxValue; y++)
@ -154,7 +139,7 @@ public class SetCreatedDate
log.Information("Nothing set!");
else
{
progressBar = new(allFiles.Count, message, options);
progressBar = new(count, message, options);
results.AddRange(SetCreatedDateForeach(progressBar, toDoCollection));
progressBar.Dispose();
log.Information("Done setting created date");

39
Shared/Models/Stateless/Methods/Container.cs
View File

@ -158,20 +158,7 @@ internal abstract class Container
return results;
}
private static void CreateShell(Properties.IPropertyConfiguration propertyConfiguration, List<string> directories)
{
string checkDirectory;
int startIndex = propertyConfiguration.RootDirectory.Length;
foreach (string directory in directories)
{
checkDirectory = directory.Insert(startIndex, "-Shell");
if (Directory.Exists(checkDirectory))
continue;
_ = Directory.CreateDirectory(checkDirectory);
}
}
internal static (int, Models.Container[]) GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory)
private static (int, Models.Container[]) GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory, string directorySearchFilter, List<string[]> filesCollection)
{
List<Models.Container> results = new();
string? directory;
@ -179,10 +166,7 @@ internal abstract class Container
Models.Container container;
const string extension = ".json";
List<string> directories = new();
const string fileSearchFilter = "*";
const string directorySearchFilter = "*";
Dictionary<string, List<Models.Item>> directoryToItems = new();
List<string[]> filesCollection = IDirectory.GetFilesCollection(propertyConfiguration.RootDirectory, directorySearchFilter, fileSearchFilter);
foreach (string[] files in filesCollection)
{
if (!files.Any())
@ -199,7 +183,6 @@ internal abstract class Container
throw new Exception();
}
}
CreateShell(propertyConfiguration, directories);
List<Models.FilePair> filePairs = GetFilePairs(propertyConfiguration, directorySearchFilter, extension, aPropertySingletonDirectory, filesCollection);
List<FilePair> collection = GetFilePairs(propertyConfiguration, aPropertySingletonDirectory, extension, filePairs);
foreach (FilePair filePair in collection)
@ -224,6 +207,26 @@ internal abstract class Container
return (collection.Count, results.ToArray());
}
internal static (int, Models.Container[]) GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory, List<string[]> filesCollection)
{
int count;
Models.Container[] results;
const string directorySearchFilter = "*";
(count, results) = GetContainers(propertyConfiguration, aPropertySingletonDirectory, directorySearchFilter, filesCollection);
return (count, results);
}
internal static (int, Models.Container[]) GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory)
{
int count;
Models.Container[] results;
const string fileSearchFilter = "*";
const string directorySearchFilter = "*";
List<string[]> filesCollection = IDirectory.GetFilesCollection(propertyConfiguration.RootDirectory, directorySearchFilter, fileSearchFilter);
(count, results) = GetContainers(propertyConfiguration, aPropertySingletonDirectory, directorySearchFilter, filesCollection);
return (count, results);
}
internal static List<int> GetFilteredDistinctIds(Properties.IPropertyConfiguration propertyConfiguration, Models.Container[] containers)
{
List<int> results = new();

5
Shared/Models/Stateless/Methods/IContainer.cs
View File

@ -33,6 +33,11 @@ public interface IContainer
static (int, Models.Container[]) GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory) =>
Container.GetContainers(propertyConfiguration, aPropertySingletonDirectory);
(int, Models.Container[]) TestStatic_GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory, List<string[]> filesCollection) =>
GetContainers(propertyConfiguration, aPropertySingletonDirectory, filesCollection);
static (int, Models.Container[]) GetContainers(Properties.IPropertyConfiguration propertyConfiguration, string aPropertySingletonDirectory, List<string[]> filesCollection) =>
Container.GetContainers(propertyConfiguration, aPropertySingletonDirectory, filesCollection);
List<int> TestStatic_GetFilteredDistinctIds(Properties.IPropertyConfiguration propertyConfiguration, Models.Container[] containers) =>
GetFilteredDistinctIds(propertyConfiguration, containers);
static List<int> GetFilteredDistinctIds(Properties.IPropertyConfiguration propertyConfiguration, Models.Container[] containers) =>

10
Shared/Models/Stateless/Methods/IDirectory.cs
View File

@ -43,4 +43,14 @@ public interface IDirectory
static void MoveFiles(List<string> files, string find, string replace) =>
XDirectory.MoveFiles(files, find, replace);
(string[], List<(Models.FileHolder, string)>) TestStatic_GetToDoCollection(Properties.IPropertyConfiguration propertyConfiguration, bool copyDuplicates, List<string[]> filesCollection, string[] directories, Action? tick) =>
GetToDoCollection(propertyConfiguration, copyDuplicates, filesCollection, directories, tick);
static (string[], List<(Models.FileHolder, string)>) GetToDoCollection(Properties.IPropertyConfiguration propertyConfiguration, bool copyDuplicates, List<string[]> filesCollection, string[] directories, Action? tick) =>
XDirectory.GetToDoCollection(propertyConfiguration, copyDuplicates, filesCollection, directories, tick);
void TestStatic_CopyOrMove(List<(Models.FileHolder, string)> toDoCollection, bool move, bool moveBack, Action? tick) =>
CopyOrMove(toDoCollection, move, moveBack, tick);
static List<string> CopyOrMove(List<(Models.FileHolder, string)> toDoCollection, bool move, bool moveBack, Action? tick) =>
XDirectory.CopyOrMove(toDoCollection, move, moveBack, tick);
}

98
Shared/Models/Stateless/Methods/XDirectory.cs
View File

@ -278,4 +278,102 @@ internal abstract partial class XDirectory
}
}
private static List<Models.FileHolder> GetFileHolders(List<string[]> filesCollection)
{
List<Models.FileHolder> results = new();
foreach (string[] files in filesCollection)
results.AddRange(files.Select(l => new Models.FileHolder(l)));
return results;
}
internal static (string[], List<(Models.FileHolder, string)>) GetToDoCollection(Properties.IPropertyConfiguration propertyConfiguration, bool copyDuplicates, List<string[]> filesCollection, string[] directories, Action? tick)
{
List<(Models.FileHolder, string)> results = new();
string checkFile;
string directory;
FileInfo fileInfo;
int directoryIndex;
string directoryName;
bool wrapped = false;
List<string> distinct = new();
List<string> distinctDirectories = new();
List<Models.FileHolder> fileHolders = GetFileHolders(filesCollection);
Models.FileHolder[] sortedFileHolders = (from l in fileHolders orderby l.LastWriteTime, l.FullName.Length descending select l).ToArray();
foreach (Models.FileHolder fileHolder in sortedFileHolders)
{
tick?.Invoke();
if (fileHolder.Name.EndsWith("len") || fileHolder.ExtensionLowered == ".id" || fileHolder.ExtensionLowered == ".lsv" || fileHolder.DirectoryName is null)
continue;
(_, directoryIndex) = IPath.GetDirectoryNameAndIndex(propertyConfiguration.ResultAllInOneSubdirectoryLength, fileHolder.NameWithoutExtension);
directoryName = Path.GetFileName(fileHolder.DirectoryName);
if (directoryName.Length < propertyConfiguration.ResultAllInOneSubdirectoryLength + 3 || !fileHolder.Name.StartsWith(directoryName))
{
if (wrapped)
continue;
directory = directories[directoryIndex];
}
else
{
if (!wrapped)
wrapped = true;
directory = Path.Combine(directories[directoryIndex], directoryName);
}
checkFile = Path.Combine(directory, $"{fileHolder.NameWithoutExtension}{fileHolder.ExtensionLowered}");
if (distinct.Contains(checkFile))
{
if (!copyDuplicates)
continue;
for (int i = 1; i < int.MaxValue; i++)
{
fileInfo = new(checkFile);
if (!fileInfo.Exists || fileHolder.Length == fileInfo.Length && fileHolder.LastWriteTime == fileInfo.LastWriteTime)
checkFile = Path.Combine(directory, $"{fileHolder.NameWithoutExtension}.{i}dup{fileHolder.ExtensionLowered}");
else
checkFile = Path.Combine(directory, $"{fileHolder.NameWithoutExtension}.{i}why{fileHolder.ExtensionLowered}");
if (distinct.Contains(checkFile))
continue;
distinct.Add(checkFile);
results.Add(new(fileHolder, checkFile));
if (!distinctDirectories.Contains(directory))
distinctDirectories.Add(directory);
break;
}
continue;
}
distinct.Add(checkFile);
results.Add(new(fileHolder, checkFile));
if (!distinctDirectories.Contains(directory))
distinctDirectories.Add(directory);
}
return (distinctDirectories.ToArray(), results);
}
internal static List<string> CopyOrMove(List<(Models.FileHolder, string)> toDoCollection, bool move, bool moveBack, Action? tick)
{
List<string> results = new();
FileInfo fileInfo;
foreach ((Models.FileHolder fileHolder, string to) in toDoCollection)
{
tick?.Invoke();
fileInfo = new(to);
if (fileInfo.Exists)
{
if (fileHolder.Length != fileInfo.Length || fileHolder.LastWriteTime != fileInfo.LastWriteTime)
fileInfo.Delete();
else
continue;
}
results.Add(fileHolder.NameWithoutExtension);
try
{
if (move || moveBack)
File.Move(fileHolder.FullName, to);
else
File.Copy(fileHolder.FullName, to);
}
catch (Exception) { }
}
return results;
}
}

3
Tests/UnitTestResize.cs
View File

@ -144,6 +144,7 @@ public class UnitTestResize
List<KeyValuePair<string, string>> metadataCollection;
int length = _PropertyConfiguration.RootDirectory.Length;
string outputResolution = _Configuration.OutputResolutions[0];
bool outputResolutionHasNumber = outputResolution.Any(l => char.IsNumber(l));
(string cResultsFullGroupDirectory, _, _) = GetResultsFullGroupDirectories(outputResolution);
(string aResultsFullGroupDirectory, string bResultsFullGroupDirectory) = GetResultsFullGroupDirectories();
_Logger.Information(_Configuration.ModelDirectory);
@ -174,7 +175,7 @@ public class UnitTestResize
}
if (property is null || item.Property is null)
throw new NullReferenceException(nameof(property));
resizedFileHolder = resize.GetResizedFileHolder(item);
resizedFileHolder = resize.GetResizedFileHolder(cResultsFullGroupDirectory, item, outputResolutionHasNumber);
Shared.Models.Methods.IBlurHasher blurHasher = new BlurHash.Models.C2_BlurHasher(_Configuration.PropertyConfiguration, resultsFullGroupDirectory: null);
_ = blurHasher.Encode(resizedFileHolder);
item.SetResizedFileHolder(resize.FileNameExtension, resizedFileHolder);

3
TestsWithFaceRecognitionDotNet/UnitTestFace.cs
View File

@ -218,6 +218,7 @@ public class UnitTestFace
List<KeyValuePair<string, string>> metadataCollection;
int length = _PropertyConfiguration.RootDirectory.Length;
string outputResolution = _Configuration.OutputResolutions[0];
bool outputResolutionHasNumber = outputResolution.Any(l => char.IsNumber(l));
(string cResultsFullGroupDirectory, _, _) = GetResultsFullGroupDirectories(outputResolution);
(string aResultsFullGroupDirectory, string bResultsFullGroupDirectory) = GetResultsFullGroupDirectories();
_Logger.Information(_Configuration.ModelDirectory);
@ -248,7 +249,7 @@ public class UnitTestFace
}
if (property is null || item.Property is null)
throw new NullReferenceException(nameof(property));
resizedFileHolder = resize.GetResizedFileHolder(item);
resizedFileHolder = resize.GetResizedFileHolder(cResultsFullGroupDirectory, item, outputResolutionHasNumber);
item.SetResizedFileHolder(resize.FileNameExtension, resizedFileHolder);
MappingFromItem mappingFromItem = IMappingFromItem.GetMappingFromItem(item);
(int _, metadataCollection) = metadata.GetMetadataCollection(subFileTuples, parseExceptions, mappingFromItem);