Save Url and Get New Root Directory

This commit is contained in:
Mike Phares 2022-09-21 13:55:57 -07:00
parent 90244811ed
commit 192d2ad776
8 changed files with 107 additions and 27 deletions

View File

@ -47,7 +47,7 @@ public class Compare
List<string> topDirectories = new(); List<string> topDirectories = new();
List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)> groupCollection; List<(int g, string sourceDirectory, string[] sourceDirectoryFiles)> groupCollection;
Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot); Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot);
Property.Models.Configuration.Verify(propertyConfiguration); Property.Models.Configuration.Verify(propertyConfiguration, requireExist: true);
Models.Configuration configuration = Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory, propertyConfiguration); Models.Configuration configuration = Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory, propertyConfiguration);
Verify(configuration); Verify(configuration);
Model? model = null; Model? model = null;

View File

@ -36,7 +36,7 @@ public class DateGroup
_FileKeyValuePairs = new List<KeyValuePair<string, string>>(); _FileKeyValuePairs = new List<KeyValuePair<string, string>>();
_FilePropertiesKeyValuePairs = new Dictionary<string, List<Tuple<string, Shared.Models.Property>>>(); _FilePropertiesKeyValuePairs = new Dictionary<string, List<Tuple<string, Shared.Models.Property>>>();
Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot); Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot);
Property.Models.Configuration.Verify(propertyConfiguration); Property.Models.Configuration.Verify(propertyConfiguration, requireExist: true);
Models.Configuration configuration = Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory, propertyConfiguration); Models.Configuration configuration = Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory, propertyConfiguration);
Verify(configuration); Verify(configuration);
bool reverse = false; bool reverse = false;

View File

@ -24,6 +24,7 @@ public partial class DlibDotNet
private readonly C_Resize _Resize; private readonly C_Resize _Resize;
private readonly F_Random _Random; private readonly F_Random _Random;
private readonly E3_Rename _Rename; private readonly E3_Rename _Rename;
private readonly IConsole _Console;
private readonly B_Metadata _Metadata; private readonly B_Metadata _Metadata;
private readonly Serilog.ILogger? _Log; private readonly Serilog.ILogger? _Log;
private readonly D2_FaceParts _FaceParts; private readonly D2_FaceParts _FaceParts;
@ -38,6 +39,7 @@ public partial class DlibDotNet
public DlibDotNet(List<string> args, IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, AppSettings appSettings, string workingDirectory, bool isSilent, IConsole console) public DlibDotNet(List<string> args, IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, AppSettings appSettings, string workingDirectory, bool isSilent, IConsole console)
{ {
_Console = console;
string argZero; string argZero;
string message; string message;
_AppSettings = appSettings; _AppSettings = appSettings;
@ -50,7 +52,7 @@ public partial class DlibDotNet
_FilePropertiesKeyValuePairs = new Dictionary<string, List<Tuple<string, Shared.Models.Property>>>(); _FilePropertiesKeyValuePairs = new Dictionary<string, List<Tuple<string, Shared.Models.Property>>>();
Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot); Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot);
_Log.Information(propertyConfiguration.RootDirectory); _Log.Information(propertyConfiguration.RootDirectory);
Property.Models.Configuration.Verify(propertyConfiguration); Property.Models.Configuration.Verify(propertyConfiguration, requireExist: false);
Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration); Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration);
Verify(configuration); Verify(configuration);
VerifyExtra(args, propertyConfiguration, configuration); VerifyExtra(args, propertyConfiguration, configuration);
@ -64,10 +66,7 @@ public partial class DlibDotNet
string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(propertyConfiguration, nameof(A_Property), create: false); string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(propertyConfiguration, nameof(A_Property), create: false);
_FirstRun = !Directory.Exists(propertyRoot); _FirstRun = !Directory.Exists(propertyRoot);
_Metadata = new(configuration.ForceMetadataLastWriteTimeToCreationTime, configuration.PropertiesChangedForMetadata); _Metadata = new(configuration.ForceMetadataLastWriteTimeToCreationTime, configuration.PropertiesChangedForMetadata);
if (args.Count > 0) argZero = args.Count > 0 ? Path.GetFullPath(args[0]) : Path.GetFullPath(propertyConfiguration.RootDirectory);
argZero = Path.GetFullPath(args[0]);
else
argZero = Path.GetFullPath(propertyConfiguration.RootDirectory);
_ArgZeroIsConfigurationRootDirectory = propertyConfiguration.RootDirectory == argZero; _ArgZeroIsConfigurationRootDirectory = propertyConfiguration.RootDirectory == argZero;
_Log.Information(configuration.ModelDirectory); _Log.Information(configuration.ModelDirectory);
(Model model, PredictorModel predictorModel, ModelParameter modelParameter) = GetModel(configuration); (Model model, PredictorModel predictorModel, ModelParameter modelParameter) = GetModel(configuration);
@ -666,7 +665,7 @@ public partial class DlibDotNet
for (int y = 0; y < int.MaxValue; y++) for (int y = 0; y < int.MaxValue; y++)
{ {
_Log.Information("Press \"Y\" key when ready to continue or close console"); _Log.Information("Press \"Y\" key when ready to continue or close console");
if (System.Console.ReadKey().Key == ConsoleKey.Y) if (_Console.ReadKey() == ConsoleKey.Y)
break; break;
} }
_Log.Information(". . ."); _Log.Information(". . .");
@ -756,6 +755,72 @@ public partial class DlibDotNet
mapLogic.SaveShortcuts(_Configuration.JuliePhares, distinctFilteredFaces); mapLogic.SaveShortcuts(_Configuration.JuliePhares, distinctFilteredFaces);
} }
private static Container? AreAllSameStartsWith(string argZero, Container[] containers)
{
Container? result = null;
string[] directoryNames = Shared.Models.Stateless.Methods.IPath.GetDirectoryNames(argZero);
if (directoryNames.Length > 2)
{
string directoryName;
string rootDirectoryName = directoryNames[^2];
if (!Directory.Exists(argZero))
throw new Exception();
foreach (Container container in containers)
{
if (container.SourceDirectory == argZero)
result = container;
directoryName = Path.GetFileName(container.SourceDirectory);
if (!directoryName.StartsWith(rootDirectoryName))
{
result = null;
break;
}
}
}
return result;
}
private string SaveUrlAndGetNewRootDirectory(Container container)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
string? result = Path.GetDirectoryName(Path.GetDirectoryName(container.SourceDirectory));
if (result is null)
throw new Exception();
Uri uri;
Item item;
string? line;
string fileName;
Task<byte[]> task;
string relativePath;
string extensionLowered;
string sourceDirectoryFile;
int length = result.Length;
HttpClient httpClient = new();
bool isValidImageFormatExtension;
for (int y = 0; y < int.MaxValue; y++)
{
_Log.Information("Enter a url for a image");
line = _Console.ReadLine();
if (string.IsNullOrEmpty(line))
break;
uri = new(line);
if (uri.HostNameType != UriHostNameType.Dns)
continue;
task = httpClient.GetByteArrayAsync(uri);
fileName = Path.GetFileName(uri.LocalPath);
sourceDirectoryFile = Path.Combine(container.SourceDirectory, fileName);
File.WriteAllBytes(sourceDirectoryFile, task.Result);
extensionLowered = Path.GetExtension(uri.LocalPath);
relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFile, length, forceExtensionToLower: true);
isValidImageFormatExtension = _Configuration.PropertyConfiguration.ValidImageFormatExtensions.Contains(extensionLowered);
item = new(sourceDirectoryFile, relativePath, isValidImageFormatExtension);
container.Items.Add(item);
}
_Log.Information(". . .");
return result;
}
private void Search(long ticks, Model? model, PredictorModel? predictorModel, string argZero, string propertyRoot, PersonContainer[] personContainers) private void Search(long ticks, Model? model, PredictorModel? predictorModel, string argZero, string propertyRoot, PersonContainer[] personContainers)
{ {
int j; int j;
@ -768,22 +833,30 @@ public partial class DlibDotNet
string dResultsFullGroupDirectory; string dResultsFullGroupDirectory;
string eResultsFullGroupDirectory; string eResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory; string d2ResultsFullGroupDirectory;
A_Property propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, model, predictorModel);
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds); int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string message = $") Building Container(s) - {totalSeconds} total second(s)"; string message = $") Building Container(s) - {totalSeconds} total second(s)";
A_Property propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, model, predictorModel);
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }; ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
using (ProgressBar progressBar = new(1, message, options)) using (ProgressBar progressBar = new(1, message, options))
{ {
progressBar.Tick(); progressBar.Tick();
if (string.IsNullOrEmpty(_Configuration.PropertyConfiguration.RootDirectory)) (j, f, t, containers) = Property.Models.Stateless.Container.GetContainers(_Configuration.PropertyConfiguration, _FirstRun, propertyLogic);
(j, f, t, containers) = A_Property.Get(_Configuration.PropertyConfiguration, propertyLogic); }
else Container? container = AreAllSameStartsWith(argZero, containers);
(j, f, t, containers) = Property.Models.Stateless.Container.GetContainers(_Configuration.PropertyConfiguration, _FirstRun, propertyLogic); if (_ArgZeroIsConfigurationRootDirectory && container is not null)
{
string? newRootDirectory = SaveUrlAndGetNewRootDirectory(container);
for (int i = 1; i < 10; i++)
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, string.Empty, create: true));
argZero = newRootDirectory;
_Configuration.PropertyConfiguration.ChangeRootDirectory(newRootDirectory);
propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false);
propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, model, predictorModel);
} }
FullDoWork(argZero, model, predictorModel, propertyRoot, ticks, propertyLogic, t, containers); FullDoWork(argZero, model, predictorModel, propertyRoot, ticks, propertyLogic, t, containers);
foreach (string outputResolution in _Configuration.OutputResolutions) foreach (string outputResolution in _Configuration.OutputResolutions)
{ {
if (_FirstRun) if (_FirstRun || container is not null)
break; break;
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory) = GetResultsFullGroupDirectories(model, predictorModel, outputResolution); (aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, eResultsFullGroupDirectory) = GetResultsFullGroupDirectories(model, predictorModel, outputResolution);
if (_ArgZeroIsConfigurationRootDirectory && _Configuration.SaveResizedSubfiles && outputResolution == _Configuration.OutputResolutions[0] && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution) && _Exceptions.Count == 0) if (_ArgZeroIsConfigurationRootDirectory && _Configuration.SaveResizedSubfiles && outputResolution == _Configuration.OutputResolutions[0] && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution) && _Exceptions.Count == 0)

View File

@ -27,7 +27,7 @@ public class NotCopyCopy
_IsEnvironment = isEnvironment; _IsEnvironment = isEnvironment;
_Log = Serilog.Log.ForContext<NotCopyCopy>(); _Log = Serilog.Log.ForContext<NotCopyCopy>();
Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot); Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot);
Property.Models.Configuration.Verify(propertyConfiguration); Property.Models.Configuration.Verify(propertyConfiguration, requireExist: true);
Models.Configuration configuration = Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory, propertyConfiguration); Models.Configuration configuration = Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory, propertyConfiguration);
Verify(configuration); Verify(configuration);
bool reverse = false; bool reverse = false;

View File

@ -35,7 +35,7 @@ public class PrepareForOld
_FileKeyValuePairs = new List<KeyValuePair<string, string>>(); _FileKeyValuePairs = new List<KeyValuePair<string, string>>();
_FilePropertiesKeyValuePairs = new Dictionary<string, List<Tuple<string, Shared.Models.Property>>>(); _FilePropertiesKeyValuePairs = new Dictionary<string, List<Tuple<string, Shared.Models.Property>>>();
Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot); Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot);
Property.Models.Configuration.Verify(propertyConfiguration); Property.Models.Configuration.Verify(propertyConfiguration, requireExist: true);
Configuration configuration = Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory, propertyConfiguration); Configuration configuration = Models.Stateless.Configuration.Get(isEnvironment, configurationRoot, workingDirectory, propertyConfiguration);
Verify(configuration); Verify(configuration);
if (propertyConfiguration.IgnoreExtensions is null) if (propertyConfiguration.IgnoreExtensions is null)

View File

@ -77,7 +77,7 @@ public class Configuration : Shared.Models.Properties.IPropertyConfiguration
public void ChangeRootDirectory(string rootDirectory) => _RootDirectory = rootDirectory; public void ChangeRootDirectory(string rootDirectory) => _RootDirectory = rootDirectory;
public static void Verify(Configuration propertyConfiguration) public static void Verify(Configuration propertyConfiguration, bool requireExist)
{ {
if (propertyConfiguration is null) if (propertyConfiguration is null)
throw new NullReferenceException(nameof(propertyConfiguration)); throw new NullReferenceException(nameof(propertyConfiguration));
@ -101,7 +101,7 @@ public class Configuration : Shared.Models.Properties.IPropertyConfiguration
throw new NullReferenceException(nameof(propertyConfiguration.FileNameDirectorySeparator)); throw new NullReferenceException(nameof(propertyConfiguration.FileNameDirectorySeparator));
if (string.IsNullOrEmpty(propertyConfiguration.Pattern)) if (string.IsNullOrEmpty(propertyConfiguration.Pattern))
throw new NullReferenceException(nameof(propertyConfiguration.Pattern)); throw new NullReferenceException(nameof(propertyConfiguration.Pattern));
if (string.IsNullOrEmpty(propertyConfiguration.RootDirectory) || !Directory.Exists(propertyConfiguration.RootDirectory)) if (string.IsNullOrEmpty(propertyConfiguration.RootDirectory) || (requireExist && !Directory.Exists(propertyConfiguration.RootDirectory)))
throw new NullReferenceException(nameof(propertyConfiguration.RootDirectory)); throw new NullReferenceException(nameof(propertyConfiguration.RootDirectory));
} }

View File

@ -15,7 +15,11 @@ public class Container
string[] sourceDirectoryFiles; string[] sourceDirectoryFiles;
List<string[]> fileCollections = new(); List<string[]> fileCollections = new();
if (!topDirectories.Any()) if (!topDirectories.Any())
{
if (!Directory.Exists(rootDirectory))
_ = Directory.CreateDirectory(rootDirectory);
topDirectories.AddRange(from l in Directory.GetDirectories(rootDirectory, "*", SearchOption.TopDirectoryOnly) select Path.GetFullPath(l)); topDirectories.AddRange(from l in Directory.GetDirectories(rootDirectory, "*", SearchOption.TopDirectoryOnly) select Path.GetFullPath(l));
}
for (int g = 1; g < 5; g++) for (int g = 1; g < 5; g++)
{ {
if (g == 4) if (g == 4)
@ -76,14 +80,8 @@ public class Container
{ {
if (sourceDirectoryFiles.Any() || Directory.GetDirectories(subDirectory, "*", SearchOption.TopDirectoryOnly).Any()) if (sourceDirectoryFiles.Any() || Directory.GetDirectories(subDirectory, "*", SearchOption.TopDirectoryOnly).Any())
results.Add(new(g, subDirectory, sourceDirectoryFiles)); results.Add(new(g, subDirectory, sourceDirectoryFiles));
else if (searchPattern == "*") else if (searchPattern == "*" && subDirectory != rootDirectory)
{
sourceDirectoryFiles = Directory.GetFiles(subDirectory, searchPattern, SearchOption.TopDirectoryOnly);
result += sourceDirectoryFiles.Length;
foreach (string subFile in sourceDirectoryFiles)
File.Delete(subFile);
Directory.Delete(subDirectory); Directory.Delete(subDirectory);
}
} }
} }
fileCollections.Reverse(); fileCollections.Reverse();
@ -172,6 +170,11 @@ public class Container
{ {
foreach (FileHolder sourceDirectoryFileHolder in sourceDirectoryFileHolderCollection) foreach (FileHolder sourceDirectoryFileHolder in sourceDirectoryFileHolderCollection)
{ {
if (sourceDirectory.Contains("Facebook"))
{
if (sourceDirectory.Contains("Facebook"))
{ }
}
relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFileHolder.FullName, length, forceExtensionToLower: true); relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFileHolder.FullName, length, forceExtensionToLower: true);
fileHolderKeyValuePairs.Add(relativePath, new(sourceDirectory, sourceDirectoryFileHolder)); fileHolderKeyValuePairs.Add(relativePath, new(sourceDirectory, sourceDirectoryFileHolder));
} }
@ -247,7 +250,7 @@ public class Container
item = new(sourceDirectoryFileHolder.FullName, relativePath, sourceDirectoryFileHolder, isValidImageFormatExtension, null, null, null); item = new(sourceDirectoryFileHolder.FullName, relativePath, sourceDirectoryFileHolder, isValidImageFormatExtension, null, null, null);
items.Add(item); items.Add(item);
} }
if (items.Any()) if (sourceDirectory == configuration.RootDirectory || items.Any())
{ {
if (!keyValuePairs.ContainsKey(sourceDirectory)) if (!keyValuePairs.ContainsKey(sourceDirectory))
{ {

View File

@ -45,7 +45,7 @@ public class Item : Properties.IItem
_ValidImageFormatExtension = validImageFormatExtension; _ValidImageFormatExtension = validImageFormatExtension;
} }
public Item(string sourceDirectoryFile, string relativePath, FileHolder? imageFileInfo, bool isValidImageFormatExtension, Property? property, bool? abandoned, bool? changed) public Item(string sourceDirectoryFile, string relativePath, FileHolder imageFileInfo, bool isValidImageFormatExtension, Property? property, bool? abandoned, bool? changed)
{ {
_Faces = new(); _Faces = new();
_Changed = changed; _Changed = changed;
@ -62,6 +62,10 @@ public class Item : Properties.IItem
throw new ArgumentException("Can not be a *.json file!"); throw new ArgumentException("Can not be a *.json file!");
} }
public Item(string sourceDirectoryFile, string relativePath, bool isValidImageFormatExtension) :
this(sourceDirectoryFile, relativePath, new(sourceDirectoryFile), isValidImageFormatExtension, null, null, null)
{ }
public override string ToString() public override string ToString()
{ {
string result = JsonSerializer.Serialize(this, new JsonSerializerOptions() { WriteIndented = true }); string result = JsonSerializer.Serialize(this, new JsonSerializerOptions() { WriteIndented = true });