phares3757/aa
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Ready to test DownloadFile

Browse Source 
DirectoryDictionary

Add http file

.7-Question

JustMediaDate

CombinedEnumAndIndex
This commit is contained in:
Mike Phares
2025-03-08 17:28:30 -07:00
parent ef4672aaf0
commit c15c854481
13 changed files with 464 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files

205
Windows/Windows.cs
View File

@ -50,6 +50,15 @@ public partial class Windows : IWindows, IDisposable
GC.SuppressFinalize(this);
}
private static void DownloadFile(HttpClient httpClient, FilePath filePath)
{
FileStream fileStream = new(filePath.FullName, FileMode.Truncate);
Task<HttpResponseMessage> httpResponseMessage = httpClient.GetAsync(filePath.FullName);
httpResponseMessage.Wait();
Task task = httpResponseMessage.Result.Content.CopyToAsync(fileStream);
task.Wait();
}
ReadOnlyCollection<string> IWindows.ConvertAndGetFastForwardMovingPictureExpertsGroupFiles(IWindowsSettings WindowsSettings, HttpClient? httpClient, FilePath filePath)
{
List<string> results = [];
@ -57,6 +66,8 @@ public partial class Windows : IWindows, IDisposable
if (isValidVideoFormatExtensions)
{
bool check;
if (httpClient is not null)
DownloadFile(httpClient, filePath);
try
{
CommandTask<CommandResult> commandTask = Cli.Wrap("L:/Git/ffmpeg-2024-10-02-git-358fdf3083-full_build/bin/ffmpeg.exe")
@ -240,10 +251,10 @@ public partial class Windows : IWindows, IDisposable
return results;
}
private List<FirstPass> GetCollection(ILogger<Program>? logger, AppSettings appSettings, IWindows windows, IEnumerable<string> files, A_Metadata metadata)
private static int WindowsSynchronousWork(ILogger<Program>? logger, AppSettings appSettings, IWindows windows, A_Metadata metadata, List<FirstPass> results, int index, KeyValuePair<string, List<FileHolder>> keyValuePair)
{
List<FirstPass> results = [];
int index = -1;
int result = index + 1;
windows.Tick();
FilePath filePath;
FirstPass firstPass;
string directoryName;
@ -253,9 +264,65 @@ public partial class Windows : IWindows, IDisposable
bool fastForwardMovingPictureExpertsGroupUsed;
MinimumYearAndPathCombined minimumYearAndPathCombined;
FilePath? fastForwardMovingPictureExpertsGroupFilePath;
ReadOnlyDictionary<string, List<FileHolder>> keyValuePairs;
ReadOnlyCollection<string>? fastForwardMovingPictureExpertsGroupFiles;
HttpClient? httpClient = string.IsNullOrEmpty(appSettings.WindowsSettings.Host) || string.IsNullOrEmpty(appSettings.WindowsSettings.Page) ? null : new();
foreach (FileHolder fileHolder in keyValuePair.Value)
{
if (appSettings.WindowsSettings.SidecarExtensions.Contains(fileHolder.ExtensionLowered))
continue;
if (appSettings.WindowsSettings.IgnoreExtensions.Contains(fileHolder.ExtensionLowered))
continue;
filePath = FilePath.Get(appSettings.ResultSettings, appSettings.MetadataSettings, fileHolder, result);
if (filePath.Id is not null && (filePath.IsIntelligentIdFormat || filePath.SortOrder is not null))
continue;
if (filePath.Id is not null)
{
fastForwardMovingPictureExpertsGroupFiles = null;
deterministicHashCode = new(null, filePath.Id, null);
directoryName = Path.GetFileName(filePath.DirectoryFullPath);
if (directoryName.EndsWith(filePath.Id.Value.ToString()))
continue;
}
else
{
fastForwardMovingPictureExpertsGroupFiles = windows.ConvertAndGetFastForwardMovingPictureExpertsGroupFiles(appSettings.WindowsSettings, httpClient, filePath);
fastForwardMovingPictureExpertsGroupFilePath = fastForwardMovingPictureExpertsGroupFiles.Count == 0 ? null : FilePath.Get(appSettings.ResultSettings, appSettings.MetadataSettings, FileHolder.Get(fastForwardMovingPictureExpertsGroupFiles[0]), result);
deterministicHashCode = fastForwardMovingPictureExpertsGroupFilePath is null ? windows.GetDeterministicHashCode(httpClient, filePath) : windows.GetDeterministicHashCode(httpClient, fastForwardMovingPictureExpertsGroupFilePath);
}
sidecarFiles = [];
filePath = FilePath.Get(filePath, deterministicHashCode);
for (int i = 0; i < keyValuePair.Value.Count; i++)
{
if (keyValuePair.Value[i].ExtensionLowered == fileHolder.ExtensionLowered)
continue;
sidecarFiles.Add(keyValuePair.Value[i]);
}
try
{ (minimumYearAndPathCombined, exifDirectory) = metadata.GetMetadataCollection(appSettings.ResultSettings, appSettings.MetadataSettings, httpClient, filePath); }
catch (Exception)
{
logger?.LogWarning("<{filePath}>", filePath.FullName);
continue;
}
fastForwardMovingPictureExpertsGroupUsed = fastForwardMovingPictureExpertsGroupFiles is not null && fastForwardMovingPictureExpertsGroupFiles.Count > 0;
if (fastForwardMovingPictureExpertsGroupUsed && fastForwardMovingPictureExpertsGroupFiles is not null)
{
foreach (string fastForwardMovingPictureExpertsGroupFile in fastForwardMovingPictureExpertsGroupFiles)
File.Delete(fastForwardMovingPictureExpertsGroupFile);
}
if (!fastForwardMovingPictureExpertsGroupUsed && appSettings.WindowsSettings.ValidVideoFormatExtensions.Contains(filePath.ExtensionLowered))
fastForwardMovingPictureExpertsGroupUsed = true;
firstPass = new(exifDirectory, fastForwardMovingPictureExpertsGroupUsed, minimumYearAndPathCombined, sidecarFiles.ToArray());
results.Add(firstPass);
}
return result;
}
private static ReadOnlyCollection<FirstPass> WindowsSynchronousWork(ILogger<Program>? logger, AppSettings appSettings, IWindows windows, IEnumerable<string> files, A_Metadata metadata)
{
List<FirstPass> results = [];
int index = -1;
ReadOnlyDictionary<string, List<FileHolder>> keyValuePairs;
if (string.IsNullOrEmpty(appSettings.WindowsSettings.Host) || string.IsNullOrEmpty(appSettings.WindowsSettings.Page))
keyValuePairs = IMetadata.GetKeyValuePairs(files);
else
@ -265,61 +332,53 @@ public partial class Windows : IWindows, IDisposable
}
foreach (KeyValuePair<string, List<FileHolder>> keyValuePair in keyValuePairs)
{
index += 1;
windows.Tick();
if (keyValuePair.Value.Count > 2)
throw new NotSupportedException("Too many sidecar files!");
foreach (FileHolder fileHolder in keyValuePair.Value)
{
if (appSettings.WindowsSettings.SidecarExtensions.Contains(fileHolder.ExtensionLowered))
continue;
if (appSettings.WindowsSettings.IgnoreExtensions.Contains(fileHolder.ExtensionLowered))
continue;
filePath = FilePath.Get(appSettings.ResultSettings, appSettings.MetadataSettings, fileHolder, index);
if (filePath.Id is not null && (filePath.IsIntelligentIdFormat || filePath.SortOrder is not null))
continue;
if (filePath.Id is not null)
{
fastForwardMovingPictureExpertsGroupFiles = null;
deterministicHashCode = new(null, filePath.Id, null);
directoryName = Path.GetFileName(filePath.DirectoryFullPath);
if (directoryName.EndsWith(filePath.Id.Value.ToString()))
continue;
}
else
{
fastForwardMovingPictureExpertsGroupFiles = windows.ConvertAndGetFastForwardMovingPictureExpertsGroupFiles(appSettings.WindowsSettings, httpClient, filePath);
fastForwardMovingPictureExpertsGroupFilePath = fastForwardMovingPictureExpertsGroupFiles.Count == 0 ? null : FilePath.Get(appSettings.ResultSettings, appSettings.MetadataSettings, FileHolder.Get(fastForwardMovingPictureExpertsGroupFiles[0]), index);
deterministicHashCode = fastForwardMovingPictureExpertsGroupFilePath is null ? windows.GetDeterministicHashCode(httpClient, filePath) : windows.GetDeterministicHashCode(httpClient, fastForwardMovingPictureExpertsGroupFilePath);
}
sidecarFiles = [];
filePath = FilePath.Get(filePath, deterministicHashCode);
for (int i = 0; i < keyValuePair.Value.Count; i++)
{
if (keyValuePair.Value[i].ExtensionLowered == fileHolder.ExtensionLowered)
continue;
sidecarFiles.Add(keyValuePair.Value[i]);
}
try
{ (minimumYearAndPathCombined, exifDirectory) = metadata.GetMetadataCollection(appSettings.ResultSettings, appSettings.MetadataSettings, httpClient, filePath); }
catch (Exception)
{
logger?.LogWarning("<{filePath}>", filePath.FullName);
continue;
}
fastForwardMovingPictureExpertsGroupUsed = fastForwardMovingPictureExpertsGroupFiles is not null && fastForwardMovingPictureExpertsGroupFiles.Count > 0;
if (fastForwardMovingPictureExpertsGroupUsed && fastForwardMovingPictureExpertsGroupFiles is not null)
{
foreach (string fastForwardMovingPictureExpertsGroupFile in fastForwardMovingPictureExpertsGroupFiles)
File.Delete(fastForwardMovingPictureExpertsGroupFile);
}
if (!fastForwardMovingPictureExpertsGroupUsed && appSettings.WindowsSettings.ValidVideoFormatExtensions.Contains(filePath.ExtensionLowered))
fastForwardMovingPictureExpertsGroupUsed = true;
firstPass = new(exifDirectory, fastForwardMovingPictureExpertsGroupUsed, minimumYearAndPathCombined, sidecarFiles.ToArray());
results.Add(firstPass);
}
index = WindowsSynchronousWork(logger, appSettings, windows, metadata, results, index, keyValuePair);
}
return results;
return results.AsReadOnly();
}
private ReadOnlyCollection<FirstPass> WindowsAsynchronousWork(AppSettings appSettings, IWindows windows, ReadOnlyCollection<string> files, A_Metadata metadata, int appSettingsMaxDegreeOfParallelism)
{
List<FirstPass> results = [];
FirstPass firstPass;
List<string> distinct = [];
List<MetadataGroup> metadataGroups = [];
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = appSettingsMaxDegreeOfParallelism };
files.AsParallel().ForAll(IMetadata.SetExifDirectoryCollection(windows, appSettings.ResultSettings, appSettings.MetadataSettings, appSettings.WindowsSettings, metadata, distinct, metadataGroups));
if (_ProgressBar?.CurrentTick != results.Count)
throw new NotSupportedException();
foreach (MetadataGroup metadataGroup in metadataGroups)
{
if (metadataGroup.FastForwardMovingPictureExpertsGroupUsed || !appSettings.WindowsSettings.ValidVideoFormatExtensions.Contains(metadataGroup.FilePath.ExtensionLowered))
firstPass = new(metadataGroup.ExifDirectory, metadataGroup.FastForwardMovingPictureExpertsGroupUsed, metadataGroup.MinimumYearAndPathCombined, metadataGroup.SidecarFiles.ToArray());
else
firstPass = new(metadataGroup.ExifDirectory, FastForwardMovingPictureExpertsGroupUsed: true, metadataGroup.MinimumYearAndPathCombined, metadataGroup.SidecarFiles.ToArray());
results.Add(firstPass);
}
return results.AsReadOnly();
}
private void WindowsWork(ILogger<Program>? logger, AppSettings appSettings, IWindows windows, long ticks, string sourceDirectory)
{
ReadOnlyCollection<FirstPass> results;
if (!Directory.Exists(sourceDirectory))
_ = Directory.CreateDirectory(sourceDirectory);
logger?.LogInformation("{Ticks} {RootDirectory}", ticks, sourceDirectory);
ReadOnlyCollection<string> files = Directory.GetFiles(sourceDirectory, "*", SearchOption.AllDirectories).ToArray().AsReadOnly();
if (files.Count > 0)
_ = IPath.DeleteEmptyDirectories(appSettings.ResultSettings.RootDirectory);
A_Metadata metadata = new(appSettings.ResultSettings, appSettings.MetadataSettings);
int appSettingsMaxDegreeOfParallelism = appSettings.WindowsSettings.MaxDegreeOfParallelism;
int filesCount = appSettingsMaxDegreeOfParallelism == 1 ? files.Count : 123000;
windows.ConstructProgressBar(filesCount, "EnumerateFiles load");
if (appSettingsMaxDegreeOfParallelism == 1)
results = WindowsSynchronousWork(logger, appSettings, windows, files, metadata);
else
results = WindowsAsynchronousWork(appSettings, windows, files, metadata, appSettingsMaxDegreeOfParallelism);
string json = JsonSerializer.Serialize(results, FirstPassCollectionSourceGenerationContext.Default.ListFirstPass);
File.WriteAllText(Path.Combine(sourceDirectory, $"{ticks}.json"), json);
}
private void WindowsWork(ILogger<Program>? logger, AppSettings appSettings, IWindows windows, long ticks)
@ -328,40 +387,8 @@ public partial class Windows : IWindows, IDisposable
Verify(logger, appSettings, windows, appSettings.WindowsSettings.Host, appSettings.WindowsSettings.Page);
else
{
FirstPass firstPass;
List<FirstPass> results = [];
string sourceDirectory = Path.GetFullPath(appSettings.ResultSettings.RootDirectory);
if (!Directory.Exists(sourceDirectory))
_ = Directory.CreateDirectory(sourceDirectory);
logger?.LogInformation("{Ticks} {RootDirectory}", ticks, sourceDirectory);
ReadOnlyCollection<string> files = Directory.GetFiles(sourceDirectory, "*", SearchOption.AllDirectories).ToArray().AsReadOnly();
if (files.Count > 0)
_ = IPath.DeleteEmptyDirectories(appSettings.ResultSettings.RootDirectory);
A_Metadata metadata = new(appSettings.ResultSettings, appSettings.MetadataSettings);
int appSettingsMaxDegreeOfParallelism = appSettings.WindowsSettings.MaxDegreeOfParallelism;
int filesCount = appSettingsMaxDegreeOfParallelism == 1 ? files.Count : 123000;
windows.ConstructProgressBar(filesCount, "EnumerateFiles load");
if (appSettingsMaxDegreeOfParallelism == 1)
results.AddRange(GetCollection(logger, appSettings, windows, files, metadata));
else
{
List<string> distinct = [];
List<MetadataGroup> metadataGroups = [];
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = appSettingsMaxDegreeOfParallelism };
files.AsParallel().ForAll(IMetadata.SetExifDirectoryCollection(windows, appSettings.ResultSettings, appSettings.MetadataSettings, appSettings.WindowsSettings, metadata, distinct, metadataGroups));
if (_ProgressBar?.CurrentTick != results.Count)
throw new NotSupportedException();
foreach (MetadataGroup metadataGroup in metadataGroups)
{
if (metadataGroup.FastForwardMovingPictureExpertsGroupUsed || !appSettings.WindowsSettings.ValidVideoFormatExtensions.Contains(metadataGroup.FilePath.ExtensionLowered))
firstPass = new(metadataGroup.ExifDirectory, metadataGroup.FastForwardMovingPictureExpertsGroupUsed, metadataGroup.MinimumYearAndPathCombined, metadataGroup.SidecarFiles.ToArray());
else
firstPass = new(metadataGroup.ExifDirectory, FastForwardMovingPictureExpertsGroupUsed: true, metadataGroup.MinimumYearAndPathCombined, metadataGroup.SidecarFiles.ToArray());
results.Add(firstPass);
}
}
string json = JsonSerializer.Serialize(results, FirstPassCollectionSourceGenerationContext.Default.ListFirstPass);
File.WriteAllText(Path.Combine(sourceDirectory, $"{ticks}.json"), json);
WindowsWork(logger, appSettings, windows, ticks, sourceDirectory);
}
}