388 lines
18 KiB
C#
388 lines
18 KiB
C#
using CliWrap;
|
|
using File_Watcher.Models;
|
|
|
|
#if ShellProgressBar
|
|
|
|
using ShellProgressBar;
|
|
|
|
#endif
|
|
|
|
using System.Collections.ObjectModel;
|
|
using System.Drawing;
|
|
using System.Drawing.Imaging;
|
|
using System.IO.Compression;
|
|
using System.Runtime.InteropServices;
|
|
using System.Text.Json;
|
|
using Phares.Metadata.Models;
|
|
using Phares.Metadata.Models.Stateless;
|
|
using Phares.Shared.Models;
|
|
using Phares.Shared.Models.Stateless;
|
|
|
|
namespace File_Watcher.Helpers;
|
|
|
|
internal static partial class DeterministicHashCodeHelper
|
|
{
|
|
|
|
private class Windows : IWindows, IDisposable
|
|
{
|
|
|
|
public long Ticks { get; init; }
|
|
public int? CurrentTick =>
|
|
#if ShellProgressBar
|
|
_ProgressBar?.CurrentTick;
|
|
#else
|
|
throw new NotSupportedException("ShellProgressBar is not supported in this context.");
|
|
#endif
|
|
|
|
#if ShellProgressBar
|
|
private ProgressBar? _ProgressBar;
|
|
private readonly ProgressBarOptions _ProgressBarOptions;
|
|
#endif
|
|
|
|
public Windows() =>
|
|
#if ShellProgressBar
|
|
_ProgressBarOptions = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
|
|
#else
|
|
throw new NotSupportedException("ShellProgressBar is not supported in this context.");
|
|
#endif
|
|
|
|
DeterministicHashCode IWindows.GetDeterministicHashCode(HttpClient httpClient, Uri uri) =>
|
|
GetDeterministicHashCode(httpClient, uri);
|
|
|
|
DeterministicHashCode IWindows.GetDeterministicHashCode(HttpClient? httpClient, FilePath filePath)
|
|
{
|
|
DeterministicHashCode result;
|
|
if (httpClient is not null)
|
|
result = GetDeterministicHashCode(httpClient, new Uri(filePath.FullName));
|
|
else
|
|
{
|
|
Stream stream = File.OpenRead(filePath.FullName);
|
|
result = GetDeterministicHashCode(stream);
|
|
stream.Dispose();
|
|
}
|
|
return result;
|
|
}
|
|
|
|
private static DeterministicHashCode GetDeterministicHashCode(HttpClient httpClient, Uri uri)
|
|
{
|
|
DeterministicHashCode result;
|
|
Stream stream = GetStream(httpClient, uri);
|
|
result = GetDeterministicHashCode(stream);
|
|
stream.Dispose();
|
|
return result;
|
|
}
|
|
|
|
private static Stream GetStream(HttpClient httpClient, Uri uri)
|
|
{
|
|
Stream result;
|
|
Task<Stream> task = httpClient.GetStreamAsync(uri);
|
|
task.Wait();
|
|
result = task.Result;
|
|
return result;
|
|
}
|
|
|
|
private static DeterministicHashCode GetDeterministicHashCode(Stream stream)
|
|
{
|
|
DeterministicHashCode result;
|
|
int? id;
|
|
int? width;
|
|
int? height;
|
|
try
|
|
{
|
|
#pragma warning disable CA1416
|
|
using Image image = Image.FromStream(stream);
|
|
width = image.Width;
|
|
height = image.Height;
|
|
using Bitmap bitmap = new(image);
|
|
Rectangle rectangle = new(0, 0, image.Width, image.Height);
|
|
BitmapData bitmapData = bitmap.LockBits(rectangle, ImageLockMode.ReadOnly, bitmap.PixelFormat);
|
|
IntPtr intPtr = bitmapData.Scan0;
|
|
int length = bitmapData.Stride * bitmap.Height;
|
|
byte[] bytes = new byte[length];
|
|
Marshal.Copy(intPtr, bytes, 0, length);
|
|
bitmap.UnlockBits(bitmapData);
|
|
#pragma warning restore CA1416
|
|
id = IId.GetDeterministicHashCode(bytes);
|
|
}
|
|
catch (Exception)
|
|
{
|
|
id = null;
|
|
width = null;
|
|
height = null;
|
|
}
|
|
result = new(height, id, width);
|
|
return result;
|
|
}
|
|
|
|
void IWindows.Tick() =>
|
|
#if ShellProgressBar
|
|
_ProgressBar?.Tick();
|
|
#else
|
|
throw new NotSupportedException("ShellProgressBar is not supported in this context.");
|
|
#endif
|
|
|
|
void IDisposable.Dispose()
|
|
{
|
|
#if ShellProgressBar
|
|
_ProgressBar?.Dispose();
|
|
#endif
|
|
GC.SuppressFinalize(this);
|
|
}
|
|
|
|
void IWindows.ConstructProgressBar(int maxTicks, string message)
|
|
{
|
|
#if ShellProgressBar
|
|
_ProgressBar?.Dispose();
|
|
_ProgressBar = new(maxTicks, message, _ProgressBarOptions);
|
|
#endif
|
|
}
|
|
|
|
ReadOnlyCollection<string> IWindows.ConvertAndGetFastForwardMovingPictureExpertsGroupFiles(ResultSettings resultSettings, HttpClient? httpClient, FilePath filePath)
|
|
{
|
|
List<string> results = [];
|
|
bool isValidVideoFormatExtensions = resultSettings.ValidVideoFormatExtensions.Contains(filePath.ExtensionLowered);
|
|
if (isValidVideoFormatExtensions)
|
|
{
|
|
bool check;
|
|
if (httpClient is not null)
|
|
DownloadFile(httpClient, filePath);
|
|
try
|
|
{
|
|
CommandTask<CommandResult> commandTask = Cli.Wrap("L:/Git/ffmpeg-2024-10-02-git-358fdf3083-full_build/bin/ffmpeg.exe")
|
|
.WithArguments(["-i", filePath.FullName, "-vf", "select=eq(n\\,0)", "-q:v", "1", $"{filePath.Name}-%4d.jpg"])
|
|
.WithWorkingDirectory(filePath.DirectoryFullPath)
|
|
.ExecuteAsync();
|
|
commandTask.Task.Wait();
|
|
check = true;
|
|
}
|
|
catch (Exception)
|
|
{
|
|
check = false;
|
|
}
|
|
if (check)
|
|
{
|
|
results.AddRange(Directory.GetFiles(filePath.DirectoryFullPath, $"{filePath.Name}-*.jpg", SearchOption.TopDirectoryOnly));
|
|
if (results.Count == 0)
|
|
throw new Exception();
|
|
File.SetCreationTime(results[0], new(filePath.CreationTicks));
|
|
File.SetLastWriteTime(results[0], new(filePath.LastWriteTicks));
|
|
Thread.Sleep(100);
|
|
}
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static void DownloadFile(HttpClient httpClient, FilePath filePath)
|
|
{
|
|
FileStream fileStream = new(filePath.FullName, FileMode.Truncate);
|
|
Task<HttpResponseMessage> httpResponseMessage = httpClient.GetAsync(filePath.FullName);
|
|
httpResponseMessage.Wait();
|
|
Task task = httpResponseMessage.Result.Content.CopyToAsync(fileStream);
|
|
task.Wait();
|
|
}
|
|
|
|
}
|
|
|
|
internal static bool WindowsWork(AppSettings appSettings, ILogger<Worker>? logger)
|
|
{
|
|
string json;
|
|
string jsonFile;
|
|
string sourceDirectory;
|
|
List<string> check = [];
|
|
string archiveEntryFile;
|
|
Windows windows = new();
|
|
ReadOnlyCollection<FirstPass> collection;
|
|
string rootDirectory = Path.GetFullPath(appSettings.ResultSettings.RootDirectory);
|
|
if (!Directory.Exists(rootDirectory))
|
|
_ = Directory.CreateDirectory(rootDirectory);
|
|
string[] zipFiles = Directory.GetFiles(rootDirectory, "*.zip", SearchOption.TopDirectoryOnly);
|
|
if (zipFiles.Length > 0)
|
|
_ = IPath.DeleteEmptyDirectories(rootDirectory);
|
|
foreach (string zipFile in zipFiles)
|
|
{
|
|
check.Clear();
|
|
sourceDirectory = zipFile[..^4];
|
|
jsonFile = $"{sourceDirectory}.json";
|
|
if (Directory.Exists(sourceDirectory) || File.Exists(jsonFile))
|
|
continue;
|
|
_ = Directory.CreateDirectory(sourceDirectory);
|
|
using ZipArchive zip = ZipFile.Open(zipFile, ZipArchiveMode.Read);
|
|
foreach (ZipArchiveEntry zipArchiveEntry in zip.Entries)
|
|
{
|
|
check.Add(zipArchiveEntry.Name);
|
|
archiveEntryFile = Path.Combine(sourceDirectory, zipArchiveEntry.Name);
|
|
zipArchiveEntry.ExtractToFile(archiveEntryFile);
|
|
}
|
|
collection = WindowsWork(windows, logger, appSettings, sourceDirectory);
|
|
if (check.Count == collection.Count)
|
|
{
|
|
json = JsonSerializer.Serialize(collection.ToList(), FirstPassCollectionSourceGenerationContext.Default.ListFirstPass);
|
|
File.WriteAllText(jsonFile, json);
|
|
}
|
|
Directory.Delete(sourceDirectory, recursive: true);
|
|
}
|
|
return true;
|
|
}
|
|
|
|
private static ReadOnlyCollection<FirstPass> WindowsWork(IWindows windows, ILogger<Worker>? logger, AppSettings appSettings, string sourceDirectory)
|
|
{
|
|
ReadOnlyCollection<FirstPass> results;
|
|
IWindows windowsInterface = windows;
|
|
logger?.LogInformation("{Ticks} {RootDirectory}", windows.Ticks, sourceDirectory);
|
|
A_Metadata metadata = new(appSettings.ResultSettings, appSettings.MetadataSettings);
|
|
int appSettingsMaxDegreeOfParallelism = appSettings.DeterministicHashCodeConfiguration.MaxDegreeOfParallelism;
|
|
ReadOnlyCollection<string> files = Directory.GetFiles(sourceDirectory, "*", SearchOption.AllDirectories).AsReadOnly();
|
|
if (files.Count > 0)
|
|
_ = IPath.DeleteEmptyDirectories(sourceDirectory);
|
|
int filesCount = appSettingsMaxDegreeOfParallelism == 1 ? files.Count : 123000;
|
|
windowsInterface.ConstructProgressBar(filesCount, "EnumerateFiles load");
|
|
if (appSettingsMaxDegreeOfParallelism == 1)
|
|
results = WindowsSynchronousWork(logger, appSettings, windowsInterface, files, metadata);
|
|
else
|
|
results = WindowsAsynchronousWork(appSettings, windows, files, metadata, appSettingsMaxDegreeOfParallelism);
|
|
return results;
|
|
}
|
|
|
|
private static ReadOnlyCollection<FirstPass> WindowsSynchronousWork(ILogger<Worker>? logger, AppSettings appSettings, IWindows windows, IEnumerable<string> files, A_Metadata metadata)
|
|
{
|
|
List<FirstPass> results = [];
|
|
int index = -1;
|
|
ReadOnlyDictionary<string, List<FileHolder>> keyValuePairs = IMetadata.GetKeyValuePairs(files);
|
|
foreach (KeyValuePair<string, List<FileHolder>> keyValuePair in keyValuePairs)
|
|
{
|
|
if (keyValuePair.Value.Count > 2)
|
|
throw new NotSupportedException("Too many sidecar files!");
|
|
index = WindowsSynchronousWork(logger, appSettings, windows, metadata, results, index, keyValuePair);
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static int WindowsSynchronousWork(ILogger<Worker>? logger, AppSettings appSettings, IWindows windows, A_Metadata metadata, List<FirstPass> results, int index, KeyValuePair<string, List<FileHolder>> keyValuePair)
|
|
{
|
|
int result = index + 1;
|
|
windows.Tick();
|
|
FilePath filePath;
|
|
FirstPass firstPass;
|
|
string directoryName;
|
|
ExifDirectory exifDirectory;
|
|
HttpClient? httpClient = null;
|
|
List<FileHolder> sidecarFiles;
|
|
DeterministicHashCode deterministicHashCode;
|
|
bool fastForwardMovingPictureExpertsGroupUsed;
|
|
MinimumYearAndPathCombined minimumYearAndPathCombined;
|
|
FilePath? fastForwardMovingPictureExpertsGroupFilePath;
|
|
ReadOnlyCollection<string>? fastForwardMovingPictureExpertsGroupFiles;
|
|
foreach (FileHolder fileHolder in keyValuePair.Value)
|
|
{
|
|
if (appSettings.DeterministicHashCodeConfiguration.SidecarExtensions.Contains(fileHolder.ExtensionLowered))
|
|
continue;
|
|
if (appSettings.ResultSettings.IgnoreExtensions.Contains(fileHolder.ExtensionLowered))
|
|
continue;
|
|
filePath = FilePath.Get(appSettings.ResultSettings, appSettings.MetadataSettings, fileHolder, result);
|
|
if (filePath.Id is not null && (filePath.IsIntelligentIdFormat || filePath.SortOrder is not null))
|
|
continue;
|
|
if (filePath.Id is not null)
|
|
{
|
|
fastForwardMovingPictureExpertsGroupFiles = null;
|
|
deterministicHashCode = new(null, filePath.Id, null);
|
|
directoryName = Path.GetFileName(filePath.DirectoryFullPath);
|
|
if (directoryName.EndsWith(filePath.Id.Value.ToString()))
|
|
continue;
|
|
}
|
|
else
|
|
{
|
|
fastForwardMovingPictureExpertsGroupFiles = windows.ConvertAndGetFastForwardMovingPictureExpertsGroupFiles(appSettings.ResultSettings, httpClient, filePath);
|
|
fastForwardMovingPictureExpertsGroupFilePath = fastForwardMovingPictureExpertsGroupFiles.Count == 0 ? null : FilePath.Get(appSettings.ResultSettings, appSettings.MetadataSettings, FileHolder.Get(fastForwardMovingPictureExpertsGroupFiles[0]), result);
|
|
deterministicHashCode = fastForwardMovingPictureExpertsGroupFilePath is null ? windows.GetDeterministicHashCode(httpClient, filePath) : windows.GetDeterministicHashCode(httpClient, fastForwardMovingPictureExpertsGroupFilePath);
|
|
}
|
|
sidecarFiles = [];
|
|
filePath = FilePath.Get(filePath, deterministicHashCode);
|
|
for (int i = 0; i < keyValuePair.Value.Count; i++)
|
|
{
|
|
if (keyValuePair.Value[i].ExtensionLowered == fileHolder.ExtensionLowered)
|
|
continue;
|
|
sidecarFiles.Add(keyValuePair.Value[i]);
|
|
}
|
|
try
|
|
{ (minimumYearAndPathCombined, exifDirectory) = metadata.GetMetadataCollection(appSettings.ResultSettings, appSettings.MetadataSettings, httpClient, filePath); }
|
|
catch (Exception)
|
|
{
|
|
logger?.LogWarning("<{filePath}>", filePath.FullName);
|
|
continue;
|
|
}
|
|
fastForwardMovingPictureExpertsGroupUsed = fastForwardMovingPictureExpertsGroupFiles is not null && fastForwardMovingPictureExpertsGroupFiles.Count > 0;
|
|
if (fastForwardMovingPictureExpertsGroupUsed && fastForwardMovingPictureExpertsGroupFiles is not null)
|
|
{
|
|
foreach (string fastForwardMovingPictureExpertsGroupFile in fastForwardMovingPictureExpertsGroupFiles)
|
|
File.Delete(fastForwardMovingPictureExpertsGroupFile);
|
|
}
|
|
if (!fastForwardMovingPictureExpertsGroupUsed && appSettings.ResultSettings.ValidVideoFormatExtensions.Contains(filePath.ExtensionLowered))
|
|
fastForwardMovingPictureExpertsGroupUsed = true;
|
|
firstPass = new(exifDirectory, fastForwardMovingPictureExpertsGroupUsed, minimumYearAndPathCombined, sidecarFiles.ToArray());
|
|
results.Add(firstPass);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
private static ReadOnlyCollection<FirstPass> WindowsAsynchronousWork(AppSettings appSettings, IWindows windows, ReadOnlyCollection<string> files, A_Metadata metadata, int appSettingsMaxDegreeOfParallelism)
|
|
{
|
|
List<FirstPass> results = [];
|
|
FirstPass firstPass;
|
|
List<string> distinct = [];
|
|
List<MetadataGroup> metadataGroups = [];
|
|
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = appSettingsMaxDegreeOfParallelism };
|
|
files.AsParallel().ForAll(IMetadata.SetExifDirectoryCollection(windows, appSettings.ResultSettings, appSettings.MetadataSettings, metadata, distinct, metadataGroups));
|
|
if (windows?.CurrentTick != results.Count)
|
|
throw new NotSupportedException();
|
|
foreach (MetadataGroup metadataGroup in metadataGroups)
|
|
{
|
|
if (metadataGroup.FastForwardMovingPictureExpertsGroupUsed || !appSettings.ResultSettings.ValidVideoFormatExtensions.Contains(metadataGroup.FilePath.ExtensionLowered))
|
|
firstPass = new(metadataGroup.ExifDirectory, metadataGroup.FastForwardMovingPictureExpertsGroupUsed, metadataGroup.MinimumYearAndPathCombined, metadataGroup.SidecarFiles.ToArray());
|
|
else
|
|
firstPass = new(metadataGroup.ExifDirectory, FastForwardMovingPictureExpertsGroupUsed: true, metadataGroup.MinimumYearAndPathCombined, metadataGroup.SidecarFiles.ToArray());
|
|
results.Add(firstPass);
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static ReadOnlyCollection<NginxFileSystem>? GetRecursiveCollection(HttpClient httpClient, string host, string page)
|
|
{
|
|
List<NginxFileSystem>? results;
|
|
Uri uri = new($"http://{host}/{page}");
|
|
string format = NginxFileSystem.GetFormat();
|
|
TimeZoneInfo timeZoneInfo = TimeZoneInfo.Local;
|
|
Task<HttpResponseMessage> taskHttpResponseMessage = httpClient.GetAsync(uri);
|
|
taskHttpResponseMessage.Wait();
|
|
if (!taskHttpResponseMessage.Result.IsSuccessStatusCode)
|
|
results = null;
|
|
else
|
|
{
|
|
Task<string> taskString = taskHttpResponseMessage.Result.Content.ReadAsStringAsync();
|
|
taskString.Wait();
|
|
NginxFileSystem[]? nginxFileSystems = JsonSerializer.Deserialize(taskString.Result, NginxFileSystemCollectionSourceGenerationContext.Default.NginxFileSystemArray);
|
|
if (nginxFileSystems is null)
|
|
results = null;
|
|
else
|
|
{
|
|
results = [];
|
|
NginxFileSystem nginxFileSystem;
|
|
ReadOnlyCollection<NginxFileSystem>? directory;
|
|
for (int i = 0; i < nginxFileSystems.Length; i++)
|
|
{
|
|
nginxFileSystem = NginxFileSystem.Get(format, timeZoneInfo, uri, nginxFileSystems[i]);
|
|
if (nginxFileSystem.Type == "file")
|
|
results.Add(nginxFileSystem);
|
|
else
|
|
{
|
|
directory = GetRecursiveCollection(httpClient, host, $"{page}/{nginxFileSystem.Name}");
|
|
if (directory is null)
|
|
continue;
|
|
results.AddRange(directory);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return results?.AsReadOnly();
|
|
}
|
|
|
|
} |