deterministic-hash-code-helper
This commit is contained in:
363
Helpers/DeterministicHashCodeHelper.cs
Normal file
363
Helpers/DeterministicHashCodeHelper.cs
Normal file
@ -0,0 +1,363 @@
|
||||
using CliWrap;
|
||||
using File_Watcher.Models;
|
||||
using ShellProgressBar;
|
||||
using System.Collections.ObjectModel;
|
||||
using System.Drawing;
|
||||
using System.Drawing.Imaging;
|
||||
using System.IO.Compression;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text.Json;
|
||||
using View_by_Distance.Metadata.Models;
|
||||
using View_by_Distance.Metadata.Models.Stateless;
|
||||
using View_by_Distance.Shared.Models;
|
||||
using View_by_Distance.Shared.Models.Stateless;
|
||||
|
||||
namespace File_Watcher.Helpers;
|
||||
|
||||
internal static partial class DeterministicHashCodeHelper
|
||||
{
|
||||
|
||||
private class Windows : IWindows, IDisposable
|
||||
{
|
||||
|
||||
private ProgressBar? _ProgressBar;
|
||||
private readonly ProgressBarOptions _ProgressBarOptions;
|
||||
|
||||
public int CurrentTick { get; internal set; }
|
||||
|
||||
public Windows() =>
|
||||
_ProgressBarOptions = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
|
||||
|
||||
DeterministicHashCode IWindows.GetDeterministicHashCode(HttpClient httpClient, Uri uri) =>
|
||||
GetDeterministicHashCode(httpClient, uri);
|
||||
|
||||
DeterministicHashCode IWindows.GetDeterministicHashCode(HttpClient? httpClient, FilePath filePath)
|
||||
{
|
||||
DeterministicHashCode result;
|
||||
if (httpClient is not null)
|
||||
result = GetDeterministicHashCode(httpClient, new Uri(filePath.FullName));
|
||||
else
|
||||
{
|
||||
Stream stream = File.OpenRead(filePath.FullName);
|
||||
result = GetDeterministicHashCode(stream);
|
||||
stream.Dispose();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static DeterministicHashCode GetDeterministicHashCode(HttpClient httpClient, Uri uri)
|
||||
{
|
||||
DeterministicHashCode result;
|
||||
Stream stream = GetStream(httpClient, uri);
|
||||
result = GetDeterministicHashCode(stream);
|
||||
stream.Dispose();
|
||||
return result;
|
||||
}
|
||||
|
||||
private static Stream GetStream(HttpClient httpClient, Uri uri)
|
||||
{
|
||||
Stream result;
|
||||
Task<Stream> task = httpClient.GetStreamAsync(uri);
|
||||
task.Wait();
|
||||
result = task.Result;
|
||||
return result;
|
||||
}
|
||||
|
||||
private static DeterministicHashCode GetDeterministicHashCode(Stream stream)
|
||||
{
|
||||
DeterministicHashCode result;
|
||||
int? id;
|
||||
int? width;
|
||||
int? height;
|
||||
try
|
||||
{
|
||||
#pragma warning disable CA1416
|
||||
using Image image = Image.FromStream(stream);
|
||||
width = image.Width;
|
||||
height = image.Height;
|
||||
using Bitmap bitmap = new(image);
|
||||
Rectangle rectangle = new(0, 0, image.Width, image.Height);
|
||||
BitmapData bitmapData = bitmap.LockBits(rectangle, ImageLockMode.ReadOnly, bitmap.PixelFormat);
|
||||
IntPtr intPtr = bitmapData.Scan0;
|
||||
int length = bitmapData.Stride * bitmap.Height;
|
||||
byte[] bytes = new byte[length];
|
||||
Marshal.Copy(intPtr, bytes, 0, length);
|
||||
bitmap.UnlockBits(bitmapData);
|
||||
#pragma warning restore CA1416
|
||||
id = IId.GetDeterministicHashCode(bytes);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
id = null;
|
||||
width = null;
|
||||
height = null;
|
||||
}
|
||||
result = new(height, id, width);
|
||||
return result;
|
||||
}
|
||||
|
||||
void IWindows.Tick() =>
|
||||
_ProgressBar?.Tick();
|
||||
|
||||
void IDisposable.Dispose()
|
||||
{
|
||||
_ProgressBar?.Dispose();
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
void IWindows.ConstructProgressBar(int maxTicks, string message)
|
||||
{
|
||||
_ProgressBar?.Dispose();
|
||||
_ProgressBar = new(maxTicks, message, _ProgressBarOptions);
|
||||
}
|
||||
|
||||
ReadOnlyCollection<string> IWindows.ConvertAndGetFastForwardMovingPictureExpertsGroupFiles(ResultSettings resultSettings, HttpClient? httpClient, FilePath filePath)
|
||||
{
|
||||
List<string> results = [];
|
||||
bool isValidVideoFormatExtensions = resultSettings.ValidVideoFormatExtensions.Contains(filePath.ExtensionLowered);
|
||||
if (isValidVideoFormatExtensions)
|
||||
{
|
||||
bool check;
|
||||
if (httpClient is not null)
|
||||
DownloadFile(httpClient, filePath);
|
||||
try
|
||||
{
|
||||
CommandTask<CommandResult> commandTask = Cli.Wrap("L:/Git/ffmpeg-2024-10-02-git-358fdf3083-full_build/bin/ffmpeg.exe")
|
||||
.WithArguments(["-i", filePath.FullName, "-vf", "select=eq(n\\,0)", "-q:v", "1", $"{filePath.Name}-%4d.jpg"])
|
||||
.WithWorkingDirectory(filePath.DirectoryFullPath)
|
||||
.ExecuteAsync();
|
||||
commandTask.Task.Wait();
|
||||
check = true;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
check = false;
|
||||
}
|
||||
if (check)
|
||||
{
|
||||
results.AddRange(Directory.GetFiles(filePath.DirectoryFullPath, $"{filePath.Name}-*.jpg", SearchOption.TopDirectoryOnly));
|
||||
if (results.Count == 0)
|
||||
throw new Exception();
|
||||
File.SetCreationTime(results[0], new(filePath.CreationTicks));
|
||||
File.SetLastWriteTime(results[0], new(filePath.LastWriteTicks));
|
||||
Thread.Sleep(100);
|
||||
}
|
||||
}
|
||||
return results.AsReadOnly();
|
||||
}
|
||||
|
||||
private static void DownloadFile(HttpClient httpClient, FilePath filePath)
|
||||
{
|
||||
FileStream fileStream = new(filePath.FullName, FileMode.Truncate);
|
||||
Task<HttpResponseMessage> httpResponseMessage = httpClient.GetAsync(filePath.FullName);
|
||||
httpResponseMessage.Wait();
|
||||
Task task = httpResponseMessage.Result.Content.CopyToAsync(fileStream);
|
||||
task.Wait();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
internal static bool WindowsWork(AppSettings appSettings, ILogger<Worker>? logger)
|
||||
{
|
||||
string json;
|
||||
string jsonFile;
|
||||
string sourceDirectory;
|
||||
List<string> check = [];
|
||||
string archiveEntryFile;
|
||||
long ticks = DateTime.Now.Ticks;
|
||||
ReadOnlyCollection<FirstPass> collection;
|
||||
string rootDirectory = Path.GetFullPath(appSettings.ResultSettings.RootDirectory);
|
||||
if (!Directory.Exists(rootDirectory))
|
||||
_ = Directory.CreateDirectory(rootDirectory);
|
||||
string[] zipFiles = Directory.GetFiles(rootDirectory, "*.zip", SearchOption.TopDirectoryOnly);
|
||||
if (zipFiles.Length > 0)
|
||||
_ = IPath.DeleteEmptyDirectories(rootDirectory);
|
||||
foreach (string zipFile in zipFiles)
|
||||
{
|
||||
check.Clear();
|
||||
sourceDirectory = zipFile[..^4];
|
||||
jsonFile = $"{sourceDirectory}.json";
|
||||
if (Directory.Exists(sourceDirectory) || File.Exists(jsonFile))
|
||||
continue;
|
||||
_ = Directory.CreateDirectory(sourceDirectory);
|
||||
using ZipArchive zip = ZipFile.Open(zipFile, ZipArchiveMode.Read);
|
||||
foreach (ZipArchiveEntry zipArchiveEntry in zip.Entries)
|
||||
{
|
||||
check.Add(zipArchiveEntry.Name);
|
||||
archiveEntryFile = Path.Combine(sourceDirectory, zipArchiveEntry.Name);
|
||||
zipArchiveEntry.ExtractToFile(archiveEntryFile);
|
||||
}
|
||||
collection = WindowsWork(logger, appSettings, ticks, sourceDirectory);
|
||||
if (check.Count == collection.Count)
|
||||
{
|
||||
json = JsonSerializer.Serialize(collection.ToList(), FirstPassCollectionSourceGenerationContext.Default.ListFirstPass);
|
||||
File.WriteAllText(jsonFile, json);
|
||||
}
|
||||
Directory.Delete(sourceDirectory, recursive: true);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static ReadOnlyCollection<FirstPass> WindowsWork(ILogger<Worker>? logger, AppSettings appSettings, long ticks, string sourceDirectory)
|
||||
{
|
||||
ReadOnlyCollection<FirstPass> results;
|
||||
Windows windows = new();
|
||||
IWindows windowsInterface = windows;
|
||||
logger?.LogInformation("{Ticks} {RootDirectory}", ticks, sourceDirectory);
|
||||
A_Metadata metadata = new(appSettings.ResultSettings, appSettings.MetadataSettings);
|
||||
int appSettingsMaxDegreeOfParallelism = appSettings.DeterministicHashCodeConfiguration.MaxDegreeOfParallelism;
|
||||
ReadOnlyCollection<string> files = Directory.GetFiles(sourceDirectory, "*", SearchOption.AllDirectories).AsReadOnly();
|
||||
if (files.Count > 0)
|
||||
_ = IPath.DeleteEmptyDirectories(sourceDirectory);
|
||||
int filesCount = appSettingsMaxDegreeOfParallelism == 1 ? files.Count : 123000;
|
||||
windowsInterface.ConstructProgressBar(filesCount, "EnumerateFiles load");
|
||||
if (appSettingsMaxDegreeOfParallelism == 1)
|
||||
results = WindowsSynchronousWork(logger, appSettings, windowsInterface, files, metadata);
|
||||
else
|
||||
results = WindowsAsynchronousWork(appSettings, windows, files, metadata, appSettingsMaxDegreeOfParallelism);
|
||||
return results;
|
||||
}
|
||||
|
||||
private static ReadOnlyCollection<FirstPass> WindowsSynchronousWork(ILogger<Worker>? logger, AppSettings appSettings, IWindows windows, IEnumerable<string> files, A_Metadata metadata)
|
||||
{
|
||||
List<FirstPass> results = [];
|
||||
int index = -1;
|
||||
ReadOnlyDictionary<string, List<FileHolder>> keyValuePairs = IMetadata.GetKeyValuePairs(files);
|
||||
foreach (KeyValuePair<string, List<FileHolder>> keyValuePair in keyValuePairs)
|
||||
{
|
||||
if (keyValuePair.Value.Count > 2)
|
||||
throw new NotSupportedException("Too many sidecar files!");
|
||||
index = WindowsSynchronousWork(logger, appSettings, windows, metadata, results, index, keyValuePair);
|
||||
}
|
||||
return results.AsReadOnly();
|
||||
}
|
||||
|
||||
private static int WindowsSynchronousWork(ILogger<Worker>? logger, AppSettings appSettings, IWindows windows, A_Metadata metadata, List<FirstPass> results, int index, KeyValuePair<string, List<FileHolder>> keyValuePair)
|
||||
{
|
||||
int result = index + 1;
|
||||
windows.Tick();
|
||||
FilePath filePath;
|
||||
FirstPass firstPass;
|
||||
string directoryName;
|
||||
ExifDirectory exifDirectory;
|
||||
HttpClient? httpClient = null;
|
||||
List<FileHolder> sidecarFiles;
|
||||
DeterministicHashCode deterministicHashCode;
|
||||
bool fastForwardMovingPictureExpertsGroupUsed;
|
||||
MinimumYearAndPathCombined minimumYearAndPathCombined;
|
||||
FilePath? fastForwardMovingPictureExpertsGroupFilePath;
|
||||
ReadOnlyCollection<string>? fastForwardMovingPictureExpertsGroupFiles;
|
||||
foreach (FileHolder fileHolder in keyValuePair.Value)
|
||||
{
|
||||
if (appSettings.DeterministicHashCodeConfiguration.SidecarExtensions.Contains(fileHolder.ExtensionLowered))
|
||||
continue;
|
||||
if (appSettings.ResultSettings.IgnoreExtensions.Contains(fileHolder.ExtensionLowered))
|
||||
continue;
|
||||
filePath = FilePath.Get(appSettings.ResultSettings, appSettings.MetadataSettings, fileHolder, result);
|
||||
if (filePath.Id is not null && (filePath.IsIntelligentIdFormat || filePath.SortOrder is not null))
|
||||
continue;
|
||||
if (filePath.Id is not null)
|
||||
{
|
||||
fastForwardMovingPictureExpertsGroupFiles = null;
|
||||
deterministicHashCode = new(null, filePath.Id, null);
|
||||
directoryName = Path.GetFileName(filePath.DirectoryFullPath);
|
||||
if (directoryName.EndsWith(filePath.Id.Value.ToString()))
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
fastForwardMovingPictureExpertsGroupFiles = windows.ConvertAndGetFastForwardMovingPictureExpertsGroupFiles(appSettings.ResultSettings, httpClient, filePath);
|
||||
fastForwardMovingPictureExpertsGroupFilePath = fastForwardMovingPictureExpertsGroupFiles.Count == 0 ? null : FilePath.Get(appSettings.ResultSettings, appSettings.MetadataSettings, FileHolder.Get(fastForwardMovingPictureExpertsGroupFiles[0]), result);
|
||||
deterministicHashCode = fastForwardMovingPictureExpertsGroupFilePath is null ? windows.GetDeterministicHashCode(httpClient, filePath) : windows.GetDeterministicHashCode(httpClient, fastForwardMovingPictureExpertsGroupFilePath);
|
||||
}
|
||||
sidecarFiles = [];
|
||||
filePath = FilePath.Get(filePath, deterministicHashCode);
|
||||
for (int i = 0; i < keyValuePair.Value.Count; i++)
|
||||
{
|
||||
if (keyValuePair.Value[i].ExtensionLowered == fileHolder.ExtensionLowered)
|
||||
continue;
|
||||
sidecarFiles.Add(keyValuePair.Value[i]);
|
||||
}
|
||||
try
|
||||
{ (minimumYearAndPathCombined, exifDirectory) = metadata.GetMetadataCollection(appSettings.ResultSettings, appSettings.MetadataSettings, httpClient, filePath); }
|
||||
catch (Exception)
|
||||
{
|
||||
logger?.LogWarning("<{filePath}>", filePath.FullName);
|
||||
continue;
|
||||
}
|
||||
fastForwardMovingPictureExpertsGroupUsed = fastForwardMovingPictureExpertsGroupFiles is not null && fastForwardMovingPictureExpertsGroupFiles.Count > 0;
|
||||
if (fastForwardMovingPictureExpertsGroupUsed && fastForwardMovingPictureExpertsGroupFiles is not null)
|
||||
{
|
||||
foreach (string fastForwardMovingPictureExpertsGroupFile in fastForwardMovingPictureExpertsGroupFiles)
|
||||
File.Delete(fastForwardMovingPictureExpertsGroupFile);
|
||||
}
|
||||
if (!fastForwardMovingPictureExpertsGroupUsed && appSettings.ResultSettings.ValidVideoFormatExtensions.Contains(filePath.ExtensionLowered))
|
||||
fastForwardMovingPictureExpertsGroupUsed = true;
|
||||
firstPass = new(exifDirectory, fastForwardMovingPictureExpertsGroupUsed, minimumYearAndPathCombined, sidecarFiles.ToArray());
|
||||
results.Add(firstPass);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static ReadOnlyCollection<FirstPass> WindowsAsynchronousWork(AppSettings appSettings, Windows windows, ReadOnlyCollection<string> files, A_Metadata metadata, int appSettingsMaxDegreeOfParallelism)
|
||||
{
|
||||
List<FirstPass> results = [];
|
||||
FirstPass firstPass;
|
||||
List<string> distinct = [];
|
||||
List<MetadataGroup> metadataGroups = [];
|
||||
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = appSettingsMaxDegreeOfParallelism };
|
||||
files.AsParallel().ForAll(IMetadata.SetExifDirectoryCollection(windows, appSettings.ResultSettings, appSettings.MetadataSettings, metadata, distinct, metadataGroups));
|
||||
if (windows?.CurrentTick != results.Count)
|
||||
throw new NotSupportedException();
|
||||
foreach (MetadataGroup metadataGroup in metadataGroups)
|
||||
{
|
||||
if (metadataGroup.FastForwardMovingPictureExpertsGroupUsed || !appSettings.ResultSettings.ValidVideoFormatExtensions.Contains(metadataGroup.FilePath.ExtensionLowered))
|
||||
firstPass = new(metadataGroup.ExifDirectory, metadataGroup.FastForwardMovingPictureExpertsGroupUsed, metadataGroup.MinimumYearAndPathCombined, metadataGroup.SidecarFiles.ToArray());
|
||||
else
|
||||
firstPass = new(metadataGroup.ExifDirectory, FastForwardMovingPictureExpertsGroupUsed: true, metadataGroup.MinimumYearAndPathCombined, metadataGroup.SidecarFiles.ToArray());
|
||||
results.Add(firstPass);
|
||||
}
|
||||
return results.AsReadOnly();
|
||||
}
|
||||
|
||||
private static ReadOnlyCollection<NginxFileSystem>? GetRecursiveCollection(HttpClient httpClient, string host, string page)
|
||||
{
|
||||
List<NginxFileSystem>? results;
|
||||
Uri uri = new($"http://{host}/{page}");
|
||||
string format = NginxFileSystem.GetFormat();
|
||||
TimeZoneInfo timeZoneInfo = TimeZoneInfo.Local;
|
||||
Task<HttpResponseMessage> taskHttpResponseMessage = httpClient.GetAsync(uri);
|
||||
taskHttpResponseMessage.Wait();
|
||||
if (!taskHttpResponseMessage.Result.IsSuccessStatusCode)
|
||||
results = null;
|
||||
else
|
||||
{
|
||||
Task<string> taskString = taskHttpResponseMessage.Result.Content.ReadAsStringAsync();
|
||||
taskString.Wait();
|
||||
NginxFileSystem[]? nginxFileSystems = JsonSerializer.Deserialize(taskString.Result, NginxFileSystemCollectionSourceGenerationContext.Default.NginxFileSystemArray);
|
||||
if (nginxFileSystems is null)
|
||||
results = null;
|
||||
else
|
||||
{
|
||||
results = [];
|
||||
NginxFileSystem nginxFileSystem;
|
||||
ReadOnlyCollection<NginxFileSystem>? directory;
|
||||
for (int i = 0; i < nginxFileSystems.Length; i++)
|
||||
{
|
||||
nginxFileSystem = NginxFileSystem.Get(format, timeZoneInfo, uri, nginxFileSystems[i]);
|
||||
if (nginxFileSystem.Type == "file")
|
||||
results.Add(nginxFileSystem);
|
||||
else
|
||||
{
|
||||
directory = GetRecursiveCollection(httpClient, host, $"{page}/{nginxFileSystem.Name}");
|
||||
if (directory is null)
|
||||
continue;
|
||||
results.AddRange(directory);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return results?.AsReadOnly();
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user