diff --git a/.vscode/launch.json b/.vscode/launch.json index 6112f26..8c72f92 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -13,26 +13,13 @@ "args": [ "s", "X", - "L:/Git/pocketbase_0.25.8_windows_amd64", - "Day-Helper-2025-03-01", - "*.json", - "-", - "L:/Git/pocketbase-import", - "json.ts", - "input", - "777", - "888", - "999", + "D:/ProgramData/VisualStudioCode", + "Day-Helper-2025-03-05", + "isccvm57294f1ed.infineon.com", + ".vscode/extensions/bennycode.sort-everything-1.4.1", "s", "X", - "L:/Git/pocketbase_0.25.8_windows_amd64", - "Day-Helper-2025-02-28", - "immich-db-backup-1739350800014.sql", - "COPY_public.", - "FROM_stdin", - "s", - "X", - "\\\\mesfs.infineon.com\\EC_Characterization_Si\\Archive\\BIORAD4\\2025_Week_09\\2025-02-28\\-660626-_2025-02-28_04;49_PM_395577460", + "\\\\mesfs.infineon.com\\EC_Characterization_Si\\Archive\\BIORAD4\\2025_Week_10\\2025-03-03\\03--_2025-03-03_05;54_AM_1292405457", "Day-Helper-2025-02-19", "csv-*.pdsf", "*.pdsf", diff --git a/ADO2024/PI4/Helper-2024-12-24.cs b/ADO2024/PI4/Helper-2024-12-24.cs index 8bb5602..14fd8c3 100644 --- a/ADO2024/PI4/Helper-2024-12-24.cs +++ b/ADO2024/PI4/Helper-2024-12-24.cs @@ -67,17 +67,22 @@ internal static partial class Helper20241224 { Task taskString = taskHttpResponseMessage.Result.Content.ReadAsStringAsync(); taskString.Wait(); - NginxFileSystem[]? nginxFileSystems = JsonSerializer.Deserialize(taskString.Result, NginxFileSystemCollectionSourceGenerationContext.Default.NginxFileSystemArray); - if (nginxFileSystems is null) + if (taskString.Result.StartsWith('<')) results = null; else { - results = []; - NginxFileSystem nginxFileSystem; - for (int i = 0; i < nginxFileSystems.Length; i++) + NginxFileSystem[]? nginxFileSystems = JsonSerializer.Deserialize(taskString.Result, NginxFileSystemCollectionSourceGenerationContext.Default.NginxFileSystemArray); + if (nginxFileSystems is null) + results = null; + else { - nginxFileSystem = NginxFileSystem.Get(format, timeZoneInfo, uri, nginxFileSystems[i]); - results.Add(nginxFileSystem); + results = []; + NginxFileSystem nginxFileSystem; + for (int i = 0; i < nginxFileSystems.Length; i++) + { + nginxFileSystem = NginxFileSystem.Get(format, timeZoneInfo, uri, nginxFileSystems[i]); + results.Add(nginxFileSystem); + } } } } diff --git a/ADO2025/PI5/Helper-2025-02-19.cs b/ADO2025/PI5/Helper-2025-02-19.cs index c83a144..e6bdda3 100644 --- a/ADO2025/PI5/Helper-2025-02-19.cs +++ b/ADO2025/PI5/Helper-2025-02-19.cs @@ -220,7 +220,7 @@ internal static partial class Helper20250219 continue; if (indexOnly.Contains(jsonPropertyOld.Name) && int.TryParse(jsonPropertyOld.Name[^2..], out int index) && i != index - 1) continue; - logger.LogWarning("For [{jsonProperty.Name}] <{directory}> doesn't match ({valueNew} != {valueOld})!", jsonPropertyOld.Name, directory, valueNew, valueOld); + logger.LogWarning("For [{jsonProperty.Name}] <{directory}> doesn't match (valueNew:{valueNew} != valueOld:{valueOld})!", jsonPropertyOld.Name, directory, valueNew, valueOld); differentColumns.Add(jsonPropertyOld.Name); } } @@ -260,7 +260,10 @@ internal static partial class Helper20250219 processDataStandardFormat = GetLogisticsColumnsAndBody(match, lines: null); jsonElementsOld = GetArray(processDataStandardFormat); if (jsonElementsOld is null || jsonElementsOld.Length != jsonElementsNew.Length) + { + logger.LogWarning("! <{match}> (jsonElementsOld.Length:{jsonElementsOld} != jsonElementsNew.Length:{jsonElementsNew})", match, jsonElementsOld?.Length, jsonElementsNew.Length); continue; + } isMatch = Compare(logger, ignore, backfill, indexOnly, keyValuePairs, directorySegment, jsonElementsNew, jsonElementsOld); if (!isMatch) { diff --git a/ADO2025/PI5/Helper-2025-03-05.cs b/ADO2025/PI5/Helper-2025-03-05.cs new file mode 100644 index 0000000..1ac8f20 --- /dev/null +++ b/ADO2025/PI5/Helper-2025-03-05.cs @@ -0,0 +1,158 @@ +using File_Folder_Helper.Models; +using Microsoft.Extensions.Logging; +using System.Collections.ObjectModel; +using System.Text.Json; + +namespace File_Folder_Helper.ADO2025.PI5; + +internal static partial class Helper20250305 +{ + + private static readonly HttpClient _HttpClient = new(); + + private record Record(Uri URI, string Path, DateTime LastModified, int? TotalSeconds); + + private static ReadOnlyCollection? GetCollection(string format, TimeZoneInfo timeZoneInfo, Uri uri) + { + List? results; + Task taskHttpResponseMessage = _HttpClient.GetAsync(uri); + taskHttpResponseMessage.Wait(); + if (!taskHttpResponseMessage.Result.IsSuccessStatusCode) + results = null; + else + { + Task taskString = taskHttpResponseMessage.Result.Content.ReadAsStringAsync(); + taskString.Wait(); + if (taskString.Result.StartsWith('<')) + results = null; + else + { + NginxFileSystem[]? nginxFileSystems = JsonSerializer.Deserialize(taskString.Result, NginxFileSystemCollectionSourceGenerationContext.Default.NginxFileSystemArray); + if (nginxFileSystems is null) + results = null; + else + { + results = []; + NginxFileSystem nginxFileSystem; + for (int i = 0; i < nginxFileSystems.Length; i++) + { + nginxFileSystem = NginxFileSystem.Get(format, timeZoneInfo, uri, nginxFileSystems[i]); + results.Add(nginxFileSystem); + } + } + } + } + return results?.AsReadOnly(); + } + + private static ReadOnlyCollection GetRecords(string format, TimeZoneInfo timeZoneInfo, string host, ReadOnlyCollection directoryNames, string compareDirectory) + { + List results = []; + Uri uri = new($"https://{host}/{string.Join('/', directoryNames)}"); + ReadOnlyCollection? nginxFileSystems = GetCollection(format, timeZoneInfo, uri); + if (nginxFileSystems is not null) + { + NginxFileSystem nginxFileSystem; + ReadOnlyCollection records; + string checkDirectory = $"{compareDirectory}\\{string.Join('\\', directoryNames)}"; + if (!Directory.Exists(checkDirectory)) + _ = Directory.CreateDirectory(checkDirectory); + for (int i = 0; i < nginxFileSystems.Count; i++) + { + nginxFileSystem = NginxFileSystem.Get(format, timeZoneInfo, uri, nginxFileSystems[i]); + if (nginxFileSystem.Type == "file") + { + Record? record = CompareFile(host, directoryNames, compareDirectory, nginxFileSystem); + if (record is not null) + results.Add(record); + } + else + { + records = CompareDirectory(format, timeZoneInfo, host, directoryNames, compareDirectory, nginxFileSystem); + foreach (Record record in records) + results.Add(record); + } + } + } + return results.AsReadOnly(); + } + + private static ReadOnlyCollection CompareDirectory(string format, TimeZoneInfo timeZoneInfo, string host, ReadOnlyCollection directoryNames, string compareDirectory, NginxFileSystem nginxFileSystem) + { + ReadOnlyCollection results; + List collection = directoryNames.ToList(); + collection.Add(nginxFileSystem.Name); + results = GetRecords(format, timeZoneInfo, host, collection.AsReadOnly(), compareDirectory); + return results; + } + + private static Record? CompareFile(string host, ReadOnlyCollection directoryNames, string compareDirectory, NginxFileSystem nginxFileSystem) + { + Record? result; + if (nginxFileSystem.LastModified is null || nginxFileSystem.Length is null) + result = null; + else + { + Uri uri = new($"https://{host}/{string.Join('/', directoryNames)}/{nginxFileSystem.Name}"); + FileInfo fileInfo = new($"{compareDirectory}\\{string.Join('\\', directoryNames)}\\{nginxFileSystem.Name}"); + if (!fileInfo.Exists) + result = new(URI: uri, Path: fileInfo.FullName, LastModified: nginxFileSystem.LastModified.Value, TotalSeconds: null); + else + { + int totalSeconds = (int)new TimeSpan(fileInfo.LastWriteTime.Ticks - nginxFileSystem.LastModified.Value.Ticks).TotalSeconds; + if (totalSeconds is not < 2 or not > -2) + result = new(URI: uri, Path: fileInfo.FullName, LastModified: nginxFileSystem.LastModified.Value, TotalSeconds: totalSeconds); + else if (fileInfo.Length != nginxFileSystem.Length.Value) + result = new(URI: uri, Path: fileInfo.FullName, LastModified: nginxFileSystem.LastModified.Value, TotalSeconds: 0); + else + result = null; + } + } + return result; + } + + private static void Download(Record record) + { + Task taskHttpResponseMessage = _HttpClient.GetAsync(record.URI); + taskHttpResponseMessage.Wait(); + if (taskHttpResponseMessage.Result.IsSuccessStatusCode) + { + Task taskString = taskHttpResponseMessage.Result.Content.ReadAsStringAsync(); + taskString.Wait(); + File.WriteAllText(record.Path, taskString.Result); + File.SetLastWriteTime(record.Path, record.LastModified); + } + } + + internal static void WriteNginxFileSystemDelta(ILogger logger, List args) + { + string host = args[2]; + string rootDirectoryName = args[3]; + string format = NginxFileSystem.GetFormat(); + TimeZoneInfo timeZoneInfo = TimeZoneInfo.Local; + string compareDirectory = Path.GetFullPath(args[0]); + logger.LogInformation("Comparing files on {host}", host); + ReadOnlyCollection records = GetRecords(format, timeZoneInfo, host, new([rootDirectoryName]), compareDirectory); +#if ShellProgressBar + ProgressBar progressBar = new(records.Count, "Downloading", new ProgressBarOptions() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }); +#endif + foreach (Record record in records) + { +#if ShellProgressBar + progressBar.Tick(); +#endif + if (record.TotalSeconds is null) + Download(record); + else if (record.TotalSeconds.Value == 0) + logger.LogInformation("Different lengths"); + else if (record.TotalSeconds.Value > 0) + logger.LogInformation("Overwrite remote (https)"); + else + logger.LogInformation("Overwrite local"); + } +#if ShellProgressBar + progressBar.Dispose(); +#endif + } + +} \ No newline at end of file diff --git a/ADO2025/PI5/Helper-2025-03-06.cs b/ADO2025/PI5/Helper-2025-03-06.cs new file mode 100644 index 0000000..8b9cde7 --- /dev/null +++ b/ADO2025/PI5/Helper-2025-03-06.cs @@ -0,0 +1,57 @@ +using Microsoft.Extensions.Logging; + +namespace File_Folder_Helper.ADO2025.PI5; + +internal static partial class Helper20250306 +{ + + private static string ProcessDataStandardFormatToJson(int columnsLine, string[] columns, string[] body) + { + string result = "[\n"; + string line; + string value; + string[] segments; + if (columns.Length == 0) + columns = body[columnsLine].Trim().Split('\t'); + for (int i = columnsLine + 1; i < body.Length; i++) + { + line = "{"; + segments = body[i].Trim().Split('\t'); + if (segments.Length != columns.Length) + break; + for (int c = 1; c < segments.Length; c++) + { + value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\"); + line += '"' + columns[c].Trim('"') + '"' + ':' + '"' + value + '"' + ','; + } + line = line[..^1] + '}' + ',' + '\n'; + result += line; + } + result = result[..^2] + ']'; + return result; + } + + private static void ProcessDataStandardFormatToJson(ILogger logger, string file) + { + string[] lines = File.ReadAllLines(file); + if (lines.Length < 7) + logger.LogWarning("<{lines}>(s)", lines.Length); + else + { + string json = ProcessDataStandardFormatToJson(6, [], lines); + File.WriteAllText(".json", json); + } + } + + internal static void ProcessDataStandardFormatToJson(ILogger logger, List args) + { + string searchPattern = args[2]; + string sourceDirectory = Path.GetFullPath(args[0]); + string[] files = Directory.GetFiles(sourceDirectory, searchPattern, SearchOption.AllDirectories); + if (files.Length != 1) + logger.LogWarning("<{files}>(s)", files.Length); + else + ProcessDataStandardFormatToJson(logger, files[0]); + } + +} \ No newline at end of file diff --git a/Day/HelperDay.cs b/Day/HelperDay.cs index fe56367..56dd616 100644 --- a/Day/HelperDay.cs +++ b/Day/HelperDay.cs @@ -141,6 +141,10 @@ internal static class HelperDay ADO2025.PI5.Helper20250228.PostgresDumpToJson(logger, args); else if (args[1] == "Day-Helper-2025-03-01") ADO2025.PI5.Helper20250301.PocketBaseImportWithDeno(logger, args); + else if (args[1] == "Day-Helper-2025-03-05") + ADO2025.PI5.Helper20250305.WriteNginxFileSystemDelta(logger, args); + else if (args[1] == "Day-Helper-2025-03-06") + ADO2025.PI5.Helper20250306.ProcessDataStandardFormatToJson(logger, args); else throw new Exception(appSettings.Company); }