WriteNginxFileSystemDelta

ProcessDataStandardFormatToJson

.vscode/launch.json

@@ -13,26 +13,13 @@
             "args": [
                 "s",
                 "X",
-                "L:/Git/pocketbase_0.25.8_windows_amd64",
+                "D:/ProgramData/VisualStudioCode",
-                "Day-Helper-2025-03-01",
+                "Day-Helper-2025-03-05",
-                "*.json",
+                "isccvm57294f1ed.infineon.com",
-                "-",
+                ".vscode/extensions/bennycode.sort-everything-1.4.1",
-                "L:/Git/pocketbase-import",
-                "json.ts",
-                "input",
-                "777",
-                "888",
-                "999",
                 "s",
                 "X",
-                "L:/Git/pocketbase_0.25.8_windows_amd64",
+                "\\\\mesfs.infineon.com\\EC_Characterization_Si\\Archive\\BIORAD4\\2025_Week_10\\2025-03-03\\03--_2025-03-03_05;54_AM_1292405457",
-                "Day-Helper-2025-02-28",
-                "immich-db-backup-1739350800014.sql",
-                "COPY_public.",
-                "FROM_stdin",
-                "s",
-                "X",
-                "\\\\mesfs.infineon.com\\EC_Characterization_Si\\Archive\\BIORAD4\\2025_Week_09\\2025-02-28\\-660626-_2025-02-28_04;49_PM_395577460",
                 "Day-Helper-2025-02-19",
                 "csv-*.pdsf",
                 "*.pdsf",

ADO2024/PI4/Helper-2024-12-24.cs

@@ -67,17 +67,22 @@ internal static partial class Helper20241224
         {
             Task<string> taskString = taskHttpResponseMessage.Result.Content.ReadAsStringAsync();
             taskString.Wait();
-            NginxFileSystem[]? nginxFileSystems = JsonSerializer.Deserialize(taskString.Result, NginxFileSystemCollectionSourceGenerationContext.Default.NginxFileSystemArray);
+            if (taskString.Result.StartsWith('<'))
-            if (nginxFileSystems is null)
                 results = null;
             else
             {
-                results = [];
+                NginxFileSystem[]? nginxFileSystems = JsonSerializer.Deserialize(taskString.Result, NginxFileSystemCollectionSourceGenerationContext.Default.NginxFileSystemArray);
-                NginxFileSystem nginxFileSystem;
+                if (nginxFileSystems is null)
-                for (int i = 0; i < nginxFileSystems.Length; i++)
+                    results = null;
+                else
                 {
-                    nginxFileSystem = NginxFileSystem.Get(format, timeZoneInfo, uri, nginxFileSystems[i]);
+                    results = [];
-                    results.Add(nginxFileSystem);
+                    NginxFileSystem nginxFileSystem;
+                    for (int i = 0; i < nginxFileSystems.Length; i++)
+                    {
+                        nginxFileSystem = NginxFileSystem.Get(format, timeZoneInfo, uri, nginxFileSystems[i]);
+                        results.Add(nginxFileSystem);
+                    }
                 }
             }
         }

ADO2025/PI5/Helper-2025-02-19.cs

@@ -220,7 +220,7 @@ internal static partial class Helper20250219
                                 continue;
                             if (indexOnly.Contains(jsonPropertyOld.Name) && int.TryParse(jsonPropertyOld.Name[^2..], out int index) && i != index - 1)
                                 continue;
-                            logger.LogWarning("For [{jsonProperty.Name}] <{directory}> doesn't match ({valueNew} != {valueOld})!", jsonPropertyOld.Name, directory, valueNew, valueOld);
+                            logger.LogWarning("For [{jsonProperty.Name}] <{directory}> doesn't match (valueNew:{valueNew} != valueOld:{valueOld})!", jsonPropertyOld.Name, directory, valueNew, valueOld);
                             differentColumns.Add(jsonPropertyOld.Name);
                         }
                     }
@@ -260,7 +260,10 @@ internal static partial class Helper20250219
                 processDataStandardFormat = GetLogisticsColumnsAndBody(match, lines: null);
                 jsonElementsOld = GetArray(processDataStandardFormat);
                 if (jsonElementsOld is null || jsonElementsOld.Length != jsonElementsNew.Length)
+                {
+                    logger.LogWarning("! <{match}> (jsonElementsOld.Length:{jsonElementsOld} != jsonElementsNew.Length:{jsonElementsNew})", match, jsonElementsOld?.Length, jsonElementsNew.Length);
                     continue;
+                }
                 isMatch = Compare(logger, ignore, backfill, indexOnly, keyValuePairs, directorySegment, jsonElementsNew, jsonElementsOld);
                 if (!isMatch)
                 {

ADO2025/PI5/Helper-2025-03-05.cs

@@ -0,0 +1,158 @@
+using File_Folder_Helper.Models;
+using Microsoft.Extensions.Logging;
+using System.Collections.ObjectModel;
+using System.Text.Json;
+
+namespace File_Folder_Helper.ADO2025.PI5;
+
+internal static partial class Helper20250305
+{
+
+    private static readonly HttpClient _HttpClient = new();
+
+    private record Record(Uri URI, string Path, DateTime LastModified, int? TotalSeconds);
+
+    private static ReadOnlyCollection<NginxFileSystem>? GetCollection(string format, TimeZoneInfo timeZoneInfo, Uri uri)
+    {
+        List<NginxFileSystem>? results;
+        Task<HttpResponseMessage> taskHttpResponseMessage = _HttpClient.GetAsync(uri);
+        taskHttpResponseMessage.Wait();
+        if (!taskHttpResponseMessage.Result.IsSuccessStatusCode)
+            results = null;
+        else
+        {
+            Task<string> taskString = taskHttpResponseMessage.Result.Content.ReadAsStringAsync();
+            taskString.Wait();
+            if (taskString.Result.StartsWith('<'))
+                results = null;
+            else
+            {
+                NginxFileSystem[]? nginxFileSystems = JsonSerializer.Deserialize(taskString.Result, NginxFileSystemCollectionSourceGenerationContext.Default.NginxFileSystemArray);
+                if (nginxFileSystems is null)
+                    results = null;
+                else
+                {
+                    results = [];
+                    NginxFileSystem nginxFileSystem;
+                    for (int i = 0; i < nginxFileSystems.Length; i++)
+                    {
+                        nginxFileSystem = NginxFileSystem.Get(format, timeZoneInfo, uri, nginxFileSystems[i]);
+                        results.Add(nginxFileSystem);
+                    }
+                }
+            }
+        }
+        return results?.AsReadOnly();
+    }
+
+    private static ReadOnlyCollection<Record> GetRecords(string format, TimeZoneInfo timeZoneInfo, string host, ReadOnlyCollection<string> directoryNames, string compareDirectory)
+    {
+        List<Record> results = [];
+        Uri uri = new($"https://{host}/{string.Join('/', directoryNames)}");
+        ReadOnlyCollection<NginxFileSystem>? nginxFileSystems = GetCollection(format, timeZoneInfo, uri);
+        if (nginxFileSystems is not null)
+        {
+            NginxFileSystem nginxFileSystem;
+            ReadOnlyCollection<Record> records;
+            string checkDirectory = $"{compareDirectory}\\{string.Join('\\', directoryNames)}";
+            if (!Directory.Exists(checkDirectory))
+                _ = Directory.CreateDirectory(checkDirectory);
+            for (int i = 0; i < nginxFileSystems.Count; i++)
+            {
+                nginxFileSystem = NginxFileSystem.Get(format, timeZoneInfo, uri, nginxFileSystems[i]);
+                if (nginxFileSystem.Type == "file")
+                {
+                    Record? record = CompareFile(host, directoryNames, compareDirectory, nginxFileSystem);
+                    if (record is not null)
+                        results.Add(record);
+                }
+                else
+                {
+                    records = CompareDirectory(format, timeZoneInfo, host, directoryNames, compareDirectory, nginxFileSystem);
+                    foreach (Record record in records)
+                        results.Add(record);
+                }
+            }
+        }
+        return results.AsReadOnly();
+    }
+
+    private static ReadOnlyCollection<Record> CompareDirectory(string format, TimeZoneInfo timeZoneInfo, string host, ReadOnlyCollection<string> directoryNames, string compareDirectory, NginxFileSystem nginxFileSystem)
+    {
+        ReadOnlyCollection<Record> results;
+        List<string> collection = directoryNames.ToList();
+        collection.Add(nginxFileSystem.Name);
+        results = GetRecords(format, timeZoneInfo, host, collection.AsReadOnly(), compareDirectory);
+        return results;
+    }
+
+    private static Record? CompareFile(string host, ReadOnlyCollection<string> directoryNames, string compareDirectory, NginxFileSystem nginxFileSystem)
+    {
+        Record? result;
+        if (nginxFileSystem.LastModified is null || nginxFileSystem.Length is null)
+            result = null;
+        else
+        {
+            Uri uri = new($"https://{host}/{string.Join('/', directoryNames)}/{nginxFileSystem.Name}");
+            FileInfo fileInfo = new($"{compareDirectory}\\{string.Join('\\', directoryNames)}\\{nginxFileSystem.Name}");
+            if (!fileInfo.Exists)
+                result = new(URI: uri, Path: fileInfo.FullName, LastModified: nginxFileSystem.LastModified.Value, TotalSeconds: null);
+            else
+            {
+                int totalSeconds = (int)new TimeSpan(fileInfo.LastWriteTime.Ticks - nginxFileSystem.LastModified.Value.Ticks).TotalSeconds;
+                if (totalSeconds is not < 2 or not > -2)
+                    result = new(URI: uri, Path: fileInfo.FullName, LastModified: nginxFileSystem.LastModified.Value, TotalSeconds: totalSeconds);
+                else if (fileInfo.Length != nginxFileSystem.Length.Value)
+                    result = new(URI: uri, Path: fileInfo.FullName, LastModified: nginxFileSystem.LastModified.Value, TotalSeconds: 0);
+                else
+                    result = null;
+            }
+        }
+        return result;
+    }
+
+    private static void Download(Record record)
+    {
+        Task<HttpResponseMessage> taskHttpResponseMessage = _HttpClient.GetAsync(record.URI);
+        taskHttpResponseMessage.Wait();
+        if (taskHttpResponseMessage.Result.IsSuccessStatusCode)
+        {
+            Task<string> taskString = taskHttpResponseMessage.Result.Content.ReadAsStringAsync();
+            taskString.Wait();
+            File.WriteAllText(record.Path, taskString.Result);
+            File.SetLastWriteTime(record.Path, record.LastModified);
+        }
+    }
+
+    internal static void WriteNginxFileSystemDelta(ILogger<Worker> logger, List<string> args)
+    {
+        string host = args[2];
+        string rootDirectoryName = args[3];
+        string format = NginxFileSystem.GetFormat();
+        TimeZoneInfo timeZoneInfo = TimeZoneInfo.Local;
+        string compareDirectory = Path.GetFullPath(args[0]);
+        logger.LogInformation("Comparing files on {host}", host);
+        ReadOnlyCollection<Record> records = GetRecords(format, timeZoneInfo, host, new([rootDirectoryName]), compareDirectory);
+#if ShellProgressBar
+        ProgressBar progressBar = new(records.Count, "Downloading", new ProgressBarOptions() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true });
+#endif
+        foreach (Record record in records)
+        {
+#if ShellProgressBar
+            progressBar.Tick();
+#endif
+            if (record.TotalSeconds is null)
+                Download(record);
+            else if (record.TotalSeconds.Value == 0)
+                logger.LogInformation("Different lengths");
+            else if (record.TotalSeconds.Value > 0)
+                logger.LogInformation("Overwrite remote (https)");
+            else
+                logger.LogInformation("Overwrite local");
+        }
+#if ShellProgressBar
+        progressBar.Dispose();
+#endif
+    }
+
+}

ADO2025/PI5/Helper-2025-03-06.cs

@@ -0,0 +1,57 @@
+using Microsoft.Extensions.Logging;
+
+namespace File_Folder_Helper.ADO2025.PI5;
+
+internal static partial class Helper20250306
+{
+
+    private static string ProcessDataStandardFormatToJson(int columnsLine, string[] columns, string[] body)
+    {
+        string result = "[\n";
+        string line;
+        string value;
+        string[] segments;
+        if (columns.Length == 0)
+            columns = body[columnsLine].Trim().Split('\t');
+        for (int i = columnsLine + 1; i < body.Length; i++)
+        {
+            line = "{";
+            segments = body[i].Trim().Split('\t');
+            if (segments.Length != columns.Length)
+                break;
+            for (int c = 1; c < segments.Length; c++)
+            {
+                value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\");
+                line += '"' + columns[c].Trim('"') + '"' + ':' + '"' + value + '"' + ',';
+            }
+            line = line[..^1] + '}' + ',' + '\n';
+            result += line;
+        }
+        result = result[..^2] + ']';
+        return result;
+    }
+
+    private static void ProcessDataStandardFormatToJson(ILogger<Worker> logger, string file)
+    {
+        string[] lines = File.ReadAllLines(file);
+        if (lines.Length < 7)
+            logger.LogWarning("<{lines}>(s)", lines.Length);
+        else
+        {
+            string json = ProcessDataStandardFormatToJson(6, [], lines);
+            File.WriteAllText(".json", json);
+        }
+    }
+
+    internal static void ProcessDataStandardFormatToJson(ILogger<Worker> logger, List<string> args)
+    {
+        string searchPattern = args[2];
+        string sourceDirectory = Path.GetFullPath(args[0]);
+        string[] files = Directory.GetFiles(sourceDirectory, searchPattern, SearchOption.AllDirectories);
+        if (files.Length != 1)
+            logger.LogWarning("<{files}>(s)", files.Length);
+        else
+            ProcessDataStandardFormatToJson(logger, files[0]);
+    }
+
+}

Day/HelperDay.cs

@@ -141,6 +141,10 @@ internal static class HelperDay
             ADO2025.PI5.Helper20250228.PostgresDumpToJson(logger, args);
         else if (args[1] == "Day-Helper-2025-03-01")
             ADO2025.PI5.Helper20250301.PocketBaseImportWithDeno(logger, args);
+        else if (args[1] == "Day-Helper-2025-03-05")
+            ADO2025.PI5.Helper20250305.WriteNginxFileSystemDelta(logger, args);
+        else if (args[1] == "Day-Helper-2025-03-06")
+            ADO2025.PI5.Helper20250306.ProcessDataStandardFormatToJson(logger, args);
         else
             throw new Exception(appSettings.Company);
     }