phares3757/file-folder-helper
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WriteNginxFileSystemDelta

Browse Source 
ProcessDataStandardFormatToJson
This commit is contained in:
Mike Phares 2025-03-06 14:01:00 -07:00
parent e89e11dcf6
commit 38ab4424bc
6 changed files with 240 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
.vscode/launch.json vendored
View File

@ -13,26 +13,13 @@
"args": [
"s",
"X",
"L:/Git/pocketbase_0.25.8_windows_amd64",
"Day-Helper-2025-03-01",
"*.json",
"-",
"L:/Git/pocketbase-import",
"json.ts",
"input",
"777",
"888",
"999",
"D:/ProgramData/VisualStudioCode",
"Day-Helper-2025-03-05",
"isccvm57294f1ed.infineon.com",
".vscode/extensions/bennycode.sort-everything-1.4.1",
"s",
"X",
"L:/Git/pocketbase_0.25.8_windows_amd64",
"Day-Helper-2025-02-28",
"immich-db-backup-1739350800014.sql",
"COPY_public.",
"FROM_stdin",
"s",
"X",
"\\\\mesfs.infineon.com\\EC_Characterization_Si\\Archive\\BIORAD4\\2025_Week_09\\2025-02-28\\-660626-_2025-02-28_04;49_PM_395577460",
"\\\\mesfs.infineon.com\\EC_Characterization_Si\\Archive\\BIORAD4\\2025_Week_10\\2025-03-03\\03--_2025-03-03_05;54_AM_1292405457",
"Day-Helper-2025-02-19",
"csv-*.pdsf",
"*.pdsf",

19
ADO2024/PI4/Helper-2024-12-24.cs
View File

@ -67,17 +67,22 @@ internal static partial class Helper20241224
{
Task<string> taskString = taskHttpResponseMessage.Result.Content.ReadAsStringAsync();
taskString.Wait();
NginxFileSystem[]? nginxFileSystems = JsonSerializer.Deserialize(taskString.Result, NginxFileSystemCollectionSourceGenerationContext.Default.NginxFileSystemArray);
if (nginxFileSystems is null)
if (taskString.Result.StartsWith('<'))
results = null;
else
{
results = [];
NginxFileSystem nginxFileSystem;
for (int i = 0; i < nginxFileSystems.Length; i++)
NginxFileSystem[]? nginxFileSystems = JsonSerializer.Deserialize(taskString.Result, NginxFileSystemCollectionSourceGenerationContext.Default.NginxFileSystemArray);
if (nginxFileSystems is null)
results = null;
else
{
nginxFileSystem = NginxFileSystem.Get(format, timeZoneInfo, uri, nginxFileSystems[i]);
results.Add(nginxFileSystem);
results = [];
NginxFileSystem nginxFileSystem;
for (int i = 0; i < nginxFileSystems.Length; i++)
{
nginxFileSystem = NginxFileSystem.Get(format, timeZoneInfo, uri, nginxFileSystems[i]);
results.Add(nginxFileSystem);
}
}
}
}

5
ADO2025/PI5/Helper-2025-02-19.cs
View File

@ -220,7 +220,7 @@ internal static partial class Helper20250219
continue;
if (indexOnly.Contains(jsonPropertyOld.Name) && int.TryParse(jsonPropertyOld.Name[^2..], out int index) && i != index - 1)
continue;
logger.LogWarning("For [{jsonProperty.Name}] <{directory}> doesn't match ({valueNew} != {valueOld})!", jsonPropertyOld.Name, directory, valueNew, valueOld);
logger.LogWarning("For [{jsonProperty.Name}] <{directory}> doesn't match (valueNew:{valueNew} != valueOld:{valueOld})!", jsonPropertyOld.Name, directory, valueNew, valueOld);
differentColumns.Add(jsonPropertyOld.Name);
}
}
@ -260,7 +260,10 @@ internal static partial class Helper20250219
processDataStandardFormat = GetLogisticsColumnsAndBody(match, lines: null);
jsonElementsOld = GetArray(processDataStandardFormat);
if (jsonElementsOld is null || jsonElementsOld.Length != jsonElementsNew.Length)
{
logger.LogWarning("! <{match}> (jsonElementsOld.Length:{jsonElementsOld} != jsonElementsNew.Length:{jsonElementsNew})", match, jsonElementsOld?.Length, jsonElementsNew.Length);
continue;
}
isMatch = Compare(logger, ignore, backfill, indexOnly, keyValuePairs, directorySegment, jsonElementsNew, jsonElementsOld);
if (!isMatch)
{

158
ADO2025/PI5/Helper-2025-03-05.cs Normal file
View File

@ -0,0 +1,158 @@
using File_Folder_Helper.Models;
using Microsoft.Extensions.Logging;
using System.Collections.ObjectModel;
using System.Text.Json;
namespace File_Folder_Helper.ADO2025.PI5;
internal static partial class Helper20250305
{
private static readonly HttpClient _HttpClient = new();
private record Record(Uri URI, string Path, DateTime LastModified, int? TotalSeconds);
private static ReadOnlyCollection<NginxFileSystem>? GetCollection(string format, TimeZoneInfo timeZoneInfo, Uri uri)
{
List<NginxFileSystem>? results;
Task<HttpResponseMessage> taskHttpResponseMessage = _HttpClient.GetAsync(uri);
taskHttpResponseMessage.Wait();
if (!taskHttpResponseMessage.Result.IsSuccessStatusCode)
results = null;
else
{
Task<string> taskString = taskHttpResponseMessage.Result.Content.ReadAsStringAsync();
taskString.Wait();
if (taskString.Result.StartsWith('<'))
results = null;
else
{
NginxFileSystem[]? nginxFileSystems = JsonSerializer.Deserialize(taskString.Result, NginxFileSystemCollectionSourceGenerationContext.Default.NginxFileSystemArray);
if (nginxFileSystems is null)
results = null;
else
{
results = [];
NginxFileSystem nginxFileSystem;
for (int i = 0; i < nginxFileSystems.Length; i++)
{
nginxFileSystem = NginxFileSystem.Get(format, timeZoneInfo, uri, nginxFileSystems[i]);
results.Add(nginxFileSystem);
}
}
}
}
return results?.AsReadOnly();
}
private static ReadOnlyCollection<Record> GetRecords(string format, TimeZoneInfo timeZoneInfo, string host, ReadOnlyCollection<string> directoryNames, string compareDirectory)
{
List<Record> results = [];
Uri uri = new($"https://{host}/{string.Join('/', directoryNames)}");
ReadOnlyCollection<NginxFileSystem>? nginxFileSystems = GetCollection(format, timeZoneInfo, uri);
if (nginxFileSystems is not null)
{
NginxFileSystem nginxFileSystem;
ReadOnlyCollection<Record> records;
string checkDirectory = $"{compareDirectory}\\{string.Join('\\', directoryNames)}";
if (!Directory.Exists(checkDirectory))
_ = Directory.CreateDirectory(checkDirectory);
for (int i = 0; i < nginxFileSystems.Count; i++)
{
nginxFileSystem = NginxFileSystem.Get(format, timeZoneInfo, uri, nginxFileSystems[i]);
if (nginxFileSystem.Type == "file")
{
Record? record = CompareFile(host, directoryNames, compareDirectory, nginxFileSystem);
if (record is not null)
results.Add(record);
}
else
{
records = CompareDirectory(format, timeZoneInfo, host, directoryNames, compareDirectory, nginxFileSystem);
foreach (Record record in records)
results.Add(record);
}
}
}
return results.AsReadOnly();
}
private static ReadOnlyCollection<Record> CompareDirectory(string format, TimeZoneInfo timeZoneInfo, string host, ReadOnlyCollection<string> directoryNames, string compareDirectory, NginxFileSystem nginxFileSystem)
{
ReadOnlyCollection<Record> results;
List<string> collection = directoryNames.ToList();
collection.Add(nginxFileSystem.Name);
results = GetRecords(format, timeZoneInfo, host, collection.AsReadOnly(), compareDirectory);
return results;
}
private static Record? CompareFile(string host, ReadOnlyCollection<string> directoryNames, string compareDirectory, NginxFileSystem nginxFileSystem)
{
Record? result;
if (nginxFileSystem.LastModified is null || nginxFileSystem.Length is null)
result = null;
else
{
Uri uri = new($"https://{host}/{string.Join('/', directoryNames)}/{nginxFileSystem.Name}");
FileInfo fileInfo = new($"{compareDirectory}\\{string.Join('\\', directoryNames)}\\{nginxFileSystem.Name}");
if (!fileInfo.Exists)
result = new(URI: uri, Path: fileInfo.FullName, LastModified: nginxFileSystem.LastModified.Value, TotalSeconds: null);
else
{
int totalSeconds = (int)new TimeSpan(fileInfo.LastWriteTime.Ticks - nginxFileSystem.LastModified.Value.Ticks).TotalSeconds;
if (totalSeconds is not < 2 or not > -2)
result = new(URI: uri, Path: fileInfo.FullName, LastModified: nginxFileSystem.LastModified.Value, TotalSeconds: totalSeconds);
else if (fileInfo.Length != nginxFileSystem.Length.Value)
result = new(URI: uri, Path: fileInfo.FullName, LastModified: nginxFileSystem.LastModified.Value, TotalSeconds: 0);
else
result = null;
}
}
return result;
}
private static void Download(Record record)
{
Task<HttpResponseMessage> taskHttpResponseMessage = _HttpClient.GetAsync(record.URI);
taskHttpResponseMessage.Wait();
if (taskHttpResponseMessage.Result.IsSuccessStatusCode)
{
Task<string> taskString = taskHttpResponseMessage.Result.Content.ReadAsStringAsync();
taskString.Wait();
File.WriteAllText(record.Path, taskString.Result);
File.SetLastWriteTime(record.Path, record.LastModified);
}
}
internal static void WriteNginxFileSystemDelta(ILogger<Worker> logger, List<string> args)
{
string host = args[2];
string rootDirectoryName = args[3];
string format = NginxFileSystem.GetFormat();
TimeZoneInfo timeZoneInfo = TimeZoneInfo.Local;
string compareDirectory = Path.GetFullPath(args[0]);
logger.LogInformation("Comparing files on {host}", host);
ReadOnlyCollection<Record> records = GetRecords(format, timeZoneInfo, host, new([rootDirectoryName]), compareDirectory);
#if ShellProgressBar
ProgressBar progressBar = new(records.Count, "Downloading", new ProgressBarOptions() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true });
#endif
foreach (Record record in records)
{
#if ShellProgressBar
progressBar.Tick();
#endif
if (record.TotalSeconds is null)
Download(record);
else if (record.TotalSeconds.Value == 0)
logger.LogInformation("Different lengths");
else if (record.TotalSeconds.Value > 0)
logger.LogInformation("Overwrite remote (https)");
else
logger.LogInformation("Overwrite local");
}
#if ShellProgressBar
progressBar.Dispose();
#endif
}
}

57
ADO2025/PI5/Helper-2025-03-06.cs Normal file
View File

@ -0,0 +1,57 @@
using Microsoft.Extensions.Logging;
namespace File_Folder_Helper.ADO2025.PI5;
internal static partial class Helper20250306
{
private static string ProcessDataStandardFormatToJson(int columnsLine, string[] columns, string[] body)
{
string result = "[\n";
string line;
string value;
string[] segments;
if (columns.Length == 0)
columns = body[columnsLine].Trim().Split('\t');
for (int i = columnsLine + 1; i < body.Length; i++)
{
line = "{";
segments = body[i].Trim().Split('\t');
if (segments.Length != columns.Length)
break;
for (int c = 1; c < segments.Length; c++)
{
value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\");
line += '"' + columns[c].Trim('"') + '"' + ':' + '"' + value + '"' + ',';
}
line = line[..^1] + '}' + ',' + '\n';
result += line;
}
result = result[..^2] + ']';
return result;
}
private static void ProcessDataStandardFormatToJson(ILogger<Worker> logger, string file)
{
string[] lines = File.ReadAllLines(file);
if (lines.Length < 7)
logger.LogWarning("<{lines}>(s)", lines.Length);
else
{
string json = ProcessDataStandardFormatToJson(6, [], lines);
File.WriteAllText(".json", json);
}
}
internal static void ProcessDataStandardFormatToJson(ILogger<Worker> logger, List<string> args)
{
string searchPattern = args[2];
string sourceDirectory = Path.GetFullPath(args[0]);
string[] files = Directory.GetFiles(sourceDirectory, searchPattern, SearchOption.AllDirectories);
if (files.Length != 1)
logger.LogWarning("<{files}>(s)", files.Length);
else
ProcessDataStandardFormatToJson(logger, files[0]);
}
}

4
Day/HelperDay.cs
View File

@ -141,6 +141,10 @@ internal static class HelperDay
ADO2025.PI5.Helper20250228.PostgresDumpToJson(logger, args);
else if (args[1] == "Day-Helper-2025-03-01")
ADO2025.PI5.Helper20250301.PocketBaseImportWithDeno(logger, args);
else if (args[1] == "Day-Helper-2025-03-05")
ADO2025.PI5.Helper20250305.WriteNginxFileSystemDelta(logger, args);
else if (args[1] == "Day-Helper-2025-03-06")
ADO2025.PI5.Helper20250306.ProcessDataStandardFormatToJson(logger, args);
else
throw new Exception(appSettings.Company);
}