From e7c1fd22211e3f192291d83b512714d353eceaf3 Mon Sep 17 00:00:00 2001 From: Mike Phares Date: Thu, 3 Jul 2025 19:10:54 -0700 Subject: [PATCH] process-data-standard-format-to-java-script-object-notation (Day-Helper-2025-07-01) --- .vscode/launch.json | 11 ++ ADO2025/PI5/Helper-2025-02-19.cs | 4 +- ADO2025/PI5/Helper-2025-03-06.cs | 2 +- ADO2025/PI6/Helper-2025-07-01.cs | 313 +++++++++++++++++++++++++++++++ Day/HelperDay.cs | 2 + 5 files changed, 329 insertions(+), 3 deletions(-) create mode 100644 ADO2025/PI6/Helper-2025-07-01.cs diff --git a/.vscode/launch.json b/.vscode/launch.json index e9eeead..6beb8e0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,6 +11,17 @@ "preLaunchTask": "build", "program": "${workspaceFolder}/bin/Debug/net8.0/win-x64/File-Folder-Helper.dll", "args": [ + "s", + "X", + "\\\\mesfs.infineon.com\\EC_APC\\Production\\Traces\\DEP08CEPIEPSILON\\PollPath", + "Day-Helper-2025-07-01", + "*.pdsf", + "654321", + "Time", + "id12~CenterTemp,id13~CenterSetpt,id15~FrontTemp,id153~PPSTEPNAME,id154~SystemState,id16~FrontSetpt,id172~LVC1Ratio,id173~LVC1Carrier,id176~TotalWaferCount,i-d-1-7-8~TIME,id18~SideTemp,id183~SCRDrive4,id19~SideSetpt,id193~SCRLOAD4,id21~RearTemp,id22~RearSetpt,id221~LeftDefaultRecipe,id222~RightDefaultRecipe,id223~RecipeCompleteMsg,id25~N2H2Setpt,id26~N2H2Flow,id27~HCLSetpt,id28~HCLFlow,id29~HCLHISetpt,id30~HCLHIFlow,id37~NSRCSetpt,id38~NSRCFlow,id39~NDILSetpt,id40~NDILFlow,id41~NINJSetpt,id42~NINJFlow,id57~LVC1Setpt,id58~LVC1Flow,id61~ROTSetpt,id62~ROTSpeed,id78~LL1State,id79~LL1Init,id80~LL1Lotid,id81~LL1WafersIn,id82~LL1WfrCnt,id83~LL2State,id84~LL2Init,id85~LL2Lotid,id86~LL2WafersIn,id87~LL2WfrCnt,id93~ProcessState", + "\\\\mesfs.infineon.com\\EC_APC\\Production\\Traces\\DEP08CEPIEPSILON\\Markdown", + "\\\\mesfs.infineon.com\\EC_APC\\Production\\Traces\\DEP08CEPIEPSILON\\KeyValuePairs", + "\\\\mesfs.infineon.com\\EC_APC\\Production\\Traces\\DEP08CEPIEPSILON\\JavaScriptObjectNotation", "s", "X", "F:/0-ISO-A", diff --git a/ADO2025/PI5/Helper-2025-02-19.cs b/ADO2025/PI5/Helper-2025-02-19.cs index 12ceca9..7e69ae1 100644 --- a/ADO2025/PI5/Helper-2025-02-19.cs +++ b/ADO2025/PI5/Helper-2025-02-19.cs @@ -277,12 +277,12 @@ internal static partial class Helper20250219 { } if (!lookForNumbers) { for (int c = 0; c < segments.Length; c++) { - value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\"); + value = segments[c].Replace("\\", "\\\\").Replace("\"", "\\\""); _ = stringBuilder.Append('"').Append(processDataStandardFormat.Columns[c]).Append("\":\"").Append(value).Append("\","); } } else { for (int c = 0; c < segments.Length; c++) { - value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\"); + value = segments[c].Replace("\\", "\\\\").Replace("\"", "\\\""); if (string.IsNullOrEmpty(value)) _ = stringBuilder.Append('"').Append(processDataStandardFormat.Columns[c]).Append("\":").Append(value).Append("null,"); else if (value.All(char.IsDigit)) diff --git a/ADO2025/PI5/Helper-2025-03-06.cs b/ADO2025/PI5/Helper-2025-03-06.cs index 172892a..60b25a9 100644 --- a/ADO2025/PI5/Helper-2025-03-06.cs +++ b/ADO2025/PI5/Helper-2025-03-06.cs @@ -71,7 +71,7 @@ internal static partial class Helper20250306 { if (segments.Length != columnsLength) continue; for (int c = 1; c < segments.Length; c++) { - value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\"); + value = segments[c].Replace("\\", "\\\\").Replace("\"", "\\\""); line += '"' + columns[c].Trim('"') + '"' + ':' + '"' + value + '"' + ','; } line = line.Substring(0, line.Length - 1) + '}' + ',' + '\n'; diff --git a/ADO2025/PI6/Helper-2025-07-01.cs b/ADO2025/PI6/Helper-2025-07-01.cs new file mode 100644 index 0000000..a474eb2 --- /dev/null +++ b/ADO2025/PI6/Helper-2025-07-01.cs @@ -0,0 +1,313 @@ + +using System.Collections.ObjectModel; +using System.Text.Json; +using System.Text.Json.Serialization; + +using Microsoft.Extensions.Logging; + +namespace File_Folder_Helper.ADO2025.PI6; + +internal static partial class Helper20250701 { + + [JsonSourceGenerationOptions(WriteIndented = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] + [JsonSerializable(typeof(JsonElement))] + internal partial class JsonElementSourceGenerationContext : JsonSerializerContext { + } + + internal static void ProcessDataStandardFormatTo(ILogger logger, List args) { + logger.LogInformation(args[0]); + logger.LogInformation(args[1]); + logger.LogInformation(args[2]); + logger.LogInformation(args[3]); + logger.LogInformation(args[4]); + logger.LogInformation(args[5]); + logger.LogInformation(args[6]); + string[] segments; + string timeColumn = args[4]; + string searchPattern = args[2]; + int sizeFilter = int.Parse(args[3]); + string[] columns = args[5].Split(','); + Dictionary columnMapping = []; + string sourceDirectory = Path.GetFullPath(args[0].Split('~')[0]); + string destinationDirectory = Path.GetFullPath(args[6].Split('~')[0]); + foreach (string column in columns) { + segments = column.Split('~'); + columnMapping.Add(segments[0], segments[1]); + } + string[] directories = Directory.GetDirectories(sourceDirectory, "*", SearchOption.TopDirectoryOnly); + ProcessDataStandardFormatTo(logger, sourceDirectory, searchPattern, sizeFilter, timeColumn, columnMapping, destinationDirectory, directories); + Helpers.HelperDeleteEmptyDirectories.DeleteEmptyDirectories(logger, sourceDirectory); + } + + private static void ProcessDataStandardFormatTo(ILogger logger, string sourceDirectory, string searchPattern, int sizeFilter, string timeColumn, Dictionary columnMapping, string destinationDirectory, string[] directories) { + string? json; + string[] files; + string markdown; + string checkFile; + string[] matches; + FileInfo fileInfo; + string? pipeTable; + string? collections; + string directoryName; + string checkDirectory; + foreach (string directory in directories) { + if (Path.GetFileName(directory).Contains('-')) { + continue; + } + files = Directory.GetFiles(directory, searchPattern, SearchOption.TopDirectoryOnly); + foreach (string file in files) { + fileInfo = new(file); + if (fileInfo.LastWriteTime > DateTime.Now.AddSeconds(-5)) { + continue; + } + directoryName = Path.GetFileName(fileInfo.DirectoryName); + if (fileInfo.Length > sizeFilter && !directoryName.StartsWith('Z')) { + checkDirectory = Path.Combine(sourceDirectory, $"Z{directoryName}"); + if (!Directory.Exists(checkDirectory)) { + _ = Directory.CreateDirectory(checkDirectory); + } + checkFile = Path.Combine(checkDirectory, fileInfo.Name); + if (File.Exists(checkFile)) { + continue; + } + File.Move(file, checkFile); + continue; + } + checkDirectory = Path.Combine(destinationDirectory, directoryName); + if (!Directory.Exists(checkDirectory)) { + _ = Directory.CreateDirectory(checkDirectory); + } + checkFile = Path.Combine(checkDirectory, $"{fileInfo.Name}.md"); + if (File.Exists(checkFile)) { + continue; + } + collections = GetMarkdown(logger, timeColumn, columnMapping, file, fileInfo.Name); + if (string.IsNullOrEmpty(collections)) { + logger.LogWarning("collections is null"); + continue; + } + File.WriteAllText(checkFile, collections); + File.SetLastWriteTime(checkFile, fileInfo.LastAccessTime); + if (!string.IsNullOrEmpty(destinationDirectory)) { + continue; + } + json = GetJavaScriptObjectNotation(logger, file); + if (string.IsNullOrEmpty(json)) { + logger.LogWarning("json is null"); + continue; + } + File.WriteAllText($"{checkFile}.md", json); + File.SetLastWriteTime($"{checkFile}.md", fileInfo.LastAccessTime); + pipeTable = GetPipeTable(logger, json); + if (string.IsNullOrEmpty(pipeTable)) { + logger.LogWarning("pipeTable is null"); + continue; + } + markdown = $"# {fileInfo.Name}{Environment.NewLine}{Environment.NewLine}{pipeTable}{Environment.NewLine}"; + File.WriteAllText($"{checkFile}.md", markdown); + File.SetLastWriteTime($"{checkFile}.md", fileInfo.LastAccessTime); + logger.LogInformation("<{checkFile}> was written", checkFile); + } + } + } + + private static string? GetMarkdown(ILogger logger, string timeColumn, Dictionary columnMapping, string file, string name) { + string? result = null; + string[] lines = File.ReadAllLines(file); + int? columnTitlesLine = GetProcessDataStandardFormatColumnTitlesLine(lines); + if (columnTitlesLine is null) { + logger.LogWarning("<{columnTitlesLine}> is null", nameof(columnTitlesLine)); + } else { + if (lines.Length < columnTitlesLine.Value + 1) { + logger.LogWarning("<{lines}>(s)", lines.Length); + } else { + result = GetMarkdown(timeColumn, columnMapping, name, lines, columnTitlesLine); + } + } + return result; + } + + private static int? GetProcessDataStandardFormatColumnTitlesLine(string[] lines) { + int? result = null; + for (int i = 0; i < lines.Length; i++) { + if (lines[i].StartsWith("END_OFFSET") && i + 3 < lines.Length) { + result = i + 1; + break; + } + } + return result; + } + + private static ReadOnlyDictionary> GetKeyValuePairs(int columnTitlesLine, string[] lines) { + Dictionary> results = []; + string value; + string[] segments; + List> collections = []; + string[] columns = lines[columnTitlesLine].Split('\t'); + foreach (string column in columns) { + collections.Add([]); + } + for (int i = columnTitlesLine + 1; i < lines.Length; i++) { + if (lines[i].StartsWith("NUM_DATA_ROWS")) { + break; + } + segments = lines[i].Split('\t'); + if (segments.Length > columns.Length) { + continue; + } + for (int c = 0; c < segments.Length; c++) { + collections[c].Add(segments[c]); + } + } + for (int i = 0; i < collections.Count; i++) { + if (collections[i].Count > 1) { + if (string.IsNullOrEmpty(collections[i][0])) { + collections[i][0] = collections[i][1]; + } + if (string.IsNullOrEmpty(collections[i][^1])) { + collections[i][^1] = collections[i][^2]; + } + } + results.Add(columns[i].Trim('"'), collections[i].AsReadOnly()); + } + return results.AsReadOnly(); + } + + private static string? GetMarkdown(string timeColumn, Dictionary columnMapping, string name, string[] lines, int? columnTitlesLine) { + string? result; + List charts = []; + List results = []; + ReadOnlyDictionary> keyValuePairs = GetKeyValuePairs(columnTitlesLine.Value, lines); + string[] columns = keyValuePairs.Keys.OrderBy(l => l).ToArray(); + if (!columns.Contains(timeColumn)) { + result = null; + } else { + string? alias; + string[] mappedColumns = columnMapping.Keys.ToArray(); + string labels = string.Join(',', keyValuePairs[timeColumn].Select(l => l.Replace(',', '_'))); + foreach (string column in columns) { + if (column == timeColumn) { + continue; + } + if (!columnMapping.TryGetValue(column, out alias) || keyValuePairs[column].Count == 0 || string.IsNullOrEmpty(keyValuePairs[column][0])) { + results.Add(string.Concat("## ", column, Environment.NewLine, Environment.NewLine, + "- ", string.Join(',', keyValuePairs[column].Select(l => l.Replace(',', '_'))), Environment.NewLine + )); + } else { + charts.Add(string.Concat( + "## ", column, " - ", alias, Environment.NewLine, Environment.NewLine, + "```chart", Environment.NewLine, + "{\"type\": \"line\", \"data\": {\"datasets\": [{", Environment.NewLine, + "\"label\": \"", column, " - ", alias, "\",", Environment.NewLine, + "\"data\": [", string.Join(',', keyValuePairs[column].Select(l => l.Replace(',', '_'))), "],", Environment.NewLine, + "\"borderColor\": \"rgb(75, 192, 192)\"", Environment.NewLine, + "}],", Environment.NewLine, + "\"labels\": [", labels, "]", Environment.NewLine, + "}}", Environment.NewLine, + "```", Environment.NewLine + )); + } + } + if (results.Count == 0 && charts.Count == 0) { + result = null; + } else { + string[] segments; + results.Add($"## Footer{Environment.NewLine}"); + results.Insert(0, $"# {name}{Environment.NewLine}"); + for (int i = columnTitlesLine.Value + 1; i < lines.Length; i++) { + if (lines[i].StartsWith("NUM_DATA_ROWS")) { + for (int j = i; j < lines.Length; j++) { + results.Add($"- {lines[j]}"); + } + } + } + results.Add(string.Empty); + results.Add(string.Empty); + result = $"{string.Join(Environment.NewLine, results)}{string.Join(Environment.NewLine, charts)}"; + } + } + return result; + } + + private static string? GetJavaScriptObjectNotation(ILogger logger, string file) { + string? result = null; + string[] lines = File.ReadAllLines(file); + int? columnTitlesLine = GetProcessDataStandardFormatColumnTitlesLine(lines); + if (columnTitlesLine is null) { + logger.LogWarning("<{columnTitlesLine}> is null", nameof(columnTitlesLine)); + } else { + if (lines.Length < columnTitlesLine.Value + 1) { + logger.LogWarning("<{lines}>(s)", lines.Length); + } else { + result = GetJavaScriptObjectNotation(columnTitlesLine.Value, [], lines); + } + } + return result; + } + + private static string GetJavaScriptObjectNotation(int columnTitlesLine, string[] columns, string[] lines) { +#pragma warning disable CA1845, IDE0057 + string result = "[\n"; + string line; + string value; + string[] segments; + if (columns.Length == 0) { + columns = lines[columnTitlesLine].Split('\t'); + } + for (int i = columnTitlesLine + 1; i < lines.Length; i++) { + if (lines[i].StartsWith("NUM_DATA_ROWS")) { + break; + } + line = "{"; + segments = lines[i].Split('\t'); + if (segments.Length > columns.Length) { + continue; + } + for (int c = 0; c < segments.Length; c++) { + value = segments[c].Replace("\\", "\\\\").Replace("\"", "\\\""); + line += '"' + columns[c].Trim('"') + '"' + ':' + '"' + value + '"' + ','; + } + line = line.Substring(0, line.Length - 1) + '}' + ',' + '\n'; + result += line; + } + result = result.Substring(0, result.Length - 2) + ']'; + return result; +#pragma warning restore CA1845, IDE0057 + } + + private static string? GetPipeTable(ILogger logger, string json) { + string? result = null; + string? value; + string[]? columns = null; + List values = []; + List results = []; + Dictionary keyValuePairs = []; + JsonElement jsonElement = JsonSerializer.Deserialize(json, JsonElementSourceGenerationContext.Default.JsonElement); + JsonElement[] jsonElements = jsonElement.EnumerateArray().ToArray(); + foreach (JsonElement j in jsonElements) { + values.Clear(); + keyValuePairs.Clear(); + foreach (JsonProperty jsonProperty in j.EnumerateObject().ToArray()) { + if (columns is null) { + JsonProperty[] jsonProperties = j.EnumerateObject().OrderBy(l => l.Name).ToArray(); + columns = jsonProperties.Select(l => l.Name).ToArray(); + results.Add($"|{string.Join('|', columns)}|"); + results.Add($"|{string.Join('|', columns.Select(l => '-'))}|"); + } + keyValuePairs.Add(jsonProperty.Name, jsonProperty.Value.ToString()); + } + foreach (string column in columns) { + if (!keyValuePairs.TryGetValue(column, out value)) { + values.Add(new string(' ', column.Length)); + } else { + values.Add(value.PadLeft(column.Length, ' ')); + } + } + results.Add($"|{string.Join('|', values)}|"); + } + if (results.Count > 0) { + result = string.Join(Environment.NewLine, results); + } + return result; + } +} \ No newline at end of file diff --git a/Day/HelperDay.cs b/Day/HelperDay.cs index 820c597..f10d1d6 100644 --- a/Day/HelperDay.cs +++ b/Day/HelperDay.cs @@ -173,6 +173,8 @@ internal static class HelperDay ADO2025.PI6.Helper20250618.MoveAllButXOfEach(logger, args); else if (args[1] == "Day-Helper-2025-06-28") ADO2025.PI6.Helper20250628.LogIsoInformation(logger, args); + else if (args[1] == "Day-Helper-2025-07-01") + ADO2025.PI6.Helper20250701.ProcessDataStandardFormatTo(logger, args); else throw new Exception(appSettings.Company); }