From fc32bebeba3afce35d7aa53b9db5fab85bbf8d41 Mon Sep 17 00:00:00 2001 From: Mike Phares Date: Mon, 21 Apr 2025 13:16:34 -0700 Subject: [PATCH] process-data-standard-format with pipes --- .../MoveMatchingFiles/FileRead.cs | 4 + .../FileHandlers/OpenInsight/FileRead.cs | 29 ---- .../Shared/ProcessDataStandardFormat.cs | 139 +++++++++++------- 3 files changed, 93 insertions(+), 79 deletions(-) diff --git a/Adaptation/FileHandlers/MoveMatchingFiles/FileRead.cs b/Adaptation/FileHandlers/MoveMatchingFiles/FileRead.cs index b99eddd..e7536eb 100644 --- a/Adaptation/FileHandlers/MoveMatchingFiles/FileRead.cs +++ b/Adaptation/FileHandlers/MoveMatchingFiles/FileRead.cs @@ -301,6 +301,8 @@ public class FileRead : Shared.FileRead, IFileRead long preWait; foreach (PreWith preWith in preWithCollection) { + if (!_IsEAFHosted) + continue; if (processDataStandardFormat is null) File.Move(preWith.MatchingFile, preWith.CheckFile); else @@ -350,6 +352,8 @@ public class FileRead : Shared.FileRead, IFileRead _Logistics = new Logistics(reportFullPath, processDataStandardFormat); processDataStandardFormat = null; } + if (!_IsEAFHosted && processDataStandardFormat is not null) + ProcessDataStandardFormat.Write(".pdsf", processDataStandardFormat); SetFileParameterLotIDToLogisticsMID(); int numberLength = 2; long ticks = dateTime.Ticks; diff --git a/Adaptation/FileHandlers/OpenInsight/FileRead.cs b/Adaptation/FileHandlers/OpenInsight/FileRead.cs index 9e1089c..dd2d6af 100644 --- a/Adaptation/FileHandlers/OpenInsight/FileRead.cs +++ b/Adaptation/FileHandlers/OpenInsight/FileRead.cs @@ -136,34 +136,6 @@ public class FileRead : Shared.FileRead, IFileRead return results.ToString(); } - private static string GetJson(int columnsLine, string[] columns, string[] body) - { -#pragma warning disable CA1845, IDE0057 - string result = "[\n"; - string line; - string value; - string[] segments; - if (columns.Length == 0) - columns = body[columnsLine].Trim().Split('\t'); - for (int i = columnsLine + 1; i < body.Length; i++) - { - line = "{"; - segments = body[i].Trim().Split('\t'); - if (segments.Length != columns.Length) - break; - for (int c = 1; c < segments.Length; c++) - { - value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\"); - line += '"' + columns[c].Trim('"') + '"' + ':' + '"' + value + '"' + ','; - } - line = line.Substring(0, line.Length - 1) + '}' + ',' + '\n'; - result += line; - } - result = result.Substring(0, result.Length - 1) + ']'; - return result; -#pragma warning restore CA1845, IDE0057 - } - private void SaveOpenInsightFile(string reportFullPath, DateTime dateTime, ProcessDataStandardFormat processDataStandardFormat, List descriptions, Test[] tests) { bool isDummyRun = false; @@ -188,7 +160,6 @@ public class FileRead : Shared.FileRead, IFileRead if (!string.IsNullOrEmpty(lines)) { long? subGroupId; - _ = GetJson(0, processDataStandardFormat.Columns.ToArray(), processDataStandardFormat.Body.ToArray()); long breakAfter = dateTime.AddSeconds(_BreakAfterSeconds).Ticks; long preWait = _FileConnectorConfiguration?.FileHandleWaitTime is null ? dateTime.AddMilliseconds(1234).Ticks : dateTime.AddMilliseconds(_FileConnectorConfiguration.FileHandleWaitTime.Value).Ticks; if (string.IsNullOrEmpty(descriptions[0].Reactor) || string.IsNullOrEmpty(descriptions[0].PSN)) diff --git a/Adaptation/Shared/ProcessDataStandardFormat.cs b/Adaptation/Shared/ProcessDataStandardFormat.cs index 059d613..b306c4c 100644 --- a/Adaptation/Shared/ProcessDataStandardFormat.cs +++ b/Adaptation/Shared/ProcessDataStandardFormat.cs @@ -26,19 +26,25 @@ internal class ProcessDataStandardFormat internal long? Sequence { get; private set; } internal ReadOnlyCollection Body { get; private set; } + internal ReadOnlyCollection Footer { get; private set; } + internal ReadOnlyCollection Header { get; private set; } internal ReadOnlyCollection Columns { get; private set; } + internal ProcessDataStandardFormat? InputPDSF { get; private set; } internal ReadOnlyCollection Logistics { get; private set; } - internal ReadOnlyCollection InputLines { get; private set; } internal ProcessDataStandardFormat(ReadOnlyCollection body, ReadOnlyCollection columns, - ReadOnlyCollection inputLines, + ReadOnlyCollection footer, + ReadOnlyCollection header, + ProcessDataStandardFormat? inputPDSF, ReadOnlyCollection logistics, long? sequence) { Body = body; Columns = columns; - InputLines = inputLines; + Footer = footer; + Header = header; + InputPDSF = inputPDSF; Logistics = logistics; Sequence = sequence; } @@ -56,7 +62,7 @@ internal class ProcessDataStandardFormat GetString(SearchFor.Archive, addSpaces, separator); internal static ProcessDataStandardFormat GetEmpty() => - new(new(Array.Empty()), new(Array.Empty()), new(Array.Empty()), new(Array.Empty()), null); + new(new(Array.Empty()), new(Array.Empty()), new(Array.Empty()), new(Array.Empty()), null, new(Array.Empty()), null); internal static List PDSFToFixedWidth(string reportFullPath) { @@ -127,19 +133,26 @@ internal class ProcessDataStandardFormat return results; } - internal static ProcessDataStandardFormat GetProcessDataStandardFormat(string reportFullPath, string[]? lines = null) + internal static ProcessDataStandardFormat GetProcessDataStandardFormat(string reportFullPath, string[]? lines = null, int columnsLine = 6) { ProcessDataStandardFormat result; string segment; - List body = new(); - List logistics = new(); - lines ??= File.ReadAllLines(reportFullPath); string[] segments; - if (lines.Length < 7) + bool addToFooter = false; + List body = new(); + List header = new(); + List footer = new(); + List columns = new(); + ReadOnlyCollection logistics; + lines ??= File.ReadAllLines(reportFullPath); + if (lines.Length < columnsLine + 1) segments = Array.Empty(); else - segments = lines[6].Trim().Split('\t'); - List columns = new(); + { + segments = lines[columnsLine].Trim().Split('\t'); + for (int i = 0; i < columnsLine; i++) + header.Add(lines[i]); + } for (int c = 0; c < segments.Length; c++) { segment = segments[c].Substring(1, segments[c].Length - 2); @@ -158,37 +171,49 @@ internal class ProcessDataStandardFormat } } } - bool lookForLogistics = false; - for (int r = 7; r < lines.Length; r++) + for (int r = columnsLine + 1; r < lines.Length; r++) { if (lines[r].StartsWith("NUM_DATA_ROWS")) - lookForLogistics = true; - if (!lookForLogistics) - { + addToFooter = true; + if (!addToFooter) body.Add(lines[r]); - continue; - } - if (lines[r].StartsWith("LOGISTICS_1")) + else { - for (int i = r; i < lines.Length; i++) - { - if (!lines[i].StartsWith("LOGISTICS_") || lines[i].StartsWith("END_HEADER")) - break; - logistics.Add(lines[i]); - } - break; + footer.Add(lines[r]); + if (lines[r].StartsWith("END_HEADER")) + break; } } - if (lines.Length > 0 && body.Count == 0 && columns.Count == 0 && logistics.Count == 0) - logistics.Add(lines[1]); + string? linesOne = lines.Length > 0 && body.Count == 0 && columns.Count == 0 ? lines[1] : null; + logistics = GetLogistics(footer, linesOne: linesOne); result = new(body: body.AsReadOnly(), columns: columns.AsReadOnly(), - inputLines: lines.ToList().AsReadOnly(), - logistics: logistics.AsReadOnly(), + footer: footer.AsReadOnly(), + header: header.AsReadOnly(), + inputPDSF: null, + logistics: logistics, sequence: null); return result; } + private static ReadOnlyCollection GetLogistics(List footer, string? linesOne) + { + List results = new(); + bool foundLogistics1 = false; + foreach (string line in footer) + { + if (line.StartsWith("END_HEADER")) + break; + if (line.StartsWith("LOGISTICS_1")) + foundLogistics1 = true; + if (foundLogistics1 && line.StartsWith("LOGISTICS_")) + results.Add(line); + } + if (!string.IsNullOrEmpty(linesOne) && results.Count == 0) + results.Add(linesOne); + return results.AsReadOnly(); + } + internal static ProcessDataStandardFormat? GetProcessDataStandardFormat(string reportFullPath, ProcessDataStandardFormatMapping pdsfMapping) { ProcessDataStandardFormat? result; @@ -196,7 +221,7 @@ internal class ProcessDataStandardFormat FileInfo fileInfo = new(reportFullPath); ProcessDataStandardFormat processDataStandardFormat = GetProcessDataStandardFormat(fileInfo.LastWriteTime, pdsfMapping.NewColumnNames.Count, columnsLine, fileInfo.FullName, lines: null); JsonElement[]? jsonElements = GetArray(pdsfMapping.NewColumnNames.Count, processDataStandardFormat, lookForNumbers: false); - if (jsonElements is null || pdsfMapping.OldColumnNames.Count != pdsfMapping.ColumnIndices.Count) + if (jsonElements is null || jsonElements.Length == 0 || pdsfMapping.OldColumnNames.Count != pdsfMapping.ColumnIndices.Count) result = null; else { @@ -212,9 +237,11 @@ internal class ProcessDataStandardFormat ProcessDataStandardFormat result; long sequence; string[] segments; + bool addToFooter = false; List body = new(); - bool lookForLogistics = false; - List logistics = new(); + List header = new(); + List footer = new(); + ReadOnlyCollection logistics; lines ??= File.ReadAllLines(path); if (lines.Length <= columnsLine) segments = Array.Empty(); @@ -223,28 +250,24 @@ internal class ProcessDataStandardFormat segments = lines[columnsLine].Split('\t'); if (segments.Length != expectedColumns) segments = Array.Empty(); + for (int i = 0; i < columnsLine; i++) + header.Add(lines[i]); } string[] columns = segments.Select(l => l.Trim('"')).ToArray(); for (int r = columnsLine + 1; r < lines.Length; r++) { if (lines[r].StartsWith("NUM_DATA_ROWS")) - lookForLogistics = true; - if (!lookForLogistics) - { + addToFooter = true; + if (!addToFooter) body.Add(lines[r]); - continue; - } - if (lines[r].StartsWith("LOGISTICS_1")) + else { - for (int i = r; i < lines.Length; i++) - { - if (!lines[i].StartsWith("LOGISTICS_") || lines[i].StartsWith("END_HEADER")) - break; - logistics.Add(lines[i]); - } - break; + footer.Add(lines[r]); + if (lines[r].StartsWith("END_HEADER")) + break; } } + logistics = GetLogistics(footer, linesOne: null); if (logistics.Count == 0) sequence = lastWriteTime.Ticks; else @@ -254,8 +277,10 @@ internal class ProcessDataStandardFormat } result = new(body: body.AsReadOnly(), columns: new(columns), - inputLines: lines.ToList().AsReadOnly(), - logistics: logistics.AsReadOnly(), + footer: footer.AsReadOnly(), + header: header.AsReadOnly(), + inputPDSF: null, + logistics: logistics, sequence: sequence); return result; } @@ -346,7 +371,9 @@ internal class ProcessDataStandardFormat } result = new(body: new(results), columns: processDataStandardFormatMapping.OldColumnNames, - inputLines: processDataStandardFormat.InputLines, + footer: processDataStandardFormat.Footer, + header: processDataStandardFormat.Header, + inputPDSF: processDataStandardFormat, logistics: processDataStandardFormat.Logistics, sequence: processDataStandardFormat.Sequence); return result; @@ -379,7 +406,19 @@ internal class ProcessDataStandardFormat results.Add("LOGISTICS_COLUMN\tB_LOGISTICS"); results.AddRange(processDataStandardFormat.Logistics); results.Add("EOF"); - results.AddRange(processDataStandardFormat.InputLines.Select(l => l.Replace('\t', '|'))); + if (processDataStandardFormat.InputPDSF is not null) + { + List hyphens = new(); + results.AddRange(processDataStandardFormat.InputPDSF.Header.Select(l => l.Replace('\t', '|'))); + results.Add(string.Empty); + results.Add($"|{string.Join("|", processDataStandardFormat.InputPDSF.Columns)}|"); + for (int i = 0; i < processDataStandardFormat.InputPDSF.Columns.Count; i++) + hyphens.Add('-'); + results.Add($"|{string.Join("|", hyphens)}|"); + results.AddRange(processDataStandardFormat.InputPDSF.Body.Select(l => l.Replace('\t', '|'))); + results.Add(string.Empty); + results.AddRange(processDataStandardFormat.InputPDSF.Footer.Select(l => l.Replace('\t', '|'))); + } File.WriteAllText(path, string.Join(Environment.NewLine, results)); }