From ca968eb19ab683a6c35a38178f9ba5729e31d580 Mon Sep 17 00:00:00 2001 From: Mike Phares Date: Mon, 21 Apr 2025 13:14:46 -0700 Subject: [PATCH] process-data-standard-format with pipes EDA logic --- Adaptation/.vscode/launch.json | 6 + .../MoveMatchingFiles/FileRead.cs | 4 + .../FileHandlers/OpenInsight/FileRead.cs | 28 ---- Adaptation/FileHandlers/Processed/FileRead.cs | 2 +- .../Shared/ProcessDataStandardFormat.cs | 139 +++++++++++------- Adaptation/_Tests/Static/hgcv.js | 138 +++++++++++++++++ 6 files changed, 238 insertions(+), 79 deletions(-) create mode 100644 Adaptation/_Tests/Static/hgcv.js diff --git a/Adaptation/.vscode/launch.json b/Adaptation/.vscode/launch.json index c406c37..74b8caf 100644 --- a/Adaptation/.vscode/launch.json +++ b/Adaptation/.vscode/launch.json @@ -5,6 +5,12 @@ "type": "coreclr", "request": "attach", "processId": 10528 + }, + { + "type": "node", + "request": "launch", + "name": "node Launch Current Opened File", + "program": "${file}" } ] } diff --git a/Adaptation/FileHandlers/MoveMatchingFiles/FileRead.cs b/Adaptation/FileHandlers/MoveMatchingFiles/FileRead.cs index 726de3f..6fd5aa3 100644 --- a/Adaptation/FileHandlers/MoveMatchingFiles/FileRead.cs +++ b/Adaptation/FileHandlers/MoveMatchingFiles/FileRead.cs @@ -301,6 +301,8 @@ public class FileRead : Shared.FileRead, IFileRead long preWait; foreach (PreWith preWith in preWithCollection) { + if (!_IsEAFHosted) + continue; if (processDataStandardFormat is null) File.Move(preWith.MatchingFile, preWith.CheckFile); else @@ -350,6 +352,8 @@ public class FileRead : Shared.FileRead, IFileRead _Logistics = new Logistics(reportFullPath, processDataStandardFormat); processDataStandardFormat = null; } + if (!_IsEAFHosted && processDataStandardFormat is not null) + ProcessDataStandardFormat.Write(".pdsf", processDataStandardFormat); SetFileParameterLotIDToLogisticsMID(); int numberLength = 2; long ticks = dateTime.Ticks; diff --git a/Adaptation/FileHandlers/OpenInsight/FileRead.cs b/Adaptation/FileHandlers/OpenInsight/FileRead.cs index dc9bd63..88cd4b6 100644 --- a/Adaptation/FileHandlers/OpenInsight/FileRead.cs +++ b/Adaptation/FileHandlers/OpenInsight/FileRead.cs @@ -188,34 +188,6 @@ public class FileRead : Shared.FileRead, IFileRead return results.ToString(); } - private static string GetJson(int columnsLine, string[] columns, string[] body) - { -#pragma warning disable CA1845, IDE0057 - string result = "[\n"; - string line; - string value; - string[] segments; - if (columns.Length == 0) - columns = body[columnsLine].Trim().Split('\t'); - for (int i = columnsLine + 1; i < body.Length; i++) - { - line = "{"; - segments = body[i].Trim().Split('\t'); - if (segments.Length != columns.Length) - break; - for (int c = 1; c < segments.Length; c++) - { - value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\"); - line += '"' + columns[c].Trim('"') + '"' + ':' + '"' + value + '"' + ','; - } - line = line.Substring(0, line.Length - 1) + '}' + ',' + '\n'; - result += line; - } - result = result.Substring(0, result.Length - 1) + ']'; - return result; -#pragma warning restore CA1845, IDE0057 - } - private Tuple> GetExtractResult(string reportFullPath, DateTime dateTime) { Tuple> results; diff --git a/Adaptation/FileHandlers/Processed/FileRead.cs b/Adaptation/FileHandlers/Processed/FileRead.cs index 21a4050..e42a246 100644 --- a/Adaptation/FileHandlers/Processed/FileRead.cs +++ b/Adaptation/FileHandlers/Processed/FileRead.cs @@ -152,7 +152,7 @@ public class FileRead : Shared.FileRead, IFileRead matchDirectoryFileName = Path.GetFileName(matchDirectoryFile); if (jobIdDirectoryFileName.StartsWith(matchDirectoryFileName)) { - checkFile = Path.Combine(matchDirectory, Path.GetFileName(matchDirectoryFile)); + checkFile = Path.Combine(matchDirectory, jobIdDirectoryFileName); if (File.Exists(checkFile)) continue; File.Move(jobIdDirectoryFile, checkFile); diff --git a/Adaptation/Shared/ProcessDataStandardFormat.cs b/Adaptation/Shared/ProcessDataStandardFormat.cs index 059d613..b306c4c 100644 --- a/Adaptation/Shared/ProcessDataStandardFormat.cs +++ b/Adaptation/Shared/ProcessDataStandardFormat.cs @@ -26,19 +26,25 @@ internal class ProcessDataStandardFormat internal long? Sequence { get; private set; } internal ReadOnlyCollection Body { get; private set; } + internal ReadOnlyCollection Footer { get; private set; } + internal ReadOnlyCollection Header { get; private set; } internal ReadOnlyCollection Columns { get; private set; } + internal ProcessDataStandardFormat? InputPDSF { get; private set; } internal ReadOnlyCollection Logistics { get; private set; } - internal ReadOnlyCollection InputLines { get; private set; } internal ProcessDataStandardFormat(ReadOnlyCollection body, ReadOnlyCollection columns, - ReadOnlyCollection inputLines, + ReadOnlyCollection footer, + ReadOnlyCollection header, + ProcessDataStandardFormat? inputPDSF, ReadOnlyCollection logistics, long? sequence) { Body = body; Columns = columns; - InputLines = inputLines; + Footer = footer; + Header = header; + InputPDSF = inputPDSF; Logistics = logistics; Sequence = sequence; } @@ -56,7 +62,7 @@ internal class ProcessDataStandardFormat GetString(SearchFor.Archive, addSpaces, separator); internal static ProcessDataStandardFormat GetEmpty() => - new(new(Array.Empty()), new(Array.Empty()), new(Array.Empty()), new(Array.Empty()), null); + new(new(Array.Empty()), new(Array.Empty()), new(Array.Empty()), new(Array.Empty()), null, new(Array.Empty()), null); internal static List PDSFToFixedWidth(string reportFullPath) { @@ -127,19 +133,26 @@ internal class ProcessDataStandardFormat return results; } - internal static ProcessDataStandardFormat GetProcessDataStandardFormat(string reportFullPath, string[]? lines = null) + internal static ProcessDataStandardFormat GetProcessDataStandardFormat(string reportFullPath, string[]? lines = null, int columnsLine = 6) { ProcessDataStandardFormat result; string segment; - List body = new(); - List logistics = new(); - lines ??= File.ReadAllLines(reportFullPath); string[] segments; - if (lines.Length < 7) + bool addToFooter = false; + List body = new(); + List header = new(); + List footer = new(); + List columns = new(); + ReadOnlyCollection logistics; + lines ??= File.ReadAllLines(reportFullPath); + if (lines.Length < columnsLine + 1) segments = Array.Empty(); else - segments = lines[6].Trim().Split('\t'); - List columns = new(); + { + segments = lines[columnsLine].Trim().Split('\t'); + for (int i = 0; i < columnsLine; i++) + header.Add(lines[i]); + } for (int c = 0; c < segments.Length; c++) { segment = segments[c].Substring(1, segments[c].Length - 2); @@ -158,37 +171,49 @@ internal class ProcessDataStandardFormat } } } - bool lookForLogistics = false; - for (int r = 7; r < lines.Length; r++) + for (int r = columnsLine + 1; r < lines.Length; r++) { if (lines[r].StartsWith("NUM_DATA_ROWS")) - lookForLogistics = true; - if (!lookForLogistics) - { + addToFooter = true; + if (!addToFooter) body.Add(lines[r]); - continue; - } - if (lines[r].StartsWith("LOGISTICS_1")) + else { - for (int i = r; i < lines.Length; i++) - { - if (!lines[i].StartsWith("LOGISTICS_") || lines[i].StartsWith("END_HEADER")) - break; - logistics.Add(lines[i]); - } - break; + footer.Add(lines[r]); + if (lines[r].StartsWith("END_HEADER")) + break; } } - if (lines.Length > 0 && body.Count == 0 && columns.Count == 0 && logistics.Count == 0) - logistics.Add(lines[1]); + string? linesOne = lines.Length > 0 && body.Count == 0 && columns.Count == 0 ? lines[1] : null; + logistics = GetLogistics(footer, linesOne: linesOne); result = new(body: body.AsReadOnly(), columns: columns.AsReadOnly(), - inputLines: lines.ToList().AsReadOnly(), - logistics: logistics.AsReadOnly(), + footer: footer.AsReadOnly(), + header: header.AsReadOnly(), + inputPDSF: null, + logistics: logistics, sequence: null); return result; } + private static ReadOnlyCollection GetLogistics(List footer, string? linesOne) + { + List results = new(); + bool foundLogistics1 = false; + foreach (string line in footer) + { + if (line.StartsWith("END_HEADER")) + break; + if (line.StartsWith("LOGISTICS_1")) + foundLogistics1 = true; + if (foundLogistics1 && line.StartsWith("LOGISTICS_")) + results.Add(line); + } + if (!string.IsNullOrEmpty(linesOne) && results.Count == 0) + results.Add(linesOne); + return results.AsReadOnly(); + } + internal static ProcessDataStandardFormat? GetProcessDataStandardFormat(string reportFullPath, ProcessDataStandardFormatMapping pdsfMapping) { ProcessDataStandardFormat? result; @@ -196,7 +221,7 @@ internal class ProcessDataStandardFormat FileInfo fileInfo = new(reportFullPath); ProcessDataStandardFormat processDataStandardFormat = GetProcessDataStandardFormat(fileInfo.LastWriteTime, pdsfMapping.NewColumnNames.Count, columnsLine, fileInfo.FullName, lines: null); JsonElement[]? jsonElements = GetArray(pdsfMapping.NewColumnNames.Count, processDataStandardFormat, lookForNumbers: false); - if (jsonElements is null || pdsfMapping.OldColumnNames.Count != pdsfMapping.ColumnIndices.Count) + if (jsonElements is null || jsonElements.Length == 0 || pdsfMapping.OldColumnNames.Count != pdsfMapping.ColumnIndices.Count) result = null; else { @@ -212,9 +237,11 @@ internal class ProcessDataStandardFormat ProcessDataStandardFormat result; long sequence; string[] segments; + bool addToFooter = false; List body = new(); - bool lookForLogistics = false; - List logistics = new(); + List header = new(); + List footer = new(); + ReadOnlyCollection logistics; lines ??= File.ReadAllLines(path); if (lines.Length <= columnsLine) segments = Array.Empty(); @@ -223,28 +250,24 @@ internal class ProcessDataStandardFormat segments = lines[columnsLine].Split('\t'); if (segments.Length != expectedColumns) segments = Array.Empty(); + for (int i = 0; i < columnsLine; i++) + header.Add(lines[i]); } string[] columns = segments.Select(l => l.Trim('"')).ToArray(); for (int r = columnsLine + 1; r < lines.Length; r++) { if (lines[r].StartsWith("NUM_DATA_ROWS")) - lookForLogistics = true; - if (!lookForLogistics) - { + addToFooter = true; + if (!addToFooter) body.Add(lines[r]); - continue; - } - if (lines[r].StartsWith("LOGISTICS_1")) + else { - for (int i = r; i < lines.Length; i++) - { - if (!lines[i].StartsWith("LOGISTICS_") || lines[i].StartsWith("END_HEADER")) - break; - logistics.Add(lines[i]); - } - break; + footer.Add(lines[r]); + if (lines[r].StartsWith("END_HEADER")) + break; } } + logistics = GetLogistics(footer, linesOne: null); if (logistics.Count == 0) sequence = lastWriteTime.Ticks; else @@ -254,8 +277,10 @@ internal class ProcessDataStandardFormat } result = new(body: body.AsReadOnly(), columns: new(columns), - inputLines: lines.ToList().AsReadOnly(), - logistics: logistics.AsReadOnly(), + footer: footer.AsReadOnly(), + header: header.AsReadOnly(), + inputPDSF: null, + logistics: logistics, sequence: sequence); return result; } @@ -346,7 +371,9 @@ internal class ProcessDataStandardFormat } result = new(body: new(results), columns: processDataStandardFormatMapping.OldColumnNames, - inputLines: processDataStandardFormat.InputLines, + footer: processDataStandardFormat.Footer, + header: processDataStandardFormat.Header, + inputPDSF: processDataStandardFormat, logistics: processDataStandardFormat.Logistics, sequence: processDataStandardFormat.Sequence); return result; @@ -379,7 +406,19 @@ internal class ProcessDataStandardFormat results.Add("LOGISTICS_COLUMN\tB_LOGISTICS"); results.AddRange(processDataStandardFormat.Logistics); results.Add("EOF"); - results.AddRange(processDataStandardFormat.InputLines.Select(l => l.Replace('\t', '|'))); + if (processDataStandardFormat.InputPDSF is not null) + { + List hyphens = new(); + results.AddRange(processDataStandardFormat.InputPDSF.Header.Select(l => l.Replace('\t', '|'))); + results.Add(string.Empty); + results.Add($"|{string.Join("|", processDataStandardFormat.InputPDSF.Columns)}|"); + for (int i = 0; i < processDataStandardFormat.InputPDSF.Columns.Count; i++) + hyphens.Add('-'); + results.Add($"|{string.Join("|", hyphens)}|"); + results.AddRange(processDataStandardFormat.InputPDSF.Body.Select(l => l.Replace('\t', '|'))); + results.Add(string.Empty); + results.AddRange(processDataStandardFormat.InputPDSF.Footer.Select(l => l.Replace('\t', '|'))); + } File.WriteAllText(path, string.Join(Environment.NewLine, results)); } diff --git a/Adaptation/_Tests/Static/hgcv.js b/Adaptation/_Tests/Static/hgcv.js new file mode 100644 index 0000000..9524667 --- /dev/null +++ b/Adaptation/_Tests/Static/hgcv.js @@ -0,0 +1,138 @@ +"use strict"; + +function getCollectionParseFloat(collection) { + let result = []; + let value; + for (let i = 0; i < collection.length; i++) { + value = parseFloat(collection[i]); + result.push(value); + } + return result; +} + +function getSum(collection) { + let result = 0; + if (!collection || collection.length === 0) { + result = 0; + } + else { + for (let i = 0; i < collection.length; i++) { + result += collection[i]; + } + } + return result; +} + +function getAverage(collection) { + let result = null; + if (collection == null || collection.length === 0) + result = 0; + else { + let sum = getSum(collection); + result = sum / collection.length; + } + return result; +} + +function getVariance(collection) { + let result = null; + if (collection == null || collection.length === 0) + result = null; + else { + let variance = 0; + let t = collection[0]; + for (let i = 1; i < collection.length; i++) { + t += collection[i]; + const diff = ((i + 1) * collection[i]) - t; + variance += diff * diff / ((i + 1.0) * i); + } + result = variance / (collection.length - 1); + } + return result; +} + +function getNineEdgeMeanDelta(edge4mmRhoPoints, edge10mmRhoPoints) { + let result; + const nine4mmEdgeSum = getSum(edge4mmRhoPoints); + const nine10mmEdgeSum = getSum(edge10mmRhoPoints); + result = (nine4mmEdgeSum - nine10mmEdgeSum) / nine10mmEdgeSum * 100; + return result; +} + +function getMax(collection) { + let result = collection[0]; + for (let i = 1; i < collection.length; i++) { + if (collection[i] > result) { + result = collection[i]; + } + } + return result; +} + +function getMin(collection) { + let result = collection[0]; + for (let i = 1; i < collection.length; i++) { + if (collection[i] < result) { + result = collection[i]; + } + } + return result; +} + +function getNineResRangePercent(criticalRhoPoints) { + let result; + const nineCriticalPointsAverage = getAverage(criticalRhoPoints); + // result = (Math.max(...criticalRhoPoints) - Math.min(...criticalRhoPoints)) / nineCriticalPointsAverage * 100; + // let max = criticalRhoPoints.reduce((a, b) => Math.max(a, b)); + // let min = criticalRhoPoints.reduce((a, b) => Math.min(a, b)); + // result = (max - min) / nineCriticalPointsAverage * 100; + // let max = criticalRhoPoints.sort((a, b) => b - a); + // let min = criticalRhoPoints.sort((a, b) => a - b); + // result = (max[0] - min[0]) / nineCriticalPointsAverage * 100; + let max = getMax(criticalRhoPoints); + let min = getMin(criticalRhoPoints); + result = (max - min) / nineCriticalPointsAverage * 100; + return result; +} + +function getValue(edge10mmRhoPoints, index) { + let result = null; + if (index === 8) { + if (edge10mmRhoPoints != undefined && edge10mmRhoPoints.length > 1) { + let collection = edge10mmRhoPoints[0] === '|' ? edge10mmRhoPoints.substring(1).split('|') : edge10mmRhoPoints.split('|'); + let collectionParseFloat = getCollectionParseFloat(collection); + result = Math.round(getAverage(collectionParseFloat) * 10000000) / 10000000; + } + } + return result; +} + +// 0 1 2 3 4 5 6 7 8 +// 1 2 3 4 5 6 7 8 9 +const allRhoAvg = getCollectionParseFloat('|2.648|3.076|2.877|2.747|2.821|2.765|2.669|2.814|2.876'.substring(1).split('|')); +const edge4mmRhoPoints = getCollectionParseFloat('|2.877|2.747|2.669|2.814'.substring(1).split('|')); +const edge10mmRhoPoints = getCollectionParseFloat('|3.076|2.821|2.765|2.876'.substring(1).split('|')); +const criticalRhoPoints = getCollectionParseFloat('|2.648|3.076|2.821|2.765|2.876'.substring(1).split('|')); + +// 0 1 2 3 4 5 6 7 8 +// 1 2 3 4 5 6 7 8 9 +const allPhase = getCollectionParseFloat('|88.874|88.999|89.085|89.029|89.018|89.007|89.049|89.024|89.007'.substring(1).split('|')); +const criticalPhasePoints = getCollectionParseFloat('|88.874|88.999|89.018|89.007|89.007'.substring(1).split('|')); + +const nineMean = getAverage(allRhoAvg); +const nine4mmEdgeMean = getAverage(edge4mmRhoPoints); +const nine10mmEdgeMean = getAverage(edge10mmRhoPoints); +const nineCriticalPointsAverage = getAverage(criticalRhoPoints); +const nineResRangePercent = getNineResRangePercent(criticalRhoPoints); +const nineCriticalPointsStdDev = Math.sqrt(getVariance(criticalRhoPoints)); +const nineCriticalPointsPhaseAngleAverage = getAverage(criticalPhasePoints); +const nineEdgeMeanDelta = getNineEdgeMeanDelta(edge4mmRhoPoints, edge10mmRhoPoints); +console.log(nineCriticalPointsStdDev); + +// Sequence to date string +// getValue(self, getContextData('1', 'cds.SEQUENCE', '')); + +const index = 8; +const gv_edge10mmRhoPoints = '|3.076|2.821|2.765|2.876'; +const value = roundNumber(getValue(gv_edge10mmRhoPoints, index), 7); +console.log("value: " + value); \ No newline at end of file