phares3757/met08resihgcv
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

process-data-standard-format with pipes

Browse Source 
EDA logic
This commit is contained in:
Mike Phares 2025-04-21 13:14:46 -07:00
parent e0298fb8f5
commit ca968eb19a
6 changed files with 238 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Adaptation/.vscode/launch.json vendored
View File

@ -5,6 +5,12 @@
"type": "coreclr",
"request": "attach",
"processId": 10528
},
{
"type": "node",
"request": "launch",
"name": "node Launch Current Opened File",
"program": "${file}"
}
]
}

4
Adaptation/FileHandlers/MoveMatchingFiles/FileRead.cs
View File

@ -301,6 +301,8 @@ public class FileRead : Shared.FileRead, IFileRead
long preWait;
foreach (PreWith preWith in preWithCollection)
{
if (!_IsEAFHosted)
continue;
if (processDataStandardFormat is null)
File.Move(preWith.MatchingFile, preWith.CheckFile);
else
@ -350,6 +352,8 @@ public class FileRead : Shared.FileRead, IFileRead
_Logistics = new Logistics(reportFullPath, processDataStandardFormat);
processDataStandardFormat = null;
}
if (!_IsEAFHosted && processDataStandardFormat is not null)
ProcessDataStandardFormat.Write(".pdsf", processDataStandardFormat);
SetFileParameterLotIDToLogisticsMID();
int numberLength = 2;
long ticks = dateTime.Ticks;

28
Adaptation/FileHandlers/OpenInsight/FileRead.cs
View File

@ -188,34 +188,6 @@ public class FileRead : Shared.FileRead, IFileRead
return results.ToString();
}
private static string GetJson(int columnsLine, string[] columns, string[] body)
{
#pragma warning disable CA1845, IDE0057
string result = "[\n";
string line;
string value;
string[] segments;
if (columns.Length == 0)
columns = body[columnsLine].Trim().Split('\t');
for (int i = columnsLine + 1; i < body.Length; i++)
{
line = "{";
segments = body[i].Trim().Split('\t');
if (segments.Length != columns.Length)
break;
for (int c = 1; c < segments.Length; c++)
{
value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\");
line += '"' + columns[c].Trim('"') + '"' + ':' + '"' + value + '"' + ',';
}
line = line.Substring(0, line.Length - 1) + '}' + ',' + '\n';
result += line;
}
result = result.Substring(0, result.Length - 1) + ']';
return result;
#pragma warning restore CA1845, IDE0057
}
private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime dateTime)
{
Tuple<string, Test[], JsonElement[], List<FileInfo>> results;

2
Adaptation/FileHandlers/Processed/FileRead.cs
View File

@ -152,7 +152,7 @@ public class FileRead : Shared.FileRead, IFileRead
matchDirectoryFileName = Path.GetFileName(matchDirectoryFile);
if (jobIdDirectoryFileName.StartsWith(matchDirectoryFileName))
{
checkFile = Path.Combine(matchDirectory, Path.GetFileName(matchDirectoryFile));
checkFile = Path.Combine(matchDirectory, jobIdDirectoryFileName);
if (File.Exists(checkFile))
continue;
File.Move(jobIdDirectoryFile, checkFile);

139
Adaptation/Shared/ProcessDataStandardFormat.cs
View File

@ -26,19 +26,25 @@ internal class ProcessDataStandardFormat
internal long? Sequence { get; private set; }
internal ReadOnlyCollection<string> Body { get; private set; }
internal ReadOnlyCollection<string> Footer { get; private set; }
internal ReadOnlyCollection<string> Header { get; private set; }
internal ReadOnlyCollection<string> Columns { get; private set; }
internal ProcessDataStandardFormat? InputPDSF { get; private set; }
internal ReadOnlyCollection<string> Logistics { get; private set; }
internal ReadOnlyCollection<string> InputLines { get; private set; }
internal ProcessDataStandardFormat(ReadOnlyCollection<string> body,
ReadOnlyCollection<string> columns,
ReadOnlyCollection<string> inputLines,
ReadOnlyCollection<string> footer,
ReadOnlyCollection<string> header,
ProcessDataStandardFormat? inputPDSF,
ReadOnlyCollection<string> logistics,
long? sequence)
{
Body = body;
Columns = columns;
InputLines = inputLines;
Footer = footer;
Header = header;
InputPDSF = inputPDSF;
Logistics = logistics;
Sequence = sequence;
}
@ -56,7 +62,7 @@ internal class ProcessDataStandardFormat
GetString(SearchFor.Archive, addSpaces, separator);
internal static ProcessDataStandardFormat GetEmpty() =>
new(new(Array.Empty<string>()), new(Array.Empty<string>()), new(Array.Empty<string>()), new(Array.Empty<string>()), null);
new(new(Array.Empty<string>()), new(Array.Empty<string>()), new(Array.Empty<string>()), new(Array.Empty<string>()), null, new(Array.Empty<string>()), null);
internal static List<string> PDSFToFixedWidth(string reportFullPath)
{
@ -127,19 +133,26 @@ internal class ProcessDataStandardFormat
return results;
}
internal static ProcessDataStandardFormat GetProcessDataStandardFormat(string reportFullPath, string[]? lines = null)
internal static ProcessDataStandardFormat GetProcessDataStandardFormat(string reportFullPath, string[]? lines = null, int columnsLine = 6)
{
ProcessDataStandardFormat result;
string segment;
List<string> body = new();
List<string> logistics = new();
lines ??= File.ReadAllLines(reportFullPath);
string[] segments;
if (lines.Length < 7)
bool addToFooter = false;
List<string> body = new();
List<string> header = new();
List<string> footer = new();
List<string> columns = new();
ReadOnlyCollection<string> logistics;
lines ??= File.ReadAllLines(reportFullPath);
if (lines.Length < columnsLine + 1)
segments = Array.Empty<string>();
else
segments = lines[6].Trim().Split('\t');
List<string> columns = new();
{
segments = lines[columnsLine].Trim().Split('\t');
for (int i = 0; i < columnsLine; i++)
header.Add(lines[i]);
}
for (int c = 0; c < segments.Length; c++)
{
segment = segments[c].Substring(1, segments[c].Length - 2);
@ -158,37 +171,49 @@ internal class ProcessDataStandardFormat
}
}
}
bool lookForLogistics = false;
for (int r = 7; r < lines.Length; r++)
for (int r = columnsLine + 1; r < lines.Length; r++)
{
if (lines[r].StartsWith("NUM_DATA_ROWS"))
lookForLogistics = true;
if (!lookForLogistics)
{
addToFooter = true;
if (!addToFooter)
body.Add(lines[r]);
continue;
}
if (lines[r].StartsWith("LOGISTICS_1"))
else
{
for (int i = r; i < lines.Length; i++)
{
if (!lines[i].StartsWith("LOGISTICS_") || lines[i].StartsWith("END_HEADER"))
break;
logistics.Add(lines[i]);
}
break;
footer.Add(lines[r]);
if (lines[r].StartsWith("END_HEADER"))
break;
}
}
if (lines.Length > 0 && body.Count == 0 && columns.Count == 0 && logistics.Count == 0)
logistics.Add(lines[1]);
string? linesOne = lines.Length > 0 && body.Count == 0 && columns.Count == 0 ? lines[1] : null;
logistics = GetLogistics(footer, linesOne: linesOne);
result = new(body: body.AsReadOnly(),
columns: columns.AsReadOnly(),
inputLines: lines.ToList().AsReadOnly(),
logistics: logistics.AsReadOnly(),
footer: footer.AsReadOnly(),
header: header.AsReadOnly(),
inputPDSF: null,
logistics: logistics,
sequence: null);
return result;
}
private static ReadOnlyCollection<string> GetLogistics(List<string> footer, string? linesOne)
{
List<string> results = new();
bool foundLogistics1 = false;
foreach (string line in footer)
{
if (line.StartsWith("END_HEADER"))
break;
if (line.StartsWith("LOGISTICS_1"))
foundLogistics1 = true;
if (foundLogistics1 && line.StartsWith("LOGISTICS_"))
results.Add(line);
}
if (!string.IsNullOrEmpty(linesOne) && results.Count == 0)
results.Add(linesOne);
return results.AsReadOnly();
}
internal static ProcessDataStandardFormat? GetProcessDataStandardFormat(string reportFullPath, ProcessDataStandardFormatMapping pdsfMapping)
{
ProcessDataStandardFormat? result;
@ -196,7 +221,7 @@ internal class ProcessDataStandardFormat
FileInfo fileInfo = new(reportFullPath);
ProcessDataStandardFormat processDataStandardFormat = GetProcessDataStandardFormat(fileInfo.LastWriteTime, pdsfMapping.NewColumnNames.Count, columnsLine, fileInfo.FullName, lines: null);
JsonElement[]? jsonElements = GetArray(pdsfMapping.NewColumnNames.Count, processDataStandardFormat, lookForNumbers: false);
if (jsonElements is null || pdsfMapping.OldColumnNames.Count != pdsfMapping.ColumnIndices.Count)
if (jsonElements is null || jsonElements.Length == 0 || pdsfMapping.OldColumnNames.Count != pdsfMapping.ColumnIndices.Count)
result = null;
else
{
@ -212,9 +237,11 @@ internal class ProcessDataStandardFormat
ProcessDataStandardFormat result;
long sequence;
string[] segments;
bool addToFooter = false;
List<string> body = new();
bool lookForLogistics = false;
List<string> logistics = new();
List<string> header = new();
List<string> footer = new();
ReadOnlyCollection<string> logistics;
lines ??= File.ReadAllLines(path);
if (lines.Length <= columnsLine)
segments = Array.Empty<string>();
@ -223,28 +250,24 @@ internal class ProcessDataStandardFormat
segments = lines[columnsLine].Split('\t');
if (segments.Length != expectedColumns)
segments = Array.Empty<string>();
for (int i = 0; i < columnsLine; i++)
header.Add(lines[i]);
}
string[] columns = segments.Select(l => l.Trim('"')).ToArray();
for (int r = columnsLine + 1; r < lines.Length; r++)
{
if (lines[r].StartsWith("NUM_DATA_ROWS"))
lookForLogistics = true;
if (!lookForLogistics)
{
addToFooter = true;
if (!addToFooter)
body.Add(lines[r]);
continue;
}
if (lines[r].StartsWith("LOGISTICS_1"))
else
{
for (int i = r; i < lines.Length; i++)
{
if (!lines[i].StartsWith("LOGISTICS_") || lines[i].StartsWith("END_HEADER"))
break;
logistics.Add(lines[i]);
}
break;
footer.Add(lines[r]);
if (lines[r].StartsWith("END_HEADER"))
break;
}
}
logistics = GetLogistics(footer, linesOne: null);
if (logistics.Count == 0)
sequence = lastWriteTime.Ticks;
else
@ -254,8 +277,10 @@ internal class ProcessDataStandardFormat
}
result = new(body: body.AsReadOnly(),
columns: new(columns),
inputLines: lines.ToList().AsReadOnly(),
logistics: logistics.AsReadOnly(),
footer: footer.AsReadOnly(),
header: header.AsReadOnly(),
inputPDSF: null,
logistics: logistics,
sequence: sequence);
return result;
}
@ -346,7 +371,9 @@ internal class ProcessDataStandardFormat
}
result = new(body: new(results),
columns: processDataStandardFormatMapping.OldColumnNames,
inputLines: processDataStandardFormat.InputLines,
footer: processDataStandardFormat.Footer,
header: processDataStandardFormat.Header,
inputPDSF: processDataStandardFormat,
logistics: processDataStandardFormat.Logistics,
sequence: processDataStandardFormat.Sequence);
return result;
@ -379,7 +406,19 @@ internal class ProcessDataStandardFormat
results.Add("LOGISTICS_COLUMN\tB_LOGISTICS");
results.AddRange(processDataStandardFormat.Logistics);
results.Add("EOF");
results.AddRange(processDataStandardFormat.InputLines.Select(l => l.Replace('\t', '|')));
if (processDataStandardFormat.InputPDSF is not null)
{
List<char> hyphens = new();
results.AddRange(processDataStandardFormat.InputPDSF.Header.Select(l => l.Replace('\t', '|')));
results.Add(string.Empty);
results.Add($"|{string.Join("|", processDataStandardFormat.InputPDSF.Columns)}|");
for (int i = 0; i < processDataStandardFormat.InputPDSF.Columns.Count; i++)
hyphens.Add('-');
results.Add($"|{string.Join("|", hyphens)}|");
results.AddRange(processDataStandardFormat.InputPDSF.Body.Select(l => l.Replace('\t', '|')));
results.Add(string.Empty);
results.AddRange(processDataStandardFormat.InputPDSF.Footer.Select(l => l.Replace('\t', '|')));
}
File.WriteAllText(path, string.Join(Environment.NewLine, results));
}

138
Adaptation/_Tests/Static/hgcv.js Normal file
View File

@ -0,0 +1,138 @@
"use strict";
function getCollectionParseFloat(collection) {
let result = [];
let value;
for (let i = 0; i < collection.length; i++) {
value = parseFloat(collection[i]);
result.push(value);
}
return result;
}
function getSum(collection) {
let result = 0;
if (!collection || collection.length === 0) {
result = 0;
}
else {
for (let i = 0; i < collection.length; i++) {
result += collection[i];
}
}
return result;
}
function getAverage(collection) {
let result = null;
if (collection == null || collection.length === 0)
result = 0;
else {
let sum = getSum(collection);
result = sum / collection.length;
}
return result;
}
function getVariance(collection) {
let result = null;
if (collection == null || collection.length === 0)
result = null;
else {
let variance = 0;
let t = collection[0];
for (let i = 1; i < collection.length; i++) {
t += collection[i];
const diff = ((i + 1) * collection[i]) - t;
variance += diff * diff / ((i + 1.0) * i);
}
result = variance / (collection.length - 1);
}
return result;
}
function getNineEdgeMeanDelta(edge4mmRhoPoints, edge10mmRhoPoints) {
let result;
const nine4mmEdgeSum = getSum(edge4mmRhoPoints);
const nine10mmEdgeSum = getSum(edge10mmRhoPoints);
result = (nine4mmEdgeSum - nine10mmEdgeSum) / nine10mmEdgeSum * 100;
return result;
}
function getMax(collection) {
let result = collection[0];
for (let i = 1; i < collection.length; i++) {
if (collection[i] > result) {
result = collection[i];
}
}
return result;
}
function getMin(collection) {
let result = collection[0];
for (let i = 1; i < collection.length; i++) {
if (collection[i] < result) {
result = collection[i];
}
}
return result;
}
function getNineResRangePercent(criticalRhoPoints) {
let result;
const nineCriticalPointsAverage = getAverage(criticalRhoPoints);
// result = (Math.max(...criticalRhoPoints) - Math.min(...criticalRhoPoints)) / nineCriticalPointsAverage * 100;
// let max = criticalRhoPoints.reduce((a, b) => Math.max(a, b));
// let min = criticalRhoPoints.reduce((a, b) => Math.min(a, b));
// result = (max - min) / nineCriticalPointsAverage * 100;
// let max = criticalRhoPoints.sort((a, b) => b - a);
// let min = criticalRhoPoints.sort((a, b) => a - b);
// result = (max[0] - min[0]) / nineCriticalPointsAverage * 100;
let max = getMax(criticalRhoPoints);
let min = getMin(criticalRhoPoints);
result = (max - min) / nineCriticalPointsAverage * 100;
return result;
}
function getValue(edge10mmRhoPoints, index) {
let result = null;
if (index === 8) {
if (edge10mmRhoPoints != undefined && edge10mmRhoPoints.length > 1) {
let collection = edge10mmRhoPoints[0] === '|' ? edge10mmRhoPoints.substring(1).split('|') : edge10mmRhoPoints.split('|');
let collectionParseFloat = getCollectionParseFloat(collection);
result = Math.round(getAverage(collectionParseFloat) * 10000000) / 10000000;
}
}
return result;
}
// 0 1 2 3 4 5 6 7 8
// 1 2 3 4 5 6 7 8 9
const allRhoAvg = getCollectionParseFloat('|2.648|3.076|2.877|2.747|2.821|2.765|2.669|2.814|2.876'.substring(1).split('|'));
const edge4mmRhoPoints = getCollectionParseFloat('|2.877|2.747|2.669|2.814'.substring(1).split('|'));
const edge10mmRhoPoints = getCollectionParseFloat('|3.076|2.821|2.765|2.876'.substring(1).split('|'));
const criticalRhoPoints = getCollectionParseFloat('|2.648|3.076|2.821|2.765|2.876'.substring(1).split('|'));
// 0 1 2 3 4 5 6 7 8
// 1 2 3 4 5 6 7 8 9
const allPhase = getCollectionParseFloat('|88.874|88.999|89.085|89.029|89.018|89.007|89.049|89.024|89.007'.substring(1).split('|'));
const criticalPhasePoints = getCollectionParseFloat('|88.874|88.999|89.018|89.007|89.007'.substring(1).split('|'));
const nineMean = getAverage(allRhoAvg);
const nine4mmEdgeMean = getAverage(edge4mmRhoPoints);
const nine10mmEdgeMean = getAverage(edge10mmRhoPoints);
const nineCriticalPointsAverage = getAverage(criticalRhoPoints);
const nineResRangePercent = getNineResRangePercent(criticalRhoPoints);
const nineCriticalPointsStdDev = Math.sqrt(getVariance(criticalRhoPoints));
const nineCriticalPointsPhaseAngleAverage = getAverage(criticalPhasePoints);
const nineEdgeMeanDelta = getNineEdgeMeanDelta(edge4mmRhoPoints, edge10mmRhoPoints);
console.log(nineCriticalPointsStdDev);
// Sequence to date string
// getValue(self, getContextData('1', 'cds.SEQUENCE', ''));
const index = 8;
const gv_edge10mmRhoPoints = '|3.076|2.821|2.765|2.876';
const value = roundNumber(getValue(gv_edge10mmRhoPoints, index), 7);
console.log("value: " + value);