phares3757/met08ddupsfs6420
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Preparation to switch to xml for InfinityQS export

Browse Source
This commit is contained in:
Mike Phares
2025-08-27 12:11:16 -07:00
parent 3467fb63a0
commit f70690754f
20 changed files with 521 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

239
Adaptation/Shared/ProcessDataStandardFormat.cs
View File

@ -2,12 +2,14 @@ using Adaptation.Shared.Methods;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
namespace Adaptation.Shared;
@ -136,6 +138,7 @@ internal class ProcessDataStandardFormat
internal static ProcessDataStandardFormat GetProcessDataStandardFormat(string reportFullPath, string[]? lines = null, int columnsLine = 6)
{
ProcessDataStandardFormat result;
long? sequence;
string segment;
string[] segments;
bool addToFooter = false;
@ -186,13 +189,25 @@ internal class ProcessDataStandardFormat
}
string? linesOne = lines.Length > 0 && body.Count == 0 && columns.Count == 0 ? lines[1] : null;
logistics = GetLogistics(footer, linesOne: linesOne);
if (logistics.Count == 0)
sequence = null;
else
{
segments = logistics[0].Split(new string[] { "SEQUENCE=" }, StringSplitOptions.None);
sequence = segments.Length < 2 || !long.TryParse(segments[1].Split(';')[0], out long s) ? null : s;
}
if (sequence is null && !string.IsNullOrEmpty(reportFullPath))
{
FileInfo fileInfo = new(reportFullPath);
sequence = fileInfo.LastWriteTime.Ticks;
}
result = new(body: body.AsReadOnly(),
columns: columns.AsReadOnly(),
footer: footer.AsReadOnly(),
header: header.AsReadOnly(),
inputPDSF: null,
logistics: logistics,
sequence: null);
sequence: sequence);
return result;
}
@ -214,19 +229,19 @@ internal class ProcessDataStandardFormat
return results.AsReadOnly();
}
internal static ProcessDataStandardFormat GetProcessDataStandardFormat(string reportFullPath, ProcessDataStandardFormatMapping pdsfMapping)
internal static ProcessDataStandardFormat GetProcessDataStandardFormat(string reportFullPath, ProcessDataStandardFormatMapping processDataStandardFormatMapping)
{
ProcessDataStandardFormat result;
const int columnsLine = 6;
FileInfo fileInfo = new(reportFullPath);
ProcessDataStandardFormat processDataStandardFormat = GetProcessDataStandardFormat(fileInfo.LastWriteTime, columnsLine, fileInfo.FullName, lines: null);
JsonElement[]? jsonElements = pdsfMapping.OldColumnNames.Count != pdsfMapping.ColumnIndices.Count ? null : GetFullArray(processDataStandardFormat);
JsonElement[]? jsonElements = processDataStandardFormatMapping.OldColumnNames.Count != processDataStandardFormatMapping.ColumnIndices.Count ? null : GetFullArray(processDataStandardFormat);
JsonProperty[]? jsonProperties = jsonElements is null || jsonElements.Length == 0 ? null : jsonElements[0].EnumerateObject().ToArray();
if (jsonElements is null || jsonProperties is null || jsonProperties.Length != pdsfMapping.NewColumnNames.Count)
if (jsonElements is null || jsonProperties is null || jsonProperties.Length != processDataStandardFormatMapping.NewColumnNames.Count)
result = processDataStandardFormat;
else
{
result = GetProcessDataStandardFormat(pdsfMapping, jsonElements, processDataStandardFormat);
result = GetProcessDataStandardFormat(processDataStandardFormatMapping, jsonElements, processDataStandardFormat);
if (result.Sequence is null || result.Columns.Count == 0 || result.Body.Count == 0 || result.Logistics.Count == 0)
result = processDataStandardFormat;
}
@ -236,7 +251,7 @@ internal class ProcessDataStandardFormat
private static ProcessDataStandardFormat GetProcessDataStandardFormat(DateTime lastWriteTime, int columnsLine, string path, string[]? lines)
{
ProcessDataStandardFormat result;
long sequence;
long? sequence;
string[] segments;
bool addToFooter = false;
List<string> body = new();
@ -268,12 +283,13 @@ internal class ProcessDataStandardFormat
}
logistics = GetLogistics(footer, linesOne: null);
if (logistics.Count == 0)
sequence = lastWriteTime.Ticks;
sequence = null;
else
{
segments = logistics[0].Split(new string[] { "SEQUENCE=" }, StringSplitOptions.None);
sequence = segments.Length < 2 || !long.TryParse(segments[1].Split(';')[0], out long s) ? lastWriteTime.Ticks : s;
sequence = segments.Length < 2 || !long.TryParse(segments[1].Split(';')[0], out long s) ? null : s;
}
sequence ??= lastWriteTime.Ticks;
result = new(body: body.AsReadOnly(),
columns: new(columns),
footer: footer.AsReadOnly(),
@ -302,7 +318,7 @@ internal class ProcessDataStandardFormat
segments = bodyLine.Split('\t').ToList();
for (int c = 0; c < segments.Count; c++)
{
value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\");
value = segments[c].Replace("\\", "\\\\").Replace("\"", "\\\"");
_ = stringBuilder.Append('"').Append(processDataStandardFormat.Columns[c]).Append("\":\"").Append(value).Append("\",");
}
_ = stringBuilder.Remove(stringBuilder.Length - 1, 1);
@ -321,12 +337,14 @@ internal class ProcessDataStandardFormat
int column;
string value;
JsonProperty jsonProperty;
List<string> debug = new();
List<string> values = new();
List<string> results = new();
JsonProperty[] jsonProperties;
List<string> unknownColumns = new();
for (int i = 0; i < jsonElements.Length; i++)
{
debug.Clear();
values.Clear();
if (jsonElements[i].ValueKind != JsonValueKind.Object)
{
@ -340,16 +358,22 @@ internal class ProcessDataStandardFormat
{
column = processDataStandardFormatMapping.ColumnIndices[c];
if (column == -1)
{
value = processDataStandardFormatMapping.OldColumnNames[c];
debug.Add($"<Item C=-01 Name=\"{value}\" DataType=\"8\" XmlType=\"1\" XPath=\"//records/record/{value}\" />");
}
else
{
jsonProperty = jsonProperties[column];
value = jsonProperty.Value.ToString();
debug.Add($"<Item C={column + 2:000} Name=\"{processDataStandardFormatMapping.OldColumnNames[c]}\" DataType=\"8\" XmlType=\"1\" XPath=\"//records/record/{jsonProperty.Name}\" />");
}
values.Add(value);
}
results.Add(string.Join("\t", values));
}
if (Debugger.IsAttached)
File.WriteAllText("../../.txt", string.Join(Environment.NewLine, debug.OrderBy(l => l)));
result = new(body: new(results),
columns: processDataStandardFormatMapping.OldColumnNames,
footer: processDataStandardFormat.Footer,
@ -364,7 +388,6 @@ internal class ProcessDataStandardFormat
{
if (processDataStandardFormat.InputPDSF is null)
throw new NullReferenceException(nameof(processDataStandardFormat.InputPDSF));
#pragma warning disable CA1845, IDE0057
string result;
string line;
string value;
@ -378,19 +401,27 @@ internal class ProcessDataStandardFormat
break;
for (int c = 0; c < segments.Length; c++)
{
value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\");
value = segments[c].Replace("\\", "\\\\").Replace("\"", "\\\"");
line += string.Concat('"', processDataStandardFormat.InputPDSF.Columns[c].Trim('"'), '"', ':', '"', value, '"', ',');
}
line = string.Concat(line.Substring(0, line.Length - 1), '}');
lines.Add(line);
}
string? json = null;
if (processDataStandardFormat.Footer is not null && processDataStandardFormat.Footer.Count > 0)
{
Dictionary<string, string> footerKeyValuePairs = GetFooterKeyValuePairs(processDataStandardFormat.Footer);
Dictionary<string, Dictionary<string, string>> logisticKeyValuePairs = GetLogisticKeyValuePairs(processDataStandardFormat.Footer, footerKeyValuePairs);
json = JsonSerializer.Serialize(logisticKeyValuePairs, DictionaryStringDictionaryStringStringSourceGenerationContext.Default.DictionaryStringDictionaryStringString);
}
string footerText = string.IsNullOrEmpty(json) || json == "{}" ? string.Empty : $",{Environment.NewLine}\"PDSF\":{Environment.NewLine}{json}";
result = string.Concat(
'{',
Environment.NewLine,
'"',
"Count",
'"',
": ",
": ",
processDataStandardFormat.Body.Count,
',',
Environment.NewLine,
@ -409,17 +440,95 @@ internal class ProcessDataStandardFormat
'"',
"Sequence",
'"',
": ",
": ",
processDataStandardFormat.Sequence,
Environment.NewLine,
footerText,
Environment.NewLine,
'}');
return result;
#pragma warning restore CA1845, IDE0057
}
private static Dictionary<string, string> GetFooterKeyValuePairs(ReadOnlyCollection<string> footerLines)
{
Dictionary<string, string> results = new();
string[] segments;
foreach (string footerLine in footerLines)
{
segments = footerLine.Split('\t');
if (segments.Length != 2 || string.IsNullOrEmpty(segments[1].Trim()))
{
continue;
}
if (segments[1].Contains(';'))
{
continue;
}
else
{
if (results.ContainsKey(segments[0]))
{
continue;
}
results.Add(segments[0], segments[1]);
}
}
return results;
}
private static Dictionary<string, Dictionary<string, string>> GetLogisticKeyValuePairs(ReadOnlyCollection<string> footerLines, Dictionary<string, string> footerKeyValuePairs)
{
Dictionary<string, Dictionary<string, string>> results = new();
string[] segments;
string[] subSegments;
string[] subSubSegments;
Dictionary<string, string>? keyValue;
results.Add("Footer", footerKeyValuePairs);
foreach (string footerLine in footerLines)
{
segments = footerLine.Split('\t');
if (segments.Length != 2 || string.IsNullOrEmpty(segments[1].Trim()))
{
continue;
}
if (!segments[1].Contains(';') || !segments[1].Contains('='))
{
continue;
}
else
{
subSegments = segments[1].Split(';');
if (subSegments.Length < 1)
{
continue;
}
if (!results.TryGetValue(segments[0], out keyValue))
{
results.Add(segments[0], new());
if (!results.TryGetValue(segments[0], out keyValue))
{
throw new Exception();
}
}
foreach (string segment in subSegments)
{
subSubSegments = segment.Split('=');
if (subSubSegments.Length != 2)
{
continue;
}
keyValue.Add(subSubSegments[0], subSubSegments[1]);
}
}
}
return results;
}
internal static void Write(string path, ProcessDataStandardFormat processDataStandardFormat, List<Metrology.WS.Results>? wsResults)
{
List<string> results = new();
if (processDataStandardFormat.InputPDSF is null)
throw new NullReferenceException(nameof(processDataStandardFormat.InputPDSF));
if (processDataStandardFormat.Sequence is null)
throw new NullReferenceException(nameof(processDataStandardFormat.Sequence));
string endOffset = "E#######T";
@ -457,25 +566,25 @@ internal class ProcessDataStandardFormat
}
}
results.Add("END_HEADER");
if (processDataStandardFormat.InputPDSF is not null)
{
results.Add(string.Empty);
List<char> hyphens = new();
results.AddRange(processDataStandardFormat.InputPDSF.Header.Select(l => l.Replace('\t', '|')));
results.Add(string.Empty);
results.Add($"|{string.Join("|", processDataStandardFormat.InputPDSF.Columns)}|");
for (int i = 0; i < processDataStandardFormat.InputPDSF.Columns.Count; i++)
hyphens.Add('-');
results.Add($"|{string.Join("|", hyphens)}|");
results.AddRange(processDataStandardFormat.InputPDSF.Body.Select(l => l.Replace('\t', '|')));
results.Add(string.Empty);
results.AddRange(processDataStandardFormat.InputPDSF.Footer.Select(l => l.Replace('\t', '|')));
results.Add(string.Empty);
results.Add("EOF");
results.Add(string.Empty);
string json = GetJson(processDataStandardFormat);
results.Add(json);
}
results.Add(string.Empty);
List<char> hyphens = new();
results.AddRange(processDataStandardFormat.InputPDSF.Header.Select(l => $"|{l.Replace('\t', '|')}|"));
results.Add(string.Empty);
results.Add($"|{string.Join("|", processDataStandardFormat.InputPDSF.Columns)}|");
for (int i = 0; i < processDataStandardFormat.InputPDSF.Columns.Count; i++)
hyphens.Add('-');
results.Add($"|{string.Join("|", hyphens)}|");
results.AddRange(processDataStandardFormat.InputPDSF.Body.Select(l => $"|{l.Replace('\t', '|')}|"));
results.Add(string.Empty);
results.AddRange(processDataStandardFormat.InputPDSF.Footer.Select(l => $"|{l.Replace('\t', '|')}|"));
results.Add(string.Empty);
string xml = GetXml(processDataStandardFormat);
results.Add(xml);
results.Add(string.Empty);
results.Add("EOF");
results.Add(string.Empty);
string json = GetJson(processDataStandardFormat);
results.Add(json);
File.WriteAllText(path, string.Join(Environment.NewLine, results));
}
@ -518,7 +627,7 @@ internal class ProcessDataStandardFormat
{
for (int c = 1; c < segments.Length; c++)
{
value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\");
value = segments[c].Replace("\\", "\\\\").Replace("\"", "\\\"");
_ = stringBuilder.Append('"').Append(processDataStandardFormat.Columns[c]).Append("\":\"").Append(value).Append("\",");
}
}
@ -526,7 +635,7 @@ internal class ProcessDataStandardFormat
{
for (int c = 1; c < segments.Length; c++)
{
value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\");
value = segments[c].Replace("\\", "\\\\").Replace("\"", "\\\"");
if (string.IsNullOrEmpty(value))
_ = stringBuilder.Append('"').Append(processDataStandardFormat.Columns[c]).Append("\":").Append(value).Append("null,");
else if (value.All(char.IsDigit))
@ -757,10 +866,70 @@ internal class ProcessDataStandardFormat
return result;
}
internal static string GetXml(ProcessDataStandardFormat processDataStandardFormat)
{
string result;
string tag;
string value;
string[] segments;
ReadOnlyCollection<string> body = processDataStandardFormat.InputPDSF is null ?
processDataStandardFormat.Body : processDataStandardFormat.InputPDSF.Body;
ReadOnlyCollection<string> columns = processDataStandardFormat.InputPDSF is null ?
processDataStandardFormat.Columns : processDataStandardFormat.InputPDSF.Columns;
List<string> lines = new() { "<?xml version=\"1.0\" encoding=\"UTF-8\"?>", "<records>" };
for (int i = 0; i < body.Count; i++)
{
lines.Add(" <record>");
segments = body[i].Trim().Split('\t');
if (segments.Length != columns.Count)
break;
for (int c = 0; c < segments.Length; c++)
{
value = segments[c].Replace("&", "&amp;")
.Replace("<", "&lt;")
.Replace(">", "&gt;")
.Replace("\"", "&quot;")
.Replace("'", "&apos;");
tag = Regex.Replace(columns[c].Trim('"'), @"[^a-zA-Z0-9]", "_").Split('\r')[0].Split('\n')[0];
lines.Add(string.Concat(" <", tag, '>', value, "</", tag, '>'));
}
lines.Add(" </record>");
}
lines.Add("</records>");
result = string.Join(Environment.NewLine, lines);
return result;
}
internal static string GetXml(string reportFullPath, string[]? lines = null)
{
string result;
bool foundXml = false;
List<string> results = new();
lines ??= File.ReadAllLines(reportFullPath);
foreach (string line in lines)
{
if (line.StartsWith("<?xml"))
foundXml = true;
if (!foundXml)
continue;
if (line.StartsWith("EOF"))
break;
results.Add(line);
}
result = string.Join(Environment.NewLine, results);
return result;
}
}
[JsonSourceGenerationOptions(WriteIndented = true)]
[JsonSerializable(typeof(JsonElement[]))]
internal partial class JsonElementCollectionSourceGenerationContext : JsonSerializerContext
{
}
[JsonSourceGenerationOptions(WriteIndented = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
[JsonSerializable(typeof(Dictionary<string, Dictionary<string, string>>))]
internal partial class DictionaryStringDictionaryStringStringSourceGenerationContext : JsonSerializerContext
{
}