Refactor and clean up file handling and processing logic

- Removed the FileRead class from Adaptation.FileHandlers.SPaCe as it was no longer needed. - Updated TypeScript and JavaScript files to improve the handling of work item updates, including new checks for WeightedShortestJobFirstFibonacci. - Modified the MESAFIBACKLOG.yml to remove unnecessary file copy tasks. - Enhanced the FileRead class in Adaptation.Shared to improve directory searching logic. - Added XML generation functionality to ProcessDataStandardFormat, allowing for better data representation. - Updated ProcessDataStandardFormatMapping to streamline the creation of mapping instances. - Cleaned up the project file by removing references to obsolete file handlers.
2025-08-29 15:45:23 -07:00
parent 3c072d177a
commit 68c3a8b5f3
20 changed files with 302 additions and 2070 deletions
--- a/Adaptation/Shared/FileRead.cs
+++ b/Adaptation/Shared/FileRead.cs
@ -383,17 +383,24 @@ public class FileRead : Properties.IFileRead
        else
        {
            string[] files;
-            string logisticsSequence = _Logistics.Sequence.ToString();
-            string[] directories = Directory.GetDirectories(jobIdDirectory, $"*{logisticsSequence}*", SearchOption.TopDirectoryOnly);
-            foreach (string directory in directories)
+            string[] directories;
+            string logisticsSequence;
+            for (int i = 0; i < 10; i++)
            {
-                files = Directory.GetFiles(directory, "*", SearchOption.TopDirectoryOnly);
-                if (files.Length == 0)
-                    continue;
-                results.Add(directory);
+                logisticsSequence = (_Logistics.Sequence + -i).ToString();
+                directories = Directory.GetDirectories(jobIdDirectory, $"*{logisticsSequence}*", SearchOption.TopDirectoryOnly);
+                foreach (string directory in directories)
+                {
+                    files = Directory.GetFiles(directory, "*", SearchOption.TopDirectoryOnly);
+                    if (files.Length == 0)
+                        continue;
+                    results.Add(directory);
+                }
+                if (results.Count == 1)
+                    break;
            }
        }
-        if ((results is null) || results.Count != 1)
+        if (results.Count != 1)
            throw new Exception("Didn't find directory by logistics sequence");
        return results.ToArray();
    }
--- a/Adaptation/Shared/ProcessDataStandardFormat.cs
+++ b/Adaptation/Shared/ProcessDataStandardFormat.cs
@ -2,12 +2,14 @@ using Adaptation.Shared.Methods;
 using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
+using System.Diagnostics;
 using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Text;
 using System.Text.Json;
 using System.Text.Json.Serialization;
+using System.Text.RegularExpressions;

 namespace Adaptation.Shared;

@ -227,19 +229,19 @@ internal class ProcessDataStandardFormat
        return results.AsReadOnly();
    }

-    internal static ProcessDataStandardFormat GetProcessDataStandardFormat(string reportFullPath, ProcessDataStandardFormatMapping pdsfMapping)
+    internal static ProcessDataStandardFormat GetProcessDataStandardFormat(string reportFullPath, ProcessDataStandardFormatMapping processDataStandardFormatMapping)
    {
        ProcessDataStandardFormat result;
        const int columnsLine = 6;
        FileInfo fileInfo = new(reportFullPath);
        ProcessDataStandardFormat processDataStandardFormat = GetProcessDataStandardFormat(fileInfo.LastWriteTime, columnsLine, fileInfo.FullName, lines: null);
-        JsonElement[]? jsonElements = pdsfMapping.OldColumnNames.Count != pdsfMapping.ColumnIndices.Count ? null : GetFullArray(processDataStandardFormat);
+        JsonElement[]? jsonElements = processDataStandardFormatMapping.OldColumnNames.Count != processDataStandardFormatMapping.ColumnIndices.Count ? null : GetFullArray(processDataStandardFormat);
        JsonProperty[]? jsonProperties = jsonElements is null || jsonElements.Length == 0 ? null : jsonElements[0].EnumerateObject().ToArray();
-        if (jsonElements is null || jsonProperties is null || jsonProperties.Length != pdsfMapping.NewColumnNames.Count)
+        if (jsonElements is null || jsonProperties is null || jsonProperties.Length != processDataStandardFormatMapping.NewColumnNames.Count)
            result = processDataStandardFormat;
        else
        {
-            result = GetProcessDataStandardFormat(pdsfMapping, jsonElements, processDataStandardFormat);
+            result = GetProcessDataStandardFormat(processDataStandardFormatMapping, jsonElements, processDataStandardFormat);
            if (result.Sequence is null || result.Columns.Count == 0 || result.Body.Count == 0 || result.Logistics.Count == 0)
                result = processDataStandardFormat;
        }
@ -335,12 +337,14 @@ internal class ProcessDataStandardFormat
        int column;
        string value;
        JsonProperty jsonProperty;
+        List<string> debug = new();
        List<string> values = new();
        List<string> results = new();
        JsonProperty[] jsonProperties;
        List<string> unknownColumns = new();
        for (int i = 0; i < jsonElements.Length; i++)
        {
+            debug.Clear();
            values.Clear();
            if (jsonElements[i].ValueKind != JsonValueKind.Object)
            {
@ -354,16 +358,22 @@ internal class ProcessDataStandardFormat
            {
                column = processDataStandardFormatMapping.ColumnIndices[c];
                if (column == -1)
+                {
                    value = processDataStandardFormatMapping.OldColumnNames[c];
+                    debug.Add($"<Item C=-01 Name=\"{value}\" DataType=\"8\" XmlType=\"1\" XPath=\"//records/record/{value}\" />");
+                }
                else
                {
                    jsonProperty = jsonProperties[column];
                    value = jsonProperty.Value.ToString();
+                    debug.Add($"<Item C={column + 2:000} Name=\"{processDataStandardFormatMapping.OldColumnNames[c]}\" DataType=\"8\" XmlType=\"1\" XPath=\"//records/record/{jsonProperty.Name}\" />");
                }
                values.Add(value);
            }
            results.Add(string.Join("\t", values));
        }
+        if (Debugger.IsAttached)
+            File.WriteAllText("../../.txt", string.Join(Environment.NewLine, debug.OrderBy(l => l)));
        result = new(body: new(results),
                     columns: processDataStandardFormatMapping.OldColumnNames,
                     footer: processDataStandardFormat.Footer,
@ -378,7 +388,6 @@ internal class ProcessDataStandardFormat
    {
        if (processDataStandardFormat.InputPDSF is null)
            throw new NullReferenceException(nameof(processDataStandardFormat.InputPDSF));
-#pragma warning disable CA1845, IDE0057
        string result;
        string line;
        string value;
@ -518,6 +527,8 @@ internal class ProcessDataStandardFormat
    internal static void Write(string path, ProcessDataStandardFormat processDataStandardFormat, List<Metrology.WS.Results>? wsResults)
    {
        List<string> results = new();
+        if (processDataStandardFormat.InputPDSF is null)
+            throw new NullReferenceException(nameof(processDataStandardFormat.InputPDSF));
        if (processDataStandardFormat.Sequence is null)
            throw new NullReferenceException(nameof(processDataStandardFormat.Sequence));
        string endOffset = "E#######T";
@ -555,25 +566,25 @@ internal class ProcessDataStandardFormat
            }
        }
        results.Add("END_HEADER");
-        if (processDataStandardFormat.InputPDSF is not null)
-        {
-            results.Add(string.Empty);
-            List<char> hyphens = new();
-            results.AddRange(processDataStandardFormat.InputPDSF.Header.Select(l => l.Replace('\t', '|')));
-            results.Add(string.Empty);
-            results.Add($"|{string.Join("|", processDataStandardFormat.InputPDSF.Columns)}|");
-            for (int i = 0; i < processDataStandardFormat.InputPDSF.Columns.Count; i++)
-                hyphens.Add('-');
-            results.Add($"|{string.Join("|", hyphens)}|");
-            results.AddRange(processDataStandardFormat.InputPDSF.Body.Select(l => l.Replace('\t', '|')));
-            results.Add(string.Empty);
-            results.AddRange(processDataStandardFormat.InputPDSF.Footer.Select(l => l.Replace('\t', '|')));
-            results.Add(string.Empty);
-            results.Add("EOF");
-            results.Add(string.Empty);
-            string json = GetJson(processDataStandardFormat);
-            results.Add(json);
-        }
+        results.Add(string.Empty);
+        List<char> hyphens = new();
+        results.AddRange(processDataStandardFormat.InputPDSF.Header.Select(l => $"|{l.Replace('\t', '|')}|"));
+        results.Add(string.Empty);
+        results.Add($"|{string.Join("|", processDataStandardFormat.InputPDSF.Columns)}|");
+        for (int i = 0; i < processDataStandardFormat.InputPDSF.Columns.Count; i++)
+            hyphens.Add('-');
+        results.Add($"|{string.Join("|", hyphens)}|");
+        results.AddRange(processDataStandardFormat.InputPDSF.Body.Select(l => $"|{l.Replace('\t', '|')}|"));
+        results.Add(string.Empty);
+        results.AddRange(processDataStandardFormat.InputPDSF.Footer.Select(l => $"|{l.Replace('\t', '|')}|"));
+        results.Add(string.Empty);
+        string xml = GetXml(processDataStandardFormat);
+        results.Add(xml);
+        results.Add(string.Empty);
+        results.Add("EOF");
+        results.Add(string.Empty);
+        string json = GetJson(processDataStandardFormat);
+        results.Add(json);
        File.WriteAllText(path, string.Join(Environment.NewLine, results));
    }

@ -855,6 +866,96 @@ internal class ProcessDataStandardFormat
        return result;
    }

+    internal static string GetXml(ProcessDataStandardFormat processDataStandardFormat)
+    {
+        string result;
+        string tag;
+        string value;
+        string[] segments;
+        List<string> values;
+        Dictionary<string, List<string>> results = new();
+        ReadOnlyCollection<string> body = processDataStandardFormat.InputPDSF is null ?
+            processDataStandardFormat.Body : processDataStandardFormat.InputPDSF.Body;
+        ReadOnlyCollection<string> columns = processDataStandardFormat.InputPDSF is null ?
+            processDataStandardFormat.Columns : processDataStandardFormat.InputPDSF.Columns;
+        List<string> lines = new() { "<?xml version=\"1.0\" encoding=\"UTF-8\"?>", "<records>" };
+        for (int i = 0; i < body.Count; i++)
+        {
+            segments = body[i].Trim().Split('\t');
+            if (segments.Length != columns.Count)
+                break;
+            for (int c = 0; c < segments.Length; c++)
+            {
+                value = segments[c].Replace("&", "&amp;")
+                                   .Replace("<", "&lt;")
+                                   .Replace(">", "&gt;")
+                                   .Replace("\"", "&quot;")
+                                   .Replace("'", "&apos;");
+                tag = Regex.Replace(columns[c].Trim('"'), @"[^a-zA-Z0-9]", "_").Split('\r')[0].Split('\n')[0];
+                if (i == 0)
+                {
+                    if (results.ContainsKey(tag))
+                        continue;
+                    results.Add(tag, new List<string>());
+                }
+                results[tag].Add(value);
+            }
+        }
+        foreach (KeyValuePair<string, List<string>> keyValuePair in results)
+        {
+            if (body.Count < 3)
+                break;
+            if (keyValuePair.Value.Count != body.Count)
+                continue;
+            values = keyValuePair.Value.Distinct().ToList();
+            if (values.Count == 2 && (string.IsNullOrEmpty(values[0]) || string.IsNullOrEmpty(values[1])))
+            {
+                for (int i = 0; i < body.Count; i++)
+                    keyValuePair.Value[i] = string.Empty;
+                foreach (string v in values)
+                {
+                    if (string.IsNullOrEmpty(v))
+                        continue;
+                    keyValuePair.Value[0] = v;
+                }
+            }
+        }
+        for (int i = 0; i < body.Count; i++)
+        {
+            lines.Add("  <record>");
+            foreach (KeyValuePair<string, List<string>> keyValuePair in results)
+            {
+                if (keyValuePair.Value.Count != body.Count)
+                    continue;
+                lines.Add(string.Concat("    <", keyValuePair.Key, '>', keyValuePair.Value[i], "</", keyValuePair.Key, '>'));
+            }
+            lines.Add("  </record>");
+        }
+        lines.Add("</records>");
+        result = string.Join(Environment.NewLine, lines);
+        return result;
+    }
+
+    internal static string GetXml(string reportFullPath, string[]? lines = null)
+    {
+        string result;
+        bool foundXml = false;
+        List<string> results = new();
+        lines ??= File.ReadAllLines(reportFullPath);
+        foreach (string line in lines)
+        {
+            if (line.StartsWith("<?xml"))
+                foundXml = true;
+            if (!foundXml)
+                continue;
+            if (line.StartsWith("EOF"))
+                break;
+            results.Add(line);
+        }
+        result = string.Join(Environment.NewLine, results);
+        return result;
+    }
+
 }

 [JsonSourceGenerationOptions(WriteIndented = true)]
--- a/Adaptation/Shared/ProcessDataStandardFormatMapping.cs
+++ b/Adaptation/Shared/ProcessDataStandardFormatMapping.cs
@ -1,33 +1,34 @@
 using System.Collections.ObjectModel;
+using System.Linq;

 namespace Adaptation.Shared;

 public class ProcessDataStandardFormatMapping
 {

-    public ReadOnlyCollection<string> BackfillColumns { get; private set; }
    public ReadOnlyCollection<int> ColumnIndices { get; private set; }
-    public ReadOnlyCollection<string> IgnoreColumns { get; private set; }
-    public ReadOnlyCollection<string> IndexOnlyColumns { get; private set; }
-    public ReadOnlyDictionary<string, string> KeyValuePairs { get; private set; }
    public ReadOnlyCollection<string> NewColumnNames { get; private set; }
    public ReadOnlyCollection<string> OldColumnNames { get; private set; }

-    public ProcessDataStandardFormatMapping(ReadOnlyCollection<string> backfillColumns,
-                                            ReadOnlyCollection<int> columnIndices,
-                                            ReadOnlyCollection<string> ignoreColumns,
-                                            ReadOnlyCollection<string> indexOnlyColumns,
-                                            ReadOnlyDictionary<string, string> keyValuePairs,
+    public ProcessDataStandardFormatMapping(ReadOnlyCollection<int> columnIndices,
                                            ReadOnlyCollection<string> newColumnNames,
                                            ReadOnlyCollection<string> oldColumnNames)
    {
-        BackfillColumns = backfillColumns;
        ColumnIndices = columnIndices;
-        IgnoreColumns = ignoreColumns;
-        IndexOnlyColumns = indexOnlyColumns;
-        KeyValuePairs = keyValuePairs;
        NewColumnNames = newColumnNames;
        OldColumnNames = oldColumnNames;
    }

+    internal static ProcessDataStandardFormatMapping Get(string processDataStandardFormatMappingOldColumnNames, string processDataStandardFormatMappingNewColumnNames, string processDataStandardFormatMappingColumnIndices)
+    {
+        ProcessDataStandardFormatMapping result;
+        ReadOnlyCollection<string> newColumnNames = new(processDataStandardFormatMappingNewColumnNames.Split(','));
+        ReadOnlyCollection<string> oldColumnNames = new(processDataStandardFormatMappingOldColumnNames.Split(','));
+        ReadOnlyCollection<int> columnIndices = new(processDataStandardFormatMappingColumnIndices.Split(',').Select(int.Parse).ToArray());
+        result = new(columnIndices: columnIndices,
+                     newColumnNames: newColumnNames,
+                     oldColumnNames: oldColumnNames);
+        return result;
+    }
+
 }