Not Tested

json in process-data-standard-format

process-data-standard-format with HeaderId and SubgroupId

process-data-standard-format with pipes

UniqueId replacement for attachments

Write input PDSF in output after EOF

ProcessDataStandardFormat over Tuple

MoveMatchingFiles to use ProcessDataStandardFormatMapping

Adaptation/.editorconfig

@@ -121,6 +121,7 @@ dotnet_diagnostic.IDE0290.severity = none # Use primary constructor [Distance]cs
 dotnet_diagnostic.IDE0300.severity = none # IDE0300: Collection initialization can be simplified
 dotnet_diagnostic.IDE0301.severity = none #IDE0301: Collection initialization can be simplified
 dotnet_diagnostic.IDE0305.severity = none # IDE0305: Collection initialization can be simplified
+dotnet_diagnostic.MSTEST0037.severity = error # MSTEST0037: Use proper 'Assert' methods
 dotnet_diagnostic.SYSLIB1045.severity = none # SYSLIB1045: diagnostics for regex source generation
 dotnet_naming_rule.abstract_method_should_be_pascal_case.severity = warning
 dotnet_naming_rule.abstract_method_should_be_pascal_case.style = pascal_case

Adaptation/.vscode/tasks.json

@@ -82,6 +82,26 @@
             "command": "code ../METCLIMATEC.csproj",
             "problemMatcher": []
         },
+        {
+            "label": "Readme",
+            "type": "shell",
+            "command": "code ../README.md",
+            "problemMatcher": []
+        },
+        {
+            "label": "File-Folder-Helper AOT s X Day-Helper-2025-03-20",
+            "type": "shell",
+            "command": "L:/DevOps/Mesa_FI/File-Folder-Helper/bin/Release/net8.0/win-x64/publish/File-Folder-Helper.exe",
+            "args": [
+                "s",
+                "X",
+                "L:/DevOps/EAF-Mesa-Integration/METCLIMATEC",
+                "Day-Helper-2025-03-20",
+                "false",
+                "4"
+            ],
+            "problemMatcher": []
+        },
         {
             "label": "Git Config",
             "type": "shell",

Adaptation/FileHandlers/APC/FileRead.cs

@@ -14,7 +14,7 @@ namespace Adaptation.FileHandlers.APC;
 public class FileRead : Shared.FileRead, IFileRead
 {
 
-    public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
+    public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
         base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
     {
         _MinFileLength = 10;
@@ -120,15 +120,15 @@ public class FileRead : Shared.FileRead, IFileRead
     private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime dateTime)
     {
         Tuple<string, Test[], JsonElement[], List<FileInfo>> results;
-        Tuple<string, string[], string[]> pdsf = ProcessDataStandardFormat.GetLogisticsColumnsAndBody(reportFullPath);
+        ProcessDataStandardFormat processDataStandardFormat = ProcessDataStandardFormat.GetProcessDataStandardFormat(reportFullPath);
-        _Logistics = new Logistics(reportFullPath, pdsf.Item1);
+        _Logistics = new Logistics(reportFullPath, processDataStandardFormat);
         SetFileParameterLotIDToLogisticsMID();
-        JsonElement[] jsonElements = ProcessDataStandardFormat.GetArray(pdsf);
+        JsonElement[] jsonElements = ProcessDataStandardFormat.GetArray(processDataStandardFormat);
         List<Shared.Properties.IDescription> descriptions = GetDuplicatorDescriptions(jsonElements);
         Test[] tests = (from l in descriptions select (Test)l.Test).ToArray();
         if (_IsEAFHosted && _FileConnectorConfiguration.FileScanningIntervalInSeconds > 0)
             FileCopy(reportFullPath, dateTime, descriptions);
-        results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(pdsf.Item1, tests, jsonElements, new List<FileInfo>());
+        results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(string.Join(Environment.NewLine, processDataStandardFormat.Logistics[0]), tests, jsonElements, new List<FileInfo>());
         return results;
     }
 

Adaptation/FileHandlers/Archive/FileRead.cs

@@ -18,7 +18,7 @@ public class FileRead : Shared.FileRead, IFileRead
     private readonly string _JobIdParentDirectory;
     private readonly string _JobIdArchiveParentDirectory;
 
-    public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
+    public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
         base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
     {
         _MinFileLength = 10;
@@ -144,15 +144,15 @@ public class FileRead : Shared.FileRead, IFileRead
     private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime dateTime)
     {
         Tuple<string, Test[], JsonElement[], List<FileInfo>> results;
-        Tuple<string, string[], string[]> pdsf = ProcessDataStandardFormat.GetLogisticsColumnsAndBody(reportFullPath);
+        ProcessDataStandardFormat processDataStandardFormat = ProcessDataStandardFormat.GetProcessDataStandardFormat(reportFullPath);
-        _Logistics = new Logistics(reportFullPath, pdsf.Item1);
+        _Logistics = new Logistics(reportFullPath, processDataStandardFormat);
         SetFileParameterLotIDToLogisticsMID();
-        JsonElement[] jsonElements = ProcessDataStandardFormat.GetArray(pdsf);
+        JsonElement[] jsonElements = ProcessDataStandardFormat.GetArray(processDataStandardFormat);
         List<Shared.Properties.IDescription> descriptions = GetDuplicatorDescriptions(jsonElements);
         Test[] tests = (from l in descriptions select (Test)l.Test).ToArray();
         if (_IsEAFHosted && _FileConnectorConfiguration.FileScanningIntervalInSeconds > 0)
             MoveArchive(reportFullPath, dateTime);
-        results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(pdsf.Item1, tests, jsonElements, new List<FileInfo>());
+        results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(string.Join(Environment.NewLine, processDataStandardFormat.Logistics[0]), tests, jsonElements, new List<FileInfo>());
         return results;
     }
 

Adaptation/FileHandlers/CellInstanceConnectionName.cs

@@ -9,7 +9,7 @@ namespace Adaptation.FileHandlers;
 public class CellInstanceConnectionName
 {
 
-    internal static IFileRead Get(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, int? connectionCount)
+    internal static IFileRead Get(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, int? connectionCount)
     {
         IFileRead result = cellInstanceConnectionName switch
         {

Adaptation/FileHandlers/Dummy/FileRead.cs

@@ -23,7 +23,7 @@ public class FileRead : Shared.FileRead, IFileRead
     private int _LastDummyRunIndex;
     private readonly string[] _CellNames;
 
-    public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
+    public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
         base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
     {
         _MinFileLength = 10;

Adaptation/FileHandlers/IQSSi/FileRead.cs

@@ -14,7 +14,7 @@ namespace Adaptation.FileHandlers.IQSSi;
 public class FileRead : Shared.FileRead, IFileRead
 {
 
-    public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
+    public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
         base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
     {
         _MinFileLength = 10;
@@ -119,15 +119,15 @@ public class FileRead : Shared.FileRead, IFileRead
     private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime dateTime)
     {
         Tuple<string, Test[], JsonElement[], List<FileInfo>> results;
-        Tuple<string, string[], string[]> pdsf = ProcessDataStandardFormat.GetLogisticsColumnsAndBody(reportFullPath);
+        ProcessDataStandardFormat processDataStandardFormat = ProcessDataStandardFormat.GetProcessDataStandardFormat(reportFullPath);
-        _Logistics = new Logistics(reportFullPath, pdsf.Item1);
+        _Logistics = new Logistics(reportFullPath, processDataStandardFormat);
         SetFileParameterLotIDToLogisticsMID();
-        JsonElement[] jsonElements = ProcessDataStandardFormat.GetArray(pdsf);
+        JsonElement[] jsonElements = ProcessDataStandardFormat.GetArray(processDataStandardFormat);
         List<Shared.Properties.IDescription> descriptions = GetDuplicatorDescriptions(jsonElements);
         Test[] tests = (from l in descriptions select (Test)l.Test).ToArray();
         if (_IsEAFHosted && _FileConnectorConfiguration.FileScanningIntervalInSeconds > 0)
             FileCopy(reportFullPath, dateTime, descriptions);
-        results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(pdsf.Item1, tests, jsonElements, new List<FileInfo>());
+        results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(string.Join(Environment.NewLine, processDataStandardFormat.Logistics[0]), tests, jsonElements, new List<FileInfo>());
         return results;
     }
 

Adaptation/FileHandlers/MoveMatchingFiles/FileRead.cs

@@ -5,17 +5,75 @@ using Adaptation.Shared.Duplicator;
 using Adaptation.Shared.Methods;
 using System;
 using System.Collections.Generic;
+using System.Collections.ObjectModel;
 using System.IO;
+using System.Linq;
 using System.Text;
 using System.Text.Json;
 using System.Threading;
 
 namespace Adaptation.FileHandlers.MoveMatchingFiles;
 
+#nullable enable
+
 public class FileRead : Shared.FileRead, IFileRead
 {
 
-    public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
+    internal class PreWith
+    {
+
+        internal string ErrFile { get; private set; }
+        internal string CheckFile { get; private set; }
+        internal string MatchingFile { get; private set; }
+        internal string CheckDirectory { get; private set; }
+        internal string NoWaitDirectory { get; private set; }
+
+        internal PreWith(string checkDirectory,
+                         string checkFile,
+                         string errFile,
+                         string matchingFile,
+                         string noWaitDirectory)
+        {
+            ErrFile = errFile;
+            CheckFile = checkFile;
+            MatchingFile = matchingFile;
+            CheckDirectory = checkDirectory;
+            NoWaitDirectory = noWaitDirectory;
+        }
+
+    }
+
+    internal class Pre
+    {
+
+        internal string MatchingFile { get; private set; }
+        internal string CheckFile { get; private set; }
+
+        internal Pre(string matchingFile, string checkFile)
+        {
+            MatchingFile = matchingFile;
+            CheckFile = checkFile;
+        }
+
+    }
+
+    internal class Post
+    {
+
+        internal string ErrFile { get; private set; }
+        internal string CheckFile { get; private set; }
+
+        internal Post(string checkFile, string errFile)
+        {
+            ErrFile = errFile;
+            CheckFile = checkFile;
+        }
+
+    }
+
+    private readonly ProcessDataStandardFormatMapping _ProcessDataStandardFormatMapping;
+
+    public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
         base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
     {
         _MinFileLength = 10;
@@ -27,6 +85,12 @@ public class FileRead : Shared.FileRead, IFileRead
             throw new Exception(cellInstanceConnectionName);
         if (!_IsDuplicator)
             throw new Exception(cellInstanceConnectionName);
+        string processDataStandardFormatMappingOldColumnNames = GetPropertyValue(cellInstanceConnectionName, modelObjectParameters, "Process.Data.Standard.Format.Mapping.Old.Column.Names");
+        string processDataStandardFormatMappingNewColumnNames = GetPropertyValue(cellInstanceConnectionName, modelObjectParameters, "Process.Data.Standard.Format.Mapping.New.Column.Names");
+        string processDataStandardFormatMappingColumnIndices = GetPropertyValue(cellInstanceConnectionName, modelObjectParameters, "Process.Data.Standard.Format.Mapping.Column.Indices");
+        _ProcessDataStandardFormatMapping = GetProcessDataStandardFormatMapping(processDataStandardFormatMappingOldColumnNames,
+                                                                                processDataStandardFormatMappingNewColumnNames,
+                                                                                processDataStandardFormatMappingColumnIndices);
     }
 
     void IFileRead.Move(Tuple<string, Test[], JsonElement[], List<FileInfo>> extractResults, Exception exception)
@@ -41,7 +105,8 @@ public class FileRead : Shared.FileRead, IFileRead
         Move(extractResults);
     }
 
-    void IFileRead.WaitForThread() => WaitForThread(thread: null, threadExceptions: null);
+    void IFileRead.WaitForThread() =>
+        WaitForThread(thread: null, threadExceptions: null);
 
     string IFileRead.GetEventDescription()
     {
@@ -88,7 +153,7 @@ public class FileRead : Shared.FileRead, IFileRead
         DateTime dateTime = DateTime.Now;
         results = GetExtractResult(reportFullPath, dateTime);
         if (results.Item3 is null)
-            results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(results.Item1, Array.Empty<Test>(), JsonSerializer.Deserialize<JsonElement[]>("[]"), results.Item4);
+            results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(results.Item1, Array.Empty<Test>(), JsonSerializer.Deserialize<JsonElement[]>("[]") ?? throw new Exception(), results.Item4);
         if (results.Item3.Length > 0 && _IsEAFHosted)
             WritePDSF(this, results.Item3);
         UpdateLastTicksDuration(DateTime.Now.Ticks - dateTime.Ticks);
@@ -104,7 +169,73 @@ public class FileRead : Shared.FileRead, IFileRead
         return results;
     }
 
-    private static List<string> GetSearchDirectories(int numberLength, string parentDirectory)
+    private static ProcessDataStandardFormatMapping GetProcessDataStandardFormatMapping(string processDataStandardFormatMappingOldColumnNames, string processDataStandardFormatMappingNewColumnNames, string processDataStandardFormatMappingColumnIndices)
+    {
+        ProcessDataStandardFormatMapping result;
+        string[] segmentsB;
+        List<string> distinct = new();
+        Dictionary<string, string> keyValuePairs = new();
+        string args4 = "Time,Test,Count,MesEntity,HeaderUniqueId,UniqueId,Id,Recipe,Date,AreaDeltaFromLastRun,GLimit,HGCV1";
+        string args5 = "Nine10mmEdgeMean,Nine4mmEdgeMean,NineCriticalPointsAverage,NineCriticalPointsPhaseAngleAverage,NineCriticalPointsStdDev,NineEdgeMeanDelta,NineMean,NineResRangePercent,AreaDeltaFromLastRun,Variation,Percentage HgCV 4PP Delta,HGCV1";
+        string args6 = "RhoAvg01,RhoAvg02,RhoAvg03,RhoAvg04,RhoAvg05,RhoAvg06,RhoAvg07,RhoAvg08,RhoAvg09,HGCV1";
+        string args7 = "FlatZMean|MeanFlatZ,GradeMean|MeanGrade,NAvgMean|MeanNAvg,NslMean|MeanNsl,PhaseMean|MeanPhase,RhoAvgMean|MeanRhoAvg,RhoslMean|MeanRhosl,RsMean|MeanRs,VdMean|MeanVd,FlatZRadialGradient|RadialGradientFlatZ,GradeRadialGradient|RadialGradientGrade,NAvgRadialGradient|RadialGradientNAvg,NslRadialGradient|RadialGradientNsl,PhaseRadialGradient|RadialGradientPhase,RhoAvgRadialGradient|RadialGradientRhoAvg,RhoslRadialGradient|RadialGradientRhosl,RsRadialGradient|RadialGradientRs,VdRadialGradient|RadialGradientVd,FlatZStdDev|StandardDeviationPercentageFlatZ,GradeStdDev|StandardDeviationPercentageGrade,NAvgStdDev|StandardDeviationPercentageNAvg,NslStdDev|StandardDeviationPercentageNsl,PhaseStdDev|StandardDeviationPercentagePhase,RhoAvgStdDev|StandardDeviationPercentageRhoAvg,RhoslStdDev|StandardDeviationPercentageRhosl,RsStdDev|StandardDeviationPercentageRs,VdStdDev|StandardDeviationPercentageVd,|HGCV1";
+        // string args8 = "Time,A_LOGISTICS,B_LOGISTICS,Test,Count,Index,MesEntity,Date,Employee,Lot,PSN,Reactor,Recipe,Area,Folder,HeaderUniqueId,Id,Layer,Model,Pattern,Phase,Plan,RampRate,RDS,SetupFile,StartVoltage,StopVoltage,UniqueId,Wafer,WaferSize,Zone,Ccomp,CondType,FlatZ,FlatZMean,FlatZRadialGradient,FlatZStdDev,GLimit,Grade,GradeMean,GradeRadialGradient,GradeStdDev,NAvg,NAvgMean,NAvgRadialGradient,NAvgStdDev,Nsl,NslMean,NslRadialGradient,NslStdDev,PhaseMean,PhaseRadialGradient,PhaseStdDev,RhoAvg,RhoAvgMean,RhoAvgRadialGradient,RhoAvgStdDev,RhoMethod,Rhosl,RhoslMean,RhoslRadialGradient,RhoslStdDev,RsMean,RsRadialGradient,RsStdDev,Vd,VdMean,VdRadialGradient,VdStdDev,Variation,AreaDeltaFromLastRun,Nine10mmEdgeMean,Nine4mmEdgeMean,NineCriticalPointsAverage,NineCriticalPointsPhaseAngleAverage,NineCriticalPointsStdDev,NineEdgeMeanDelta,NineMean,NineResRangePercent,RhoAvg01,RhoAvg02,RhoAvg03,RhoAvg04,RhoAvg05,RhoAvg06,RhoAvg07,RhoAvg08,RhoAvg09";
+        // string args9 = "Time,A_LOGISTICS,B_LOGISTICS,Index,Operator,StartVoltage,Wafer,StopVoltage,Lot,RampRate,Plan,GLimit,Date,Time,SetupFile,WaferSize,Folder,Ccomp,Pattern,Area,CondType,RhoMethod,Model,MeanNAvg,MeanNsl,MeanVd,MeanFlatZ,MeanRhoAvg,MeanRhosl,MeanPhase,MeanGrade,MeanRs,StandardDeviationPercentageNAvg,StandardDeviationPercentageNsl,StandardDeviationPercentageVd,StandardDeviationPercentageFlatZ,StandardDeviationPercentageRhoAvg,StandardDeviationPercentageRhosl,StandardDeviationPercentagePhase,StandardDeviationPercentageGrade,StandardDeviationPercentageRs,RadialGradientNAvg,RadialGradientNsl,RadialGradientVd,RadialGradientFlatZ,RadialGradientRhoAvg,RadialGradientRhosl,RadialGradientPhase,RadialGradientGrade,RadialGradientRs,Site,X,Y,NAvg,RhoAvg,Nsl,Rhosl,Vd,Phase,FlatZ,Grade,XLeft,XRight,BottomY,TopY,RDS,PSN,Reactor,Layer,Zone,Employee,InferredLot,Nine10mmEdgeMean,Nine4mmEdgeMean,NineCriticalPointsAverage,NineCriticalPointsPhaseAngleAverage,NineCriticalPointsStdDev,NineEdgeMeanDelta,NineMean,NineResRangePercent,AreaDeltaFromLastRun,Variation,Percentage HgCV 4PP Delta,RhoAvg01,RhoAvg02,RhoAvg03,RhoAvg04,RhoAvg05,RhoAvg06,RhoAvg07,RhoAvg08,RhoAvg09";
+        // string args10 = "0,1,2,-1,-1,3,-1,12,70,8,66,67,-1,19,16,-1,-1,68,22,18,58,10,9,65,14,5,7,-1,6,15,69,17,20,59,26,44,35,11,60,30,48,39,53,23,41,32,55,24,42,33,29,47,38,54,27,45,36,21,56,28,46,37,31,49,40,57,25,43,34,81,80,72,73,74,75,76,77,78,79,83,84,85,86,87,88,89,90,91";
+        string[] segments = args7.Split(',');
+        ReadOnlyCollection<string> ignoreColumns = new(args4.Split(','));
+        ReadOnlyCollection<string> backfillColumns = new(args5.Split(','));
+        ReadOnlyCollection<string> indexOnlyColumns = new(args6.Split(','));
+        ReadOnlyCollection<string> newColumnNames = new(processDataStandardFormatMappingNewColumnNames.Split(','));
+        ReadOnlyCollection<string> oldColumnNames = new(processDataStandardFormatMappingOldColumnNames.Split(','));
+        ReadOnlyCollection<int> columnIndices = new(processDataStandardFormatMappingColumnIndices.Split(',').Select(int.Parse).ToArray());
+        foreach (string segment in segments)
+        {
+            segmentsB = segment.Split('|');
+            if (segmentsB.Length != 2)
+                continue;
+            if (distinct.Contains(segmentsB[0]))
+                continue;
+            distinct.Add(segmentsB[0]);
+            keyValuePairs.Add(segmentsB[0], segmentsB[1]);
+        }
+        result = new(backfillColumns: backfillColumns,
+                     columnIndices: columnIndices,
+                     newColumnNames: newColumnNames,
+                     ignoreColumns: ignoreColumns,
+                     indexOnlyColumns: indexOnlyColumns,
+                     keyValuePairs: new(keyValuePairs),
+                     oldColumnNames: oldColumnNames);
+        return result;
+    }
+
+    private static ReadOnlyCollection<PreWith> GetPreWithCollection(ReadOnlyCollection<Pre> preCollection)
+    {
+        List<PreWith> results = new();
+        string errFile;
+        PreWith preWith;
+        string? checkDirectory;
+        string noWaitDirectory;
+        foreach (Pre pre in preCollection)
+        {
+            errFile = string.Concat(pre.CheckFile, ".err");
+            checkDirectory = Path.GetDirectoryName(pre.CheckFile);
+            if (string.IsNullOrEmpty(checkDirectory))
+                continue;
+            if (!Directory.Exists(checkDirectory))
+                _ = Directory.CreateDirectory(checkDirectory);
+            noWaitDirectory = Path.Combine(checkDirectory, "NoWaitDirectory");
+            preWith = new(checkDirectory: checkDirectory,
+                          checkFile: pre.CheckFile,
+                          errFile: errFile,
+                          matchingFile: pre.MatchingFile,
+                          noWaitDirectory: noWaitDirectory);
+            results.Add(preWith);
+        }
+        return results.AsReadOnly();
+    }
+
+    private static ReadOnlyCollection<string> GetSearchDirectories(int numberLength, string parentDirectory)
     {
         List<string> results = new();
         string[] directories = Directory.GetDirectories(parentDirectory, "*", SearchOption.TopDirectoryOnly);
@@ -115,10 +246,133 @@ public class FileRead : Shared.FileRead, IFileRead
             results.Add(directory);
         }
         results.Sort();
+        return results.AsReadOnly();
+    }
+
+    private static void CreatePointerFile(int numberLength, string parentDirectory, ReadOnlyCollection<string> matchingFiles)
+    {
+        string checkFile;
+        string writeFile;
+        string? directoryName;
+        int parentDirectoryLength = parentDirectory.Length;
+        foreach (string matchingFile in matchingFiles)
+        {
+            directoryName = Path.GetDirectoryName(matchingFile);
+            if (directoryName is null)
+                continue;
+            checkFile = $"{matchingFile[0]}{directoryName.Substring(parentDirectoryLength + numberLength + 1)}";
+            writeFile = Path.Combine(parentDirectory, $"{directoryName.Substring(parentDirectory.Length + 1, numberLength)}.txt");
+            if (File.Exists(writeFile))
+                continue;
+            File.AppendAllLines(writeFile, new string[] { parentDirectory, matchingFile, directoryName, checkFile });
+        }
+    }
+
+    private static ReadOnlyCollection<Pre> GetPreCollection(int numberLength, string parentDirectory, ReadOnlyCollection<string> matchingFiles)
+    {
+        List<Pre> results = new();
+        Pre pre;
+        string checkFile;
+        int parentDirectoryLength = parentDirectory.Length;
+        foreach (string matchingFile in matchingFiles)
+        {
+            checkFile = $"{matchingFile[0]}{matchingFile.Substring(parentDirectoryLength + numberLength + 1)}";
+            pre = new(matchingFile, checkFile);
+            results.Add(pre);
+        }
+        return results.AsReadOnly();
+    }
+
+    private void MoveCollection(DateTime dateTime, ProcessDataStandardFormat processDataStandardFormat, ReadOnlyCollection<PreWith> preWithCollection)
+    {
+        ReadOnlyCollection<Post> postCollection = GetPostCollection(dateTime, processDataStandardFormat, preWithCollection);
+        if (postCollection.Count != 0)
+        {
+            Thread.Sleep(500);
+            StringBuilder stringBuilder = new();
+            foreach (Post post in postCollection)
+            {
+                if (File.Exists(post.ErrFile))
+                    _ = stringBuilder.AppendLine(File.ReadAllText(post.ErrFile));
+                if (File.Exists(post.CheckFile))
+                    _ = stringBuilder.AppendLine($"<{post.CheckFile}> was not consumed by the end!");
+            }
+            if (stringBuilder.Length > 0)
+                throw new Exception(stringBuilder.ToString());
+        }
+    }
+
+    private ReadOnlyCollection<Post> GetPostCollection(DateTime dateTime, ProcessDataStandardFormat processDataStandardFormat, ReadOnlyCollection<PreWith> preWithCollection)
+    {
+        List<Post> results = new();
+        Post post;
+        long preWait;
+        foreach (PreWith preWith in preWithCollection)
+        {
+            if (!_IsEAFHosted)
+                continue;
+            if (!_StaticRuns.TryGetValue(_Logistics.Sequence, out List<Shared.Metrology.WS.Results>? wsResults))
+                wsResults = null;
+            ProcessDataStandardFormat.Write(preWith.CheckFile, processDataStandardFormat, wsResults);
+            File.Delete(preWith.MatchingFile);
+            if (Directory.Exists(preWith.NoWaitDirectory))
+            {
+                post = new(preWith.CheckFile, preWith.ErrFile);
+                results.Add(post);
+                continue;
+            }
+            if (_FileConnectorConfiguration?.FileHandleWaitTime is null)
+                preWait = DateTime.Now.AddMilliseconds(1234).Ticks;
+            else
+                preWait = DateTime.Now.AddMilliseconds(_FileConnectorConfiguration.FileHandleWaitTime.Value).Ticks;
+            for (short i = 0; i < short.MaxValue; i++)
+            {
+                if (DateTime.Now.Ticks > preWait)
+                    break;
+                Thread.Sleep(500);
+            }
+            for (int i = 0; i < int.MaxValue; i++)
+            {
+                if (File.Exists(preWith.ErrFile))
+                    throw new Exception(File.ReadAllText(preWith.ErrFile));
+                if (!File.Exists(preWith.CheckFile))
+                    break;
+                if (new TimeSpan(DateTime.Now.Ticks - dateTime.Ticks).TotalSeconds > _BreakAfterSeconds)
+                    throw new Exception($"Not all files were consumed after {_BreakAfterSeconds} second(s)!");
+                Thread.Sleep(500);
+            }
+        }
+        return results.AsReadOnly();
+    }
+
+    private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime dateTime)
+    {
+        Tuple<string, Test[], JsonElement[], List<FileInfo>> results = new(string.Empty, Array.Empty<Test>(), Array.Empty<JsonElement>(), new List<FileInfo>());
+        ProcessDataStandardFormat processDataStandardFormat = ProcessDataStandardFormat.GetProcessDataStandardFormat(reportFullPath, _ProcessDataStandardFormatMapping);
+        _Logistics = new Logistics(reportFullPath, processDataStandardFormat);
+        if (!_IsEAFHosted)
+            ProcessDataStandardFormat.Write("../../.pdsf", processDataStandardFormat, wsResults: null);
+        SetFileParameterLotIDToLogisticsMID();
+        int numberLength = 2;
+        long ticks = dateTime.Ticks;
+        string parentParentDirectory = GetParentParent(reportFullPath);
+        ReadOnlyCollection<string> searchDirectories = GetSearchDirectories(numberLength, parentParentDirectory);
+        ReadOnlyCollection<string> matchingFiles = GetMatchingFiles(ticks, reportFullPath, searchDirectories);
+        if (matchingFiles.Count != searchDirectories.Count)
+            throw new Exception($"Didn't find all files after {_BreakAfterSeconds} second(s)!");
+        if (_IsEAFHosted)
+        {
+            try
+            { CreatePointerFile(numberLength, parentParentDirectory, matchingFiles); }
+            catch (Exception) { }
+        }
+        ReadOnlyCollection<Pre> preCollection = GetPreCollection(numberLength, parentParentDirectory, matchingFiles);
+        ReadOnlyCollection<PreWith> preWithCollection = GetPreWithCollection(preCollection);
+        MoveCollection(dateTime, processDataStandardFormat, preWithCollection);
         return results;
     }
 
-    private List<string> GetMatchingFiles(long ticks, string reportFullPath, List<string> searchDirectories)
+    private ReadOnlyCollection<string> GetMatchingFiles(long ticks, string reportFullPath, ReadOnlyCollection<string> searchDirectories)
     {
         List<string> results = new();
         string[] found;
@@ -137,131 +391,7 @@ public class FileRead : Shared.FileRead, IFileRead
                     break;
             }
         }
-        return results;
+        return results.AsReadOnly();
-    }
-
-    private static List<(string matchingFile, string checkFile)> GetCollection(int numberLength, string parentDirectory, List<string> matchingFiles)
-    {
-        List<(string matchingFile, string checkFile)> results = new();
-        string checkFile;
-        int parentDirectoryLength = parentDirectory.Length;
-        foreach (string matchingFile in matchingFiles)
-        {
-            checkFile = $"{matchingFile[0]}{matchingFile.Substring(parentDirectoryLength + numberLength + 1)}";
-            results.Add(new(matchingFile, checkFile));
-        }
-        return results;
-    }
-
-    private static List<(string, string, string, string, string)> GetCollection(List<(string matchingFile, string checkFile)> collection)
-    {
-        List<(string, string, string, string, string)> results = new();
-        string errFile;
-        string checkDirectory;
-        string noWaitDirectory;
-        foreach ((string matchingFile, string checkFile) in collection)
-        {
-            errFile = string.Concat(checkFile, ".err");
-            checkDirectory = Path.GetDirectoryName(checkFile);
-            if (!Directory.Exists(checkDirectory))
-                _ = Directory.CreateDirectory(checkDirectory);
-            noWaitDirectory = Path.Combine(checkDirectory, "NoWaitDirectory");
-            results.Add(new(matchingFile, checkFile, errFile, checkDirectory, noWaitDirectory));
-        }
-        return results;
-    }
-
-    private void MoveCollection(DateTime dateTime, List<(string matchingFile, string checkFile)> collection)
-    {
-        long preWait;
-        List<(string checkFile, string errFile)> postCollection = new();
-        foreach ((string matchingFile, string checkFile, string errFile, string checkDirectory, string noWaitDirectory) in GetCollection(collection))
-        {
-            File.Move(matchingFile, checkFile);
-            if (Directory.Exists(noWaitDirectory))
-            {
-                postCollection.Add(new(checkFile, errFile));
-                continue;
-            }
-            if (_FileConnectorConfiguration?.FileHandleWaitTime is null)
-                preWait = DateTime.Now.AddMilliseconds(1234).Ticks;
-            else
-                preWait = DateTime.Now.AddMilliseconds(_FileConnectorConfiguration.FileHandleWaitTime.Value).Ticks;
-            for (short i = 0; i < short.MaxValue; i++)
-            {
-                if (DateTime.Now.Ticks > preWait)
-                    break;
-                Thread.Sleep(500);
-            }
-            for (int i = 0; i < int.MaxValue; i++)
-            {
-                if (File.Exists(errFile))
-                    throw new Exception(File.ReadAllText(errFile));
-                if (!File.Exists(checkFile))
-                    break;
-                if (new TimeSpan(DateTime.Now.Ticks - dateTime.Ticks).TotalSeconds > _BreakAfterSeconds)
-                    throw new Exception($"Not all files were consumed after {_BreakAfterSeconds} second(s)!");
-                Thread.Sleep(500);
-            }
-        }
-        if (postCollection.Count != 0)
-        {
-            Thread.Sleep(500);
-            StringBuilder stringBuilder = new();
-            foreach ((string checkFile, string errFile) in postCollection)
-            {
-                if (File.Exists(errFile))
-                    _ = stringBuilder.AppendLine(File.ReadAllText(errFile));
-                if (File.Exists(checkFile))
-                    _ = stringBuilder.AppendLine($"<{checkFile}> was not consumed by the end!");
-            }
-            if (stringBuilder.Length > 0)
-                throw new Exception(stringBuilder.ToString());
-        }
-    }
-
-    private static void CreatePointerFile(int numberLength, string parentDirectory, List<string> matchingFiles)
-    {
-#nullable enable
-        string checkFile;
-        string writeFile;
-        string? directoryName;
-        int parentDirectoryLength = parentDirectory.Length;
-        foreach (string matchingFile in matchingFiles)
-        {
-            directoryName = Path.GetDirectoryName(matchingFile);
-            if (directoryName is null)
-                continue;
-            checkFile = $"{matchingFile[0]}{directoryName.Substring(parentDirectoryLength + numberLength + 1)}";
-            writeFile = Path.Combine(parentDirectory, $"{directoryName.Substring(parentDirectory.Length + 1, numberLength)}.txt");
-            if (File.Exists(writeFile))
-                continue;
-            File.AppendAllLines(writeFile, new string[] { parentDirectory, matchingFile, directoryName, checkFile });
-        }
-#nullable disable
-    }
-
-    private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime dateTime)
-    {
-        Tuple<string, Test[], JsonElement[], List<FileInfo>> results = new(string.Empty, null, null, new List<FileInfo>());
-        Tuple<string, string[], string[]> pdsf = ProcessDataStandardFormat.GetLogisticsColumnsAndBody(reportFullPath);
-        _Logistics = new Logistics(reportFullPath, pdsf.Item1);
-        SetFileParameterLotIDToLogisticsMID();
-        if (string.IsNullOrEmpty(_Logistics.JobID) || _Logistics.JobID == "null")
-            throw new NotSupportedException("JobId is null! Please update Logistics.");
-        int numberLength = 2;
-        long ticks = dateTime.Ticks;
-        string parentParentDirectory = GetParentParent(reportFullPath);
-        List<string> searchDirectories = GetSearchDirectories(numberLength, parentParentDirectory);
-        List<string> matchingFiles = GetMatchingFiles(ticks, reportFullPath, searchDirectories);
-        if (matchingFiles.Count != searchDirectories.Count)
-            throw new Exception($"Didn't find all files after {_BreakAfterSeconds} second(s)!");
-        try
-        { CreatePointerFile(numberLength, parentParentDirectory, matchingFiles); }
-        catch (Exception) { }
-        List<(string matchingFile, string checkFile)> collection = GetCollection(numberLength, parentParentDirectory, matchingFiles);
-        MoveCollection(dateTime, collection);
-        return results;
     }
 
 }

Adaptation/FileHandlers/Processed/FileRead.cs

@@ -17,7 +17,7 @@ public class FileRead : Shared.FileRead, IFileRead
     private readonly string _JobIdParentDirectory;
     private readonly string _JobIdProcessParentDirectory;
 
-    public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
+    public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
         base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
     {
         _MinFileLength = 10;
@@ -129,6 +129,7 @@ public class FileRead : Shared.FileRead, IFileRead
         string destinationJobIdDirectory = Path.Combine(_JobIdProcessParentDirectory, _Logistics.JobID, directoryName);
         string sequenceDirectory = Path.Combine(destinationJobIdDirectory, logisticsSequence);
         // string jsonFileName = Path.Combine(sequenceDirectory, $"{Path.GetFileNameWithoutExtension(reportFullPath)}.json");
+        MoveMatchingFile(jobIdDirectory, matchDirectories[0]);
         Directory.Move(matchDirectories[0], destinationJobIdDirectory);
         if (!Directory.Exists(sequenceDirectory))
             _ = Directory.CreateDirectory(sequenceDirectory);
@@ -136,16 +137,40 @@ public class FileRead : Shared.FileRead, IFileRead
         // File.WriteAllText(jsonFileName, json);
     }
 
+    private static void MoveMatchingFile(string jobIdDirectory, string matchDirectory)
+    {
+        string checkFile;
+        string jobIdDirectoryFileName;
+        string matchDirectoryFileName;
+        string[] jobIdDirectoryFiles = Directory.GetFiles(jobIdDirectory, "*", SearchOption.TopDirectoryOnly);
+        string[] matchDirectoryFiles = Directory.GetFiles(matchDirectory, "*", SearchOption.TopDirectoryOnly);
+        foreach (string jobIdDirectoryFile in jobIdDirectoryFiles)
+        {
+            jobIdDirectoryFileName = Path.GetFileName(jobIdDirectoryFile);
+            foreach (string matchDirectoryFile in matchDirectoryFiles)
+            {
+                matchDirectoryFileName = Path.GetFileName(matchDirectoryFile);
+                if (jobIdDirectoryFileName.StartsWith(matchDirectoryFileName))
+                {
+                    checkFile = Path.Combine(matchDirectory, jobIdDirectoryFileName);
+                    if (File.Exists(checkFile))
+                        continue;
+                    File.Move(jobIdDirectoryFile, checkFile);
+                }
+            }
+        }
+    }
+
     private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime dateTime)
     {
         Tuple<string, Test[], JsonElement[], List<FileInfo>> results;
-        Tuple<string, string[], string[]> pdsf = ProcessDataStandardFormat.GetLogisticsColumnsAndBody(reportFullPath);
+        ProcessDataStandardFormat processDataStandardFormat = ProcessDataStandardFormat.GetProcessDataStandardFormat(reportFullPath);
-        _Logistics = new Logistics(reportFullPath, pdsf.Item1);
+        _Logistics = new Logistics(reportFullPath, processDataStandardFormat);
         SetFileParameterLotIDToLogisticsMID();
-        JsonElement[] jsonElements = ProcessDataStandardFormat.GetArray(pdsf);
+        JsonElement[] jsonElements = ProcessDataStandardFormat.GetArray(processDataStandardFormat);
         List<csv.Description> descriptions = csv.ProcessData.GetDescriptions(jsonElements);
         Test[] tests = (from l in descriptions select (Test)l.Test).ToArray();
-        results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(pdsf.Item1, tests, jsonElements, new List<FileInfo>());
+        results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(string.Join(Environment.NewLine, processDataStandardFormat.Logistics[0]), tests, jsonElements, new List<FileInfo>());
         if (_IsEAFHosted && _FileConnectorConfiguration.FileScanningIntervalInSeconds > 0)
             DirectoryMove(reportFullPath, dateTime, descriptions);
         else if (!_IsEAFHosted)

Adaptation/FileHandlers/SPaCe/FileRead.cs

@@ -14,7 +14,7 @@ namespace Adaptation.FileHandlers.SPaCe;
 public class FileRead : Shared.FileRead, IFileRead
 {
 
-    public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
+    public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
         base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
     {
         _MinFileLength = 10;
@@ -117,15 +117,15 @@ public class FileRead : Shared.FileRead, IFileRead
     private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime dateTime)
     {
         Tuple<string, Test[], JsonElement[], List<FileInfo>> results;
-        Tuple<string, string[], string[]> pdsf = ProcessDataStandardFormat.GetLogisticsColumnsAndBody(reportFullPath);
+        ProcessDataStandardFormat processDataStandardFormat = ProcessDataStandardFormat.GetProcessDataStandardFormat(reportFullPath);
-        _Logistics = new Logistics(reportFullPath, pdsf.Item1);
+        _Logistics = new Logistics(reportFullPath, processDataStandardFormat);
         SetFileParameterLotIDToLogisticsMID();
-        JsonElement[] jsonElements = ProcessDataStandardFormat.GetArray(pdsf);
+        JsonElement[] jsonElements = ProcessDataStandardFormat.GetArray(processDataStandardFormat);
         List<Shared.Properties.IDescription> descriptions = GetDuplicatorDescriptions(jsonElements);
         Test[] tests = (from l in descriptions select (Test)l.Test).ToArray();
         if (_IsEAFHosted && _FileConnectorConfiguration.FileScanningIntervalInSeconds > 0)
             FileCopy(reportFullPath, dateTime, descriptions);
-        results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(pdsf.Item1, tests, jsonElements, new List<FileInfo>());
+        results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(string.Join(Environment.NewLine, processDataStandardFormat.Logistics[0]), tests, jsonElements, new List<FileInfo>());
         return results;
     }
 

Adaptation/FileHandlers/csv/Description.cs

@@ -24,7 +24,7 @@ public class Description : IDescription, Shared.Properties.IDescription
     public string MID { get; set; }
     //
     public string Date { get; set; }
-    public string Lot { get; set; }
+    public string RDS { get; set; }
     public string Part { get; set; }
     public string Process { get; set; }
     public string Recipe { get; set; }
@@ -65,7 +65,7 @@ public class Description : IDescription, Shared.Properties.IDescription
         List<string> results = new()
         {
             nameof(Date),
-            nameof(Lot),
+            nameof(RDS),
             nameof(Part),
             nameof(Process),
             nameof(Recipe)
@@ -165,7 +165,7 @@ public class Description : IDescription, Shared.Properties.IDescription
                     MID = logistics.MID,
                     //
                     Date = detail.Date,
-                    Lot = processData.MetaData.Date,
+                    RDS = processData.MetaData.Date,
                     Part = $"{processData.MetaData.DeviceId}-{processData.MetaData.DeviceType}-{processData.MetaData.DeviceNumber}",
                     Process = $"{processData.MetaData.DescriptionName}",
                     Recipe = processData.MetaData.Frequency,
@@ -204,7 +204,7 @@ public class Description : IDescription, Shared.Properties.IDescription
             MID = logistics.MID,
             //
             Date = nameof(Date),
-            Lot = nameof(Lot),
+            RDS = nameof(RDS),
             Part = nameof(Part),
             Process = nameof(Process),
             Recipe = nameof(Recipe),

Adaptation/FileHandlers/csv/FileRead.cs

@@ -15,7 +15,7 @@ public class FileRead : Shared.FileRead, IFileRead
     private long? _TickOffset;
     private readonly CommaSeparatedValuesConfiguration _CommaSeparatedValuesConfiguration;
 
-    public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
+    public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
         base(new Description(), true, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
     {
         _MinFileLength = 1000;

Adaptation/FileHandlers/csv/ProcessData.cs

@@ -25,7 +25,6 @@ public class ProcessData : IProcessData
     public ProcessData(IFileRead fileRead, Logistics logistics, List<FileInfo> fileInfoCollection, CommaSeparatedValuesConfiguration commaSeparatedValuesConfiguration, MetaData metaData)
     {
         JobID = logistics.JobID;
-        fileInfoCollection.Clear();
         _Details = new List<object>();
         MesEntity = logistics.MesEntity;
         Parse(fileRead, logistics, fileInfoCollection, commaSeparatedValuesConfiguration, metaData);
@@ -49,9 +48,13 @@ public class ProcessData : IProcessData
             if (description.Test != (int)tests[i])
                 throw new Exception();
         }
+        FileInfo fileInfo = new($"{logistics.ReportFullPath}.descriptions.json");
         List<Description> fileReadDescriptions = (from l in descriptions select (Description)l).ToList();
         string json = JsonSerializer.Serialize(fileReadDescriptions, fileReadDescriptions.GetType());
-        JsonElement[] jsonElements = JsonSerializer.Deserialize<JsonElement[]>(json);
+        File.WriteAllText(fileInfo.FullName, json);
+        File.SetLastWriteTime(fileInfo.FullName, logistics.DateTimeFromSequence);
+        fileInfoCollection.Add(fileInfo);
+        JsonElement[] jsonElements = JsonSerializer.Deserialize<JsonElement[]>(json) ?? throw new Exception();
         results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(logistics.Logistics1[0], tests.ToArray(), jsonElements, fileInfoCollection);
         return results;
     }

Adaptation/METCLIMATEC.Tests.csproj

@@ -69,7 +69,7 @@
         <PackageReference Include="System.Text.Json" Version="9.0.0" />
     </ItemGroup>
     <ItemGroup>
-        <PackageReference Include="Infineon.Mesa.PDF.Text.Stripper" Version="4.8.0.1"><NoWarn>NU1701</NoWarn></PackageReference>
+        <PackageReference Include="Infineon.Mesa.PDF.Text.Stripper" Version="4.8.0.2"><NoWarn>NU1701</NoWarn></PackageReference>
     </ItemGroup>
     <ItemGroup>
         <PackageReference Include="Infineon.Yoda.DotNetCore" Version="5.4.3" />

Adaptation/Shared/Duplicator/Description.cs

@@ -12,7 +12,7 @@ public class Description : IDescription, Properties.IDescription
     public int Test { get; set; }
     public int Count { get; set; }
     public int Index { get; set; }
-    public string Lot { get; set; }
+    public string RDS { get; set; }
     //
     public string EventName { get; set; }
     public string NullData { get; set; }
@@ -141,7 +141,7 @@ public class Description : IDescription, Properties.IDescription
                     MID = logistics.MID,
                     //
                     Date = DateTime.Now.ToString(GetDateFormat()),
-                    Lot = string.Empty,
+                    RDS = string.Empty,
                 };
                 results.Add(description);
             }

Adaptation/Shared/Logistics.cs

@@ -35,6 +35,9 @@ public class Logistics : ILogistics
     public long Sequence => _Sequence;
     public double TotalSecondsSinceLastWriteTimeFromSequence => _TotalSecondsSinceLastWriteTimeFromSequence;
 
+    private static string DefaultMesEntity(DateTime dateTime) =>
+        string.Concat(dateTime.Ticks, "_MES_ENTITY");
+
     public Logistics(IFileRead fileRead)
     {
         DateTime dateTime = DateTime.Now;
@@ -84,13 +87,13 @@ public class Logistics : ILogistics
         _Logistics2 = new List<Logistics2>();
     }
 
-    public Logistics(string reportFullPath, string logistics)
+    internal Logistics(string reportFullPath, ProcessDataStandardFormat processDataStandardFormat)
     {
         string key;
         DateTime dateTime;
         string[] segments;
         _FileInfo = new(reportFullPath);
-        _Logistics1 = logistics.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries).ToList();
+        _Logistics1 = processDataStandardFormat.Logistics.ToList();
         if (Logistics1.Count == 0 || !Logistics1[0].StartsWith("LOGISTICS_1"))
         {
             _NullData = null;
@@ -190,8 +193,6 @@ public class Logistics : ILogistics
         }
     }
 
-    private static string DefaultMesEntity(DateTime dateTime) => string.Concat(dateTime.Ticks, "_MES_ENTITY");
-
     internal void Update(string mid, string processJobID)
     {
         _MID = mid;

Adaptation/Shared/Metrology/WS.Attachment.cs

@@ -6,23 +6,25 @@ public partial class WS
     public class Attachment
     {
 
-        public string SubGroupId { get; set; }
+#nullable enable
-        public long HeaderId { get; set; }
-        public string HeaderIdDirectory { get; set; }
-        public string UniqueId { get; set; }
-        public string DestinationFileName { get; set; }
-        public string SourceFileName { get; set; }
-        public string AttachmentId { get; set; }
 
-        public Attachment(string subGroupId, long headerId, string headerIdDirectory, string uniqueId, string destinationFileName, string sourceFileName)
+        public long HeaderId { get; set; }
+        public string UniqueId { get; set; }
+        public string SubGroupId { get; set; }
+        public string AttachmentId { get; set; }
+        public string SourceFileName { get; set; }
+        public string HeaderIdDirectory { get; set; }
+        public string DestinationFileName { get; set; }
+
+        public Attachment(Results? results, string headerIdDirectory, string uniqueId, string destinationFileName, string sourceFileName)
         {
-            SubGroupId = subGroupId;
-            HeaderId = headerId;
-            HeaderIdDirectory = headerIdDirectory;
             UniqueId = uniqueId;
-            DestinationFileName = destinationFileName;
             SourceFileName = sourceFileName;
+            HeaderIdDirectory = headerIdDirectory;
+            DestinationFileName = destinationFileName;
             AttachmentId = System.Guid.NewGuid().ToString();
+            HeaderId = results?.HeaderId is null ? -1 : results.HeaderId.Value;
+            SubGroupId = results?.SubgroupId is null ? string.Empty : results.SubgroupId.Value.ToString();
         }
 
     }

Adaptation/Shared/Metrology/WS.Results.cs

@@ -1,27 +1,75 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
+using System.Text;
 using System.Text.Json;
+using System.Text.Json.Serialization;
 
 namespace Adaptation.Shared.Metrology;
 
 public partial class WS
 {
-    // this class represents the response from the Inbound API endpoint
+
     public class Results
     {
-        // true or false if data was written to the database
-        public bool Success { get; set; }
 
-        // if true, contains ID of the Header record in the database
+#nullable enable
-        public long HeaderID { get; set; }
 
-        // if false, this collection will contain a list of errors
+        [JsonConstructor]
-        public List<string> Errors { get; set; }
+        public Results(List<string>? errors,
+                       long? headerId,
+                       long? subgroupId,
+                       bool? success,
+                       List<string>? warnings)
+        {
+            Errors = errors;
+            Success = success;
+            HeaderId = headerId;
+            Warnings = warnings;
+            SubgroupId = subgroupId;
+        }
 
-        // this collection will contain a list of warnings, they will not prevent data from being saved
+        [JsonPropertyName("errors")] public List<string>? Errors { get; set; }
-        public List<string> Warnings { get; set; }
+        [JsonPropertyName("headerID")] public long? HeaderId { get; set; }
+        [JsonPropertyName("subgroupId")] public long? SubgroupId { get; set; }
+        [JsonPropertyName("success")] public bool? Success { get; set; }
+        [JsonPropertyName("warnings")] public List<string>? Warnings { get; set; }
+
+        public override string ToString()
+        {
+            string result = JsonSerializer.Serialize(this, ResultsSourceGenerationContext.Default.Results);
+            return result;
+        }
+
+        internal static Results Get(Results results, long? subgroupId) =>
+            new(results.Errors, results.HeaderId, subgroupId, results.Success, results.Warnings);
+
+        internal static Results Get(string resultsJson, Exception e)
+        {
+            Results results;
+            Exception? exception = e;
+            List<string> errors = new();
+            StringBuilder stringBuilder = new();
+            while (exception is not null)
+            {
+                _ = stringBuilder.AppendLine(exception.Message);
+                exception = exception.InnerException;
+            }
+            errors.Add(resultsJson);
+            errors.Add(stringBuilder.ToString());
+            results = new(errors: errors,
+                          headerId: null,
+                          subgroupId: null,
+                          success: false,
+                          warnings: new());
+            return results;
+        }
 
-        // this is just a helper function to make displaying the results easier
-        public override string ToString() => JsonSerializer.Serialize(this, GetType());
     }
 
 }
+
+[JsonSourceGenerationOptions(WriteIndented = true)]
+[JsonSerializable(typeof(WS.Results))]
+internal partial class ResultsSourceGenerationContext : JsonSerializerContext
+{
+}

Adaptation/Shared/Metrology/WS.cs

@@ -10,9 +10,11 @@ namespace Adaptation.Shared.Metrology;
 public partial class WS
 {
 
+#nullable enable
+
     public static (string, Results) SendData(string url, long sequence, string directory, object payload, int timeoutSeconds = 120)
     {
-        Results results = new();
+        Results? wsResults = null;
         string resultsJson = string.Empty;
         try
         {
@@ -30,29 +32,20 @@ public partial class WS
                 };
                 HttpResponseMessage httpResponseMessage = httpClient.SendAsync(httpRequestMessage, HttpCompletionOption.ResponseContentRead).Result;
                 resultsJson = httpResponseMessage.Content.ReadAsStringAsync().Result;
-                results = JsonSerializer.Deserialize<Results>(resultsJson, new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
+                wsResults = JsonSerializer.Deserialize(resultsJson, ResultsSourceGenerationContext.Default.Results);
-                string checkDirectory = Path.Combine(directory, $"-{results.HeaderID}");
+                if (wsResults is null)
+                    throw new NullReferenceException(nameof(wsResults));
+                string checkDirectory = Path.Combine(directory, $"-{wsResults.HeaderId}");
                 if (!Directory.Exists(checkDirectory))
                     _ = Directory.CreateDirectory(checkDirectory);
                 File.WriteAllText(Path.Combine(checkDirectory, $"{sequence}.json"), json);
             }
-            if (!results.Success)
+            if (wsResults.Success is null || !wsResults.Success.Value)
-                results.Errors.Add(results.ToString());
+                wsResults.Errors?.Add(wsResults.ToString());
         }
         catch (Exception e)
-        {
+        { wsResults ??= Results.Get(resultsJson, e); }
-            Exception exception = e;
+        return new(resultsJson, wsResults);
-            StringBuilder stringBuilder = new();
-            while (exception is not null)
-            {
-                _ = stringBuilder.AppendLine(exception.Message);
-                exception = exception.InnerException;
-            }
-            results.Errors ??= new List<string>();
-            results.Errors.Add(resultsJson);
-            results.Errors.Add(stringBuilder.ToString());
-        }
-        return new(resultsJson, results);
     }
 
     public static void AttachFile(string url, Attachment attachment, int timeoutSeconds = 60)
@@ -69,16 +62,20 @@ public partial class WS
         }
     }
 
-    public static void AttachFiles(string url, List<Attachment> headerAttachments = null, List<Attachment> dataAttachments = null)
+    public static void AttachFiles(string url, List<Attachment>? headerAttachments = null, List<Attachment>? dataAttachments = null)
     {
         string directory;
         try
         {
+            string? directoryName;
             if (headerAttachments is not null)
             {
                 foreach (Attachment attachment in headerAttachments)
                 {
-                    directory = Path.Combine(Path.GetDirectoryName(attachment.HeaderIdDirectory), attachment.AttachmentId) ?? throw new Exception();
+                    directoryName = Path.GetDirectoryName(attachment.HeaderIdDirectory);
+                    if (string.IsNullOrEmpty(directoryName))
+                        continue;
+                    directory = Path.Combine(directoryName, attachment.AttachmentId) ?? throw new Exception();
                     if (!Directory.Exists(directory))
                         _ = Directory.CreateDirectory(directory);
                     File.Copy(attachment.SourceFileName, Path.Combine(directory, attachment.DestinationFileName), overwrite: true);
@@ -88,7 +85,10 @@ public partial class WS
             {
                 foreach (Attachment attachment in dataAttachments)
                 {
-                    directory = Path.Combine(Path.GetDirectoryName(attachment.HeaderIdDirectory.Replace("Header", "Data")), attachment.AttachmentId) ?? throw new Exception();
+                    directoryName = Path.GetDirectoryName(attachment.HeaderIdDirectory.Replace("Header", "Data"));
+                    if (string.IsNullOrEmpty(directoryName))
+                        continue;
+                    directory = Path.Combine(directoryName, attachment.AttachmentId) ?? throw new Exception();
                     if (!Directory.Exists(directory))
                         _ = Directory.CreateDirectory(directory);
                     File.Copy(attachment.SourceFileName, Path.Combine(directory, attachment.DestinationFileName), overwrite: true);
@@ -108,7 +108,7 @@ public partial class WS
         }
         catch (Exception e)
         {
-            Exception exception = e;
+            Exception? exception = e;
             StringBuilder stringBuilder = new();
             while (exception is not null)
             {

Adaptation/Shared/ProcessDataStandardFormatMapping.cs

@@ -0,0 +1,33 @@
+using System.Collections.ObjectModel;
+
+namespace Adaptation.Shared;
+
+public class ProcessDataStandardFormatMapping
+{
+
+    public ReadOnlyCollection<string> BackfillColumns { get; private set; }
+    public ReadOnlyCollection<int> ColumnIndices { get; private set; }
+    public ReadOnlyCollection<string> IgnoreColumns { get; private set; }
+    public ReadOnlyCollection<string> IndexOnlyColumns { get; private set; }
+    public ReadOnlyDictionary<string, string> KeyValuePairs { get; private set; }
+    public ReadOnlyCollection<string> NewColumnNames { get; private set; }
+    public ReadOnlyCollection<string> OldColumnNames { get; private set; }
+
+    public ProcessDataStandardFormatMapping(ReadOnlyCollection<string> backfillColumns,
+                                            ReadOnlyCollection<int> columnIndices,
+                                            ReadOnlyCollection<string> ignoreColumns,
+                                            ReadOnlyCollection<string> indexOnlyColumns,
+                                            ReadOnlyDictionary<string, string> keyValuePairs,
+                                            ReadOnlyCollection<string> newColumnNames,
+                                            ReadOnlyCollection<string> oldColumnNames)
+    {
+        BackfillColumns = backfillColumns;
+        ColumnIndices = columnIndices;
+        IgnoreColumns = ignoreColumns;
+        IndexOnlyColumns = indexOnlyColumns;
+        KeyValuePairs = keyValuePairs;
+        NewColumnNames = newColumnNames;
+        OldColumnNames = oldColumnNames;
+    }
+
+}

Adaptation/Shared/Properties/IDescription.cs

@@ -6,6 +6,6 @@ public interface IDescription
     int Test { get; }
     int Count { get; }
     int Index { get; }
-    string Lot { get; }
+    string RDS { get; }
 
 }

Adaptation/_Tests/Static/METCLIMATEC.cs

@@ -51,7 +51,7 @@ public class METCLIMATEC : LoggingUnitTesting, IDisposable
     public void TestDateTime()
     {
         DateTime dateTime = DateTime.Now;
-        Assert.IsTrue(dateTime.ToString("M/d/yyyy h:mm:ss tt") == dateTime.ToString());
+        Assert.AreEqual(dateTime.ToString(), dateTime.ToString("M/d/yyyy h:mm:ss tt"));
     }
 
 #if DEBUG

Adaptation/_Tests/Static/csv.cs

@@ -51,7 +51,7 @@ public class CSV : LoggingUnitTesting, IDisposable
     public void TestDateTime()
     {
         DateTime dateTime = DateTime.Now;
-        Assert.IsTrue(dateTime.ToString("M/d/yyyy h:mm:ss tt") == dateTime.ToString());
+        Assert.AreEqual(dateTime.ToString(), dateTime.ToString("M/d/yyyy h:mm:ss tt"));
     }
 
 #if DEBUG

FileHandlers/FileRead.cs

@@ -37,7 +37,7 @@ public partial class FileRead : FileReaderHandler, ISMTP
     private FilePathGenerator _FilePathGeneratorForTarget;
     private readonly List<EquipmentParameter> _EquipmentParameters;
     private static readonly Dictionary<string, List<long>> _DummyRuns;
-    private static readonly Dictionary<long, List<string>> _StaticRuns;
+    private static readonly Dictionary<long, List<Adaptation.Shared.Metrology.WS.Results>> _StaticRuns;
 
     static FileRead()
     {

METCLIMATEC.csproj

@@ -147,6 +147,7 @@
     <Compile Include="Adaptation\Shared\Metrology\WS.Results.cs" />
     <Compile Include="Adaptation\Shared\ParameterType.cs" />
     <Compile Include="Adaptation\Shared\ProcessDataStandardFormat.cs" />
+    <Compile Include="Adaptation\Shared\ProcessDataStandardFormatMapping.cs" />
     <Compile Include="Adaptation\Shared\Properties\IDescription.cs" />
     <Compile Include="Adaptation\Shared\Properties\IFileRead.cs" />
     <Compile Include="Adaptation\Shared\Properties\ILogistics.cs" />