Refactor file handling and add Log.FileRead implementation

This commit is contained in:
2025-09-22 09:20:41 -07:00
parent f15c525ab5
commit 9d0d24fb83
9 changed files with 257 additions and 91 deletions

View File

@ -41,24 +41,6 @@ stages:
displayName: "Nuget Clear"
enabled: false
- task: CopyFiles@2
displayName: 'Copy GhostPCL Files to: D:\EAF-Mesa-Integration\copy'
inputs:
Contents: "*"
SourceFolder: '\\mesfs.infineon.com\EC_EAFRepository\Staging\DeploymentStorage\GhostPCL'
TargetFolder: 'D:\EAF-Mesa-Integration\copy\GhostPCL'
OverWrite: true
enabled: true
- task: CopyFiles@2
displayName: 'Copy LincPDFC Files to: D:\EAF-Mesa-Integration\copy'
inputs:
Contents: "*"
SourceFolder: '\\mesfs.infineon.com\EC_EAFRepository\Staging\DeploymentStorage\LincPDFC'
TargetFolder: 'D:\EAF-Mesa-Integration\copy\LincPDFC'
OverWrite: true
enabled: true
- script: |
"C:\program files\dotnet\dotnet.exe" user-secrets init
"C:\program files\dotnet\dotnet.exe" user-secrets set "BuildNumber" "$(Build.BuildId)"
@ -202,24 +184,6 @@ stages:
displayName: "Nuget Clear"
enabled: false
- task: CopyFiles@2
displayName: 'Copy GhostPCL Files to: D:\EAF-Mesa-Integration\copy'
inputs:
Contents: "*"
SourceFolder: '\\mestsa003.infineon.com\EC_EAFRepository\Staging\DeploymentStorage\GhostPCL'
TargetFolder: 'D:\EAF-Mesa-Integration\copy\GhostPCL'
OverWrite: true
enabled: true
- task: CopyFiles@2
displayName: 'Copy LincPDFC Files to: D:\EAF-Mesa-Integration\copy'
inputs:
Contents: "*"
SourceFolder: '\\mestsa003.infineon.com\EC_EAFRepository\Staging\DeploymentStorage\LincPDFC'
TargetFolder: 'D:\EAF-Mesa-Integration\copy\LincPDFC'
OverWrite: true
enabled: true
- script: |
"C:\program files\dotnet\dotnet.exe" user-secrets init
"C:\program files\dotnet\dotnet.exe" user-secrets set "BuildNumber" "$(Build.BuildId)"

View File

@ -117,43 +117,6 @@ public class FileRead : Shared.FileRead, IFileRead
return results;
}
private static int? GetKeyColumnIndex(string[] columns, string keyColumn)
{
#nullable enable
int? result = null;
for (int i = 0; i < columns.Length; i++)
{
if (columns[i] != keyColumn)
continue;
result = i;
break;
}
return result;
}
private ReadOnlyCollection<string> GetSystemStateValues(List<string> lines, string[] columns, int keyColumnIndex)
{
List<string> results = new();
string[] values;
string? systemState;
string keyColumnValue;
for (int i = 7; i < lines.Count; i++)
{
values = lines[i].Split('\t');
if (values.Length != columns.Length)
continue;
keyColumnValue = values[keyColumnIndex];
if (string.IsNullOrEmpty(keyColumnValue))
continue;
if (!_SystemStateToNames.TryGetValue(keyColumnValue, out systemState))
continue;
if (results.Contains(systemState))
continue;
results.Add(systemState);
}
return new(results);
}
private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime _)
{
Tuple<string, Test[], JsonElement[], List<FileInfo>> results;

View File

@ -16,6 +16,7 @@ public class CellInstanceConnectionName
nameof(Aggregation) => new Aggregation.FileRead(smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null),
nameof(Dummy) => new Dummy.FileRead(smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null),
nameof(InfinityQS) => new InfinityQS.FileRead(smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null),
nameof(Log) => new Log.FileRead(smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null),
nameof(R29) => new R29.FileRead(smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null),
nameof(R30) => new R30.FileRead(smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null),
nameof(R32) => new R32.FileRead(smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null),

View File

@ -0,0 +1,133 @@
using Adaptation.Eaf.Management.ConfigurationData.CellAutomation;
using Adaptation.Ifx.Eaf.EquipmentConnector.File.Configuration;
using Adaptation.Shared;
using Adaptation.Shared.Duplicator;
using Adaptation.Shared.Methods;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.IO;
using System.Linq;
using System.Text.Json;
namespace Adaptation.FileHandlers.Log;
public class FileRead : Shared.FileRead, IFileRead
{
private readonly ReadOnlyDictionary<string, string> _SystemStateToNames;
public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
{
_MinFileLength = 10;
_NullData = string.Empty;
_Logistics = new(this);
if (_FileParameter is null)
throw new Exception(cellInstanceConnectionName);
if (_ModelObjectParameterDefinitions is null)
throw new Exception(cellInstanceConnectionName);
if (!_IsDuplicator)
throw new Exception(cellInstanceConnectionName);
string[] segments;
Dictionary<string, string> systemStateToNames = new();
ModelObjectParameterDefinition[] systemStates = GetProperties(cellInstanceConnectionName, modelObjectParameters, "ProcessDataStandardFormat.SystemState");
foreach (ModelObjectParameterDefinition modelObjectParameterDefinition in systemStates)
{
segments = modelObjectParameterDefinition.Value.Split('|');
if (segments.Length != 2)
continue;
systemStateToNames.Add(segments[0], segments[1]);
}
_SystemStateToNames = new(systemStateToNames);
}
void IFileRead.Move(Tuple<string, Test[], JsonElement[], List<FileInfo>> extractResults, Exception exception)
{
bool isErrorFile = exception is not null;
if (!isErrorFile && !string.IsNullOrEmpty(_Logistics.ReportFullPath))
{
FileInfo fileInfo = new(_Logistics.ReportFullPath);
if (fileInfo.Exists && fileInfo.LastWriteTime < fileInfo.CreationTime)
File.SetLastWriteTime(_Logistics.ReportFullPath, fileInfo.CreationTime);
}
Move(extractResults);
}
void IFileRead.WaitForThread() => WaitForThread(thread: null, threadExceptions: null);
string IFileRead.GetEventDescription()
{
string result = _Description.GetEventDescription();
return result;
}
List<string> IFileRead.GetHeaderNames()
{
List<string> results = _Description.GetHeaderNames();
return results;
}
string[] IFileRead.Move(Tuple<string, Test[], JsonElement[], List<FileInfo>> extractResults, string to, string from, string resolvedFileLocation, Exception exception)
{
string[] results = Move(extractResults, to, from, resolvedFileLocation, exception);
return results;
}
JsonProperty[] IFileRead.GetDefault()
{
JsonProperty[] results = _Description.GetDefault(this, _Logistics);
return results;
}
Dictionary<string, string> IFileRead.GetDisplayNamesJsonElement()
{
Dictionary<string, string> results = _Description.GetDisplayNamesJsonElement(this);
return results;
}
List<IDescription> IFileRead.GetDescriptions(IFileRead fileRead, List<Test> tests, IProcessData processData)
{
List<IDescription> results = _Description.GetDescriptions(fileRead, _Logistics, tests, processData);
return results;
}
Tuple<string, Test[], JsonElement[], List<FileInfo>> IFileRead.GetExtractResult(string reportFullPath, string eventName)
{
Tuple<string, Test[], JsonElement[], List<FileInfo>> results;
if (string.IsNullOrEmpty(eventName))
throw new Exception();
_ReportFullPath = reportFullPath;
DateTime dateTime = DateTime.Now;
results = GetExtractResult(reportFullPath, dateTime);
if (results.Item3 is null)
results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(results.Item1, Array.Empty<Test>(), JsonSerializer.Deserialize<JsonElement[]>("[]"), results.Item4);
if (results.Item3.Length > 0 && _IsEAFHosted)
WritePDSF(this, results.Item3);
UpdateLastTicksDuration(DateTime.Now.Ticks - dateTime.Ticks);
return results;
}
Tuple<string, Test[], JsonElement[], List<FileInfo>> IFileRead.ReExtract()
{
Tuple<string, Test[], JsonElement[], List<FileInfo>> results;
List<string> headerNames = _Description.GetHeaderNames();
Dictionary<string, string> keyValuePairs = _Description.GetDisplayNamesJsonElement(this);
results = ReExtract(this, headerNames, keyValuePairs);
return results;
}
private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime _)
{
Tuple<string, Test[], JsonElement[], List<FileInfo>> results;
ProcessDataStandardFormat processDataStandardFormat = ProcessDataStandardFormat.GetProcessDataStandardFormat(reportFullPath);
_Logistics = new Logistics(reportFullPath, processDataStandardFormat);
SetFileParameterLotIDToLogisticsMID();
JsonElement[] jsonElements = ProcessDataStandardFormat.GetArray(processDataStandardFormat);
List<Shared.Properties.IDescription> descriptions = GetDuplicatorDescriptions(jsonElements);
Test[] tests = (from l in descriptions select (Test)l.Test).ToArray();
results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(string.Join(Environment.NewLine, processDataStandardFormat.Logistics), tests, jsonElements, new List<FileInfo>());
return results;
}
}

View File

@ -3,6 +3,7 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Adaptation.Shared.Duplicator;
@ -178,4 +179,16 @@ public class Description : IDescription, Properties.IDescription
internal static string GetDateFormat() => "MM/dd/yyyy hh:mm:ss tt";
}
[JsonSourceGenerationOptions(WriteIndented = true, NumberHandling = JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString)]
[JsonSerializable(typeof(Description))]
internal partial class SharedDescriptionSourceGenerationContext : JsonSerializerContext
{
}
[JsonSourceGenerationOptions(WriteIndented = true, NumberHandling = JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString)]
[JsonSerializable(typeof(Description[]))]
internal partial class SharedDescriptionArraySourceGenerationContext : JsonSerializerContext
{
}

View File

@ -9,7 +9,6 @@ using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
namespace Adaptation.Shared;
@ -447,12 +446,13 @@ public class FileRead : Properties.IFileRead
{
List<Properties.IDescription> results = new();
Duplicator.Description description;
JsonSerializerOptions jsonSerializerOptions = new() { NumberHandling = JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString };
foreach (JsonElement jsonElement in jsonElements)
{
if (jsonElement.ValueKind != JsonValueKind.Object)
throw new Exception();
description = JsonSerializer.Deserialize<Duplicator.Description>(jsonElement.ToString(), jsonSerializerOptions);
description = JsonSerializer.Deserialize(jsonElement.ToString(), Duplicator.SharedDescriptionSourceGenerationContext.Default.Description);
if (description is null)
continue;
results.Add(description);
}
return results;

View File

@ -578,6 +578,9 @@ internal class ProcessDataStandardFormat
results.Add(string.Empty);
results.AddRange(processDataStandardFormat.InputPDSF.Footer.Select(l => $"|{l.Replace('\t', '|')}|"));
results.Add(string.Empty);
string xml = GetXml(processDataStandardFormat);
results.Add(xml);
results.Add(string.Empty);
results.Add("EOF");
results.Add(string.Empty);
string json = GetJson(processDataStandardFormat);
@ -651,6 +654,17 @@ internal class ProcessDataStandardFormat
return results;
}
internal static JsonElement[] GetArray(string reportFullPath, string[] lines, ProcessDataStandardFormat processDataStandardFormat)
{
JsonElement[] results;
string? json = GetRecordsJson(reportFullPath, lines);
if (string.IsNullOrEmpty(json))
results = GetArray(processDataStandardFormat);
else
results = JsonSerializer.Deserialize(json, JsonElementCollectionSourceGenerationContext.Default.JsonElementArray) ?? throw new Exception();
return results;
}
internal static string GetPDSFText(IFileRead fileRead, Logistics logistics, JsonElement[] jsonElements, string logisticsText)
{
string result;
@ -869,6 +883,8 @@ internal class ProcessDataStandardFormat
string tag;
string value;
string[] segments;
List<string> values;
Dictionary<string, List<string>> results = new();
ReadOnlyCollection<string> body = processDataStandardFormat.InputPDSF is null ?
processDataStandardFormat.Body : processDataStandardFormat.InputPDSF.Body;
ReadOnlyCollection<string> columns = processDataStandardFormat.InputPDSF is null ?
@ -876,7 +892,6 @@ internal class ProcessDataStandardFormat
List<string> lines = new() { "<?xml version=\"1.0\" encoding=\"UTF-8\"?>", "<records>" };
for (int i = 0; i < body.Count; i++)
{
lines.Add(" <record>");
segments = body[i].Trim().Split('\t');
if (segments.Length != columns.Count)
break;
@ -888,7 +903,42 @@ internal class ProcessDataStandardFormat
.Replace("\"", "&quot;")
.Replace("'", "&apos;");
tag = Regex.Replace(columns[c].Trim('"'), @"[^a-zA-Z0-9]", "_").Split('\r')[0].Split('\n')[0];
lines.Add(string.Concat(" <", tag, '>', value, "</", tag, '>'));
if (i == 0)
{
if (results.ContainsKey(tag))
continue;
results.Add(tag, new List<string>());
}
results[tag].Add(value);
}
}
foreach (KeyValuePair<string, List<string>> keyValuePair in results)
{
if (body.Count < 2)
break;
if (keyValuePair.Value.Count != body.Count)
continue;
values = keyValuePair.Value.Distinct().ToList();
if (values.Count == 2 && (string.IsNullOrEmpty(values[0]) || string.IsNullOrEmpty(values[1])))
{
for (int i = 0; i < body.Count; i++)
keyValuePair.Value[i] = string.Empty;
foreach (string v in values)
{
if (string.IsNullOrEmpty(v))
continue;
keyValuePair.Value[0] = v;
}
}
}
for (int i = 0; i < body.Count; i++)
{
lines.Add(" <record>");
foreach (KeyValuePair<string, List<string>> keyValuePair in results)
{
if (keyValuePair.Value.Count != body.Count)
continue;
lines.Add(string.Concat(" <", keyValuePair.Key, '>', keyValuePair.Value[i], "</", keyValuePair.Key, '>'));
}
lines.Add(" </record>");
}
@ -897,6 +947,46 @@ internal class ProcessDataStandardFormat
return result;
}
internal static string GetXml(string reportFullPath, string[]? lines = null)
{
string result;
bool foundXml = false;
List<string> results = new();
lines ??= File.ReadAllLines(reportFullPath);
foreach (string line in lines)
{
if (line.StartsWith("<?xml"))
foundXml = true;
if (!foundXml)
continue;
if (line.StartsWith("EOF"))
break;
results.Add(line);
}
result = string.Join(Environment.NewLine, results);
return result;
}
private static string? GetRecordsJson(string reportFullPath, string[] lines)
{
string? result;
bool foundRecords = false;
List<string> results = new();
lines ??= File.ReadAllLines(reportFullPath);
foreach (string line in lines)
{
if (line.StartsWith("\"Records\""))
foundRecords = true;
if (!foundRecords)
continue;
if (line == "],")
break;
results.Add(line);
}
result = results.Count == 0 ? null : $"{string.Join(Environment.NewLine, results.Skip(1))}{Environment.NewLine}]";
return result;
}
}
[JsonSourceGenerationOptions(WriteIndented = true)]

View File

@ -1,33 +1,34 @@
using System.Collections.ObjectModel;
using System.Linq;
namespace Adaptation.Shared;
public class ProcessDataStandardFormatMapping
{
public ReadOnlyCollection<string> BackfillColumns { get; private set; }
public ReadOnlyCollection<int> ColumnIndices { get; private set; }
public ReadOnlyCollection<string> IgnoreColumns { get; private set; }
public ReadOnlyCollection<string> IndexOnlyColumns { get; private set; }
public ReadOnlyDictionary<string, string> KeyValuePairs { get; private set; }
public ReadOnlyCollection<string> NewColumnNames { get; private set; }
public ReadOnlyCollection<string> OldColumnNames { get; private set; }
public ProcessDataStandardFormatMapping(ReadOnlyCollection<string> backfillColumns,
ReadOnlyCollection<int> columnIndices,
ReadOnlyCollection<string> ignoreColumns,
ReadOnlyCollection<string> indexOnlyColumns,
ReadOnlyDictionary<string, string> keyValuePairs,
public ProcessDataStandardFormatMapping(ReadOnlyCollection<int> columnIndices,
ReadOnlyCollection<string> newColumnNames,
ReadOnlyCollection<string> oldColumnNames)
{
BackfillColumns = backfillColumns;
ColumnIndices = columnIndices;
IgnoreColumns = ignoreColumns;
IndexOnlyColumns = indexOnlyColumns;
KeyValuePairs = keyValuePairs;
NewColumnNames = newColumnNames;
OldColumnNames = oldColumnNames;
}
internal static ProcessDataStandardFormatMapping Get(string processDataStandardFormatMappingOldColumnNames, string processDataStandardFormatMappingNewColumnNames, string processDataStandardFormatMappingColumnIndices)
{
ProcessDataStandardFormatMapping result;
ReadOnlyCollection<string> newColumnNames = new(processDataStandardFormatMappingNewColumnNames.Split(','));
ReadOnlyCollection<string> oldColumnNames = new(processDataStandardFormatMappingOldColumnNames.Split(','));
ReadOnlyCollection<int> columnIndices = new(processDataStandardFormatMappingColumnIndices.Split(',').Select(int.Parse).ToArray());
result = new(columnIndices: columnIndices,
newColumnNames: newColumnNames,
oldColumnNames: oldColumnNames);
return result;
}
}

View File

@ -108,6 +108,7 @@
<Compile Include="Adaptation\FileHandlers\Dummy\FileRead.cs" />
<Compile Include="Adaptation\FileHandlers\InfinityQS\FileRead.cs" />
<Compile Include="Adaptation\FileHandlers\InfinityQS\Six.cs" />
<Compile Include="Adaptation\FileHandlers\Log\FileRead.cs" />
<Compile Include="Adaptation\FileHandlers\R29\FileRead.cs" />
<Compile Include="Adaptation\FileHandlers\R30\FileRead.cs" />
<Compile Include="Adaptation\FileHandlers\R32\FileRead.cs" />