From 9d0d24fb83370d0c47c9e00c7f53fedb05ed2a8a Mon Sep 17 00:00:00 2001 From: "phares@iscn5cg20977xq" Date: Mon, 22 Sep 2025 09:20:41 -0700 Subject: [PATCH] Refactor file handling and add Log.FileRead implementation --- Adaptation/DEP08CEPIEPSILON.yml | 36 ----- .../FileHandlers/Aggregation/FileRead.cs | 37 ----- .../CellInstanceConnectionName.cs | 1 + Adaptation/FileHandlers/Log/FileRead.cs | 133 ++++++++++++++++++ Adaptation/Shared/Duplicator/Description.cs | 13 ++ Adaptation/Shared/FileRead.cs | 6 +- .../Shared/ProcessDataStandardFormat.cs | 94 ++++++++++++- .../ProcessDataStandardFormatMapping.cs | 27 ++-- DEP08CEPIEPSILON.csproj | 1 + 9 files changed, 257 insertions(+), 91 deletions(-) create mode 100644 Adaptation/FileHandlers/Log/FileRead.cs diff --git a/Adaptation/DEP08CEPIEPSILON.yml b/Adaptation/DEP08CEPIEPSILON.yml index 5962376..857c0cd 100644 --- a/Adaptation/DEP08CEPIEPSILON.yml +++ b/Adaptation/DEP08CEPIEPSILON.yml @@ -41,24 +41,6 @@ stages: displayName: "Nuget Clear" enabled: false - - task: CopyFiles@2 - displayName: 'Copy GhostPCL Files to: D:\EAF-Mesa-Integration\copy' - inputs: - Contents: "*" - SourceFolder: '\\mesfs.infineon.com\EC_EAFRepository\Staging\DeploymentStorage\GhostPCL' - TargetFolder: 'D:\EAF-Mesa-Integration\copy\GhostPCL' - OverWrite: true - enabled: true - - - task: CopyFiles@2 - displayName: 'Copy LincPDFC Files to: D:\EAF-Mesa-Integration\copy' - inputs: - Contents: "*" - SourceFolder: '\\mesfs.infineon.com\EC_EAFRepository\Staging\DeploymentStorage\LincPDFC' - TargetFolder: 'D:\EAF-Mesa-Integration\copy\LincPDFC' - OverWrite: true - enabled: true - - script: | "C:\program files\dotnet\dotnet.exe" user-secrets init "C:\program files\dotnet\dotnet.exe" user-secrets set "BuildNumber" "$(Build.BuildId)" @@ -202,24 +184,6 @@ stages: displayName: "Nuget Clear" enabled: false - - task: CopyFiles@2 - displayName: 'Copy GhostPCL Files to: D:\EAF-Mesa-Integration\copy' - inputs: - Contents: "*" - SourceFolder: '\\mestsa003.infineon.com\EC_EAFRepository\Staging\DeploymentStorage\GhostPCL' - TargetFolder: 'D:\EAF-Mesa-Integration\copy\GhostPCL' - OverWrite: true - enabled: true - - - task: CopyFiles@2 - displayName: 'Copy LincPDFC Files to: D:\EAF-Mesa-Integration\copy' - inputs: - Contents: "*" - SourceFolder: '\\mestsa003.infineon.com\EC_EAFRepository\Staging\DeploymentStorage\LincPDFC' - TargetFolder: 'D:\EAF-Mesa-Integration\copy\LincPDFC' - OverWrite: true - enabled: true - - script: | "C:\program files\dotnet\dotnet.exe" user-secrets init "C:\program files\dotnet\dotnet.exe" user-secrets set "BuildNumber" "$(Build.BuildId)" diff --git a/Adaptation/FileHandlers/Aggregation/FileRead.cs b/Adaptation/FileHandlers/Aggregation/FileRead.cs index 6031283..833eeff 100644 --- a/Adaptation/FileHandlers/Aggregation/FileRead.cs +++ b/Adaptation/FileHandlers/Aggregation/FileRead.cs @@ -117,43 +117,6 @@ public class FileRead : Shared.FileRead, IFileRead return results; } - private static int? GetKeyColumnIndex(string[] columns, string keyColumn) - { -#nullable enable - int? result = null; - for (int i = 0; i < columns.Length; i++) - { - if (columns[i] != keyColumn) - continue; - result = i; - break; - } - return result; - } - - private ReadOnlyCollection GetSystemStateValues(List lines, string[] columns, int keyColumnIndex) - { - List results = new(); - string[] values; - string? systemState; - string keyColumnValue; - for (int i = 7; i < lines.Count; i++) - { - values = lines[i].Split('\t'); - if (values.Length != columns.Length) - continue; - keyColumnValue = values[keyColumnIndex]; - if (string.IsNullOrEmpty(keyColumnValue)) - continue; - if (!_SystemStateToNames.TryGetValue(keyColumnValue, out systemState)) - continue; - if (results.Contains(systemState)) - continue; - results.Add(systemState); - } - return new(results); - } - private Tuple> GetExtractResult(string reportFullPath, DateTime _) { Tuple> results; diff --git a/Adaptation/FileHandlers/CellInstanceConnectionName.cs b/Adaptation/FileHandlers/CellInstanceConnectionName.cs index 9e92df0..561a69b 100644 --- a/Adaptation/FileHandlers/CellInstanceConnectionName.cs +++ b/Adaptation/FileHandlers/CellInstanceConnectionName.cs @@ -16,6 +16,7 @@ public class CellInstanceConnectionName nameof(Aggregation) => new Aggregation.FileRead(smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null), nameof(Dummy) => new Dummy.FileRead(smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null), nameof(InfinityQS) => new InfinityQS.FileRead(smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null), + nameof(Log) => new Log.FileRead(smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null), nameof(R29) => new R29.FileRead(smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null), nameof(R30) => new R30.FileRead(smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null), nameof(R32) => new R32.FileRead(smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null), diff --git a/Adaptation/FileHandlers/Log/FileRead.cs b/Adaptation/FileHandlers/Log/FileRead.cs new file mode 100644 index 0000000..b168424 --- /dev/null +++ b/Adaptation/FileHandlers/Log/FileRead.cs @@ -0,0 +1,133 @@ +using Adaptation.Eaf.Management.ConfigurationData.CellAutomation; +using Adaptation.Ifx.Eaf.EquipmentConnector.File.Configuration; +using Adaptation.Shared; +using Adaptation.Shared.Duplicator; +using Adaptation.Shared.Methods; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.IO; +using System.Linq; +using System.Text.Json; + +namespace Adaptation.FileHandlers.Log; + +public class FileRead : Shared.FileRead, IFileRead +{ + + private readonly ReadOnlyDictionary _SystemStateToNames; + + public FileRead(ISMTP smtp, Dictionary fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList modelObjectParameters, string equipmentDictionaryName, Dictionary> dummyRuns, Dictionary> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) : + base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null) + { + _MinFileLength = 10; + _NullData = string.Empty; + _Logistics = new(this); + if (_FileParameter is null) + throw new Exception(cellInstanceConnectionName); + if (_ModelObjectParameterDefinitions is null) + throw new Exception(cellInstanceConnectionName); + if (!_IsDuplicator) + throw new Exception(cellInstanceConnectionName); + string[] segments; + Dictionary systemStateToNames = new(); + ModelObjectParameterDefinition[] systemStates = GetProperties(cellInstanceConnectionName, modelObjectParameters, "ProcessDataStandardFormat.SystemState"); + foreach (ModelObjectParameterDefinition modelObjectParameterDefinition in systemStates) + { + segments = modelObjectParameterDefinition.Value.Split('|'); + if (segments.Length != 2) + continue; + systemStateToNames.Add(segments[0], segments[1]); + } + _SystemStateToNames = new(systemStateToNames); + } + + void IFileRead.Move(Tuple> extractResults, Exception exception) + { + bool isErrorFile = exception is not null; + if (!isErrorFile && !string.IsNullOrEmpty(_Logistics.ReportFullPath)) + { + FileInfo fileInfo = new(_Logistics.ReportFullPath); + if (fileInfo.Exists && fileInfo.LastWriteTime < fileInfo.CreationTime) + File.SetLastWriteTime(_Logistics.ReportFullPath, fileInfo.CreationTime); + } + Move(extractResults); + } + + void IFileRead.WaitForThread() => WaitForThread(thread: null, threadExceptions: null); + + string IFileRead.GetEventDescription() + { + string result = _Description.GetEventDescription(); + return result; + } + + List IFileRead.GetHeaderNames() + { + List results = _Description.GetHeaderNames(); + return results; + } + + string[] IFileRead.Move(Tuple> extractResults, string to, string from, string resolvedFileLocation, Exception exception) + { + string[] results = Move(extractResults, to, from, resolvedFileLocation, exception); + return results; + } + + JsonProperty[] IFileRead.GetDefault() + { + JsonProperty[] results = _Description.GetDefault(this, _Logistics); + return results; + } + + Dictionary IFileRead.GetDisplayNamesJsonElement() + { + Dictionary results = _Description.GetDisplayNamesJsonElement(this); + return results; + } + + List IFileRead.GetDescriptions(IFileRead fileRead, List tests, IProcessData processData) + { + List results = _Description.GetDescriptions(fileRead, _Logistics, tests, processData); + return results; + } + + Tuple> IFileRead.GetExtractResult(string reportFullPath, string eventName) + { + Tuple> results; + if (string.IsNullOrEmpty(eventName)) + throw new Exception(); + _ReportFullPath = reportFullPath; + DateTime dateTime = DateTime.Now; + results = GetExtractResult(reportFullPath, dateTime); + if (results.Item3 is null) + results = new Tuple>(results.Item1, Array.Empty(), JsonSerializer.Deserialize("[]"), results.Item4); + if (results.Item3.Length > 0 && _IsEAFHosted) + WritePDSF(this, results.Item3); + UpdateLastTicksDuration(DateTime.Now.Ticks - dateTime.Ticks); + return results; + } + + Tuple> IFileRead.ReExtract() + { + Tuple> results; + List headerNames = _Description.GetHeaderNames(); + Dictionary keyValuePairs = _Description.GetDisplayNamesJsonElement(this); + results = ReExtract(this, headerNames, keyValuePairs); + return results; + } + + private Tuple> GetExtractResult(string reportFullPath, DateTime _) + { + Tuple> results; + ProcessDataStandardFormat processDataStandardFormat = ProcessDataStandardFormat.GetProcessDataStandardFormat(reportFullPath); + _Logistics = new Logistics(reportFullPath, processDataStandardFormat); + SetFileParameterLotIDToLogisticsMID(); + JsonElement[] jsonElements = ProcessDataStandardFormat.GetArray(processDataStandardFormat); + List descriptions = GetDuplicatorDescriptions(jsonElements); + Test[] tests = (from l in descriptions select (Test)l.Test).ToArray(); + results = new Tuple>(string.Join(Environment.NewLine, processDataStandardFormat.Logistics), tests, jsonElements, new List()); + return results; + } + +} \ No newline at end of file diff --git a/Adaptation/Shared/Duplicator/Description.cs b/Adaptation/Shared/Duplicator/Description.cs index 964612e..d9bb3b8 100644 --- a/Adaptation/Shared/Duplicator/Description.cs +++ b/Adaptation/Shared/Duplicator/Description.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Linq; using System.Text.Json; +using System.Text.Json.Serialization; namespace Adaptation.Shared.Duplicator; @@ -178,4 +179,16 @@ public class Description : IDescription, Properties.IDescription internal static string GetDateFormat() => "MM/dd/yyyy hh:mm:ss tt"; +} + +[JsonSourceGenerationOptions(WriteIndented = true, NumberHandling = JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString)] +[JsonSerializable(typeof(Description))] +internal partial class SharedDescriptionSourceGenerationContext : JsonSerializerContext +{ +} + +[JsonSourceGenerationOptions(WriteIndented = true, NumberHandling = JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString)] +[JsonSerializable(typeof(Description[]))] +internal partial class SharedDescriptionArraySourceGenerationContext : JsonSerializerContext +{ } \ No newline at end of file diff --git a/Adaptation/Shared/FileRead.cs b/Adaptation/Shared/FileRead.cs index 48aacab..aaa6958 100644 --- a/Adaptation/Shared/FileRead.cs +++ b/Adaptation/Shared/FileRead.cs @@ -9,7 +9,6 @@ using System.IO; using System.Linq; using System.Text; using System.Text.Json; -using System.Text.Json.Serialization; using System.Threading; namespace Adaptation.Shared; @@ -447,12 +446,13 @@ public class FileRead : Properties.IFileRead { List results = new(); Duplicator.Description description; - JsonSerializerOptions jsonSerializerOptions = new() { NumberHandling = JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString }; foreach (JsonElement jsonElement in jsonElements) { if (jsonElement.ValueKind != JsonValueKind.Object) throw new Exception(); - description = JsonSerializer.Deserialize(jsonElement.ToString(), jsonSerializerOptions); + description = JsonSerializer.Deserialize(jsonElement.ToString(), Duplicator.SharedDescriptionSourceGenerationContext.Default.Description); + if (description is null) + continue; results.Add(description); } return results; diff --git a/Adaptation/Shared/ProcessDataStandardFormat.cs b/Adaptation/Shared/ProcessDataStandardFormat.cs index 350b406..9d48a31 100644 --- a/Adaptation/Shared/ProcessDataStandardFormat.cs +++ b/Adaptation/Shared/ProcessDataStandardFormat.cs @@ -578,6 +578,9 @@ internal class ProcessDataStandardFormat results.Add(string.Empty); results.AddRange(processDataStandardFormat.InputPDSF.Footer.Select(l => $"|{l.Replace('\t', '|')}|")); results.Add(string.Empty); + string xml = GetXml(processDataStandardFormat); + results.Add(xml); + results.Add(string.Empty); results.Add("EOF"); results.Add(string.Empty); string json = GetJson(processDataStandardFormat); @@ -651,6 +654,17 @@ internal class ProcessDataStandardFormat return results; } + internal static JsonElement[] GetArray(string reportFullPath, string[] lines, ProcessDataStandardFormat processDataStandardFormat) + { + JsonElement[] results; + string? json = GetRecordsJson(reportFullPath, lines); + if (string.IsNullOrEmpty(json)) + results = GetArray(processDataStandardFormat); + else + results = JsonSerializer.Deserialize(json, JsonElementCollectionSourceGenerationContext.Default.JsonElementArray) ?? throw new Exception(); + return results; + } + internal static string GetPDSFText(IFileRead fileRead, Logistics logistics, JsonElement[] jsonElements, string logisticsText) { string result; @@ -869,6 +883,8 @@ internal class ProcessDataStandardFormat string tag; string value; string[] segments; + List values; + Dictionary> results = new(); ReadOnlyCollection body = processDataStandardFormat.InputPDSF is null ? processDataStandardFormat.Body : processDataStandardFormat.InputPDSF.Body; ReadOnlyCollection columns = processDataStandardFormat.InputPDSF is null ? @@ -876,7 +892,6 @@ internal class ProcessDataStandardFormat List lines = new() { "", "" }; for (int i = 0; i < body.Count; i++) { - lines.Add(" "); segments = body[i].Trim().Split('\t'); if (segments.Length != columns.Count) break; @@ -888,7 +903,42 @@ internal class ProcessDataStandardFormat .Replace("\"", """) .Replace("'", "'"); tag = Regex.Replace(columns[c].Trim('"'), @"[^a-zA-Z0-9]", "_").Split('\r')[0].Split('\n')[0]; - lines.Add(string.Concat(" <", tag, '>', value, "')); + if (i == 0) + { + if (results.ContainsKey(tag)) + continue; + results.Add(tag, new List()); + } + results[tag].Add(value); + } + } + foreach (KeyValuePair> keyValuePair in results) + { + if (body.Count < 2) + break; + if (keyValuePair.Value.Count != body.Count) + continue; + values = keyValuePair.Value.Distinct().ToList(); + if (values.Count == 2 && (string.IsNullOrEmpty(values[0]) || string.IsNullOrEmpty(values[1]))) + { + for (int i = 0; i < body.Count; i++) + keyValuePair.Value[i] = string.Empty; + foreach (string v in values) + { + if (string.IsNullOrEmpty(v)) + continue; + keyValuePair.Value[0] = v; + } + } + } + for (int i = 0; i < body.Count; i++) + { + lines.Add(" "); + foreach (KeyValuePair> keyValuePair in results) + { + if (keyValuePair.Value.Count != body.Count) + continue; + lines.Add(string.Concat(" <", keyValuePair.Key, '>', keyValuePair.Value[i], "')); } lines.Add(" "); } @@ -897,6 +947,46 @@ internal class ProcessDataStandardFormat return result; } + internal static string GetXml(string reportFullPath, string[]? lines = null) + { + string result; + bool foundXml = false; + List results = new(); + lines ??= File.ReadAllLines(reportFullPath); + foreach (string line in lines) + { + if (line.StartsWith(" results = new(); + lines ??= File.ReadAllLines(reportFullPath); + foreach (string line in lines) + { + if (line.StartsWith("\"Records\"")) + foundRecords = true; + if (!foundRecords) + continue; + if (line == "],") + break; + results.Add(line); + } + result = results.Count == 0 ? null : $"{string.Join(Environment.NewLine, results.Skip(1))}{Environment.NewLine}]"; + return result; + } + } [JsonSourceGenerationOptions(WriteIndented = true)] diff --git a/Adaptation/Shared/ProcessDataStandardFormatMapping.cs b/Adaptation/Shared/ProcessDataStandardFormatMapping.cs index c5a75ec..f99db0a 100644 --- a/Adaptation/Shared/ProcessDataStandardFormatMapping.cs +++ b/Adaptation/Shared/ProcessDataStandardFormatMapping.cs @@ -1,33 +1,34 @@ using System.Collections.ObjectModel; +using System.Linq; namespace Adaptation.Shared; public class ProcessDataStandardFormatMapping { - public ReadOnlyCollection BackfillColumns { get; private set; } public ReadOnlyCollection ColumnIndices { get; private set; } - public ReadOnlyCollection IgnoreColumns { get; private set; } - public ReadOnlyCollection IndexOnlyColumns { get; private set; } - public ReadOnlyDictionary KeyValuePairs { get; private set; } public ReadOnlyCollection NewColumnNames { get; private set; } public ReadOnlyCollection OldColumnNames { get; private set; } - public ProcessDataStandardFormatMapping(ReadOnlyCollection backfillColumns, - ReadOnlyCollection columnIndices, - ReadOnlyCollection ignoreColumns, - ReadOnlyCollection indexOnlyColumns, - ReadOnlyDictionary keyValuePairs, + public ProcessDataStandardFormatMapping(ReadOnlyCollection columnIndices, ReadOnlyCollection newColumnNames, ReadOnlyCollection oldColumnNames) { - BackfillColumns = backfillColumns; ColumnIndices = columnIndices; - IgnoreColumns = ignoreColumns; - IndexOnlyColumns = indexOnlyColumns; - KeyValuePairs = keyValuePairs; NewColumnNames = newColumnNames; OldColumnNames = oldColumnNames; } + internal static ProcessDataStandardFormatMapping Get(string processDataStandardFormatMappingOldColumnNames, string processDataStandardFormatMappingNewColumnNames, string processDataStandardFormatMappingColumnIndices) + { + ProcessDataStandardFormatMapping result; + ReadOnlyCollection newColumnNames = new(processDataStandardFormatMappingNewColumnNames.Split(',')); + ReadOnlyCollection oldColumnNames = new(processDataStandardFormatMappingOldColumnNames.Split(',')); + ReadOnlyCollection columnIndices = new(processDataStandardFormatMappingColumnIndices.Split(',').Select(int.Parse).ToArray()); + result = new(columnIndices: columnIndices, + newColumnNames: newColumnNames, + oldColumnNames: oldColumnNames); + return result; + } + } \ No newline at end of file diff --git a/DEP08CEPIEPSILON.csproj b/DEP08CEPIEPSILON.csproj index ece16f3..6e2187a 100644 --- a/DEP08CEPIEPSILON.csproj +++ b/DEP08CEPIEPSILON.csproj @@ -108,6 +108,7 @@ +