using Adaptation.Eaf.Management.ConfigurationData.CellAutomation; using Adaptation.Ifx.Eaf.EquipmentConnector.File.Configuration; using Adaptation.Shared; using Adaptation.Shared.Duplicator; using Adaptation.Shared.Methods; using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.IO; using System.Linq; using System.Text; using System.Text.Json; using System.Threading; namespace Adaptation.FileHandlers.MoveMatchingFiles; #nullable enable public class FileRead : Shared.FileRead, IFileRead { internal class PreWith { internal string ErrFile { get; private set; } internal string CheckFile { get; private set; } internal string MatchingFile { get; private set; } internal string CheckDirectory { get; private set; } internal string NoWaitDirectory { get; private set; } internal PreWith(string checkDirectory, string checkFile, string errFile, string matchingFile, string noWaitDirectory) { ErrFile = errFile; CheckFile = checkFile; MatchingFile = matchingFile; CheckDirectory = checkDirectory; NoWaitDirectory = noWaitDirectory; } } internal class Pre { internal string MatchingFile { get; private set; } internal string CheckFile { get; private set; } internal Pre(string matchingFile, string checkFile) { MatchingFile = matchingFile; CheckFile = checkFile; } } internal class Post { internal string ErrFile { get; private set; } internal string CheckFile { get; private set; } internal Post(string checkFile, string errFile) { ErrFile = errFile; CheckFile = checkFile; } } private readonly ProcessDataStandardFormatMapping _ProcessDataStandardFormatMapping; public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) : base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null) { _MinFileLength = 10; _NullData = string.Empty; _Logistics = new(this); if (_FileParameter is null) throw new Exception(cellInstanceConnectionName); if (_ModelObjectParameterDefinitions is null) throw new Exception(cellInstanceConnectionName); if (!_IsDuplicator) throw new Exception(cellInstanceConnectionName); string processDataStandardFormatMappingOldColumnNames = GetPropertyValue(cellInstanceConnectionName, modelObjectParameters, "Process.Data.Standard.Format.Mapping.Old.Column.Names"); string processDataStandardFormatMappingNewColumnNames = GetPropertyValue(cellInstanceConnectionName, modelObjectParameters, "Process.Data.Standard.Format.Mapping.New.Column.Names"); string processDataStandardFormatMappingColumnIndices = GetPropertyValue(cellInstanceConnectionName, modelObjectParameters, "Process.Data.Standard.Format.Mapping.Column.Indices"); _ProcessDataStandardFormatMapping = GetProcessDataStandardFormatMapping(processDataStandardFormatMappingOldColumnNames, processDataStandardFormatMappingNewColumnNames, processDataStandardFormatMappingColumnIndices); } void IFileRead.Move(Tuple<string, Test[], JsonElement[], List<FileInfo>> extractResults, Exception exception) { bool isErrorFile = exception is not null; if (!isErrorFile && !string.IsNullOrEmpty(_Logistics.ReportFullPath)) { FileInfo fileInfo = new(_Logistics.ReportFullPath); if (fileInfo.Exists && fileInfo.LastWriteTime < fileInfo.CreationTime) File.SetLastWriteTime(_Logistics.ReportFullPath, fileInfo.CreationTime); } Move(extractResults); } void IFileRead.WaitForThread() => WaitForThread(thread: null, threadExceptions: null); string IFileRead.GetEventDescription() { string result = _Description.GetEventDescription(); return result; } List<string> IFileRead.GetHeaderNames() { List<string> results = _Description.GetHeaderNames(); return results; } string[] IFileRead.Move(Tuple<string, Test[], JsonElement[], List<FileInfo>> extractResults, string to, string from, string resolvedFileLocation, Exception exception) { string[] results = Move(extractResults, to, from, resolvedFileLocation, exception); return results; } JsonProperty[] IFileRead.GetDefault() { JsonProperty[] results = _Description.GetDefault(this, _Logistics); return results; } Dictionary<string, string> IFileRead.GetDisplayNamesJsonElement() { Dictionary<string, string> results = _Description.GetDisplayNamesJsonElement(this); return results; } List<IDescription> IFileRead.GetDescriptions(IFileRead fileRead, List<Test> tests, IProcessData processData) { List<IDescription> results = _Description.GetDescriptions(fileRead, _Logistics, tests, processData); return results; } Tuple<string, Test[], JsonElement[], List<FileInfo>> IFileRead.GetExtractResult(string reportFullPath, string eventName) { Tuple<string, Test[], JsonElement[], List<FileInfo>> results; if (string.IsNullOrEmpty(eventName)) throw new Exception(); _ReportFullPath = reportFullPath; DateTime dateTime = DateTime.Now; results = GetExtractResult(reportFullPath, dateTime); if (results.Item3 is null) results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(results.Item1, Array.Empty<Test>(), JsonSerializer.Deserialize<JsonElement[]>("[]") ?? throw new Exception(), results.Item4); if (results.Item3.Length > 0 && _IsEAFHosted) WritePDSF(this, results.Item3); UpdateLastTicksDuration(DateTime.Now.Ticks - dateTime.Ticks); return results; } Tuple<string, Test[], JsonElement[], List<FileInfo>> IFileRead.ReExtract() { Tuple<string, Test[], JsonElement[], List<FileInfo>> results; List<string> headerNames = _Description.GetHeaderNames(); Dictionary<string, string> keyValuePairs = _Description.GetDisplayNamesJsonElement(this); results = ReExtract(this, headerNames, keyValuePairs); return results; } private static ProcessDataStandardFormatMapping GetProcessDataStandardFormatMapping(string processDataStandardFormatMappingOldColumnNames, string processDataStandardFormatMappingNewColumnNames, string processDataStandardFormatMappingColumnIndices) { ProcessDataStandardFormatMapping result; string[] segmentsB; List<string> distinct = new(); Dictionary<string, string> keyValuePairs = new(); string args4 = "Time,HeaderUniqueId,UniqueId,Id,Recipe,Date,AreaDeltaFromLastRun,GLimit"; string args5 = "Nine10mmEdgeMean,Nine4mmEdgeMean,NineCriticalPointsAverage,NineCriticalPointsPhaseAngleAverage,NineCriticalPointsStdDev,NineEdgeMeanDelta,NineMean,NineResRangePercent,AreaDeltaFromLastRun,Variation,Percentage HgCV 4PP Delta"; string args6 = "RhoAvg01,RhoAvg02,RhoAvg03,RhoAvg04,RhoAvg05,RhoAvg06,RhoAvg07,RhoAvg08,RhoAvg09"; string args7 = "Test|EventId,FlatZMean|MeanFlatZ,GradeMean|MeanGrade,NAvgMean|MeanNAvg,NslMean|MeanNsl,PhaseMean|MeanPhase,RhoAvgMean|MeanRhoAvg,RhoslMean|MeanRhosl,RsMean|MeanRs,VdMean|MeanVd,FlatZRadialGradient|RadialGradientFlatZ,GradeRadialGradient|RadialGradientGrade,NAvgRadialGradient|RadialGradientNAvg,NslRadialGradient|RadialGradientNsl,PhaseRadialGradient|RadialGradientPhase,RhoAvgRadialGradient|RadialGradientRhoAvg,RhoslRadialGradient|RadialGradientRhosl,RsRadialGradient|RadialGradientRs,VdRadialGradient|RadialGradientVd,FlatZStdDev|StandardDeviationPercentageFlatZ,GradeStdDev|StandardDeviationPercentageGrade,NAvgStdDev|StandardDeviationPercentageNAvg,NslStdDev|StandardDeviationPercentageNsl,PhaseStdDev|StandardDeviationPercentagePhase,RhoAvgStdDev|StandardDeviationPercentageRhoAvg,RhoslStdDev|StandardDeviationPercentageRhosl,RsStdDev|StandardDeviationPercentageRs,VdStdDev|StandardDeviationPercentageVd"; // string args8 = "Time,A_LOGISTICS,B_LOGISTICS,Test,Count,Index,MesEntity,Date,Employee,Lot,PSN,Reactor,Recipe,Area,Folder,HeaderUniqueId,Id,Layer,Model,Pattern,Phase,Plan,RampRate,RDS,SetupFile,StartVoltage,StopVoltage,UniqueId,Wafer,WaferSize,Zone,Ccomp,CondType,FlatZ,FlatZMean,FlatZRadialGradient,FlatZStdDev,GLimit,Grade,GradeMean,GradeRadialGradient,GradeStdDev,NAvg,NAvgMean,NAvgRadialGradient,NAvgStdDev,Nsl,NslMean,NslRadialGradient,NslStdDev,PhaseMean,PhaseRadialGradient,PhaseStdDev,RhoAvg,RhoAvgMean,RhoAvgRadialGradient,RhoAvgStdDev,RhoMethod,Rhosl,RhoslMean,RhoslRadialGradient,RhoslStdDev,RsMean,RsRadialGradient,RsStdDev,Vd,VdMean,VdRadialGradient,VdStdDev,Variation,AreaDeltaFromLastRun,Nine10mmEdgeMean,Nine4mmEdgeMean,NineCriticalPointsAverage,NineCriticalPointsPhaseAngleAverage,NineCriticalPointsStdDev,NineEdgeMeanDelta,NineMean,NineResRangePercent,RhoAvg01,RhoAvg02,RhoAvg03,RhoAvg04,RhoAvg05,RhoAvg06,RhoAvg07,RhoAvg08,RhoAvg09"; // string args9 = "Time,A_LOGISTICS,B_LOGISTICS,Count,Sequence,MesEntity,Index,Operator,StartVoltage,Wafer,StopVoltage,Lot,RampRate,Plan,GLimit,Date,Time,SetupFile,WaferSize,Folder,Ccomp,Pattern,Area,CondType,RhoMethod,Model,MeanNAvg,MeanNsl,MeanVd,MeanFlatZ,MeanRhoAvg,MeanRhosl,MeanPhase,MeanGrade,MeanRs,StandardDeviationPercentageNAvg,StandardDeviationPercentageNsl,StandardDeviationPercentageVd,StandardDeviationPercentageFlatZ,StandardDeviationPercentageRhoAvg,StandardDeviationPercentageRhosl,StandardDeviationPercentagePhase,StandardDeviationPercentageGrade,StandardDeviationPercentageRs,RadialGradientNAvg,RadialGradientNsl,RadialGradientVd,RadialGradientFlatZ,RadialGradientRhoAvg,RadialGradientRhosl,RadialGradientPhase,RadialGradientGrade,RadialGradientRs,Site,X,Y,NAvg,RhoAvg,Nsl,Rhosl,Vd,Phase,FlatZ,Grade,XLeft,XRight,BottomY,TopY,RDS,PSN,Reactor,Layer,Zone,Employee,InferredLot,Nine10mmEdgeMean,Nine4mmEdgeMean,NineCriticalPointsAverage,NineCriticalPointsPhaseAngleAverage,NineCriticalPointsStdDev,NineEdgeMeanDelta,NineMean,NineResRangePercent,AreaDeltaFromLastRun,Variation,Percentage HgCV 4PP Delta,RhoAvg01,RhoAvg02,RhoAvg03,RhoAvg04,RhoAvg05,RhoAvg06,RhoAvg07,RhoAvg08,RhoAvg09,EventId"; // string args10 = "0,1,2,95,3,6,5,15,73,11,69,70,-1,22,19,-1,-1,71,25,21,61,13,12,68,17,8,10,-1,9,18,72,20,23,62,29,47,38,14,63,33,51,42,56,26,44,35,58,27,45,36,32,50,41,57,30,48,39,24,59,31,49,40,34,52,43,60,28,46,37,84,83,75,76,77,78,79,80,81,82,86,87,88,89,90,91,92,93,94"; string[] segments = args7.Split(','); ReadOnlyCollection<string> ignoreColumns = new(args4.Split(',')); ReadOnlyCollection<string> backfillColumns = new(args5.Split(',')); ReadOnlyCollection<string> indexOnlyColumns = new(args6.Split(',')); ReadOnlyCollection<string> newColumnNames = new(processDataStandardFormatMappingNewColumnNames.Split(',')); ReadOnlyCollection<string> oldColumnNames = new(processDataStandardFormatMappingOldColumnNames.Split(',')); ReadOnlyCollection<int> columnIndices = new(processDataStandardFormatMappingColumnIndices.Split(',').Select(int.Parse).ToArray()); foreach (string segment in segments) { segmentsB = segment.Split('|'); if (segmentsB.Length != 2) continue; if (distinct.Contains(segmentsB[0])) continue; distinct.Add(segmentsB[0]); keyValuePairs.Add(segmentsB[0], segmentsB[1]); } result = new(backfillColumns: backfillColumns, columnIndices: columnIndices, newColumnNames: newColumnNames, ignoreColumns: ignoreColumns, indexOnlyColumns: indexOnlyColumns, keyValuePairs: new(keyValuePairs), oldColumnNames: oldColumnNames); return result; } private static ReadOnlyCollection<PreWith> GetPreWithCollection(ReadOnlyCollection<Pre> preCollection) { List<PreWith> results = new(); string errFile; PreWith preWith; string? checkDirectory; string noWaitDirectory; foreach (Pre pre in preCollection) { errFile = string.Concat(pre.CheckFile, ".err"); checkDirectory = Path.GetDirectoryName(pre.CheckFile); if (string.IsNullOrEmpty(checkDirectory)) continue; if (!Directory.Exists(checkDirectory)) _ = Directory.CreateDirectory(checkDirectory); noWaitDirectory = Path.Combine(checkDirectory, "NoWaitDirectory"); preWith = new(checkDirectory: checkDirectory, checkFile: pre.CheckFile, errFile: errFile, matchingFile: pre.MatchingFile, noWaitDirectory: noWaitDirectory); results.Add(preWith); } return results.AsReadOnly(); } private static ReadOnlyCollection<string> GetSearchDirectories(int numberLength, string parentDirectory) { List<string> results = new(); string[] directories = Directory.GetDirectories(parentDirectory, "*", SearchOption.TopDirectoryOnly); foreach (string directory in directories) { if (Path.GetFileName(directory).Length != numberLength) continue; results.Add(directory); } results.Sort(); return results.AsReadOnly(); } private static void CreatePointerFile(int numberLength, string parentDirectory, ReadOnlyCollection<string> matchingFiles) { string checkFile; string writeFile; string? directoryName; int parentDirectoryLength = parentDirectory.Length; foreach (string matchingFile in matchingFiles) { directoryName = Path.GetDirectoryName(matchingFile); if (directoryName is null) continue; checkFile = $"{matchingFile[0]}{directoryName.Substring(parentDirectoryLength + numberLength + 1)}"; writeFile = Path.Combine(parentDirectory, $"{directoryName.Substring(parentDirectory.Length + 1, numberLength)}.txt"); if (File.Exists(writeFile)) continue; File.AppendAllLines(writeFile, new string[] { parentDirectory, matchingFile, directoryName, checkFile }); } } private static ReadOnlyCollection<Pre> GetPreCollection(int numberLength, string parentDirectory, ReadOnlyCollection<string> matchingFiles) { List<Pre> results = new(); Pre pre; string checkFile; int parentDirectoryLength = parentDirectory.Length; foreach (string matchingFile in matchingFiles) { checkFile = $"{matchingFile[0]}{matchingFile.Substring(parentDirectoryLength + numberLength + 1)}"; pre = new(matchingFile, checkFile); results.Add(pre); } return results.AsReadOnly(); } private void MoveCollection(DateTime dateTime, ProcessDataStandardFormat processDataStandardFormat, ReadOnlyCollection<PreWith> preWithCollection) { ReadOnlyCollection<Post> postCollection = GetPostCollection(dateTime, processDataStandardFormat, preWithCollection); if (postCollection.Count != 0) { Thread.Sleep(500); StringBuilder stringBuilder = new(); foreach (Post post in postCollection) { if (File.Exists(post.ErrFile)) _ = stringBuilder.AppendLine(File.ReadAllText(post.ErrFile)); if (File.Exists(post.CheckFile)) _ = stringBuilder.AppendLine($"<{post.CheckFile}> was not consumed by the end!"); } if (stringBuilder.Length > 0) throw new Exception(stringBuilder.ToString()); } } private ReadOnlyCollection<Post> GetPostCollection(DateTime dateTime, ProcessDataStandardFormat processDataStandardFormat, ReadOnlyCollection<PreWith> preWithCollection) { List<Post> results = new(); Post post; long preWait; foreach (PreWith preWith in preWithCollection) { if (!_IsEAFHosted) continue; if (!_StaticRuns.TryGetValue(_Logistics.Sequence, out List<Shared.Metrology.WS.Results>? wsResults)) wsResults = null; ProcessDataStandardFormat.Write(preWith.CheckFile, processDataStandardFormat, wsResults); File.Delete(preWith.MatchingFile); if (Directory.Exists(preWith.NoWaitDirectory)) { post = new(preWith.CheckFile, preWith.ErrFile); results.Add(post); continue; } if (_FileConnectorConfiguration?.FileHandleWaitTime is null) preWait = DateTime.Now.AddMilliseconds(1234).Ticks; else preWait = DateTime.Now.AddMilliseconds(_FileConnectorConfiguration.FileHandleWaitTime.Value).Ticks; for (short i = 0; i < short.MaxValue; i++) { if (DateTime.Now.Ticks > preWait) break; Thread.Sleep(500); } for (int i = 0; i < int.MaxValue; i++) { if (File.Exists(preWith.ErrFile)) throw new Exception(File.ReadAllText(preWith.ErrFile)); if (!File.Exists(preWith.CheckFile)) break; if (new TimeSpan(DateTime.Now.Ticks - dateTime.Ticks).TotalSeconds > _BreakAfterSeconds) throw new Exception($"Not all files were consumed after {_BreakAfterSeconds} second(s)!"); Thread.Sleep(500); } } return results.AsReadOnly(); } private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime dateTime) { Tuple<string, Test[], JsonElement[], List<FileInfo>> results = new(string.Empty, Array.Empty<Test>(), Array.Empty<JsonElement>(), new List<FileInfo>()); ProcessDataStandardFormat processDataStandardFormat = ProcessDataStandardFormat.GetProcessDataStandardFormat(reportFullPath, _ProcessDataStandardFormatMapping); _Logistics = new Logistics(reportFullPath, processDataStandardFormat); if (!_IsEAFHosted) ProcessDataStandardFormat.Write("../../.pdsf", processDataStandardFormat, wsResults: null); SetFileParameterLotIDToLogisticsMID(); int numberLength = 2; long ticks = dateTime.Ticks; string parentParentDirectory = GetParentParent(reportFullPath); ReadOnlyCollection<string> searchDirectories = GetSearchDirectories(numberLength, parentParentDirectory); ReadOnlyCollection<string> matchingFiles = GetMatchingFiles(ticks, reportFullPath, searchDirectories); if (matchingFiles.Count != searchDirectories.Count) throw new Exception($"Didn't find all files after {_BreakAfterSeconds} second(s)!"); if (_IsEAFHosted) { try { CreatePointerFile(numberLength, parentParentDirectory, matchingFiles); } catch (Exception) { } } ReadOnlyCollection<Pre> preCollection = GetPreCollection(numberLength, parentParentDirectory, matchingFiles); ReadOnlyCollection<PreWith> preWithCollection = GetPreWithCollection(preCollection); MoveCollection(dateTime, processDataStandardFormat, preWithCollection); return results; } private ReadOnlyCollection<string> GetMatchingFiles(long ticks, string reportFullPath, ReadOnlyCollection<string> searchDirectories) { List<string> results = new(); string[] found; string fileName = Path.GetFileName(reportFullPath); foreach (string searchDirectory in searchDirectories) { for (int i = 0; i < int.MaxValue; i++) { found = Directory.GetFiles(searchDirectory, fileName, SearchOption.AllDirectories); if (found.Length != 0) { results.AddRange(found); break; } if (new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds > _BreakAfterSeconds) break; } } return results.AsReadOnly(); } }