using Adaptation.Shared; using Adaptation.Shared.Methods; using log4net; using System; using System.Collections.Generic; using System.Data; using System.Diagnostics; using System.IO; using System.Linq; using System.Text.Json; using System.Text.Json.Serialization; using System.Text.RegularExpressions; namespace Adaptation.FileHandlers.pcl; public class ProcessData : IProcessData { private int _I; private string _Data; private readonly ILog _Log; private readonly List _Details; public string JobID { get; set; } public string MesEntity { get; set; } public string Area { get; set; } public string Ccomp { get; set; } public string CondType { get; set; } public DateTime Date { get; set; } public string Employee { get; set; } public string FlatZMean { get; set; } public string FlatZRadialGradient { get; set; } public string FlatZStdDev { get; set; } public string Folder { get; set; } public string GLimit { get; set; } public string GradeMean { get; set; } public string GradeRadialGradient { get; set; } public string GradeStdDev { get; set; } public string Layer { get; set; } public string Lot { get; set; } public string Model { get; set; } public string NAvgMean { get; set; } public string NAvgRadialGradient { get; set; } public string NAvgStdDev { get; set; } public string NslMean { get; set; } public string NslRadialGradient { get; set; } public string NslStdDev { get; set; } public string PSN { get; set; } public string Pattern { get; set; } public string PhaseMean { get; set; } public string PhaseRadialGradient { get; set; } public string PhaseStdDev { get; set; } public string Plan { get; set; } public string RDS { get; set; } public string RampRate { get; set; } public string Reactor { get; set; } public string RhoAvgMean { get; set; } public string RhoAvgRadialGradient { get; set; } public string RhoAvgStdDev { get; set; } public string RhoMethod { get; set; } public string RhoslMean { get; set; } public string RhoslRadialGradient { get; set; } public string RhoslStdDev { get; set; } public string RsMean { get; set; } public string RsRadialGradient { get; set; } public string RsStdDev { get; set; } public string SetupFile { get; set; } public string StartVoltage { get; set; } public string StopVoltage { get; set; } public string UniqueId { get; set; } public string VdMean { get; set; } public string VdRadialGradient { get; set; } public string VdStdDev { get; set; } public string Wafer { get; set; } public string WaferSize { get; set; } public string Zone { get; set; } List Shared.Properties.IProcessData.Details => _Details; public ProcessData(IFileRead fileRead, Logistics logistics, List fileInfoCollection, string ghostPCLFileName, string pdfTextStripperFileName) { fileInfoCollection.Clear(); _Details = new List(); _I = 0; _Data = string.Empty; JobID = logistics.JobID; Date = GetDateTime(logistics); MesEntity = logistics.MesEntity; _Log = LogManager.GetLogger(typeof(ProcessData)); Parse(fileRead, logistics, fileInfoCollection, ghostPCLFileName, pdfTextStripperFileName); } private static DateTime GetDateTime(Logistics logistics) => logistics.DateTimeFromSequence; string IProcessData.GetCurrentReactor(IFileRead fileRead, Logistics logistics, Dictionary reactors) => throw new Exception(string.Concat("See ", nameof(Parse))); Tuple> IProcessData.GetResults(IFileRead fileRead, Logistics logistics, List fileInfoCollection) { Tuple> results; List tests = new(); foreach (object item in _Details) tests.Add(Test.HgCV); List descriptions = fileRead.GetDescriptions(fileRead, tests, this); if (tests.Count != descriptions.Count) throw new Exception(); for (int i = 0; i < tests.Count; i++) { if (descriptions[i] is not Description description) throw new Exception(); if (description.Test != (int)tests[i]) throw new Exception(); } List fileReadDescriptions = (from l in descriptions select (Description)l).ToList(); string json = JsonSerializer.Serialize(fileReadDescriptions, fileReadDescriptions.GetType()); JsonElement[] jsonElements = JsonSerializer.Deserialize(json); results = new Tuple>(logistics.Logistics1[0], tests.ToArray(), jsonElements, fileInfoCollection); return results; } /// /// Convert the raw data file to parsable file format - in this case from PCL to PDF /// /// source file to be converted to PDF /// private static string ConvertSourceFileToPdf(Logistics logistics, string ghostPCLFileName) { string result = Path.ChangeExtension(logistics.ReportFullPath, ".pdf"); if (!File.Exists(result)) { //string arguments = string.Concat("-i \"", sourceFile, "\" -o \"", result, "\""); string arguments = string.Concat("-dSAFER -dBATCH -dNOPAUSE -dFIXEDMEDIA -dFitPage -dAutoRotatePages=/All -dDEVICEWIDTHPOINTS=792 -dDEVICEHEIGHTPOINTS=612 -sOutputFile=\"", result, "\" -sDEVICE=pdfwrite \"", logistics.ReportFullPath, "\""); //Process process = Process.Start(configData.LincPDFCFileName, arguments); Process process = Process.Start(ghostPCLFileName, arguments); _ = process.WaitForExit(30000); if (!File.Exists(result)) throw new Exception("PDF file wasn't created"); } return result; } private void ScanPast(string text) { int num = _Data.IndexOf(text, _I); if (num > -1) { _I = num + text.Length; } else { _I = _Data.Length; } } private string GetBefore(string text) { int num = _Data.IndexOf(text, _I); string text2; if (num > -1) { text2 = _Data.Substring(_I, num - _I); _I = num + text.Length; return text2.Trim(); } text2 = _Data.Substring(_I); _I = _Data.Length; return text2.Trim(); } private static bool IsNullOrWhiteSpace(string text) { for (int i = 0; i < text.Length; i++) { if (!char.IsWhiteSpace(text[i])) { return false; } } return true; } private bool IsBlankLine() { int num = _Data.IndexOf("\n", _I); return IsNullOrWhiteSpace((num > -1) ? _Data.Substring(_I, num - _I) : _Data.Substring(_I)); } private string GetToEOL() => GetBefore("\n"); private string GetToken() { while (_I < _Data.Length && IsNullOrWhiteSpace(_Data.Substring(_I, 1))) { _I++; } int j; for (j = _I; j < _Data.Length && !IsNullOrWhiteSpace(_Data.Substring(j, 1)); j++) { } string text = _Data.Substring(_I, j - _I); _I = j; return text.Trim(); } private static string GetTextFromPDF(string pdfTextStripperFileName, string sourceFileNamePdf, string altHeaderFileName) { string result; ProcessStartInfo processStartInfo = new(pdfTextStripperFileName, $"s \"{sourceFileNamePdf}\"") { UseShellExecute = false, RedirectStandardError = true, RedirectStandardOutput = true, }; Process process = Process.Start(processStartInfo); _ = process.WaitForExit(30000); if (!File.Exists(altHeaderFileName)) result = string.Empty; else result = File.ReadAllText(altHeaderFileName); return result; } private static (string, string) GetReactorAndRDS(string defaultReactor, string defaultRDS, string text, string formattedText, string[] segments) { string rds; string reactor; if (string.IsNullOrEmpty(text) || segments.Length == 0 || string.IsNullOrEmpty(formattedText)) reactor = defaultReactor; else reactor = segments[0]; if (segments.Length <= 1 || !int.TryParse(segments[1], out int rdsValue) || rdsValue < 99) rds = defaultRDS; else rds = segments[1]; if (reactor.Length > 3) { rds = reactor; reactor = defaultReactor; } return new(reactor, rds); } private static (string, string) GetLayerAndPSN(string defaultLayer, string defaultPSN, string[] segments) { string psn; string layer; if (segments.Length <= 2) { psn = defaultPSN; layer = defaultLayer; } else { string[] segmentsB = segments[2].Split('.'); psn = segmentsB[0]; if (segmentsB.Length <= 1) layer = defaultLayer; else { layer = segmentsB[1]; if (layer.Length > 1 && layer[0] == '0') layer = layer.Substring(1); } } return (layer, psn); } private static string GetZone(string[] segments) { string result; if (segments.Length <= 3) result = string.Empty; else { result = segments[3]; if (result.Length > 1 && result[0] == '0') result = result.Substring(1); } return result; } public static Descriptor GetDescriptor(string text) { Descriptor result; string lot; string psn; string rds; string zone; string layer; string reactor; string employee; string defaultPSN = string.Empty; string defaultRDS = string.Empty; string defaultZone = string.Empty; string defaultLayer = string.Empty; string defaultReactor = string.Empty; string defaultEmployee = string.Empty; if (Regex.IsMatch(text, @"^[a-zA-z][0-9]{2,4}$")) { lot = text.ToUpper(); psn = defaultPSN; rds = defaultRDS; zone = defaultZone; layer = defaultLayer; reactor = defaultReactor; employee = defaultEmployee; } else if (string.IsNullOrEmpty(text) || (text.Length is 2 or 3 && Regex.IsMatch(text, "^[a-zA-z]{2,3}"))) { lot = text; employee = lot; psn = defaultPSN; rds = defaultRDS; zone = defaultZone; layer = defaultLayer; reactor = defaultReactor; } else if (Regex.IsMatch(text, @"^[0-9]{2}[.][0-9]{1}[.]?[0-9]{0,1}")) { string[] segments = text.Split('.'); lot = text; psn = defaultPSN; rds = defaultRDS; layer = segments[1]; reactor = segments[0]; employee = defaultEmployee; if (segments.Length <= 2) zone = defaultZone; else zone = segments[2]; } else { // Remove illegal characters \/:*?"<>| found in the Lot. lot = Regex.Replace(text, @"[\\,\/,\:,\*,\?,\"",\<,\>,\|]", "_").Split('\r')[0].Split('\n')[0]; if (lot.Length > 2 && lot[0] == '1' && (lot[1] == 'T' || lot[1] == 't')) lot = lot.Substring(2); string[] segments = lot.Split('-'); // bool hasRDS = Regex.IsMatch(lot, "[-]?([QP][0-9]{4,}|[0-9]{5,})[-]?"); (reactor, rds) = GetReactorAndRDS(defaultReactor, defaultRDS, text, lot, segments); (layer, psn) = GetLayerAndPSN(defaultLayer, defaultPSN, segments); zone = GetZone(segments); if (segments.Length <= 4) employee = defaultEmployee; else employee = segments[4]; } result = new(employee, layer, lot, psn, rds, reactor, zone); return result; } private void Set(Logistics logistics, string headerText) { string lot; string rds; string psn; string zone; string layer; string reactor; string employee; ScanPast("Lot :"); if (headerText.Contains("Ramp Rate :")) lot = GetBefore("Ramp Rate :"); else if (headerText.Contains("Forward Rate :")) lot = GetBefore("Forward Rate :"); else if (headerText.Contains("Conduct Type:")) lot = GetBefore("Conduct Type:"); else lot = string.Empty; Descriptor descriptor = GetDescriptor(lot); lot = descriptor.Lot; psn = descriptor.PSN; rds = descriptor.RDS; zone = descriptor.Zone; layer = descriptor.Layer; reactor = descriptor.Reactor; employee = descriptor.Employee; Lot = lot; PSN = psn; RDS = rds; Zone = zone; Layer = layer; Reactor = reactor; Employee = employee; UniqueId = string.Format("{0}_{1}_{2}", logistics.JobID, lot, Path.GetFileNameWithoutExtension(logistics.ReportFullPath)); } #pragma warning disable IDE0060 private void Parse(IFileRead fileRead, Logistics logistics, List fileInfoCollection, string ghostPCLFileName, string pdfTextStripperFileName) #pragma warning restore IDE0060 { string headerText; string sourceFileNamePdf = ConvertSourceFileToPdf(logistics, ghostPCLFileName); fileInfoCollection.Add(new FileInfo(sourceFileNamePdf)); string altHeaderFileName = Path.ChangeExtension(logistics.ReportFullPath, ".txt"); if (File.Exists(altHeaderFileName)) { headerText = File.ReadAllText(altHeaderFileName); fileInfoCollection.Add(new FileInfo(altHeaderFileName)); } else { try { //Pdfbox, IKVM.AWT.WinForms org.apache.pdfbox.pdmodel.PDDocument pdfDocument = org.apache.pdfbox.pdmodel.PDDocument.load(sourceFileNamePdf); org.apache.pdfbox.util.PDFTextStripper stripper = new(); headerText = stripper.getText(pdfDocument); pdfDocument.close(); File.AppendAllText(altHeaderFileName, headerText); fileInfoCollection.Add(new FileInfo(altHeaderFileName)); } catch (Exception) { if (!File.Exists(pdfTextStripperFileName)) throw; headerText = GetTextFromPDF(pdfTextStripperFileName, sourceFileNamePdf, altHeaderFileName); if (string.IsNullOrEmpty(headerText)) throw; fileInfoCollection.Add(new FileInfo(altHeaderFileName)); } } if (headerText.Contains("G A T E V O L T A G E")) throw new Exception("Ignore: GATEVOLTAGE runs are not parsed."); if (!string.IsNullOrEmpty(headerText)) { headerText = headerText.Replace("box", ""); headerText = headerText.Replace("bar", ""); headerText = headerText.Replace("horiz", ""); headerText = headerText.Replace("center", ""); headerText = headerText.Replace("upper", ""); headerText = headerText.Replace("lower", ""); headerText = headerText.Replace("right", ""); headerText = headerText.Replace("left", ""); headerText = headerText.Replace("thin", ""); headerText = headerText.Replace("vertical", ""); headerText = headerText.Replace("line", ""); headerText = headerText.Replace("middle", ""); headerText = headerText.Replace("side", ""); headerText = headerText.Replace("top", ""); // This will change "Stop Voltage" to "S Voltage" headerText = headerText.Replace("corner", ""); headerText = headerText.Replace("bottom", ""); headerText = headerText.Replace("ruleunder", "_"); headerText = headerText.Replace("@", ""); headerText = headerText.Replace("*", ""); _I = 0; _Data = headerText; _Log.Debug($"****MERCURY-DATA [002]= {headerText}"); ScanPast("Operator:"); _ = GetBefore("Start Voltage:"); StartVoltage = GetBefore("V"); ScanPast("Wafer :"); Wafer = GetBefore("S Voltage :"); // This is actually "Stop Voltage" StopVoltage = GetBefore("V"); Set(logistics, headerText); RampRate = GetBefore("mV/sec"); ScanPast("Plan :"); Plan = GetBefore("G limit :"); //GLimit = GetBefore("S "); GLimit = GetBefore("S"); ScanPast("Setup File:"); //SetupFile = GetBefore("O O"); SetupFile = GetBefore("O O"); ScanPast("Wafer size :"); WaferSize = GetBefore("mm"); ScanPast("Folder :"); //Folder = GetBefore("N N"); Folder = GetBefore("N N"); ScanPast("Ccomp : "); Ccomp = GetBefore("pF"); ScanPast("Pattern :"); //Pattern = GetBefore("C C"); Pattern = GetBefore("C C"); ScanPast("Area:"); Area = GetBefore("cm2"); ScanPast("Cond Type :"); CondType = GetBefore("Rho Method:"); //RhoMethod = GetBefore("N N"); RhoMethod = GetBefore("N N"); ScanPast("Model :"); //Model = GetBefore("T T"); Model = GetBefore("T T"); ScanPast("Navg :"); NAvgMean = GetToken(); NAvgStdDev = GetToken(); NAvgRadialGradient = GetToken(); ScanPast("Nsl :"); NslMean = GetToken(); NslStdDev = GetToken(); NslRadialGradient = GetToken(); ScanPast("Vd :"); VdMean = GetToken(); VdStdDev = GetToken(); VdRadialGradient = GetToken(); ScanPast("Flat Z:"); FlatZMean = GetToken(); FlatZStdDev = GetToken(); FlatZRadialGradient = GetToken(); ScanPast("Rhoavg:"); RhoAvgMean = GetToken(); RhoAvgStdDev = GetToken(); RhoAvgRadialGradient = GetToken(); ScanPast("Rhosl :"); RhoslMean = GetToken(); RhoslStdDev = GetToken(); RhoslRadialGradient = GetToken(); ScanPast("Phase :"); PhaseMean = GetToken(); PhaseStdDev = GetToken(); PhaseRadialGradient = GetToken(); ScanPast("Grade :"); GradeMean = GetToken(); GradeStdDev = GetToken(); GradeRadialGradient = GetToken(); ScanPast("Rs :"); RsMean = GetToken(); RsStdDev = GetToken(); RsRadialGradient = GetToken(); //ScanPast("Flat Z: Grade : % Flat Z: Grade : % Flat Z: Grade : %"); ScanPast("Flat Z: Grade : % Flat Z: Grade : % Flat Z: Grade : %"); string token = GetToken(); while (!string.IsNullOrEmpty(token)) { Detail hgProbeDetail = new() { NAvg = token }; _ = GetToEOL(); hgProbeDetail.Nsl = GetToken(); _ = GetToEOL(); hgProbeDetail.Vd = GetToken(); _ = GetToEOL(); hgProbeDetail.FlatZ = GetToken(); _ = GetToEOL(); hgProbeDetail.RhoAvg = GetToken(); _ = GetToEOL(); hgProbeDetail.Rhosl = GetToken(); _ = GetToEOL(); hgProbeDetail.Phase = GetToken(); _ = GetToEOL(); hgProbeDetail.Grade = GetToken(); hgProbeDetail.UniqueId = string.Concat("_Point-", _Details.Count + 1); _Details.Add(hgProbeDetail); _ = GetToken(); _ = GetToken(); _ = GetToken(); _ = GetToken(); token = GetToken(); //if (token.Contains("LincPDF") || token.Contains("MULTIPLE")) if (token.Contains("MULTIPLE")) { //ScanPast("Flat Z: Grade : % Flat Z: Grade : % Flat Z: Grade : %"); //ScanPast("Flat Z: Grade : % Flat Z: Grade : % Flat Z: Grade : %"); ScanPast("Flat Z: Grade : % Flat Z: Grade : % Flat Z: Grade : %"); ScanPast("Flat Z: Grade : % Flat Z: Grade : % Flat Z: Grade : %"); token = GetToken(); } } } foreach (Detail detail in _Details.Cast()) { detail.HeaderUniqueId = UniqueId; detail.UniqueId = string.Concat(detail, detail.UniqueId); } fileInfoCollection.Add(logistics.FileInfo); } #nullable enable internal static List GetDescriptions(JsonElement[] jsonElements) { List results = new(); Description? description; JsonSerializerOptions jsonSerializerOptions = new() { NumberHandling = JsonNumberHandling.AllowReadingFromString | JsonNumberHandling.WriteAsString }; foreach (JsonElement jsonElement in jsonElements) { if (jsonElement.ValueKind != JsonValueKind.Object) throw new Exception(); description = JsonSerializer.Deserialize(jsonElement.ToString(), jsonSerializerOptions); if (description is null) continue; results.Add(description); } return results; } }