diff --git a/Adaptation/FileHandlers/pcl/Complete.cs b/Adaptation/FileHandlers/pcl/Complete.cs index 9e2ee68..55983fe 100644 --- a/Adaptation/FileHandlers/pcl/Complete.cs +++ b/Adaptation/FileHandlers/pcl/Complete.cs @@ -1,6 +1,9 @@ +using Adaptation.Shared; using System.Collections.Generic; using System.Collections.ObjectModel; +using System.IO; using System.Linq; +using System.Text.Json; using System.Text.Json.Serialization; namespace Adaptation.FileHandlers.pcl; @@ -19,19 +22,28 @@ internal class Complete public Header Header { get; } public Wafer[] Wafers { get; } - internal static Complete? Get(string headerFileName, Dictionary pages, Constant constant) + internal static Complete? Get(Logistics logistics, List fileInfoCollection, ReadOnlyDictionary pages) { Complete? result; - Header? header = Header.Get(headerFileName, pages, constant); + Constant constant = new(); + string headerFileName = pages.ElementAt(pages.Count - 1).Key; + Header? header = Header.Get(pages, constant, headerFileName); if (header is null) result = null; else { - ReadOnlyCollection wafers = Wafer.Get(headerFileName, pages, constant); + ReadOnlyCollection wafers = Wafer.Get(pages, constant, headerFileName); if (wafers.Count == 0) result = null; else + { result = new(header, wafers.ToArray()); + FileInfo fileInfo = new($"{logistics.ReportFullPath}.json"); + string json = JsonSerializer.Serialize(result, CompleteSourceGenerationContext.Default.Complete); + File.WriteAllText(fileInfo.FullName, json); + File.SetLastWriteTime(fileInfo.FullName, logistics.DateTimeFromSequence); + fileInfoCollection.Add(fileInfo); + } } return result; } diff --git a/Adaptation/FileHandlers/pcl/Constant.cs b/Adaptation/FileHandlers/pcl/Constant.cs index 85ed1cb..c8fa9f2 100644 --- a/Adaptation/FileHandlers/pcl/Constant.cs +++ b/Adaptation/FileHandlers/pcl/Constant.cs @@ -3,6 +3,7 @@ namespace Adaptation.FileHandlers.pcl; internal class Constant { + public string Id { get; } = "ID#"; public string Max { get; } = "Max:"; public string Min { get; } = "Min:"; public string Date { get; } = "Date:"; diff --git a/Adaptation/FileHandlers/pcl/Convert.cs b/Adaptation/FileHandlers/pcl/Convert.cs new file mode 100644 index 0000000..6368aec --- /dev/null +++ b/Adaptation/FileHandlers/pcl/Convert.cs @@ -0,0 +1,112 @@ +using Adaptation.Shared; +using System; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics; +using System.IO; + +namespace Adaptation.FileHandlers.pcl; + +internal class Convert +{ + + /// + /// Convert the raw data file to parsable file format - in this case from PCL to PDF + /// + /// source file to be converted to PDF + /// + private static string ConvertSourceFileToPdf(string ghostPCLFileName, Logistics logistics) + { + string result = Path.ChangeExtension(logistics.ReportFullPath, ".pdf"); + if (!File.Exists(result)) + { + //string arguments = string.Concat("-i \"", sourceFile, "\" -o \"", result, "\""); + string arguments = string.Concat("-dSAFER -dBATCH -dNOPAUSE -sOutputFile=\"", result, "\" -sDEVICE=pdfwrite \"", logistics.ReportFullPath, "\""); + //Process process = Process.Start(configData.LincPDFCFileName, arguments); + Process process = Process.Start(ghostPCLFileName, arguments); + _ = process.WaitForExit(30000); + if (!File.Exists(result)) + throw new Exception("PDF file wasn't created"); + } + return result; + } + + internal static ReadOnlyDictionary PDF(Logistics logistics, string ghostPCLFileName, List fileInfoCollection) + { + Dictionary results = new(); + object item; + string pageText; + string pagePDFFile; + string pageTextFile; + List sourceFiles = new(); + string sourceFileNamePdf = ConvertSourceFileToPdf(ghostPCLFileName, logistics); + sourceFiles.Add(sourceFileNamePdf); + string sourcePath = Path.GetDirectoryName(logistics.ReportFullPath) ?? throw new Exception(); + string sourceFileNameWithoutExtension = Path.GetFileNameWithoutExtension(logistics.ReportFullPath); + string[] txtFiles = Directory.GetFiles(sourcePath, $"{sourceFileNameWithoutExtension}_*.txt", SearchOption.TopDirectoryOnly); + if (txtFiles.Length != 0) + { + foreach (string txtFile in txtFiles) + { + sourceFiles.Add(txtFile); + pageText = File.ReadAllText(txtFile); + pagePDFFile = Path.ChangeExtension(txtFile, ".pdf"); + if (!File.Exists(pagePDFFile)) + continue; + results.Add(pagePDFFile, pageText); + } + } + if (results.Count == 0) + { + java.io.File file = new(sourceFileNamePdf); + org.apache.pdfbox.util.Splitter splitter = new(); + org.apache.pdfbox.pdmodel.PDDocument pdDocument = org.apache.pdfbox.pdmodel.PDDocument.load(file); + java.util.List list = splitter.split(pdDocument); + java.util.ListIterator iterator = list.listIterator(); + org.apache.pdfbox.util.PDFTextStripper dataStripper = new(); + for (short i = 1; i < short.MaxValue; i++) + { + if (!iterator.hasNext()) + break; + item = iterator.next(); + pagePDFFile = string.Concat(sourcePath, @"\", sourceFileNameWithoutExtension, "_", i, ".pdf"); + pageTextFile = Path.ChangeExtension(pagePDFFile, ".txt"); + if (File.Exists(pageTextFile)) + { + pageText = File.ReadAllText(pageTextFile); + sourceFiles.Add(pageTextFile); + if (item is not org.apache.pdfbox.pdmodel.PDDocument pd) + continue; + pd.close(); + } + else if (File.Exists(pagePDFFile)) + { + org.apache.pdfbox.pdmodel.PDDocument document = org.apache.pdfbox.pdmodel.PDDocument.load(pagePDFFile); + pageText = dataStripper.getText(document); + document.close(); + sourceFiles.Add(pagePDFFile); + if (item is not org.apache.pdfbox.pdmodel.PDDocument pd) + continue; + pd.close(); + } + else + { + if (item is not org.apache.pdfbox.pdmodel.PDDocument pd) + continue; + pageText = dataStripper.getText(pd); + pd.save(pagePDFFile); + sourceFiles.Add(pagePDFFile); + pd.close(); + File.WriteAllText(pageTextFile, pageText); + sourceFiles.Add(pageTextFile); + } + results.Add(pagePDFFile, pageText); + } + pdDocument.close(); + } + foreach (string sourceFile in sourceFiles) + fileInfoCollection.Add(new FileInfo(sourceFile)); + return new(results); + } + +} \ No newline at end of file diff --git a/Adaptation/FileHandlers/pcl/FileRead.cs b/Adaptation/FileHandlers/pcl/FileRead.cs index d057939..ca1d11f 100644 --- a/Adaptation/FileHandlers/pcl/FileRead.cs +++ b/Adaptation/FileHandlers/pcl/FileRead.cs @@ -4,6 +4,7 @@ using Adaptation.Shared; using Adaptation.Shared.Methods; using System; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.IO; using System.Text.Json; using System.Text.RegularExpressions; @@ -100,9 +101,11 @@ public class FileRead : Shared.FileRead, IFileRead return results; } +#nullable enable + private Tuple> GetExtractResult(string reportFullPath, DateTime dateTime) { - Tuple> results = new(string.Empty, null, null, new List()); + Tuple> results = new(string.Empty, Array.Empty(), Array.Empty(), new List()); _TickOffset ??= 0; // new FileInfo(reportFullPath).LastWriteTime.Ticks - dateTime.Ticks; _Logistics = new Logistics(this, _TickOffset.Value, reportFullPath, useSplitForMID: true); SetFileParameterLotIDToLogisticsMID(); @@ -110,9 +113,13 @@ public class FileRead : Shared.FileRead, IFileRead results.Item4.Add(_Logistics.FileInfo); else { - IProcessData iProcessData = new ProcessData(this, _Logistics, results.Item4, _GhostPCLFileName); - if (iProcessData is not ProcessData processData) + ReadOnlyDictionary pages = Convert.PDF(_Logistics, _GhostPCLFileName, results.Item4); + Complete? complete = Complete.Get(_Logistics, results.Item4, pages); + if (complete is null) throw new Exception(string.Concat("A) No Data - ", dateTime.Ticks)); + IProcessData iProcessData = new ProcessData(this, _Logistics, results.Item4, pages, complete); + if (iProcessData is not ProcessData processData) + throw new Exception(string.Concat("B) No Data - ", dateTime.Ticks)); string mid; if (!string.IsNullOrEmpty(processData.Lot) && string.IsNullOrEmpty(processData.Reactor) && string.IsNullOrEmpty(processData.RDS) && string.IsNullOrEmpty(processData.PSN)) mid = processData.Lot; @@ -126,7 +133,7 @@ public class FileRead : Shared.FileRead, IFileRead SetFileParameterLotID(mid); _Logistics.Update(mid, processData.Reactor); if (iProcessData.Details.Count == 0) - throw new Exception(string.Concat("B) No Data - ", dateTime.Ticks)); + throw new Exception(string.Concat("C) No Data - ", dateTime.Ticks)); results = iProcessData.GetResults(this, _Logistics, results.Item4); } return results; diff --git a/Adaptation/FileHandlers/pcl/Header.cs b/Adaptation/FileHandlers/pcl/Header.cs index cf92d3a..f0ab878 100644 --- a/Adaptation/FileHandlers/pcl/Header.cs +++ b/Adaptation/FileHandlers/pcl/Header.cs @@ -215,7 +215,7 @@ public class Header return GetBefore(text, i, "\n", false); } - internal static Header Get(string headerFileName, Dictionary pages, Constant constant) + internal static Header Get(ReadOnlyDictionary pages, Constant constant, string headerFileName) { Header? result; string id; diff --git a/Adaptation/FileHandlers/pcl/ProcessData.cs b/Adaptation/FileHandlers/pcl/ProcessData.cs index 20d11d2..25dbc0a 100644 --- a/Adaptation/FileHandlers/pcl/ProcessData.cs +++ b/Adaptation/FileHandlers/pcl/ProcessData.cs @@ -3,6 +3,7 @@ using Adaptation.Shared.Methods; using log4net; using System; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.Data; using System.Diagnostics; using System.IO; @@ -72,9 +73,8 @@ public class ProcessData : IProcessData List Shared.Properties.IProcessData.Details => _Details; - public ProcessData(IFileRead fileRead, Logistics logistics, List fileInfoCollection, string ghostPCLFileName) + internal ProcessData(IFileRead fileRead, Logistics logistics, List fileInfoCollection, ReadOnlyDictionary pages, Complete complete) { - fileInfoCollection.Clear(); _Details = new List(); _I = 0; _Data = string.Empty; @@ -82,7 +82,7 @@ public class ProcessData : IProcessData Date = GetDateTime(logistics); MesEntity = logistics.MesEntity; _Log = LogManager.GetLogger(typeof(ProcessData)); - Parse(fileRead, logistics, fileInfoCollection, ghostPCLFileName); + Parse(fileRead, logistics, fileInfoCollection, pages, complete); } private static DateTime GetDateTime(Logistics logistics) => @@ -381,7 +381,7 @@ public class ProcessData : IProcessData return result; } - private void Set(ILogistics logistics) + private void Set(ILogistics logistics, Complete complete) { string lot; string rds; @@ -390,14 +390,9 @@ public class ProcessData : IProcessData string reactor; string employee; ScanPast("Recipe ID:"); - recipe = GetBefore("LotID:"); - recipe = recipe.Replace(";", ""); - if (_Data.Contains("[]")) - lot = GetBefore("[]"); - else if (_Data.Contains("[7]")) - lot = GetBefore("[7]"); - else - lot = GetBefore("["); + _ = GetBefore("LotID:"); + lot = complete.Header.Id; + recipe = complete.Header.Recipe; Descriptor descriptor = GetDescriptor(lot); lot = descriptor.Lot; psn = descriptor.PSN; @@ -413,10 +408,8 @@ public class ProcessData : IProcessData UniqueId = string.Format("{0}_{1}_{2}", logistics.JobID, lot, Path.GetFileNameWithoutExtension(logistics.ReportFullPath)); } - private void ParseLotSummary(IFileRead fileRead, ILogistics logistics, string headerFileName, Dictionary pages, Dictionary> slots) + private void ParseLotSummary(ILogistics logistics, ReadOnlyDictionary pages, Complete complete, string headerFileName, Dictionary> slots) { - if (fileRead is null) - throw new ArgumentNullException(nameof(fileRead)); _I = 0; ParseErrorText = string.Empty; if (!pages.TryGetValue(headerFileName, out string value)) @@ -425,7 +418,7 @@ public class ProcessData : IProcessData _Data = value; ScanPast("Date:"); _ = GetToEOL(); - Set(logistics); + Set(logistics, complete); // determine number of wafers and their slot numbers _Log.Debug(_Data.Substring(_I)); string slot; @@ -501,7 +494,7 @@ public class ProcessData : IProcessData HazeAverageStdDev = toEol4[8].Trim(); } - private Detail ParseWaferSummary(string waferFileName, Dictionary pages) + private Detail ParseWaferSummary(string waferFileName, ReadOnlyDictionary pages) { Detail result = new() { Data = "*Data*", i = -1, }; _I = 0; @@ -604,110 +597,19 @@ public class ProcessData : IProcessData #nullable enable - private static Complete? GetComplete(string headerFileName, Dictionary pages) + private void Parse(IFileRead fileRead, Logistics logistics, List fileInfoCollection, ReadOnlyDictionary pages, Complete complete) { - Complete? result; - Constant constant = new(); - result = Complete.Get(headerFileName, pages, constant); - return result; - } - - private void Parse(IFileRead fileRead, Logistics logistics, List fileInfoCollection, string ghostPCLFileName) - { - object item; - string pageText; - string pagePDFFile; - string pageTextFile; + if (!fileRead.IsEAFHosted) + fileInfoCollection.Clear(); List sourceFiles = new(); List missingSlots = new(); - Dictionary pages = new(); - string sourceFileNamePdf = ConvertSourceFileToPdf(ghostPCLFileName, logistics); - sourceFiles.Add(sourceFileNamePdf); + Dictionary> slots = new(); + List> pageMapping = new(); + _Log.Debug($"****ParseData - Parsing lot summary"); + string headerFileName = pages.ElementAt(pages.Count - 1).Key; + ParseLotSummary(logistics, pages, complete, headerFileName, slots); string sourcePath = Path.GetDirectoryName(logistics.ReportFullPath) ?? throw new Exception(); string sourceFileNameWithoutExtension = Path.GetFileNameWithoutExtension(logistics.ReportFullPath); - string[] txtFiles = Directory.GetFiles(sourcePath, $"{sourceFileNameWithoutExtension}_*.txt", SearchOption.TopDirectoryOnly); - if (txtFiles.Length != 0) - { - foreach (string txtFile in txtFiles) - { - sourceFiles.Add(txtFile); - pageText = File.ReadAllText(txtFile); - pagePDFFile = Path.ChangeExtension(txtFile, ".pdf"); - if (!File.Exists(pagePDFFile)) - continue; - pages.Add(pagePDFFile, pageText); - } - } - if (pages.Count == 0) - { - java.io.File file = new(sourceFileNamePdf); - org.apache.pdfbox.util.Splitter splitter = new(); - org.apache.pdfbox.pdmodel.PDDocument pdDocument = org.apache.pdfbox.pdmodel.PDDocument.load(file); - java.util.List list = splitter.split(pdDocument); - java.util.ListIterator iterator = list.listIterator(); - org.apache.pdfbox.util.PDFTextStripper dataStripper = new(); - for (short i = 1; i < short.MaxValue; i++) - { - if (!iterator.hasNext()) - break; - item = iterator.next(); - pagePDFFile = string.Concat(sourcePath, @"\", sourceFileNameWithoutExtension, "_", i, ".pdf"); - pageTextFile = Path.ChangeExtension(pagePDFFile, ".txt"); - if (File.Exists(pageTextFile)) - { - pageText = File.ReadAllText(pageTextFile); - sourceFiles.Add(pageTextFile); - if (item is not org.apache.pdfbox.pdmodel.PDDocument pd) - continue; - pd.close(); - } - else if (File.Exists(pagePDFFile)) - { - org.apache.pdfbox.pdmodel.PDDocument document = org.apache.pdfbox.pdmodel.PDDocument.load(pagePDFFile); - pageText = dataStripper.getText(document); - document.close(); - sourceFiles.Add(pagePDFFile); - if (item is not org.apache.pdfbox.pdmodel.PDDocument pd) - continue; - pd.close(); - } - else - { - if (item is not org.apache.pdfbox.pdmodel.PDDocument pd) - continue; - pageText = dataStripper.getText(pd); - pd.save(pagePDFFile); - sourceFiles.Add(pagePDFFile); - pd.close(); - File.WriteAllText(pageTextFile, pageText); - sourceFiles.Add(pageTextFile); - } - pages.Add(pagePDFFile, pageText); - } - pdDocument.close(); - } - _Log.Debug($"****ParseData - Parsing lot summary"); - List> pageMapping = new(); - string headerFileName = string.Concat(sourcePath, @"\", sourceFileNameWithoutExtension, "_", pages.Count, ".pdf"); - try - { - Complete? complete = GetComplete(headerFileName, pages); - if (complete is null) - _Log.Warn($"Could not get Complete from {sourceFileNameWithoutExtension}"); - else - { - FileInfo fileInfo = new($"{sourceFileNameWithoutExtension}.json"); - string json = JsonSerializer.Serialize(complete, CompleteSourceGenerationContext.Default.Complete); - File.WriteAllText(fileInfo.FullName, json); - fileInfoCollection.Add(fileInfo); - } - } - catch (Exception ex) - { - _Log.Error($"Error in {nameof(GetComplete)}", ex); - } - Dictionary> slots = new(); - ParseLotSummary(fileRead, logistics, headerFileName, pages, slots); foreach (KeyValuePair keyValuePair in pages) { if (keyValuePair.Key == headerFileName) @@ -793,8 +695,6 @@ public class ProcessData : IProcessData fileInfoCollection.Add(logistics.FileInfo); } -#nullable enable - internal static List GetDescriptions(JsonElement[] jsonElements) { List results = new(); diff --git a/Adaptation/FileHandlers/pcl/Wafer.cs b/Adaptation/FileHandlers/pcl/Wafer.cs index dc3333c..fc9dd09 100644 --- a/Adaptation/FileHandlers/pcl/Wafer.cs +++ b/Adaptation/FileHandlers/pcl/Wafer.cs @@ -79,7 +79,7 @@ public class Wafer public string Thruput { get; } public string Recipe { get; } - internal static ReadOnlyCollection Get(string headerFileName, Dictionary pages, Constant constant) + internal static ReadOnlyCollection Get(ReadOnlyDictionary pages, Constant constant, string headerFileName) { List results = new(); Wafer wafer; @@ -96,9 +96,11 @@ public class Wafer stringList = new(); if (!pages.TryGetValue(keyValuePair.Key, out text)) throw new Exception(); + if (string.IsNullOrEmpty(text) || !text.Contains(constant.Id)) + continue; Header.ScanPast(text, i, constant.Date); string date = Header.GetToEOL(text, i); - Header.ScanPast(text, i, "ID#"); + Header.ScanPast(text, i, constant.Id); string id = Header.GetToEOL(text, i); if (id.Length > 5) id = string.Concat(id.Substring(0, 5), "... - ***"); diff --git a/MET08DDUPSFS6420.csproj b/MET08DDUPSFS6420.csproj index 9eefecc..baf83e4 100644 --- a/MET08DDUPSFS6420.csproj +++ b/MET08DDUPSFS6420.csproj @@ -122,6 +122,7 @@ +