phares3757/met08ddupsfs6420
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reqiure Complete

Browse Source
This commit is contained in:
Mike Phares 2024-11-20 08:16:28 -07:00
parent 0f6308ace5
commit bd8552edea
8 changed files with 163 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Adaptation/FileHandlers/pcl/Complete.cs
View File

@ -1,6 +1,9 @@
using Adaptation.Shared;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Adaptation.FileHandlers.pcl;
@ -19,19 +22,28 @@ internal class Complete
public Header Header { get; }
public Wafer[] Wafers { get; }
internal static Complete? Get(string headerFileName, Dictionary<string, string> pages, Constant constant)
internal static Complete? Get(Logistics logistics, List<FileInfo> fileInfoCollection, ReadOnlyDictionary<string, string> pages)
{
Complete? result;
Header? header = Header.Get(headerFileName, pages, constant);
Constant constant = new();
string headerFileName = pages.ElementAt(pages.Count - 1).Key;
Header? header = Header.Get(pages, constant, headerFileName);
if (header is null)
result = null;
else
{
ReadOnlyCollection<Wafer> wafers = Wafer.Get(headerFileName, pages, constant);
ReadOnlyCollection<Wafer> wafers = Wafer.Get(pages, constant, headerFileName);
if (wafers.Count == 0)
result = null;
else
{
result = new(header, wafers.ToArray());
FileInfo fileInfo = new($"{logistics.ReportFullPath}.json");
string json = JsonSerializer.Serialize(result, CompleteSourceGenerationContext.Default.Complete);
File.WriteAllText(fileInfo.FullName, json);
File.SetLastWriteTime(fileInfo.FullName, logistics.DateTimeFromSequence);
fileInfoCollection.Add(fileInfo);
}
}
return result;
}

1
Adaptation/FileHandlers/pcl/Constant.cs
View File

@ -3,6 +3,7 @@ namespace Adaptation.FileHandlers.pcl;
internal class Constant
{
public string Id { get; } = "ID#";
public string Max { get; } = "Max:";
public string Min { get; } = "Min:";
public string Date { get; } = "Date:";

112
Adaptation/FileHandlers/pcl/Convert.cs Normal file
View File

@ -0,0 +1,112 @@
using Adaptation.Shared;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.IO;
namespace Adaptation.FileHandlers.pcl;
internal class Convert
{
/// <summary>
/// Convert the raw data file to parsable file format - in this case from PCL to PDF
/// </summary>
/// <param name="sourceFile">source file to be converted to PDF</param>
/// <returns></returns>
private static string ConvertSourceFileToPdf(string ghostPCLFileName, Logistics logistics)
{
string result = Path.ChangeExtension(logistics.ReportFullPath, ".pdf");
if (!File.Exists(result))
{
//string arguments = string.Concat("-i \"", sourceFile, "\" -o \"", result, "\"");
string arguments = string.Concat("-dSAFER -dBATCH -dNOPAUSE -sOutputFile=\"", result, "\" -sDEVICE=pdfwrite \"", logistics.ReportFullPath, "\"");
//Process process = Process.Start(configData.LincPDFCFileName, arguments);
Process process = Process.Start(ghostPCLFileName, arguments);
_ = process.WaitForExit(30000);
if (!File.Exists(result))
throw new Exception("PDF file wasn't created");
}
return result;
}
internal static ReadOnlyDictionary<string, string> PDF(Logistics logistics, string ghostPCLFileName, List<FileInfo> fileInfoCollection)
{
Dictionary<string, string> results = new();
object item;
string pageText;
string pagePDFFile;
string pageTextFile;
List<string> sourceFiles = new();
string sourceFileNamePdf = ConvertSourceFileToPdf(ghostPCLFileName, logistics);
sourceFiles.Add(sourceFileNamePdf);
string sourcePath = Path.GetDirectoryName(logistics.ReportFullPath) ?? throw new Exception();
string sourceFileNameWithoutExtension = Path.GetFileNameWithoutExtension(logistics.ReportFullPath);
string[] txtFiles = Directory.GetFiles(sourcePath, $"{sourceFileNameWithoutExtension}_*.txt", SearchOption.TopDirectoryOnly);
if (txtFiles.Length != 0)
{
foreach (string txtFile in txtFiles)
{
sourceFiles.Add(txtFile);
pageText = File.ReadAllText(txtFile);
pagePDFFile = Path.ChangeExtension(txtFile, ".pdf");
if (!File.Exists(pagePDFFile))
continue;
results.Add(pagePDFFile, pageText);
}
}
if (results.Count == 0)
{
java.io.File file = new(sourceFileNamePdf);
org.apache.pdfbox.util.Splitter splitter = new();
org.apache.pdfbox.pdmodel.PDDocument pdDocument = org.apache.pdfbox.pdmodel.PDDocument.load(file);
java.util.List list = splitter.split(pdDocument);
java.util.ListIterator iterator = list.listIterator();
org.apache.pdfbox.util.PDFTextStripper dataStripper = new();
for (short i = 1; i < short.MaxValue; i++)
{
if (!iterator.hasNext())
break;
item = iterator.next();
pagePDFFile = string.Concat(sourcePath, @"\", sourceFileNameWithoutExtension, "_", i, ".pdf");
pageTextFile = Path.ChangeExtension(pagePDFFile, ".txt");
if (File.Exists(pageTextFile))
{
pageText = File.ReadAllText(pageTextFile);
sourceFiles.Add(pageTextFile);
if (item is not org.apache.pdfbox.pdmodel.PDDocument pd)
continue;
pd.close();
}
else if (File.Exists(pagePDFFile))
{
org.apache.pdfbox.pdmodel.PDDocument document = org.apache.pdfbox.pdmodel.PDDocument.load(pagePDFFile);
pageText = dataStripper.getText(document);
document.close();
sourceFiles.Add(pagePDFFile);
if (item is not org.apache.pdfbox.pdmodel.PDDocument pd)
continue;
pd.close();
}
else
{
if (item is not org.apache.pdfbox.pdmodel.PDDocument pd)
continue;
pageText = dataStripper.getText(pd);
pd.save(pagePDFFile);
sourceFiles.Add(pagePDFFile);
pd.close();
File.WriteAllText(pageTextFile, pageText);
sourceFiles.Add(pageTextFile);
}
results.Add(pagePDFFile, pageText);
}
pdDocument.close();
}
foreach (string sourceFile in sourceFiles)
fileInfoCollection.Add(new FileInfo(sourceFile));
return new(results);
}
}

15
Adaptation/FileHandlers/pcl/FileRead.cs
View File

@ -4,6 +4,7 @@ using Adaptation.Shared;
using Adaptation.Shared.Methods;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.IO;
using System.Text.Json;
using System.Text.RegularExpressions;
@ -100,9 +101,11 @@ public class FileRead : Shared.FileRead, IFileRead
return results;
}
#nullable enable
private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime dateTime)
{
Tuple<string, Test[], JsonElement[], List<FileInfo>> results = new(string.Empty, null, null, new List<FileInfo>());
Tuple<string, Test[], JsonElement[], List<FileInfo>> results = new(string.Empty, Array.Empty<Test>(), Array.Empty<JsonElement>(), new List<FileInfo>());
_TickOffset ??= 0; // new FileInfo(reportFullPath).LastWriteTime.Ticks - dateTime.Ticks;
_Logistics = new Logistics(this, _TickOffset.Value, reportFullPath, useSplitForMID: true);
SetFileParameterLotIDToLogisticsMID();
@ -110,9 +113,13 @@ public class FileRead : Shared.FileRead, IFileRead
results.Item4.Add(_Logistics.FileInfo);
else
{
IProcessData iProcessData = new ProcessData(this, _Logistics, results.Item4, _GhostPCLFileName);
if (iProcessData is not ProcessData processData)
ReadOnlyDictionary<string, string> pages = Convert.PDF(_Logistics, _GhostPCLFileName, results.Item4);
Complete? complete = Complete.Get(_Logistics, results.Item4, pages);
if (complete is null)
throw new Exception(string.Concat("A) No Data - ", dateTime.Ticks));
IProcessData iProcessData = new ProcessData(this, _Logistics, results.Item4, pages, complete);
if (iProcessData is not ProcessData processData)
throw new Exception(string.Concat("B) No Data - ", dateTime.Ticks));
string mid;
if (!string.IsNullOrEmpty(processData.Lot) && string.IsNullOrEmpty(processData.Reactor) && string.IsNullOrEmpty(processData.RDS) && string.IsNullOrEmpty(processData.PSN))
mid = processData.Lot;
@ -126,7 +133,7 @@ public class FileRead : Shared.FileRead, IFileRead
SetFileParameterLotID(mid);
_Logistics.Update(mid, processData.Reactor);
if (iProcessData.Details.Count == 0)
throw new Exception(string.Concat("B) No Data - ", dateTime.Ticks));
throw new Exception(string.Concat("C) No Data - ", dateTime.Ticks));
results = iProcessData.GetResults(this, _Logistics, results.Item4);
}
return results;

2
Adaptation/FileHandlers/pcl/Header.cs
View File

@ -215,7 +215,7 @@ public class Header
return GetBefore(text, i, "\n", false);
}
internal static Header Get(string headerFileName, Dictionary<string, string> pages, Constant constant)
internal static Header Get(ReadOnlyDictionary<string, string> pages, Constant constant, string headerFileName)
{
Header? result;
string id;

136
Adaptation/FileHandlers/pcl/ProcessData.cs
View File

@ -3,6 +3,7 @@ using Adaptation.Shared.Methods;
using log4net;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Data;
using System.Diagnostics;
using System.IO;
@ -72,9 +73,8 @@ public class ProcessData : IProcessData
List<object> Shared.Properties.IProcessData.Details => _Details;
public ProcessData(IFileRead fileRead, Logistics logistics, List<FileInfo> fileInfoCollection, string ghostPCLFileName)
internal ProcessData(IFileRead fileRead, Logistics logistics, List<FileInfo> fileInfoCollection, ReadOnlyDictionary<string, string> pages, Complete complete)
{
fileInfoCollection.Clear();
_Details = new List<object>();
_I = 0;
_Data = string.Empty;
@ -82,7 +82,7 @@ public class ProcessData : IProcessData
Date = GetDateTime(logistics);
MesEntity = logistics.MesEntity;
_Log = LogManager.GetLogger(typeof(ProcessData));
Parse(fileRead, logistics, fileInfoCollection, ghostPCLFileName);
Parse(fileRead, logistics, fileInfoCollection, pages, complete);
}
private static DateTime GetDateTime(Logistics logistics) =>
@ -381,7 +381,7 @@ public class ProcessData : IProcessData
return result;
}
private void Set(ILogistics logistics)
private void Set(ILogistics logistics, Complete complete)
{
string lot;
string rds;
@ -390,14 +390,9 @@ public class ProcessData : IProcessData
string reactor;
string employee;
ScanPast("Recipe ID:");
recipe = GetBefore("LotID:");
recipe = recipe.Replace(";", "");
if (_Data.Contains("[]"))
lot = GetBefore("[]");
else if (_Data.Contains("[7]"))
lot = GetBefore("[7]");
else
lot = GetBefore("[");
_ = GetBefore("LotID:");
lot = complete.Header.Id;
recipe = complete.Header.Recipe;
Descriptor descriptor = GetDescriptor(lot);
lot = descriptor.Lot;
psn = descriptor.PSN;
@ -413,10 +408,8 @@ public class ProcessData : IProcessData
UniqueId = string.Format("{0}_{1}_{2}", logistics.JobID, lot, Path.GetFileNameWithoutExtension(logistics.ReportFullPath));
}
private void ParseLotSummary(IFileRead fileRead, ILogistics logistics, string headerFileName, Dictionary<string, string> pages, Dictionary<string, List<Detail>> slots)
private void ParseLotSummary(ILogistics logistics, ReadOnlyDictionary<string, string> pages, Complete complete, string headerFileName, Dictionary<string, List<Detail>> slots)
{
if (fileRead is null)
throw new ArgumentNullException(nameof(fileRead));
_I = 0;
ParseErrorText = string.Empty;
if (!pages.TryGetValue(headerFileName, out string value))
@ -425,7 +418,7 @@ public class ProcessData : IProcessData
_Data = value;
ScanPast("Date:");
_ = GetToEOL();
Set(logistics);
Set(logistics, complete);
// determine number of wafers and their slot numbers
_Log.Debug(_Data.Substring(_I));
string slot;
@ -501,7 +494,7 @@ public class ProcessData : IProcessData
HazeAverageStdDev = toEol4[8].Trim();
}
private Detail ParseWaferSummary(string waferFileName, Dictionary<string, string> pages)
private Detail ParseWaferSummary(string waferFileName, ReadOnlyDictionary<string, string> pages)
{
Detail result = new() { Data = "*Data*", i = -1, };
_I = 0;
@ -604,110 +597,19 @@ public class ProcessData : IProcessData
#nullable enable
private static Complete? GetComplete(string headerFileName, Dictionary<string, string> pages)
private void Parse(IFileRead fileRead, Logistics logistics, List<FileInfo> fileInfoCollection, ReadOnlyDictionary<string, string> pages, Complete complete)
{
Complete? result;
Constant constant = new();
result = Complete.Get(headerFileName, pages, constant);
return result;
}
private void Parse(IFileRead fileRead, Logistics logistics, List<FileInfo> fileInfoCollection, string ghostPCLFileName)
{
object item;
string pageText;
string pagePDFFile;
string pageTextFile;
if (!fileRead.IsEAFHosted)
fileInfoCollection.Clear();
List<string> sourceFiles = new();
List<string> missingSlots = new();
Dictionary<string, string> pages = new();
string sourceFileNamePdf = ConvertSourceFileToPdf(ghostPCLFileName, logistics);
sourceFiles.Add(sourceFileNamePdf);
Dictionary<string, List<Detail>> slots = new();
List<Tuple<string, string>> pageMapping = new();
_Log.Debug($"****ParseData - Parsing lot summary");
string headerFileName = pages.ElementAt(pages.Count - 1).Key;
ParseLotSummary(logistics, pages, complete, headerFileName, slots);
string sourcePath = Path.GetDirectoryName(logistics.ReportFullPath) ?? throw new Exception();
string sourceFileNameWithoutExtension = Path.GetFileNameWithoutExtension(logistics.ReportFullPath);
string[] txtFiles = Directory.GetFiles(sourcePath, $"{sourceFileNameWithoutExtension}_*.txt", SearchOption.TopDirectoryOnly);
if (txtFiles.Length != 0)
{
foreach (string txtFile in txtFiles)
{
sourceFiles.Add(txtFile);
pageText = File.ReadAllText(txtFile);
pagePDFFile = Path.ChangeExtension(txtFile, ".pdf");
if (!File.Exists(pagePDFFile))
continue;
pages.Add(pagePDFFile, pageText);
}
}
if (pages.Count == 0)
{
java.io.File file = new(sourceFileNamePdf);
org.apache.pdfbox.util.Splitter splitter = new();
org.apache.pdfbox.pdmodel.PDDocument pdDocument = org.apache.pdfbox.pdmodel.PDDocument.load(file);
java.util.List list = splitter.split(pdDocument);
java.util.ListIterator iterator = list.listIterator();
org.apache.pdfbox.util.PDFTextStripper dataStripper = new();
for (short i = 1; i < short.MaxValue; i++)
{
if (!iterator.hasNext())
break;
item = iterator.next();
pagePDFFile = string.Concat(sourcePath, @"\", sourceFileNameWithoutExtension, "_", i, ".pdf");
pageTextFile = Path.ChangeExtension(pagePDFFile, ".txt");
if (File.Exists(pageTextFile))
{
pageText = File.ReadAllText(pageTextFile);
sourceFiles.Add(pageTextFile);
if (item is not org.apache.pdfbox.pdmodel.PDDocument pd)
continue;
pd.close();
}
else if (File.Exists(pagePDFFile))
{
org.apache.pdfbox.pdmodel.PDDocument document = org.apache.pdfbox.pdmodel.PDDocument.load(pagePDFFile);
pageText = dataStripper.getText(document);
document.close();
sourceFiles.Add(pagePDFFile);
if (item is not org.apache.pdfbox.pdmodel.PDDocument pd)
continue;
pd.close();
}
else
{
if (item is not org.apache.pdfbox.pdmodel.PDDocument pd)
continue;
pageText = dataStripper.getText(pd);
pd.save(pagePDFFile);
sourceFiles.Add(pagePDFFile);
pd.close();
File.WriteAllText(pageTextFile, pageText);
sourceFiles.Add(pageTextFile);
}
pages.Add(pagePDFFile, pageText);
}
pdDocument.close();
}
_Log.Debug($"****ParseData - Parsing lot summary");
List<Tuple<string, string>> pageMapping = new();
string headerFileName = string.Concat(sourcePath, @"\", sourceFileNameWithoutExtension, "_", pages.Count, ".pdf");
try
{
Complete? complete = GetComplete(headerFileName, pages);
if (complete is null)
_Log.Warn($"Could not get Complete from {sourceFileNameWithoutExtension}");
else
{
FileInfo fileInfo = new($"{sourceFileNameWithoutExtension}.json");
string json = JsonSerializer.Serialize(complete, CompleteSourceGenerationContext.Default.Complete);
File.WriteAllText(fileInfo.FullName, json);
fileInfoCollection.Add(fileInfo);
}
}
catch (Exception ex)
{
_Log.Error($"Error in {nameof(GetComplete)}", ex);
}
Dictionary<string, List<Detail>> slots = new();
ParseLotSummary(fileRead, logistics, headerFileName, pages, slots);
foreach (KeyValuePair<string, string> keyValuePair in pages)
{
if (keyValuePair.Key == headerFileName)
@ -793,8 +695,6 @@ public class ProcessData : IProcessData
fileInfoCollection.Add(logistics.FileInfo);
}
#nullable enable
internal static List<Description> GetDescriptions(JsonElement[] jsonElements)
{
List<Description> results = new();

6
Adaptation/FileHandlers/pcl/Wafer.cs
View File

@ -79,7 +79,7 @@ public class Wafer
public string Thruput { get; }
public string Recipe { get; }
internal static ReadOnlyCollection<Wafer> Get(string headerFileName, Dictionary<string, string> pages, Constant constant)
internal static ReadOnlyCollection<Wafer> Get(ReadOnlyDictionary<string, string> pages, Constant constant, string headerFileName)
{
List<Wafer> results = new();
Wafer wafer;
@ -96,9 +96,11 @@ public class Wafer
stringList = new();
if (!pages.TryGetValue(keyValuePair.Key, out text))
throw new Exception();
if (string.IsNullOrEmpty(text) || !text.Contains(constant.Id))
continue;
Header.ScanPast(text, i, constant.Date);
string date = Header.GetToEOL(text, i);
Header.ScanPast(text, i, "ID#");
Header.ScanPast(text, i, constant.Id);
string id = Header.GetToEOL(text, i);
if (id.Length > 5)
id = string.Concat(id.Substring(0, 5), "... - ***");

1
MET08DDUPSFS6420.csproj
View File

@ -122,6 +122,7 @@
<Compile Include="Adaptation\FileHandlers\OpenInsight\Root.cs" />
<Compile Include="Adaptation\FileHandlers\pcl\Complete.cs" />
<Compile Include="Adaptation\FileHandlers\pcl\Constant.cs" />
<Compile Include="Adaptation\FileHandlers\pcl\Convert.cs" />
<Compile Include="Adaptation\FileHandlers\pcl\Description.cs" />
<Compile Include="Adaptation\FileHandlers\pcl\Descriptor.cs" />
<Compile Include="Adaptation\FileHandlers\pcl\Detail.cs" />