phares3757/met08thftirstratus
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
met08thftirstratus/Adaptation/FileHandlers/txt/ProcessData.cs

557 lines
30 KiB
C#
Raw Blame History

using Adaptation.Shared;
using Adaptation.Shared.Duplicator;
using Adaptation.Shared.Methods;
using log4net;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
namespace Adaptation.FileHandlers.txt;
public partial class ProcessData : IProcessData
{
private readonly ILog _Log;
private readonly List<object> _Details;
public string JobID { get; set; }
public string MesEntity { get; set; }
List<object> Shared.Properties.IProcessData.Details => _Details;
public ProcessData(IFileRead fileRead, Logistics logistics, long tickOffset, List<FileInfo> fileInfoCollection, string originalDataBioRad)
{
JobID = logistics.JobID;
fileInfoCollection.Clear();
_Details = new List<object>();
MesEntity = logistics.MesEntity;
_Log = LogManager.GetLogger(typeof(ProcessData));
List<Tuple<string, bool, DateTime, string>> tuples = Parse(fileRead, logistics, tickOffset, fileInfoCollection, originalDataBioRad);
_Details.AddRange(tuples);
}
string IProcessData.GetCurrentReactor(IFileRead fileRead, Logistics logistics, Dictionary<string, string> reactors) => throw new Exception(string.Concat("See ", nameof(Parse)));
Tuple<string, Test[], JsonElement[], List<FileInfo>> IProcessData.GetResults(IFileRead fileRead, Logistics logistics, List<FileInfo> fileInfoCollection)
{
Tuple<string, Test[], JsonElement[], List<FileInfo>> results;
List<Test> tests = new();
foreach (object item in _Details)
tests.Add(Test.BioRadStratus);
List<IDescription> descriptions = fileRead.GetDescriptions(fileRead, tests, this);
if (tests.Count != descriptions.Count)
throw new Exception();
for (int i = 0; i < tests.Count; i++)
{
if (descriptions[i] is not Description description)
throw new Exception();
if (description.Test != (int)tests[i])
throw new Exception();
}
List<Description> fileReadDescriptions = (from l in descriptions select (Description)l).ToList();
string json = JsonSerializer.Serialize(fileReadDescriptions, fileReadDescriptions.GetType());
JsonElement[] jsonElements = JsonSerializer.Deserialize<JsonElement[]>(json);
results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(logistics.Logistics1[0], tests.ToArray(), jsonElements, fileInfoCollection);
return results;
}
#nullable enable
private List<Tuple<string, bool, DateTime, string>> Parse(IFileRead fileRead, Logistics logistics, long tickOffset, List<FileInfo> fileInfoCollection, string originalDataBioRad)
{
List<Tuple<string, bool, DateTime, string>> results = new();
string[] reportFullPathlines = File.ReadAllLines(logistics.ReportFullPath);
// ***********************************************************************************
// * Step #2 - Verify completeness of each cassette scan in the raw data source file *
// ***********************************************************************************
bool? cassetteScanCompleted = null;
// Scrub the source file to verify that for each cassette, present in the file, there is a complete
// data set (i.e., that is there is a start and finished statement).
//
// Scenario #1 - Normal
// For every cassette "started" there must be a matching cassette "finished".
// Scenario #2 - Only Cassette "finished" (with or witout additional cassette complete data sets)
// Incomplete data file. File will be process and generate error for the incomplete portion.
// Scenario #3 - Only Cassette "Started"
// Bail out of the solution. Source data file not ready to be processed.
foreach (string line in reportFullPathlines)
{
if (line is null)
break;
if (line.Contains("Cassette") && line.Contains("started") && (cassetteScanCompleted is null || cassetteScanCompleted.Value))
{
cassetteScanCompleted = false;
_Log.Debug("****Extract() - CassetteScanCompleted = FALSE");
}
else if (line.Contains("Cassette") && line.Contains("finished") && (cassetteScanCompleted is null || !cassetteScanCompleted.Value))
{
cassetteScanCompleted = true;
_Log.Debug("****Extract() - CassetteScanCompleted = TRUE");
}
}
Dictionary<string, List<string>> cassetteIDAndDataSets;
if (string.IsNullOrEmpty(logistics.ReportFullPath))
cassetteIDAndDataSets = new();
else if (cassetteScanCompleted is null || !cassetteScanCompleted.Value)
{
cassetteIDAndDataSets = new();
// Raw source file has an incomplete data set or it only contains a "Process failed" and should not be
// processed /split yet. Simply get out of this routine until enough data has been appended to the file.
_Log.Debug($"****Extract() - Raw source file has an incomplete data set and should not be processed yet.");
}
else
cassetteIDAndDataSets = GetCassetteIDAndDataSets(reportFullPathlines);
if (cassetteIDAndDataSets.Count != 0)
{
int wafer;
string user;
string runID;
bool isBioRad;
string recipe;
int count = -1;
int stringIndex;
string dataText;
string dataType;
string[] segments;
string cassetteID;
string recipeName;
IProcessData iProcessData;
DateTime cassetteDateTime;
string recipeSearch = "Recipe";
string toolType = string.Empty;
StringBuilder contents = new();
Stratus.ProcessData processData;
foreach (KeyValuePair<string, List<string>> keyValuePair in cassetteIDAndDataSets)
{
isBioRad = false;
dataType = string.Empty;
cassetteID = keyValuePair.Key;
for (int i = 0; i < keyValuePair.Value.Count; i++)
{
dataText = keyValuePair.Value[i];
// Finished capturing the complete cassette scan data information. Release the cassette file.
if (dataText.Contains("Cassette") &&
dataText.Contains("Wafer") &&
dataText.Contains("Slot") &&
dataText.Contains("Recipe") &&
dataText.Contains("Points") &&
dataText.Contains("Thickness") &&
dataText.Contains("Mean") &&
dataText.Contains("Source:") &&
dataText.Contains("Destination:"))
{
// Extract the recipe name
runID = string.Empty;
recipeName = string.Empty;
stringIndex = dataText.IndexOf(recipeSearch);
recipeName = dataText.Substring(stringIndex + recipeSearch.Length);
_Log.Debug($"****Extract(FDR): recipeName = {recipeName}");
#pragma warning disable CA2249
if (!string.IsNullOrEmpty(recipeName) && (recipeName.IndexOf("center", StringComparison.CurrentCultureIgnoreCase) >= 0))
#pragma warning restore CA2249
{
/***************************************/
/* STRATUS Measurement = FQA Thickness */
/***************************************/
// Recipes that contains the substring "Center" are STRATUS centerpoint recipes. They are used for Inspection and FQA measurements.
// measurement. The data from these scans should be uploaded to the Metrology Viewer database as STRATUS and uploaded to the
// OpenInsight [FQA Thickness - Post Epi - QA Metrology / Thk/RHO Value for each slotID] automatically.
isBioRad = false;
toolType = "STRATUS";
dataType = "FQA Thickness";
}
#pragma warning disable CA2249
else if (!string.IsNullOrEmpty(recipeName) && (recipeName.IndexOf("prod_", StringComparison.CurrentCultureIgnoreCase) >= 0))
#pragma warning restore CA2249
{
/******************************************/
/* BIORAD Measurement = Product Thickness */
/******************************************/
// Recipes that contains the substring "Center" are STRATUS centerpoint recipes. They are used for Inspection and FQA measurements.
// measurement. The data from these scans should be uploaded to the Metrology Viewer database as STRATUS and uploaded to the
// OpenInsight [FQA Thickness - Post Epi - QA Metrology / Thk/RHO Value for each slotID] automatically.
isBioRad = true;
toolType = "BIORAD";
dataType = "Product Thickness";
}
else if (!string.IsNullOrEmpty(recipeName) &&
#pragma warning disable CA2249
((recipeName.IndexOf("T-Low", StringComparison.CurrentCultureIgnoreCase) >= 0) ||
(recipeName.IndexOf("T_Low", StringComparison.CurrentCultureIgnoreCase) >= 0) ||
(recipeName.IndexOf("T-Mid", StringComparison.CurrentCultureIgnoreCase) >= 0) ||
(recipeName.IndexOf("T_Mid", StringComparison.CurrentCultureIgnoreCase) >= 0) ||
(recipeName.IndexOf("T-High", StringComparison.CurrentCultureIgnoreCase) >= 0) ||
(recipeName.IndexOf("T_High", StringComparison.CurrentCultureIgnoreCase) >= 0)))
#pragma warning restore CA2249
{
/*************************************/
/* BIORAD Measurement = No Uploading */
/*************************************/
// Recipes that contains the substring "T-Low, T_Low, T-Mid, T_Mid and T-High, T_High" are BIORAD verification recipe. The information
// should be uploaded to the Metrology Viewer database as BIORAD. No OpenInsight.
isBioRad = true;
toolType = "BIORAD";
dataType = "Verification";
}
else
{
// Count the number of wafers (ref. "Source: Slot") in the cassette
int waferCount = Regex.Matches(dataText, "Source: Slot").Count;
if (waferCount == 1)
{
// Metrology Thickness. Upload to OpenInsight same as BR2 and BR3
isBioRad = true;
toolType = "BIORAD";
dataType = "Metrology Thickness";
}
else if (waferCount > 1)
{
// Inspection Measurement. Do not upload to OpenInsight.
isBioRad = true;
toolType = "BIORAD";
dataType = "Inspection";
}
}
}
_Log.Debug($"****Extract(FDR): ToolType = {toolType}");
_Log.Debug($"****Extract(FDR): DataType = {dataType}");
if (!isBioRad)
{
cassetteDateTime = logistics.DateTimeFromSequence.AddTicks(i * -1);
results.Add(new Tuple<string, bool, DateTime, string>(cassetteID, isBioRad, cassetteDateTime, dataText));
}
else
{
Stratus.Run? complete = null;
processData = new Stratus.ProcessData(fileRead, logistics, fileInfoCollection, originalDataBioRad, complete, dataText: dataText);
iProcessData = processData;
if (iProcessData.Details.Count == 0)
_Log.Warn("No Details!");
else
{
foreach (object item in iProcessData.Details)
{
if (item is not Stratus.Detail detail)
throw new Exception();
count += 1;
_ = contents.Clear();
cassetteDateTime = logistics.DateTimeFromSequence.AddTicks(count * -1);
user = processData.Employee?.ToString() ?? "";
recipe = detail.Recipe?.ToString() ?? "";
if (isBioRad)
recipe = recipe.Split(' ').First();
_ = contents.Append("Bio-Rad ").Append("QS400MEPI".PadRight(17)).Append("Recipe: ").Append(recipe.PadRight(25)).AppendLine(processData.Date.ToString(Stratus.Description.GetDateFormat()));
_ = contents.Append("operator: ").Append(user.PadRight(22)).Append("batch: BIORAD #").AppendLine(logistics.JobID.Substring(6, 1));
_ = contents.Append("cassette: ").Append("".PadRight(22)).Append("wafer: ").AppendLine(processData.Cassette);
_ = contents.AppendLine("--------------------------------------------------------------------------------");
_ = contents.AppendLine(" position thickness position thickness position thickness");
segments = detail.Thickness.Split(',');
for (int j = 0; j < segments.Length; j++)
{
wafer = j + 1;
_ = contents.Append(wafer.ToString().PadLeft(11));
if ((wafer % 3) > 0)
_ = contents.Append(segments[j].PadLeft(10));
else
_ = contents.AppendLine(segments[j].PadLeft(10));
}
if ((segments.Length % 3) > 0)
_ = contents.AppendLine();
_ = contents.Append(" wafer mean thickness = ").Append(detail.Mean).Append(", std. dev = ").Append(detail.StdDev).Append(' ').AppendLine(detail.PassFail);
_ = contents.AppendLine("================================================================================");
_ = contents.AppendLine("");
_ = contents.AppendLine("Radial variation (computation B) PASS:");
_ = contents.AppendLine("");
_ = contents.AppendLine(" thickness 0.0000");
_ = contents.AppendLine("");
_ = contents.Append(" Slot:").Append(detail.Slot).AppendLine(";");
results.Add(new Tuple<string, bool, DateTime, string>(cassetteID, isBioRad, cassetteDateTime, contents.ToString()));
}
}
}
}
}
}
// **********************************************
// * Step #3 - Protect the raw data source file *
// **********************************************
// The multi-cassettes raw source file is ready to be splitted. Each cassette scan set has
// been determined to be complete (i.e., has the started & finished statements). At this point
// it is important to rename the multi-cassette raw data source file, located in the RawData
// folder, to a different name so that the tool does not attempt to update the file while being
// processed by the EAF cell instance.
// Get the last date/time the DataBioRad.txt file was updated
DateTime afterCheck = new(File.GetLastWriteTime(logistics.ReportFullPath).Ticks + tickOffset);
// Ensure that the DataBioRad.txt file has not been updated since the FileReader began the healthcheck
// If the date/time values are different between the "Before" and "After" checks then let it go. The
// tool is still busy trying to update the file. The FileReader will try to catch the data on the
// next update.
if (logistics.DateTimeFromSequence != afterCheck)
{
results.Clear();
_Log.Debug($"****Extract() - DataBioRad.txt file is getting updated fast");
_Log.Debug($"****Extract() - DataBioRadDateTime_AfterCheck = {afterCheck.Ticks}");
_Log.Debug($"****Extract() - DataBioRadDateTime_BeforeCheck = {logistics.Sequence}");
}
return results;
}
private static Dictionary<string, List<string>> GetCassetteIDAndDataSets(string[] reportFullPathlines)
{
Dictionary<string, List<string>> results = new();
string line;
string[] segments;
int cassetteEndIndex;
int thicknessCounter;
string thicknessHead;
string thicknessInfo;
string thicknessTail;
int cassetteStartIndex;
StringBuilder lines = new();
string slotID = string.Empty;
string cassetteID = string.Empty;
string batchHeader = string.Empty;
bool finishedReadingThicknessInfo;
bool slotInformationCaptured = false;
bool pointsInformationCaptured = false;
bool sourceInformationCaptured = false;
bool waferWaferInformationCaptured = false;
bool destinationInformationCaptured = false;
List<Tuple<string, int, int>> cassetteStartAndEnds = new();
for (int i = 0; i < reportFullPathlines.Length; i++)
{
line = reportFullPathlines[i].Trim();
if (string.IsNullOrEmpty(line))
continue;
if (line.StartsWith("Batch") && line.Contains("started"))
batchHeader = line;
if (i + 1 == reportFullPathlines.Length)
continue;
if (line.StartsWith("Cassette") && line.Contains("started"))
{
for (int j = i + 1; j < reportFullPathlines.Length; j++)
{
if (j + 1 == reportFullPathlines.Length)
cassetteStartAndEnds.Add(new Tuple<string, int, int>(batchHeader, i, j));
else
{
line = reportFullPathlines[j].Trim();
if (line.StartsWith("Cassette") && line.Contains("started"))
{
cassetteStartAndEnds.Add(new Tuple<string, int, int>(batchHeader, i, j - 1));
break;
}
}
}
}
}
foreach (Tuple<string, int, int> tuple in cassetteStartAndEnds)
{
_ = lines.Clear();
batchHeader = tuple.Item1;
cassetteEndIndex = tuple.Item3;
cassetteStartIndex = tuple.Item2;
for (int l = cassetteStartIndex; l <= cassetteEndIndex; l++)
{
line = reportFullPathlines[l].Trim();
if (string.IsNullOrEmpty(line))
continue;
if (l == cassetteStartIndex)
{
// Save the previously saved "Batch Header"
_ = lines.AppendLine(batchHeader);
// Save the first line of the cassette scan information
_ = lines.AppendLine(line);
// Each new cassette initialize the WaferWafer information flag
waferWaferInformationCaptured = false;
slotInformationCaptured = false;
if (line.Length > 9)
{
// Detected a new cassette data scan. Extract the cassette ID.
// Example: "Cassette 47-241330-4238 started."
segments = line.Substring(9).Split(new string[] { "started" }, StringSplitOptions.RemoveEmptyEntries);
if (segments.Length != 0)
// Remove illegal characters \/:*?"<>| found in the Cassette.
cassetteID = Regex.Replace(segments[0].Trim(), @"[\\,\/,\:,\*,\?,\"",\<,\>,\|]", "_").Split('\r')[0].Split('\n')[0];
}
}
// Continue reading and saving the cassette scan information, into the cassette
// scan output file, until the end of the cassette scan "Finished" statement has
// been detected.
// Maintain standard for mat between various BioRad tools. The "Points" and "Thickness"
// values between various BioRad tools might be spread over multiple lines. The following
// is simply to regroup the "Points" and "Thickness" information on the same line accordingly.
if (line.StartsWith("Wafer Wafer"))
{
_ = lines.AppendLine(line);
slotInformationCaptured = false;
waferWaferInformationCaptured = true;
}
else if (line.StartsWith("Slot"))
{
slotID = string.Empty;
segments = line.Split(' ');
if (segments.Length > 1)
slotID = segments[1];
// There are cases where the WaferWafer information is missing. Create a
// WaferWafer entry based off the slot number.
if (!waferWaferInformationCaptured)
{
waferWaferInformationCaptured = true;
_ = lines.AppendLine("Wafer Wafer " + slotID + ".");
}
_ = lines.AppendLine(line);
slotInformationCaptured = true;
}
else if (line.StartsWith("Recipe"))
{
_ = lines.AppendLine(line);
pointsInformationCaptured = false;
}
else if (line.StartsWith("Points"))
{
_ = lines.AppendLine(line);
pointsInformationCaptured = true;
}
else if (line.Contains("Thickness"))
{
// Before addressing the "Thickness" section, ensure that the "Points" section
// has been found. Otherwise, we need to write out a default value.
if (!pointsInformationCaptured)
{
// No "Points" information has been capture. Default to "Points : 0 0"
_ = lines.AppendLine("Points : 0 0");
pointsInformationCaptured = true;
}
// The "Thickness" output section comes out differently between various Stratus tools. In some
// cases, the thickness values are either empty (no values), on the same line or on different lines.
// Below are examples of how the data needs to be formatted after being parsed:
// Thickness, um 1 - 1 0
// Thickness, um 1 - 1 13.630
// Thickness, um 1 - 9 1.197 1.231 1.248 1.235 1.199 1.202 1.236 1.242 1.212
thicknessCounter = 0;
thicknessHead = line;
thicknessInfo = "";
thicknessTail = "";
finishedReadingThicknessInfo = false;
for (int t = l + 1; t <= cassetteEndIndex; t++)
{
l = t;
line = reportFullPathlines[l].Trim();
if (string.IsNullOrEmpty(line))
continue;
if (!line.StartsWith("Slot"))
{
thicknessCounter++;
thicknessTail = string.Concat(thicknessTail, " ", line);
}
else
{
finishedReadingThicknessInfo = true;
if (thicknessCounter != 0)
thicknessInfo = string.Concat(" 1 - ", thicknessCounter);
else
{
// Two possible formatting scenarios at this point. Either the data was already
// formatted properly on one line. Or the Thickness value was missing, in which
// case we need to default the thickness value to zero (0).
segments = thicknessHead.Split(' ');
if (segments.Length > 2)
{
// The "Thickness" raw data if formatted as a normal single line format and
// already include the Header + Info + Tail
}
else
{
// The "Thickness raw data has no values. Formatting the output with zero.
thicknessInfo = " 1 - 1";
thicknessTail = " 0";
}
}
_ = lines.AppendLine(string.Concat(thicknessHead, thicknessInfo, thicknessTail));
// The "Slot" keyword is the tag that determines the end of the Thickness section. The "Slot"
// information has already been ready. Simply write it back.
_ = lines.AppendLine(line);
}
if (finishedReadingThicknessInfo)
break;
}
}
else if (line.StartsWith("Mean"))
{
_ = lines.AppendLine(line);
sourceInformationCaptured = false;
destinationInformationCaptured = false;
}
else if (line.StartsWith("Source:") && slotInformationCaptured)
{
_ = lines.AppendLine(line);
sourceInformationCaptured = true;
}
else if (line.StartsWith("Destination:") && slotInformationCaptured)
{
if (!sourceInformationCaptured)
{
sourceInformationCaptured = true;
_ = lines.AppendLine(string.Concat("Source: Slot ", slotID, ", Cassette"));
}
_ = lines.AppendLine(line);
destinationInformationCaptured = true;
// Each time a cassette slot section has been completed, we must reinitialize
// the "Wafer Wafer" information flag in case there are multiple slots in the
// same cassette
slotInformationCaptured = false;
waferWaferInformationCaptured = false;
}
else if (line.StartsWith("Cassette") && line.Contains("finished."))
{
// Reach the end of the cassette data set information
if (!sourceInformationCaptured)
{
sourceInformationCaptured = true;
_ = lines.AppendLine(string.Concat("Source: Slot ", slotID, ", Cassette"));
}
if (!destinationInformationCaptured)
{
destinationInformationCaptured = true;
_ = lines.AppendLine(string.Concat("Destination: Slot ", slotID, ", Cassette"));
// Each time a cassette slot section has been completed, we must reinitialize
// the "Wafer Wafer" information flag in case there are multiple slots in the
// same cassette
slotInformationCaptured = false;
waferWaferInformationCaptured = false;
}
// Write the end of cassette statement to the output file
_ = lines.AppendLine(line);
// Read the Mean-Average line information, post the cassette "Finished" statement
for (int a = l + 1; a <= cassetteEndIndex; a++)
{
l = a;
line = reportFullPathlines[l].Trim();
if (string.IsNullOrEmpty(line))
continue;
// There are many blank lines in the source file. Search for the first
// occurrence of the string "Mean".
if (line.StartsWith("Mean"))
{
_ = lines.AppendLine(line);
break;
}
// The mean Average information is missing. We are done reading the cassette information.
if (line.StartsWith("Batch"))
break;
}
if (!results.ContainsKey(cassetteID))
results.Add(cassetteID, new List<string>());
results[cassetteID].Add(lines.ToString());
}
}
}
return results;
}
}
Reference in New Issue View Git Blame Copy Permalink