diff --git a/Adaptation/FileHandlers/Stratus/FileRead.cs b/Adaptation/FileHandlers/Stratus/FileRead.cs index 5db0bcf..6b96cbe 100644 --- a/Adaptation/FileHandlers/Stratus/FileRead.cs +++ b/Adaptation/FileHandlers/Stratus/FileRead.cs @@ -111,7 +111,9 @@ public class FileRead : Shared.FileRead, IFileRead if (iProcessData is not ProcessData processData) throw new Exception(string.Concat("A) No Data - ", dateTime.Ticks)); string mid; - if (!string.IsNullOrEmpty(processData.Employee) && string.IsNullOrEmpty(processData.Reactor) && string.IsNullOrEmpty(processData.RDS) && string.IsNullOrEmpty(processData.PSN)) + if (!string.IsNullOrEmpty(processData.Cassette) && string.IsNullOrEmpty(processData.Reactor) && string.IsNullOrEmpty(processData.RDS) && string.IsNullOrEmpty(processData.PSN)) + mid = processData.Cassette; + else if (!string.IsNullOrEmpty(processData.Employee) && string.IsNullOrEmpty(processData.Reactor) && string.IsNullOrEmpty(processData.RDS) && string.IsNullOrEmpty(processData.PSN)) mid = processData.Employee; else { diff --git a/Adaptation/FileHandlers/Stratus/ProcessData.cs b/Adaptation/FileHandlers/Stratus/ProcessData.cs index 0adc58f..f04cc2d 100644 --- a/Adaptation/FileHandlers/Stratus/ProcessData.cs +++ b/Adaptation/FileHandlers/Stratus/ProcessData.cs @@ -321,7 +321,7 @@ public partial class ProcessData : IProcessData string defaultLayer = string.Empty; string defaultReactor = string.Empty; string defaultEmployee = string.Empty; - if (Regex.IsMatch(text, @"^[a-zA-z][0-9]{4}$")) + if (Regex.IsMatch(text, @"^[a-zA-z][0-9]{2,4}$")) { cassette = text.ToUpper(); psn = defaultPSN; @@ -423,7 +423,6 @@ public partial class ProcessData : IProcessData if (dateTimeText.EndsWith(".")) dateTimeText = dateTimeText.Remove(dateTimeText.Length - 1, 1); date = GetDateTime(logistics, dateTimeText); - ; Descriptor descriptor = GetDescriptor(text); cassette = descriptor.Cassette; psn = descriptor.PSN; @@ -457,9 +456,13 @@ public partial class ProcessData : IProcessData receivedData = File.ReadAllText(logistics.ReportFullPath); _Log.Debug($"****ParseData - Source file contents:"); _Log.Debug(receivedData); - string[] files = Directory.GetFiles(Path.GetDirectoryName(logistics.ReportFullPath), string.Concat(originalDataBioRad, logistics.Sequence, "*"), SearchOption.TopDirectoryOnly); - foreach (string file in files) - fileInfoCollection.Add(new FileInfo(file)); + List moveFiles = new(); + string directoryName = Path.GetDirectoryName(logistics.ReportFullPath); + string fileNameWithoutExtension = Path.GetFileNameWithoutExtension(logistics.ReportFullPath); + moveFiles.AddRange(Directory.GetFiles(directoryName, string.Concat(originalDataBioRad, "*", logistics.Sequence, "*"), SearchOption.TopDirectoryOnly)); + moveFiles.AddRange(Directory.GetFiles(directoryName, string.Concat(originalDataBioRad, "*", fileNameWithoutExtension.Split('_').Last(), "*"), SearchOption.TopDirectoryOnly)); + foreach (string moveFile in moveFiles.Distinct()) + fileInfoCollection.Add(new FileInfo(moveFile)); if (!string.IsNullOrEmpty(receivedData)) { int i; diff --git a/Adaptation/FileHandlers/txt/FileRead.cs b/Adaptation/FileHandlers/txt/FileRead.cs index e65dbeb..c449f1b 100644 --- a/Adaptation/FileHandlers/txt/FileRead.cs +++ b/Adaptation/FileHandlers/txt/FileRead.cs @@ -115,8 +115,8 @@ public class FileRead : Shared.FileRead, IFileRead string tupleFileName; DateTime cassetteDateTime; string directoryName = Path.GetDirectoryName(reportFullPath); - string sequenceDirectoryName = string.Concat(Path.GetDirectoryName(reportFullPath), @"\", _Logistics.Sequence); - string originalDataBioRad = string.Concat(Path.GetDirectoryName(reportFullPath), @"\", _OriginalDataBioRad, _Logistics.Sequence, ".txt"); + string sequenceDirectoryName = Path.Combine(directoryName, _Logistics.Sequence.ToString()); + string originalDataBioRad = Path.Combine(directoryName, $"{_OriginalDataBioRad}{_Logistics.Sequence}.txt"); List> tuples = ProcessData.GetTuples(this, _Logistics, _TickOffset.Value, results.Item4, _OriginalDataBioRad); if (_IsEAFHosted) { @@ -125,7 +125,7 @@ public class FileRead : Shared.FileRead, IFileRead if (!Directory.Exists(sequenceDirectoryName)) _ = Directory.CreateDirectory(sequenceDirectoryName); File.Move(reportFullPath, originalDataBioRad); - _Log.Debug(string.Concat("****Extract() - Renamed [", reportFullPath, "] to [", originalDataBioRad, "] ", dateTime.Ticks)); + _Log.Debug(string.Concat("****Extract() - Renamed [", reportFullPath, "] to [", originalDataBioRad, "]")); } foreach (Tuple tuple in tuples) { @@ -137,12 +137,12 @@ public class FileRead : Shared.FileRead, IFileRead tupleFileName = string.Concat("DetailDataBioRad_", cassetteID, "_", cassetteDateTime.Ticks, ".txt"); else tupleFileName = string.Concat("CassetteDataBioRad_", cassetteID, "_", cassetteDateTime.Ticks, ".txt"); - fileNameTemp = string.Concat(sequenceDirectoryName, @"\", tupleFileName); + fileNameTemp = Path.Combine(sequenceDirectoryName, tupleFileName); File.WriteAllText(fileNameTemp, dataText); File.SetLastWriteTime(fileNameTemp, cassetteDateTime); if (_Logistics.Sequence != cassetteDateTime.Ticks && File.Exists(originalDataBioRad)) - File.Copy(originalDataBioRad, string.Concat(Path.GetDirectoryName(reportFullPath), @"\", _OriginalDataBioRad, cassetteDateTime.Ticks, ".txt")); - File.Move(fileNameTemp, string.Concat(directoryName, @"\", tupleFileName)); + File.Copy(originalDataBioRad, Path.Combine(directoryName, $"{_OriginalDataBioRad}{cassetteDateTime.Ticks}.txt")); + File.Move(fileNameTemp, Path.Combine(directoryName, tupleFileName)); } if (Directory.Exists(sequenceDirectoryName)) Directory.Delete(sequenceDirectoryName); diff --git a/Adaptation/FileHandlers/txt/ProcessData.cs b/Adaptation/FileHandlers/txt/ProcessData.cs index efcf861..b99c9f8 100644 --- a/Adaptation/FileHandlers/txt/ProcessData.cs +++ b/Adaptation/FileHandlers/txt/ProcessData.cs @@ -17,11 +17,10 @@ public partial class ProcessData { List> results = new(); ILog log = LogManager.GetLogger(typeof(ProcessData)); + string[] reportFullPathlines = File.ReadAllLines(logistics.ReportFullPath); // *********************************************************************************** // * Step #2 - Verify completeness of each cassette scan in the raw data source file * // *********************************************************************************** - string line; - StreamReader rawDataFilePtr; bool? cassetteScanCompleted = null; // Scrub the source file to verify that for each cassette, present in the file, there is a complete // data set (i.e., that is there is a start and finished statement). @@ -32,454 +31,195 @@ public partial class ProcessData // Incomplete data file. File will be process and generate error for the incomplete portion. // Scenario #3 - Only Cassette "Started" // Bail out of the solution. Source data file not ready to be processed. - using (rawDataFilePtr = new StreamReader(logistics.ReportFullPath)) + foreach (string line in reportFullPathlines) { - for (short i = 0; i < short.MaxValue; i++) + if (line is null) + break; + if (line.Contains("Cassette") && line.Contains("started") && (cassetteScanCompleted is null || cassetteScanCompleted.Value)) { - line = rawDataFilePtr.ReadLine(); - if (line is null) - break; - if (line.Contains("Cassette") && line.Contains("started") && (cassetteScanCompleted is null || cassetteScanCompleted.Value)) - { - cassetteScanCompleted = false; - log.Debug("****Extract() - CassetteScanCompleted = FALSE"); - } - else if (line.Contains("Cassette") && line.Contains("finished") && (cassetteScanCompleted is null || !cassetteScanCompleted.Value)) - { - cassetteScanCompleted = true; - log.Debug("****Extract() - CassetteScanCompleted = TRUE"); - } + cassetteScanCompleted = false; + log.Debug("****Extract() - CassetteScanCompleted = FALSE"); + } + else if (line.Contains("Cassette") && line.Contains("finished") && (cassetteScanCompleted is null || !cassetteScanCompleted.Value)) + { + cassetteScanCompleted = true; + log.Debug("****Extract() - CassetteScanCompleted = TRUE"); } - // Making sure that the file has been released - rawDataFilePtr.Close(); - rawDataFilePtr?.Dispose(); } - if (cassetteScanCompleted is null || !cassetteScanCompleted.Value) + Dictionary> cassetteIDAndDataSets; + if (string.IsNullOrEmpty(logistics.ReportFullPath)) + cassetteIDAndDataSets = new(); + else if (cassetteScanCompleted is null || !cassetteScanCompleted.Value) + { + cassetteIDAndDataSets = new(); // Raw source file has an incomplete data set or it only contains a "Process failed" and should not be // processed /split yet. Simply get out of this routine until enough data has been appended to the file. log.Debug($"****Extract() - Raw source file has an incomplete data set and should not be processed yet."); + } else + cassetteIDAndDataSets = GetCassetteIDAndDataSets(reportFullPathlines); + if (cassetteIDAndDataSets.Any()) { - Dictionary> cassetteIDAndDataSets = new(); - if (!string.IsNullOrEmpty(logistics.ReportFullPath)) + int wafer; + string user; + string runID; + bool isBioRad; + string recipe; + int count = -1; + int stringIndex; + string dataText; + string dataType; + string[] segments; + string cassetteID; + string recipeName; + IProcessData iProcessData; + DateTime cassetteDateTime; + string recipeSearch = "Recipe"; + string toolType = string.Empty; + StringBuilder contents = new(); + Stratus.ProcessData processData; + foreach (KeyValuePair> keyValuePair in cassetteIDAndDataSets) { - string[] segments; - int cassetteEndIndex; - int thicknessCounter; - string thicknessHead; - string thicknessInfo; - string thicknessTail; - int cassetteStartIndex; - StringBuilder lines = new(); - string slotID = string.Empty; - string cassetteID = string.Empty; - string batchHeader = string.Empty; - bool finishedReadingThicknessInfo; - bool slotInformationCaptured = false; - bool pointsInformationCaptured = false; - bool sourceInformationCaptured = false; - bool waferWaferInformationCaptured = false; - bool destinationInformationCaptured = false; - string[] reportFullPathlines = File.ReadAllLines(logistics.ReportFullPath); - List> cassetteStartAndEnds = new(); - for (int i = 0; i < reportFullPathlines.Length; i++) + isBioRad = false; + dataType = string.Empty; + cassetteID = keyValuePair.Key; + for (int i = 0; i < keyValuePair.Value.Count; i++) { - line = reportFullPathlines[i].Trim(); - if (string.IsNullOrEmpty(line)) - continue; - if (line.StartsWith("Batch") && line.Contains("started")) - batchHeader = line; - if (i + 1 == reportFullPathlines.Length) - continue; - if (line.StartsWith("Cassette") && line.Contains("started")) + dataText = keyValuePair.Value[i]; + // Finished capturing the complete cassette scan data information. Release the cassette file. + if (dataText.Contains("Cassette") && + dataText.Contains("Wafer") && + dataText.Contains("Slot") && + dataText.Contains("Recipe") && + dataText.Contains("Points") && + dataText.Contains("Thickness") && + dataText.Contains("Mean") && + dataText.Contains("Source:") && + dataText.Contains("Destination:")) { - for (int j = i + 1; j < reportFullPathlines.Length; j++) - { - if (j + 1 == reportFullPathlines.Length) - cassetteStartAndEnds.Add(new Tuple(batchHeader, i, j)); - else - { - line = reportFullPathlines[j].Trim(); - if (line.StartsWith("Cassette") && line.Contains("started")) - { - cassetteStartAndEnds.Add(new Tuple(batchHeader, i, j - 1)); - break; - } - } - } - } - } - foreach (Tuple tuple in cassetteStartAndEnds) - { - _ = lines.Clear(); - batchHeader = tuple.Item1; - cassetteEndIndex = tuple.Item3; - cassetteStartIndex = tuple.Item2; - for (int l = cassetteStartIndex; l <= cassetteEndIndex; l++) - { - line = reportFullPathlines[l].Trim(); - if (string.IsNullOrEmpty(line)) - continue; - if (l == cassetteStartIndex) - { - // Save the previously saved "Batch Header" - _ = lines.AppendLine(batchHeader); - // Save the first line of the cassette scan information - _ = lines.AppendLine(line); - // Each new cassette initialize the WaferWafer information flag - waferWaferInformationCaptured = false; - slotInformationCaptured = false; - if (line.Length > 9) - { - // Detected a new cassette data scan. Extract the cassette ID. - // Example: "Cassette 47-241330-4238 started." - segments = line.Substring(9).Split(new string[] { "started" }, StringSplitOptions.RemoveEmptyEntries); - if (segments.Any()) - { - // Detected a new cassette scan in the raw source file - cassetteID = segments[0].Trim(); - cassetteID = cassetteID.Replace(":", string.Empty); - cassetteID = cassetteID.Replace("*", string.Empty); - cassetteID = cassetteID.Replace("\\", string.Empty); - } - } - } - // Continue reading and saving the cassette scan information, into the cassette - // scan output file, until the end of the cassette scan "Finished" statement has - // been detected. - // Maintain standard for mat between various BioRad tools. The "Points" and "Thickness" - // values between various BioRad tools might be spread over multiple lines. The following - // is simply to regroup the "Points" and "Thickness" information on the same line accordingly. - if (line.StartsWith("Wafer Wafer")) - { - _ = lines.AppendLine(line); - slotInformationCaptured = false; - waferWaferInformationCaptured = true; - } - else if (line.StartsWith("Slot")) - { - slotID = string.Empty; - segments = line.Split(' '); - if (segments.Length > 1) - slotID = segments[1]; - // There are cases where the WaferWafer information is missing. Create a - // WaferWafer entry based off the slot number. - if (!waferWaferInformationCaptured) - { - waferWaferInformationCaptured = true; - _ = lines.AppendLine("Wafer Wafer " + slotID + "."); - } - _ = lines.AppendLine(line); - slotInformationCaptured = true; - } - else if (line.StartsWith("Recipe")) - { - _ = lines.AppendLine(line); - pointsInformationCaptured = false; - } - else if (line.StartsWith("Points")) - { - _ = lines.AppendLine(line); - pointsInformationCaptured = true; - } - else if (line.Contains("Thickness")) - { - // Before addressing the "Thickness" section, ensure that the "Points" section - // has been found. Otherwise, we need to write out a default value. - if (!pointsInformationCaptured) - { - // No "Points" information has been capture. Default to "Points : 0 0" - _ = lines.AppendLine("Points : 0 0"); - pointsInformationCaptured = true; - } - // The "Thickness" output section comes out differently between various Stratus tools. In some - // cases, the thickness values are either empty (no values), on the same line or on different lines. - // Below are examples of how the data needs to be formatted after being parsed: - // Thickness, um 1 - 1 0 - // Thickness, um 1 - 1 13.630 - // Thickness, um 1 - 9 1.197 1.231 1.248 1.235 1.199 1.202 1.236 1.242 1.212 - thicknessCounter = 0; - thicknessHead = line; - thicknessInfo = ""; - thicknessTail = ""; - finishedReadingThicknessInfo = false; - for (int t = l + 1; t <= cassetteEndIndex; t++) - { - l = t; - line = reportFullPathlines[l].Trim(); - if (string.IsNullOrEmpty(line)) - continue; - if (!line.StartsWith("Slot")) - { - thicknessCounter++; - thicknessTail = string.Concat(thicknessTail, " ", line); - } - else - { - finishedReadingThicknessInfo = true; - if (thicknessCounter != 0) - thicknessInfo = string.Concat(" 1 - ", thicknessCounter); - else - { - // Two possible formatting scenarios at this point. Either the data was already - // formatted properly on one line. Or the Thickness value was missing, in which - // case we need to default the thickness value to zero (0). - segments = thicknessHead.Split(' '); - if (segments.Length > 2) - { - // The "Thickness" raw data if formatted as a normal single line format and - // already include the Header + Info + Tail - } - else - { - // The "Thickness raw data has no values. Formatting the output with zero. - thicknessInfo = " 1 - 1"; - thicknessTail = " 0"; - } - } - _ = lines.AppendLine(string.Concat(thicknessHead, thicknessInfo, thicknessTail)); - // The "Slot" keyword is the tag that determines the end of the Thickness section. The "Slot" - // information has already been ready. Simply write it back. - _ = lines.AppendLine(line); - } - if (finishedReadingThicknessInfo) - break; - } - } - else if (line.StartsWith("Mean")) - { - _ = lines.AppendLine(line); - sourceInformationCaptured = false; - destinationInformationCaptured = false; - } - else if (line.StartsWith("Source:") && slotInformationCaptured) - { - _ = lines.AppendLine(line); - sourceInformationCaptured = true; - } - else if (line.StartsWith("Destination:") && slotInformationCaptured) - { - if (!sourceInformationCaptured) - { - sourceInformationCaptured = true; - _ = lines.AppendLine(string.Concat("Source: Slot ", slotID, ", Cassette")); - } - _ = lines.AppendLine(line); - destinationInformationCaptured = true; - // Each time a cassette slot section has been completed, we must reinitialize - // the "Wafer Wafer" information flag in case there are multiple slots in the - // same cassette - slotInformationCaptured = false; - waferWaferInformationCaptured = false; - } - else if (line.StartsWith("Cassette") && line.Contains("finished.")) - { - // Reach the end of the cassette data set information - if (!sourceInformationCaptured) - { - sourceInformationCaptured = true; - _ = lines.AppendLine(string.Concat("Source: Slot ", slotID, ", Cassette")); - } - if (!destinationInformationCaptured) - { - destinationInformationCaptured = true; - _ = lines.AppendLine(string.Concat("Destination: Slot ", slotID, ", Cassette")); - // Each time a cassette slot section has been completed, we must reinitialize - // the "Wafer Wafer" information flag in case there are multiple slots in the - // same cassette - slotInformationCaptured = false; - waferWaferInformationCaptured = false; - } - // Write the end of cassette statement to the output file - _ = lines.AppendLine(line); - // Read the Mean-Average line information, post the cassette "Finished" statement - for (int a = l + 1; a <= cassetteEndIndex; a++) - { - l = a; - line = reportFullPathlines[l].Trim(); - if (string.IsNullOrEmpty(line)) - continue; - // There are many blank lines in the source file. Search for the first - // occurrence of the string "Mean". - if (line.StartsWith("Mean")) - { - _ = lines.AppendLine(line); - break; - } - // The mean Average information is missing. We are done reading the cassette information. - if (line.StartsWith("Batch")) - break; - } - if (!cassetteIDAndDataSets.ContainsKey(cassetteID)) - cassetteIDAndDataSets.Add(cassetteID, new List()); - cassetteIDAndDataSets[cassetteID].Add(lines.ToString()); - } - } - } - if (cassetteStartAndEnds is null) - { } - } - if (cassetteIDAndDataSets.Any()) - { - int wafer; - string user; - string runID; - bool isBioRad; - string recipe; - int count = -1; - int stringIndex; - string dataText; - string dataType; - string[] segments; - string cassetteID; - string recipeName; - IProcessData iProcessData; - DateTime cassetteDateTime; - string recipeSearch = "Recipe"; - string toolType = string.Empty; - StringBuilder contents = new(); - Stratus.ProcessData processData; - foreach (KeyValuePair> keyValuePair in cassetteIDAndDataSets) - { - isBioRad = false; - dataType = string.Empty; - cassetteID = keyValuePair.Key; - for (int i = 0; i < keyValuePair.Value.Count; i++) - { - dataText = keyValuePair.Value[i]; - // Finished capturing the complete cassette scan data information. Release the cassette file. - if (dataText.Contains("Cassette") && - dataText.Contains("Wafer") && - dataText.Contains("Slot") && - dataText.Contains("Recipe") && - dataText.Contains("Points") && - dataText.Contains("Thickness") && - dataText.Contains("Mean") && - dataText.Contains("Source:") && - dataText.Contains("Destination:")) - { - // Extract the recipe name - runID = string.Empty; - recipeName = string.Empty; - stringIndex = dataText.IndexOf(recipeSearch); - recipeName = dataText.Substring(stringIndex + recipeSearch.Length); - log.Debug($"****Extract(FDR): recipeName = {recipeName}"); + // Extract the recipe name + runID = string.Empty; + recipeName = string.Empty; + stringIndex = dataText.IndexOf(recipeSearch); + recipeName = dataText.Substring(stringIndex + recipeSearch.Length); + log.Debug($"****Extract(FDR): recipeName = {recipeName}"); #pragma warning disable CA2249 - if (!string.IsNullOrEmpty(recipeName) && (recipeName.IndexOf("center", StringComparison.CurrentCultureIgnoreCase) >= 0)) + if (!string.IsNullOrEmpty(recipeName) && (recipeName.IndexOf("center", StringComparison.CurrentCultureIgnoreCase) >= 0)) #pragma warning restore CA2249 - { - /***************************************/ - /* STRATUS Measurement = FQA Thickness */ - /***************************************/ - // Recipes that contains the substring "Center" are STRATUS centerpoint recipes. They are used for Inspection and FQA measurements. - // measurement. The data from these scans should be uploaded to the Metrology Viewer database as STRATUS and uploaded to the - // OpenInsight [FQA Thickness - Post Epi - QA Metrology / Thk/RHO Value for each slotID] automatically. - isBioRad = false; - toolType = "STRATUS"; - dataType = "FQA Thickness"; - } -#pragma warning disable CA2249 - else if (!string.IsNullOrEmpty(recipeName) && (recipeName.IndexOf("prod_", StringComparison.CurrentCultureIgnoreCase) >= 0)) -#pragma warning restore CA2249 - { - /******************************************/ - /* BIORAD Measurement = Product Thickness */ - /******************************************/ - // Recipes that contains the substring "Center" are STRATUS centerpoint recipes. They are used for Inspection and FQA measurements. - // measurement. The data from these scans should be uploaded to the Metrology Viewer database as STRATUS and uploaded to the - // OpenInsight [FQA Thickness - Post Epi - QA Metrology / Thk/RHO Value for each slotID] automatically. - isBioRad = true; - toolType = "BIORAD"; - dataType = "Product Thickness"; - } - else if (!string.IsNullOrEmpty(recipeName) && -#pragma warning disable CA2249 - ((recipeName.IndexOf("T-Low", StringComparison.CurrentCultureIgnoreCase) >= 0) || - (recipeName.IndexOf("T_Low", StringComparison.CurrentCultureIgnoreCase) >= 0) || - (recipeName.IndexOf("T-Mid", StringComparison.CurrentCultureIgnoreCase) >= 0) || - (recipeName.IndexOf("T_Mid", StringComparison.CurrentCultureIgnoreCase) >= 0) || - (recipeName.IndexOf("T-High", StringComparison.CurrentCultureIgnoreCase) >= 0) || - (recipeName.IndexOf("T_High", StringComparison.CurrentCultureIgnoreCase) >= 0))) -#pragma warning restore CA2249 - { - /*************************************/ - /* BIORAD Measurement = No Uploading */ - /*************************************/ - // Recipes that contains the substring "T-Low, T_Low, T-Mid, T_Mid and T-High, T_High" are BIORAD verification recipe. The information - // should be uploaded to the Metrology Viewer database as BIORAD. No OpenInsight. - isBioRad = true; - toolType = "BIORAD"; - dataType = "Verification"; - } - else - { - // Count the number of wafers (ref. "Source: Slot") in the cassette - int waferCount = Regex.Matches(dataText, "Source: Slot").Count; - if (waferCount == 1) - { - // Metrology Thickness. Upload to OpenInsight same as BR2 and BR3 - isBioRad = true; - toolType = "BIORAD"; - dataType = "Metrology Thickness"; - } - else if (waferCount > 1) - { - // Inspection Measurement. Do not upload to OpenInsight. - isBioRad = true; - toolType = "BIORAD"; - dataType = "Inspection"; - } - } - } - log.Debug($"****Extract(FDR): ToolType = {toolType}"); - log.Debug($"****Extract(FDR): DataType = {dataType}"); - if (!isBioRad) { - cassetteDateTime = logistics.DateTimeFromSequence.AddTicks(i * -1); - results.Add(new Tuple(cassetteID, isBioRad, cassetteDateTime, dataText)); + /***************************************/ + /* STRATUS Measurement = FQA Thickness */ + /***************************************/ + // Recipes that contains the substring "Center" are STRATUS centerpoint recipes. They are used for Inspection and FQA measurements. + // measurement. The data from these scans should be uploaded to the Metrology Viewer database as STRATUS and uploaded to the + // OpenInsight [FQA Thickness - Post Epi - QA Metrology / Thk/RHO Value for each slotID] automatically. + isBioRad = false; + toolType = "STRATUS"; + dataType = "FQA Thickness"; + } +#pragma warning disable CA2249 + else if (!string.IsNullOrEmpty(recipeName) && (recipeName.IndexOf("prod_", StringComparison.CurrentCultureIgnoreCase) >= 0)) +#pragma warning restore CA2249 + { + /******************************************/ + /* BIORAD Measurement = Product Thickness */ + /******************************************/ + // Recipes that contains the substring "Center" are STRATUS centerpoint recipes. They are used for Inspection and FQA measurements. + // measurement. The data from these scans should be uploaded to the Metrology Viewer database as STRATUS and uploaded to the + // OpenInsight [FQA Thickness - Post Epi - QA Metrology / Thk/RHO Value for each slotID] automatically. + isBioRad = true; + toolType = "BIORAD"; + dataType = "Product Thickness"; + } + else if (!string.IsNullOrEmpty(recipeName) && +#pragma warning disable CA2249 + ((recipeName.IndexOf("T-Low", StringComparison.CurrentCultureIgnoreCase) >= 0) || + (recipeName.IndexOf("T_Low", StringComparison.CurrentCultureIgnoreCase) >= 0) || + (recipeName.IndexOf("T-Mid", StringComparison.CurrentCultureIgnoreCase) >= 0) || + (recipeName.IndexOf("T_Mid", StringComparison.CurrentCultureIgnoreCase) >= 0) || + (recipeName.IndexOf("T-High", StringComparison.CurrentCultureIgnoreCase) >= 0) || + (recipeName.IndexOf("T_High", StringComparison.CurrentCultureIgnoreCase) >= 0))) +#pragma warning restore CA2249 + { + /*************************************/ + /* BIORAD Measurement = No Uploading */ + /*************************************/ + // Recipes that contains the substring "T-Low, T_Low, T-Mid, T_Mid and T-High, T_High" are BIORAD verification recipe. The information + // should be uploaded to the Metrology Viewer database as BIORAD. No OpenInsight. + isBioRad = true; + toolType = "BIORAD"; + dataType = "Verification"; } else { - processData = new Stratus.ProcessData(fileRead, logistics, fileInfoCollection, originalDataBioRad, dataText: dataText); - iProcessData = processData; - if (!iProcessData.Details.Any()) - log.Warn("No Details!"); - else + // Count the number of wafers (ref. "Source: Slot") in the cassette + int waferCount = Regex.Matches(dataText, "Source: Slot").Count; + if (waferCount == 1) { - foreach (object item in iProcessData.Details) + // Metrology Thickness. Upload to OpenInsight same as BR2 and BR3 + isBioRad = true; + toolType = "BIORAD"; + dataType = "Metrology Thickness"; + } + else if (waferCount > 1) + { + // Inspection Measurement. Do not upload to OpenInsight. + isBioRad = true; + toolType = "BIORAD"; + dataType = "Inspection"; + } + } + } + log.Debug($"****Extract(FDR): ToolType = {toolType}"); + log.Debug($"****Extract(FDR): DataType = {dataType}"); + if (!isBioRad) + { + cassetteDateTime = logistics.DateTimeFromSequence.AddTicks(i * -1); + results.Add(new Tuple(cassetteID, isBioRad, cassetteDateTime, dataText)); + } + else + { + processData = new Stratus.ProcessData(fileRead, logistics, fileInfoCollection, originalDataBioRad, dataText: dataText); + iProcessData = processData; + if (!iProcessData.Details.Any()) + log.Warn("No Details!"); + else + { + foreach (object item in iProcessData.Details) + { + if (item is not Stratus.Detail detail) + throw new Exception(); + count += 1; + _ = contents.Clear(); + cassetteDateTime = logistics.DateTimeFromSequence.AddTicks(count * -1); + user = processData.Employee?.ToString() ?? ""; + recipe = detail.Recipe?.ToString() ?? ""; + _ = contents.Append("Bio-Rad ").Append("QS400MEPI".PadRight(17)).Append("Recipe: ").Append(recipe.PadRight(25)).AppendLine(processData.Date.ToString(Stratus.Description.GetDateFormat())); + _ = contents.Append("operator: ").Append(user.PadRight(22)).Append("batch: BIORAD #").AppendLine(logistics.JobID.Substring(6, 1)); + _ = contents.Append("cassette: ").Append("".PadRight(22)).Append("wafer: ").AppendLine(processData.Cassette); + _ = contents.AppendLine("--------------------------------------------------------------------------------"); + _ = contents.AppendLine(" position thickness position thickness position thickness"); + segments = detail.Thickness.Split(','); + for (int j = 0; j < segments.Length; j++) { - if (item is not Stratus.Detail detail) - throw new Exception(); - count += 1; - _ = contents.Clear(); - cassetteDateTime = logistics.DateTimeFromSequence.AddTicks(count * -1); - user = processData.Employee?.ToString() ?? ""; - recipe = detail.Recipe?.ToString() ?? ""; - _ = contents.Append("Bio-Rad ").Append("QS400MEPI".PadRight(17)).Append("Recipe: ").Append(recipe.PadRight(25)).AppendLine(processData.Date.ToString(Stratus.Description.GetDateFormat())); - _ = contents.Append("operator: ").Append(user.PadRight(22)).Append("batch: BIORAD #").AppendLine(logistics.JobID.Substring(6, 1)); - _ = contents.Append("cassette: ").Append("".PadRight(22)).Append("wafer: ").AppendLine(processData.Cassette); - _ = contents.AppendLine("--------------------------------------------------------------------------------"); - _ = contents.AppendLine(" position thickness position thickness position thickness"); - segments = detail.Thickness.Split(','); - for (int j = 0; j < segments.Length; j++) - { - wafer = j + 1; - _ = contents.Append(wafer.ToString().PadLeft(11)); - if ((wafer % 3) > 0) - _ = contents.Append(segments[j].PadLeft(10)); - else - _ = contents.AppendLine(segments[j].PadLeft(10)); - } - if ((segments.Length % 3) > 0) - _ = contents.AppendLine(); - _ = contents.Append(" wafer mean thickness = ").Append(detail.Mean).Append(", std. dev = ").Append(detail.StdDev).Append(' ').AppendLine(detail.PassFail); - _ = contents.AppendLine("================================================================================"); - _ = contents.AppendLine(""); - _ = contents.AppendLine("Radial variation (computation B) PASS:"); - _ = contents.AppendLine(""); - _ = contents.AppendLine(" thickness 0.0000"); - results.Add(new Tuple(cassetteID, isBioRad, cassetteDateTime, contents.ToString())); + wafer = j + 1; + _ = contents.Append(wafer.ToString().PadLeft(11)); + if ((wafer % 3) > 0) + _ = contents.Append(segments[j].PadLeft(10)); + else + _ = contents.AppendLine(segments[j].PadLeft(10)); } + if ((segments.Length % 3) > 0) + _ = contents.AppendLine(); + _ = contents.Append(" wafer mean thickness = ").Append(detail.Mean).Append(", std. dev = ").Append(detail.StdDev).Append(' ').AppendLine(detail.PassFail); + _ = contents.AppendLine("================================================================================"); + _ = contents.AppendLine(""); + _ = contents.AppendLine("Radial variation (computation B) PASS:"); + _ = contents.AppendLine(""); + _ = contents.AppendLine(" thickness 0.0000"); + _ = contents.AppendLine(""); + _ = contents.Append(" Slot:").Append(detail.Slot).AppendLine(";"); + results.Add(new Tuple(cassetteID, isBioRad, cassetteDateTime, contents.ToString())); } } } @@ -509,4 +249,263 @@ public partial class ProcessData } return results; } + + private static Dictionary> GetCassetteIDAndDataSets(string[] reportFullPathlines) + { + Dictionary> results = new(); + string line; + string[] segments; + int cassetteEndIndex; + int thicknessCounter; + string thicknessHead; + string thicknessInfo; + string thicknessTail; + int cassetteStartIndex; + StringBuilder lines = new(); + string slotID = string.Empty; + string cassetteID = string.Empty; + string batchHeader = string.Empty; + bool finishedReadingThicknessInfo; + bool slotInformationCaptured = false; + bool pointsInformationCaptured = false; + bool sourceInformationCaptured = false; + bool waferWaferInformationCaptured = false; + bool destinationInformationCaptured = false; + List> cassetteStartAndEnds = new(); + for (int i = 0; i < reportFullPathlines.Length; i++) + { + line = reportFullPathlines[i].Trim(); + if (string.IsNullOrEmpty(line)) + continue; + if (line.StartsWith("Batch") && line.Contains("started")) + batchHeader = line; + if (i + 1 == reportFullPathlines.Length) + continue; + if (line.StartsWith("Cassette") && line.Contains("started")) + { + for (int j = i + 1; j < reportFullPathlines.Length; j++) + { + if (j + 1 == reportFullPathlines.Length) + cassetteStartAndEnds.Add(new Tuple(batchHeader, i, j)); + else + { + line = reportFullPathlines[j].Trim(); + if (line.StartsWith("Cassette") && line.Contains("started")) + { + cassetteStartAndEnds.Add(new Tuple(batchHeader, i, j - 1)); + break; + } + } + } + } + } + foreach (Tuple tuple in cassetteStartAndEnds) + { + _ = lines.Clear(); + batchHeader = tuple.Item1; + cassetteEndIndex = tuple.Item3; + cassetteStartIndex = tuple.Item2; + for (int l = cassetteStartIndex; l <= cassetteEndIndex; l++) + { + line = reportFullPathlines[l].Trim(); + if (string.IsNullOrEmpty(line)) + continue; + if (l == cassetteStartIndex) + { + // Save the previously saved "Batch Header" + _ = lines.AppendLine(batchHeader); + // Save the first line of the cassette scan information + _ = lines.AppendLine(line); + // Each new cassette initialize the WaferWafer information flag + waferWaferInformationCaptured = false; + slotInformationCaptured = false; + if (line.Length > 9) + { + // Detected a new cassette data scan. Extract the cassette ID. + // Example: "Cassette 47-241330-4238 started." + segments = line.Substring(9).Split(new string[] { "started" }, StringSplitOptions.RemoveEmptyEntries); + if (segments.Any()) + { + // Detected a new cassette scan in the raw source file + cassetteID = segments[0].Trim(); + cassetteID = cassetteID.Replace(":", string.Empty); + cassetteID = cassetteID.Replace("*", string.Empty); + cassetteID = cassetteID.Replace("\\", string.Empty); + } + } + } + // Continue reading and saving the cassette scan information, into the cassette + // scan output file, until the end of the cassette scan "Finished" statement has + // been detected. + // Maintain standard for mat between various BioRad tools. The "Points" and "Thickness" + // values between various BioRad tools might be spread over multiple lines. The following + // is simply to regroup the "Points" and "Thickness" information on the same line accordingly. + if (line.StartsWith("Wafer Wafer")) + { + _ = lines.AppendLine(line); + slotInformationCaptured = false; + waferWaferInformationCaptured = true; + } + else if (line.StartsWith("Slot")) + { + slotID = string.Empty; + segments = line.Split(' '); + if (segments.Length > 1) + slotID = segments[1]; + // There are cases where the WaferWafer information is missing. Create a + // WaferWafer entry based off the slot number. + if (!waferWaferInformationCaptured) + { + waferWaferInformationCaptured = true; + _ = lines.AppendLine("Wafer Wafer " + slotID + "."); + } + _ = lines.AppendLine(line); + slotInformationCaptured = true; + } + else if (line.StartsWith("Recipe")) + { + _ = lines.AppendLine(line); + pointsInformationCaptured = false; + } + else if (line.StartsWith("Points")) + { + _ = lines.AppendLine(line); + pointsInformationCaptured = true; + } + else if (line.Contains("Thickness")) + { + // Before addressing the "Thickness" section, ensure that the "Points" section + // has been found. Otherwise, we need to write out a default value. + if (!pointsInformationCaptured) + { + // No "Points" information has been capture. Default to "Points : 0 0" + _ = lines.AppendLine("Points : 0 0"); + pointsInformationCaptured = true; + } + // The "Thickness" output section comes out differently between various Stratus tools. In some + // cases, the thickness values are either empty (no values), on the same line or on different lines. + // Below are examples of how the data needs to be formatted after being parsed: + // Thickness, um 1 - 1 0 + // Thickness, um 1 - 1 13.630 + // Thickness, um 1 - 9 1.197 1.231 1.248 1.235 1.199 1.202 1.236 1.242 1.212 + thicknessCounter = 0; + thicknessHead = line; + thicknessInfo = ""; + thicknessTail = ""; + finishedReadingThicknessInfo = false; + for (int t = l + 1; t <= cassetteEndIndex; t++) + { + l = t; + line = reportFullPathlines[l].Trim(); + if (string.IsNullOrEmpty(line)) + continue; + if (!line.StartsWith("Slot")) + { + thicknessCounter++; + thicknessTail = string.Concat(thicknessTail, " ", line); + } + else + { + finishedReadingThicknessInfo = true; + if (thicknessCounter != 0) + thicknessInfo = string.Concat(" 1 - ", thicknessCounter); + else + { + // Two possible formatting scenarios at this point. Either the data was already + // formatted properly on one line. Or the Thickness value was missing, in which + // case we need to default the thickness value to zero (0). + segments = thicknessHead.Split(' '); + if (segments.Length > 2) + { + // The "Thickness" raw data if formatted as a normal single line format and + // already include the Header + Info + Tail + } + else + { + // The "Thickness raw data has no values. Formatting the output with zero. + thicknessInfo = " 1 - 1"; + thicknessTail = " 0"; + } + } + _ = lines.AppendLine(string.Concat(thicknessHead, thicknessInfo, thicknessTail)); + // The "Slot" keyword is the tag that determines the end of the Thickness section. The "Slot" + // information has already been ready. Simply write it back. + _ = lines.AppendLine(line); + } + if (finishedReadingThicknessInfo) + break; + } + } + else if (line.StartsWith("Mean")) + { + _ = lines.AppendLine(line); + sourceInformationCaptured = false; + destinationInformationCaptured = false; + } + else if (line.StartsWith("Source:") && slotInformationCaptured) + { + _ = lines.AppendLine(line); + sourceInformationCaptured = true; + } + else if (line.StartsWith("Destination:") && slotInformationCaptured) + { + if (!sourceInformationCaptured) + { + sourceInformationCaptured = true; + _ = lines.AppendLine(string.Concat("Source: Slot ", slotID, ", Cassette")); + } + _ = lines.AppendLine(line); + destinationInformationCaptured = true; + // Each time a cassette slot section has been completed, we must reinitialize + // the "Wafer Wafer" information flag in case there are multiple slots in the + // same cassette + slotInformationCaptured = false; + waferWaferInformationCaptured = false; + } + else if (line.StartsWith("Cassette") && line.Contains("finished.")) + { + // Reach the end of the cassette data set information + if (!sourceInformationCaptured) + { + sourceInformationCaptured = true; + _ = lines.AppendLine(string.Concat("Source: Slot ", slotID, ", Cassette")); + } + if (!destinationInformationCaptured) + { + destinationInformationCaptured = true; + _ = lines.AppendLine(string.Concat("Destination: Slot ", slotID, ", Cassette")); + // Each time a cassette slot section has been completed, we must reinitialize + // the "Wafer Wafer" information flag in case there are multiple slots in the + // same cassette + slotInformationCaptured = false; + waferWaferInformationCaptured = false; + } + // Write the end of cassette statement to the output file + _ = lines.AppendLine(line); + // Read the Mean-Average line information, post the cassette "Finished" statement + for (int a = l + 1; a <= cassetteEndIndex; a++) + { + l = a; + line = reportFullPathlines[l].Trim(); + if (string.IsNullOrEmpty(line)) + continue; + // There are many blank lines in the source file. Search for the first + // occurrence of the string "Mean". + if (line.StartsWith("Mean")) + { + _ = lines.AppendLine(line); + break; + } + // The mean Average information is missing. We are done reading the cassette information. + if (line.StartsWith("Batch")) + break; + } + if (!results.ContainsKey(cassetteID)) + results.Add(cassetteID, new List()); + results[cassetteID].Add(lines.ToString()); + } + } + } + return results; + } } \ No newline at end of file diff --git a/Adaptation/Shared/Test.cs b/Adaptation/Shared/Test.cs index aeb4bea..3aa3552 100644 --- a/Adaptation/Shared/Test.cs +++ b/Adaptation/Shared/Test.cs @@ -21,7 +21,7 @@ public enum Test Denton = 9, DiffusionLength = 45, GRATXTCenter = 51, - GRATXTEdge = 52, //Largest + GRATXTEdge = 52, GrowthRateXML = 50, Hall = 10, HgCV = 23, @@ -38,6 +38,7 @@ public enum Test RPMPLRatio = 17, RPMXY = 15, SP1 = 8, + SRP2100 = 53, //Largest Tencor = 7, UV = 35, VerificationLehighton = 14, diff --git a/Adaptation/_Tests/Extract/Staging/v2.49.0/BIORAD4.cs b/Adaptation/_Tests/Extract/Staging/v2.49.0/BIORAD4.cs index 2970ee5..07c0594 100644 --- a/Adaptation/_Tests/Extract/Staging/v2.49.0/BIORAD4.cs +++ b/Adaptation/_Tests/Extract/Staging/v2.49.0/BIORAD4.cs @@ -77,6 +77,22 @@ public class BIORAD4 Shared.AdaptationTesting.UpdatePassDirectory(variables[2]); } +#if DEBUG + [Ignore] +#endif + [TestMethod] + public void Staging__v2_49_0__BIORAD4__txt638187028378748930__THigh() + { + bool validatePDSF = false; + string check = "*DataBioRad.txt"; + _BIORAD4.Staging__v2_49_0__BIORAD4__txt(); + MethodBase methodBase = new StackFrame().GetMethod(); + string[] variables = _BIORAD4.AdaptationTesting.GetVariables(methodBase, check, validatePDSF); + IFileRead fileRead = _BIORAD4.AdaptationTesting.Get(methodBase, sourceFileLocation: variables[2], sourceFileFilter: variables[3], useCyclicalForDescription: false); + _ = fileRead.ReExtract(); + Shared.AdaptationTesting.UpdatePassDirectory(variables[2]); + } + #if DEBUG [Ignore] #endif diff --git a/Adaptation/_Tests/Static/MET08THFTIRSTRATUS.cs b/Adaptation/_Tests/Static/MET08THFTIRSTRATUS.cs index e3910c4..0bc1752 100644 --- a/Adaptation/_Tests/Static/MET08THFTIRSTRATUS.cs +++ b/Adaptation/_Tests/Static/MET08THFTIRSTRATUS.cs @@ -45,7 +45,9 @@ public class MET08THFTIRSTRATUS : LoggingUnitTesting, IDisposable Assert.IsTrue(dateTime.ToString("M/d/yyyy h:mm:ss tt") == dateTime.ToString()); } +#if DEBUG [Ignore] +#endif [TestMethod] public void Staging() { diff --git a/Adaptation/_Tests/Static/Stratus.cs b/Adaptation/_Tests/Static/Stratus.cs index fb7a8d6..158e7d9 100644 --- a/Adaptation/_Tests/Static/Stratus.cs +++ b/Adaptation/_Tests/Static/Stratus.cs @@ -38,6 +38,13 @@ public class Stratus : LoggingUnitTesting, IDisposable LoggingUnitTesting?.Dispose(); } + private static void NonThrowTryCatch() + { + try + { throw new Exception(); } + catch (Exception) { } + } + [TestMethod] public void TestDateTime() { @@ -171,10 +178,10 @@ public class Stratus : LoggingUnitTesting, IDisposable Assert.IsTrue(descriptor.Reactor is "75"); Assert.IsTrue(string.IsNullOrEmpty(descriptor.Zone)); Assert.IsTrue(string.IsNullOrEmpty(descriptor.Employee)); - descriptor = FileHandlers.Stratus.ProcessData.GetDescriptor("p5801"); - Assert.IsTrue(descriptor.Cassette == "P5801"); - descriptor = FileHandlers.Stratus.ProcessData.GetDescriptor("P5801"); - Assert.IsTrue(descriptor.Cassette == "P5801"); + descriptor = FileHandlers.Stratus.ProcessData.GetDescriptor("B48"); + Assert.IsTrue(descriptor.Cassette == "B48"); + descriptor = FileHandlers.Stratus.ProcessData.GetDescriptor("B48"); + Assert.IsTrue(descriptor.Cassette == "B48"); Assert.IsTrue(string.IsNullOrEmpty(descriptor.Layer)); Assert.IsTrue(string.IsNullOrEmpty(descriptor.PSN)); Assert.IsTrue(string.IsNullOrEmpty(descriptor.PSN)); @@ -182,9 +189,12 @@ public class Stratus : LoggingUnitTesting, IDisposable Assert.IsTrue(string.IsNullOrEmpty(descriptor.Zone)); Assert.IsTrue(string.IsNullOrEmpty(descriptor.Employee)); LoggingUnitTesting.Logger.LogInformation(string.Concat(methodBase.Name, " - Exit")); + NonThrowTryCatch(); } +#if DEBUG [Ignore] +#endif [TestMethod] public void Staging() { diff --git a/FileHandlers/FileRead.cs b/FileHandlers/FileRead.cs index dd38d02..e30c434 100644 --- a/FileHandlers/FileRead.cs +++ b/FileHandlers/FileRead.cs @@ -164,7 +164,7 @@ public partial class FileRead : FileReaderHandler, ISMTP Equipment.SelfDescriptionBuilder.AddParameterTypeDefinition(structuredType); } if (!parameterTypeDefinitions.ContainsKey(jsonProperty.Value.ValueKind)) - throw new Exception(string.Concat('{', jsonProperty.Value.ValueKind, "} is not mapped!")); + throw new Exception(string.Concat('<', jsonProperty.Name, "> {", jsonProperty.Value.ValueKind, "} is not mapped!")); } foreach (JsonProperty jsonProperty in jsonProperties) {