met08thftirstratus/Adaptation/FileHandlers/txt/ProcessData.cs

using Adaptation.Shared;
using Adaptation.Shared.Duplicator;
using Adaptation.Shared.Methods;
using log4net;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;

namespace Adaptation.FileHandlers.txt;

public partial class ProcessData : IProcessData
{

    private readonly ILog _Log;
    private readonly List<object> _Details;

    public string JobID { get; set; }
    public string MesEntity { get; set; }

    List<object> Shared.Properties.IProcessData.Details => _Details;

    internal ProcessData(IFileRead fileRead, Logistics logistics, long tickOffset, List<FileInfo> fileInfoCollection, string originalDataBioRad)
    {
        JobID = logistics.JobID;
        _Details = new List<object>();
        MesEntity = logistics.MesEntity;
        _Log = LogManager.GetLogger(typeof(ProcessData));
        List<Tuple<string, bool, DateTime, string>> tuples = Parse(fileRead, logistics, tickOffset, fileInfoCollection, originalDataBioRad);
        _Details.AddRange(tuples);
    }

    string IProcessData.GetCurrentReactor(IFileRead fileRead, Logistics logistics, Dictionary<string, string> reactors) => throw new Exception(string.Concat("See ", nameof(Parse)));

    Tuple<string, Test[], JsonElement[], List<FileInfo>> IProcessData.GetResults(IFileRead fileRead, Logistics logistics, List<FileInfo> fileInfoCollection)
    {
        Tuple<string, Test[], JsonElement[], List<FileInfo>> results;
        List<Test> tests = new();
        foreach (object item in _Details)
            tests.Add(Test.BioRadStratus);
        List<IDescription> descriptions = fileRead.GetDescriptions(fileRead, tests, this);
        if (tests.Count != descriptions.Count)
            throw new Exception();
        for (int i = 0; i < tests.Count; i++)
        {
            if (descriptions[i] is not Description description)
                throw new Exception();
            if (description.Test != (int)tests[i])
                throw new Exception();
        }
        FileInfo fileInfo = new($"{logistics.ReportFullPath}.descriptions.json");
        List<Description> fileReadDescriptions = (from l in descriptions select (Description)l).ToList();
        string json = JsonSerializer.Serialize(fileReadDescriptions, fileReadDescriptions.GetType());
        File.WriteAllText(fileInfo.FullName, json);
        File.SetLastWriteTime(fileInfo.FullName, logistics.DateTimeFromSequence);
        fileInfoCollection.Add(fileInfo);
        JsonElement[] jsonElements = JsonSerializer.Deserialize<JsonElement[]>(json) ?? throw new Exception();
        results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(logistics.Logistics1[0], tests.ToArray(), jsonElements, fileInfoCollection);
        return results;
    }

#nullable enable

    private List<Tuple<string, bool, DateTime, string>> Parse(IFileRead fileRead, Logistics logistics, long tickOffset, List<FileInfo> fileInfoCollection, string originalDataBioRad)
    {
        List<Tuple<string, bool, DateTime, string>> results = new();
        string[] reportFullPathlines = File.ReadAllLines(logistics.ReportFullPath);
        // ***********************************************************************************
        // * Step #2 - Verify completeness of each cassette scan in the raw data source file *
        // ***********************************************************************************
        bool? cassetteScanCompleted = null;
        // Scrub the source file to verify that for each cassette, present in the file, there is a complete
        // data set (i.e., that is there is a start and finished statement).
        //
        // Scenario #1 - Normal
        //      For every cassette "started" there must be a matching cassette "finished".
        // Scenario #2 - Only Cassette "finished" (with or witout additional cassette complete data sets)
        //      Incomplete data file.  File will be process and generate error for the incomplete portion.
        // Scenario #3 - Only Cassette "Started"
        //      Bail out of the solution.  Source data file not ready to be processed.
        foreach (string line in reportFullPathlines)
        {
            if (line is null)
                break;
            if (line.Contains("Cassette") && line.Contains("started") && (cassetteScanCompleted is null || cassetteScanCompleted.Value))
            {
                cassetteScanCompleted = false;
                _Log.Debug("****Extract() - CassetteScanCompleted = FALSE");
            }
            else if (line.Contains("Cassette") && line.Contains("finished") && (cassetteScanCompleted is null || !cassetteScanCompleted.Value))
            {
                cassetteScanCompleted = true;
                _Log.Debug("****Extract() - CassetteScanCompleted = TRUE");
            }
        }
        Dictionary<string, List<string>> cassetteIDAndDataSets;
        if (string.IsNullOrEmpty(logistics.ReportFullPath))
            cassetteIDAndDataSets = new();
        else if (cassetteScanCompleted is null || !cassetteScanCompleted.Value)
        {
            cassetteIDAndDataSets = new();
            // Raw source file has an incomplete data set or it only contains a "Process failed" and should not be
            // processed /split yet.  Simply get out of this routine until enough data has been appended to the file.
            _Log.Debug($"****Extract() - Raw source file has an incomplete data set and should not be processed yet.");
        }
        else
            cassetteIDAndDataSets = GetCassetteIDAndDataSets(reportFullPathlines);
        if (cassetteIDAndDataSets.Count != 0)
        {
            int wafer;
            string user;
            string runID;
            bool isBioRad;
            string recipe;
            int count = -1;
            int stringIndex;
            string dataText;
            string dataType;
            string[] segments;
            string cassetteID;
            string recipeName;
            IProcessData iProcessData;
            DateTime cassetteDateTime;
            string recipeSearch = "Recipe";
            string toolType = string.Empty;
            StringBuilder contents = new();
            Stratus.ProcessData processData;
            foreach (KeyValuePair<string, List<string>> keyValuePair in cassetteIDAndDataSets)
            {
                isBioRad = false;
                dataType = string.Empty;
                cassetteID = keyValuePair.Key;
                for (int i = 0; i < keyValuePair.Value.Count; i++)
                {
                    dataText = keyValuePair.Value[i];
                    // Finished capturing the complete cassette scan data information.  Release the cassette file.
                    if (dataText.Contains("Cassette") &&
                        dataText.Contains("Wafer") &&
                        dataText.Contains("Slot") &&
                        dataText.Contains("Recipe") &&
                        dataText.Contains("Points") &&
                        dataText.Contains("Thickness") &&
                        dataText.Contains("Mean") &&
                        dataText.Contains("Source:") &&
                        dataText.Contains("Destination:"))
                    {
                        // Extract the recipe name
                        runID = string.Empty;
                        recipeName = string.Empty;
                        stringIndex = dataText.IndexOf(recipeSearch);
                        recipeName = dataText.Substring(stringIndex + recipeSearch.Length);
                        _Log.Debug($"****Extract(FDR): recipeName = {recipeName}");
#pragma warning disable CA2249
                        if (!string.IsNullOrEmpty(recipeName) && (recipeName.IndexOf("center", StringComparison.CurrentCultureIgnoreCase) >= 0))
#pragma warning restore CA2249
                        {
                            /***************************************/
                            /* STRATUS Measurement = FQA Thickness */
                            /***************************************/
                            // Recipes that contains the substring "Center" are STRATUS centerpoint recipes.  They are used for Inspection and FQA measurements.
                            // measurement.  The data from these scans should be uploaded to the Metrology Viewer database as STRATUS and uploaded to the 
                            // OpenInsight [FQA Thickness - Post Epi - QA Metrology / Thk/RHO Value for each slotID] automatically.
                            isBioRad = false;
                            toolType = "STRATUS";
                            dataType = "FQA Thickness";
                        }
#pragma warning disable CA2249
                        else if (!string.IsNullOrEmpty(recipeName) && (recipeName.IndexOf("prod_", StringComparison.CurrentCultureIgnoreCase) >= 0))
#pragma warning restore CA2249
                        {
                            /******************************************/
                            /* BIORAD Measurement = Product Thickness */
                            /******************************************/
                            // Recipes that contains the substring "Center" are STRATUS centerpoint recipes.  They are used for Inspection and FQA measurements.
                            // measurement.  The data from these scans should be uploaded to the Metrology Viewer database as STRATUS and uploaded to the 
                            // OpenInsight [FQA Thickness - Post Epi - QA Metrology / Thk/RHO Value for each slotID] automatically.
                            isBioRad = true;
                            toolType = "BIORAD";
                            dataType = "Product Thickness";
                        }
                        else if (!string.IsNullOrEmpty(recipeName) &&
#pragma warning disable CA2249
                                    ((recipeName.IndexOf("T-Low", StringComparison.CurrentCultureIgnoreCase) >= 0) ||
                                     (recipeName.IndexOf("T_Low", StringComparison.CurrentCultureIgnoreCase) >= 0) ||
                                     (recipeName.IndexOf("T-Mid", StringComparison.CurrentCultureIgnoreCase) >= 0) ||
                                     (recipeName.IndexOf("T_Mid", StringComparison.CurrentCultureIgnoreCase) >= 0) ||
                                     (recipeName.IndexOf("T-High", StringComparison.CurrentCultureIgnoreCase) >= 0) ||
                                     (recipeName.IndexOf("T_High", StringComparison.CurrentCultureIgnoreCase) >= 0)))
#pragma warning restore CA2249
                        {
                            /*************************************/
                            /* BIORAD Measurement = No Uploading */
                            /*************************************/
                            // Recipes that contains the substring "T-Low, T_Low, T-Mid, T_Mid and T-High, T_High" are BIORAD verification recipe.  The information
                            // should be uploaded to the Metrology Viewer database as BIORAD.  No OpenInsight.
                            isBioRad = true;
                            toolType = "BIORAD";
                            dataType = "Verification";
                        }
                        else
                        {
                            // Count the number of wafers (ref. "Source: Slot") in the cassette
                            int waferCount = Regex.Matches(dataText, "Source: Slot").Count;
                            if (waferCount == 1)
                            {
                                // Metrology Thickness.  Upload to OpenInsight same as BR2 and BR3
                                isBioRad = true;
                                toolType = "BIORAD";
                                dataType = "Metrology Thickness";
                            }
                            else if (waferCount > 1)
                            {
                                // Inspection Measurement.  Do not upload to OpenInsight.
                                isBioRad = true;
                                toolType = "BIORAD";
                                dataType = "Inspection";
                            }
                        }
                    }
                    _Log.Debug($"****Extract(FDR): ToolType = {toolType}");
                    _Log.Debug($"****Extract(FDR): DataType = {dataType}");
                    if (!isBioRad)
                    {
                        cassetteDateTime = logistics.DateTimeFromSequence.AddTicks(i * -1);
                        results.Add(new Tuple<string, bool, DateTime, string>(cassetteID, isBioRad, cassetteDateTime, dataText));
                    }
                    else
                    {
                        Stratus.Run? complete = null;
                        processData = new Stratus.ProcessData(fileRead, logistics, fileInfoCollection, originalDataBioRad, complete, dataText: dataText);
                        iProcessData = processData;
                        if (iProcessData.Details.Count == 0)
                            _Log.Warn("No Details!");
                        else
                        {
                            foreach (object item in iProcessData.Details)
                            {
                                if (item is not Stratus.Detail detail)
                                    throw new Exception();
                                count += 1;
                                _ = contents.Clear();
                                cassetteDateTime = logistics.DateTimeFromSequence.AddTicks(count * -1);
                                user = processData.Employee?.ToString() ?? "";
                                recipe = detail.Recipe?.ToString() ?? "";
                                if (isBioRad)
                                    recipe = recipe.Split(' ').First();
                                _ = contents.Append("Bio-Rad ").Append("QS400MEPI".PadRight(17)).Append("Recipe: ").Append(recipe.PadRight(25)).AppendLine(processData.Date.ToString(Stratus.Description.GetDateFormat()));
                                _ = contents.Append("operator: ").Append(user.PadRight(22)).Append("batch:     BIORAD #").AppendLine(logistics.JobID.Substring(6, 1));
                                _ = contents.Append("cassette: ").Append("".PadRight(22)).Append("wafer:     ").AppendLine(processData.Cassette);
                                _ = contents.AppendLine("--------------------------------------------------------------------------------");
                                _ = contents.AppendLine("   position thickness   position thickness   position thickness");
                                segments = detail.Thickness.Split(',');
                                for (int j = 0; j < segments.Length; j++)
                                {
                                    wafer = j + 1;
                                    _ = contents.Append(wafer.ToString().PadLeft(11));
                                    if ((wafer % 3) > 0)
                                        _ = contents.Append(segments[j].PadLeft(10));
                                    else
                                        _ = contents.AppendLine(segments[j].PadLeft(10));
                                }
                                if ((segments.Length % 3) > 0)
                                    _ = contents.AppendLine();
                                _ = contents.Append("   wafer mean thickness =    ").Append(detail.Mean).Append(", std. dev =    ").Append(detail.StdDev).Append(' ').AppendLine(detail.PassFail);
                                _ = contents.AppendLine("================================================================================");
                                _ = contents.AppendLine("");
                                _ = contents.AppendLine("Radial variation (computation B) PASS:");
                                _ = contents.AppendLine("");
                                _ = contents.AppendLine("     thickness      0.0000");
                                _ = contents.AppendLine("");
                                _ = contents.Append("     Slot:").Append(detail.Slot).AppendLine(";");
                                results.Add(new Tuple<string, bool, DateTime, string>(cassetteID, isBioRad, cassetteDateTime, contents.ToString()));
                            }
                        }
                    }
                }
            }
        }
        // **********************************************
        // * Step #3 - Protect the raw data source file *
        // **********************************************
        // The multi-cassettes raw source file is ready to be splitted.  Each cassette scan set has
        // been determined to be complete (i.e., has the started & finished statements).  At this point
        // it is important to rename the multi-cassette raw data source file, located in the RawData
        // folder, to a different name so that the tool does not attempt to update the file while being
        // processed by the EAF cell instance.
        // Get the last date/time the DataBioRad.txt file was updated
        DateTime afterCheck = new(File.GetLastWriteTime(logistics.ReportFullPath).Ticks + tickOffset);
        // Ensure that the DataBioRad.txt file has not been updated since the FileReader began the healthcheck
        // If the date/time values are different between the "Before" and "After" checks then let it go.  The
        // tool is still busy trying to update the file.  The FileReader will try to catch the data on the 
        // next update.
        if (logistics.DateTimeFromSequence != afterCheck)
        {
            results.Clear();
            _Log.Debug($"****Extract() - DataBioRad.txt file is getting updated fast");
            _Log.Debug($"****Extract() - DataBioRadDateTime_AfterCheck  = {afterCheck.Ticks}");
            _Log.Debug($"****Extract() - DataBioRadDateTime_BeforeCheck = {logistics.Sequence}");
        }
        return results;
    }

    private static Dictionary<string, List<string>> GetCassetteIDAndDataSets(string[] reportFullPathlines)
    {
        Dictionary<string, List<string>> results = new();
        string line;
        string[] segments;
        int cassetteEndIndex;
        int thicknessCounter;
        string thicknessHead;
        string thicknessInfo;
        string thicknessTail;
        int cassetteStartIndex;
        StringBuilder lines = new();
        string slotID = string.Empty;
        string cassetteID = string.Empty;
        string batchHeader = string.Empty;
        bool finishedReadingThicknessInfo;
        bool slotInformationCaptured = false;
        bool pointsInformationCaptured = false;
        bool sourceInformationCaptured = false;
        bool waferWaferInformationCaptured = false;
        bool destinationInformationCaptured = false;
        List<Tuple<string, int, int>> cassetteStartAndEnds = new();
        for (int i = 0; i < reportFullPathlines.Length; i++)
        {
            line = reportFullPathlines[i].Trim();
            if (string.IsNullOrEmpty(line))
                continue;
            if (line.StartsWith("Batch") && line.Contains("started"))
                batchHeader = line;
            if (i + 1 == reportFullPathlines.Length)
                continue;
            if (line.StartsWith("Cassette") && line.Contains("started"))
            {
                for (int j = i + 1; j < reportFullPathlines.Length; j++)
                {
                    if (j + 1 == reportFullPathlines.Length)
                        cassetteStartAndEnds.Add(new Tuple<string, int, int>(batchHeader, i, j));
                    else
                    {
                        line = reportFullPathlines[j].Trim();
                        if (line.StartsWith("Cassette") && line.Contains("started"))
                        {
                            cassetteStartAndEnds.Add(new Tuple<string, int, int>(batchHeader, i, j - 1));
                            break;
                        }
                    }
                }
            }
        }
        foreach (Tuple<string, int, int> tuple in cassetteStartAndEnds)
        {
            _ = lines.Clear();
            batchHeader = tuple.Item1;
            cassetteEndIndex = tuple.Item3;
            cassetteStartIndex = tuple.Item2;
            for (int l = cassetteStartIndex; l <= cassetteEndIndex; l++)
            {
                line = reportFullPathlines[l].Trim();
                if (string.IsNullOrEmpty(line))
                    continue;
                if (l == cassetteStartIndex)
                {
                    // Save the previously saved "Batch Header"
                    _ = lines.AppendLine(batchHeader);
                    // Save the first line of the cassette scan information
                    _ = lines.AppendLine(line);
                    // Each new cassette initialize the WaferWafer information flag
                    waferWaferInformationCaptured = false;
                    slotInformationCaptured = false;
                    if (line.Length > 9)
                    {
                        // Detected a new cassette data scan.  Extract the cassette ID.
                        // Example: "Cassette 47-241330-4238 started."
                        segments = line.Substring(9).Split(new string[] { "started" }, StringSplitOptions.RemoveEmptyEntries);
                        if (segments.Length != 0)
                            // Remove illegal characters \/:*?"<>| found in the Cassette.
                            cassetteID = Regex.Replace(segments[0].Trim(), @"[\\,\/,\:,\*,\?,\"",\<,\>,\|]", "_").Split('\r')[0].Split('\n')[0];
                    }
                }
                // Continue reading and saving the cassette scan information, into the cassette
                // scan output file, until the end of the cassette scan "Finished" statement has
                // been detected.
                // Maintain standard for mat between various BioRad tools.  The "Points" and "Thickness"
                // values between various BioRad tools might be spread over multiple lines.  The following
                // is simply to regroup the "Points" and "Thickness" information on the same line accordingly.
                if (line.StartsWith("Wafer Wafer"))
                {
                    _ = lines.AppendLine(line);
                    slotInformationCaptured = false;
                    waferWaferInformationCaptured = true;
                }
                else if (line.StartsWith("Slot"))
                {
                    slotID = string.Empty;
                    segments = line.Split(' ');
                    if (segments.Length > 1)
                        slotID = segments[1];
                    // There are cases where the WaferWafer information is missing.  Create a
                    // WaferWafer entry based off the slot number.
                    if (!waferWaferInformationCaptured)
                    {
                        waferWaferInformationCaptured = true;
                        _ = lines.AppendLine("Wafer Wafer " + slotID + ".");
                    }
                    _ = lines.AppendLine(line);
                    slotInformationCaptured = true;
                }
                else if (line.StartsWith("Recipe"))
                {
                    _ = lines.AppendLine(line);
                    pointsInformationCaptured = false;
                }
                else if (line.StartsWith("Points"))
                {
                    _ = lines.AppendLine(line);
                    pointsInformationCaptured = true;
                }
                else if (line.Contains("Thickness"))
                {
                    // Before addressing the "Thickness" section, ensure that the "Points" section
                    // has been found.  Otherwise, we need to write out a default value.
                    if (!pointsInformationCaptured)
                    {
                        // No "Points" information has been capture.  Default to "Points : 0 0"
                        _ = lines.AppendLine("Points : 0 0");
                        pointsInformationCaptured = true;
                    }
                    // The "Thickness" output section comes out differently between various Stratus tools.  In some
                    // cases, the thickness values are either empty (no values), on the same line or on different lines.
                    // Below are examples of how the data needs to be formatted after being parsed:
                    //      Thickness, um 1 - 1 0
                    //      Thickness, um 1 - 1 13.630
                    //      Thickness, um 1 - 9 1.197 1.231 1.248 1.235 1.199 1.202 1.236 1.242 1.212
                    thicknessCounter = 0;
                    thicknessHead = line;
                    thicknessInfo = "";
                    thicknessTail = "";
                    finishedReadingThicknessInfo = false;
                    for (int t = l + 1; t <= cassetteEndIndex; t++)
                    {
                        l = t;
                        line = reportFullPathlines[l].Trim();
                        if (string.IsNullOrEmpty(line))
                            continue;
                        if (!line.StartsWith("Slot"))
                        {
                            thicknessCounter++;
                            thicknessTail = string.Concat(thicknessTail, " ", line);
                        }
                        else
                        {
                            finishedReadingThicknessInfo = true;
                            if (thicknessCounter != 0)
                                thicknessInfo = string.Concat(" 1 - ", thicknessCounter);
                            else
                            {
                                // Two possible formatting scenarios at this point.  Either the data was already
                                // formatted properly on one line.  Or the Thickness value was missing, in which
                                // case we need to default the thickness value to zero (0).
                                segments = thicknessHead.Split(' ');
                                if (segments.Length > 2)
                                {
                                    // The "Thickness" raw data if formatted as a normal single line format and
                                    // already include the Header + Info + Tail
                                }
                                else
                                {
                                    // The "Thickness raw data has no values.  Formatting the output with zero.
                                    thicknessInfo = " 1 - 1";
                                    thicknessTail = " 0";
                                }
                            }
                            _ = lines.AppendLine(string.Concat(thicknessHead, thicknessInfo, thicknessTail));
                            // The "Slot" keyword is the tag that determines the end of the Thickness section.  The "Slot"
                            // information has already been ready.  Simply write it back.
                            _ = lines.AppendLine(line);
                        }
                        if (finishedReadingThicknessInfo)
                            break;
                    }
                }
                else if (line.StartsWith("Mean"))
                {
                    _ = lines.AppendLine(line);
                    sourceInformationCaptured = false;
                    destinationInformationCaptured = false;
                }
                else if (line.StartsWith("Source:") && slotInformationCaptured)
                {
                    _ = lines.AppendLine(line);
                    sourceInformationCaptured = true;
                }
                else if (line.StartsWith("Destination:") && slotInformationCaptured)
                {
                    if (!sourceInformationCaptured)
                    {
                        sourceInformationCaptured = true;
                        _ = lines.AppendLine(string.Concat("Source: Slot ", slotID, ", Cassette"));
                    }
                    _ = lines.AppendLine(line);
                    destinationInformationCaptured = true;
                    // Each time a cassette slot section has been completed, we must reinitialize
                    // the "Wafer Wafer" information flag in case there are multiple slots in the 
                    // same cassette
                    slotInformationCaptured = false;
                    waferWaferInformationCaptured = false;
                }
                else if (line.StartsWith("Cassette") && line.Contains("finished."))
                {
                    // Reach the end of the cassette data set information
                    if (!sourceInformationCaptured)
                    {
                        sourceInformationCaptured = true;
                        _ = lines.AppendLine(string.Concat("Source: Slot ", slotID, ", Cassette"));
                    }
                    if (!destinationInformationCaptured)
                    {
                        destinationInformationCaptured = true;
                        _ = lines.AppendLine(string.Concat("Destination: Slot ", slotID, ", Cassette"));
                        // Each time a cassette slot section has been completed, we must reinitialize
                        // the "Wafer Wafer" information flag in case there are multiple slots in the 
                        // same cassette
                        slotInformationCaptured = false;
                        waferWaferInformationCaptured = false;
                    }
                    // Write the end of cassette statement to the output file
                    _ = lines.AppendLine(line);
                    // Read the Mean-Average line information, post the cassette "Finished" statement
                    for (int a = l + 1; a <= cassetteEndIndex; a++)
                    {
                        l = a;
                        line = reportFullPathlines[l].Trim();
                        if (string.IsNullOrEmpty(line))
                            continue;
                        // There are many blank lines in the source file.  Search for the first
                        // occurrence of the string "Mean".
                        if (line.StartsWith("Mean"))
                        {
                            _ = lines.AppendLine(line);
                            break;
                        }
                        // The mean Average information is missing.  We are done reading the cassette information.
                        if (line.StartsWith("Batch"))
                            break;
                    }
                    if (!results.ContainsKey(cassetteID))
                        results.Add(cassetteID, new List<string>());
                    results[cassetteID].Add(lines.ToString());
                }
            }
        }
        return results;
    }

}