Initial Commit

This commit is contained in:
2023-05-08 15:54:57 -07:00
commit 5924860f6c
166 changed files with 19013 additions and 0 deletions

View File

@ -0,0 +1,186 @@
using Adaptation.Eaf.Management.ConfigurationData.CellAutomation;
using Adaptation.Ifx.Eaf.EquipmentConnector.File.Configuration;
using Adaptation.Shared;
using Adaptation.Shared.Duplicator;
using Adaptation.Shared.Methods;
using Adaptation.Shared.Metrology;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
namespace Adaptation.FileHandlers.OpenInsight;
public class FileRead : Shared.FileRead, IFileRead
{
private readonly string _IqsConnectionString;
private readonly string _OpenInsightFilePattern;
private readonly string _OpenInsightApiECDirectory;
private readonly string _OpenInsightApiIFXDirectory;
public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
{
_MinFileLength = 10;
_NullData = string.Empty;
_Logistics = new(this);
if (_FileParameter is null)
throw new Exception(cellInstanceConnectionName);
if (_ModelObjectParameterDefinitions is null)
throw new Exception(cellInstanceConnectionName);
if (!_IsDuplicator)
throw new Exception(cellInstanceConnectionName);
_OpenInsightApiIFXDirectory = @"\\messdv002.na.infineon.com\Candela\Archive\API";
_OpenInsightApiECDirectory = @"\\messv02ecc1.ec.local\EC_Metrology_Si\Archive\API";
_IqsConnectionString = GetPropertyValue(cellInstanceConnectionName, modelObjectParameters, "IQS.ConnectionString");
_OpenInsightFilePattern = GetPropertyValue(cellInstanceConnectionName, modelObjectParameters, "OpenInsight.FilePattern");
}
void IFileRead.Move(Tuple<string, Test[], JsonElement[], List<FileInfo>> extractResults, Exception exception)
{
bool isErrorFile = exception is not null;
if (!isErrorFile && !string.IsNullOrEmpty(_Logistics.ReportFullPath))
{
FileInfo fileInfo = new(_Logistics.ReportFullPath);
if (fileInfo.Exists && fileInfo.LastWriteTime < fileInfo.CreationTime)
File.SetLastWriteTime(_Logistics.ReportFullPath, fileInfo.CreationTime);
}
Move(extractResults);
}
void IFileRead.WaitForThread() => WaitForThread(thread: null, threadExceptions: null);
string IFileRead.GetEventDescription()
{
string result = _Description.GetEventDescription();
return result;
}
List<string> IFileRead.GetHeaderNames()
{
List<string> results = _Description.GetHeaderNames();
return results;
}
string[] IFileRead.Move(Tuple<string, Test[], JsonElement[], List<FileInfo>> extractResults, string to, string from, string resolvedFileLocation, Exception exception)
{
string[] results = Move(extractResults, to, from, resolvedFileLocation, exception);
return results;
}
JsonProperty[] IFileRead.GetDefault()
{
JsonProperty[] results = _Description.GetDefault(this, _Logistics);
return results;
}
Dictionary<string, string> IFileRead.GetDisplayNamesJsonElement()
{
Dictionary<string, string> results = _Description.GetDisplayNamesJsonElement(this);
return results;
}
List<IDescription> IFileRead.GetDescriptions(IFileRead fileRead, List<Test> tests, IProcessData processData)
{
List<IDescription> results = _Description.GetDescriptions(fileRead, _Logistics, tests, processData);
return results;
}
Tuple<string, Test[], JsonElement[], List<FileInfo>> IFileRead.GetExtractResult(string reportFullPath, string eventName)
{
Tuple<string, Test[], JsonElement[], List<FileInfo>> results;
if (string.IsNullOrEmpty(eventName))
throw new Exception();
_ReportFullPath = reportFullPath;
DateTime dateTime = DateTime.Now;
results = GetExtractResult(reportFullPath, dateTime);
if (results.Item3 is null)
results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(results.Item1, Array.Empty<Test>(), JsonSerializer.Deserialize<JsonElement[]>("[]"), results.Item4);
if (results.Item3.Length > 0 && _IsEAFHosted)
WritePDSF(this, results.Item3);
UpdateLastTicksDuration(DateTime.Now.Ticks - dateTime.Ticks);
return results;
}
Tuple<string, Test[], JsonElement[], List<FileInfo>> IFileRead.ReExtract()
{
Tuple<string, Test[], JsonElement[], List<FileInfo>> results;
List<string> headerNames = _Description.GetHeaderNames();
Dictionary<string, string> keyValuePairs = _Description.GetDisplayNamesJsonElement(this);
results = ReExtract(this, headerNames, keyValuePairs);
return results;
}
private static string GetLines(List<json.Description> descriptions)
{
StringBuilder results = new();
json.Description x = descriptions[0];
char del = x.MesEntity == x.Reactor ? '\t' : '~';
_ = results.Append(x.Date).Append(del).
Append(x.Employee).Append(del).
Append(x.Recipe).Append(del).
Append(x.Reactor).Append(del).
Append(x.RDS).Append(del).
Append(x.PSN).Append(del).
Append(x.Layer).Append(del).
Append(x.Zone);
return results.ToString();
}
private void SaveOpenInsightFile(string reportFullPath, DateTime dateTime, string logistics, List<json.Description> descriptions, Test[] tests)
{
bool isDummyRun = false;
List<(Shared.Properties.IScopeInfo, string)> collection = new();
string successDirectory = _FileConnectorConfiguration.AlternateTargetFolder;
string parentParent = GetParentParent(_FileConnectorConfiguration.SourceFileLocation);
if (parentParent.Contains(_CellInstanceName))
parentParent = Path.GetDirectoryName(parentParent);
string duplicateDirectory = Path.Combine(parentParent, "Data");
if (!Directory.Exists(duplicateDirectory))
_ = Directory.CreateDirectory(duplicateDirectory);
string duplicateFile = Path.Combine(duplicateDirectory, Path.GetFileName(reportFullPath));
if (descriptions.Any() && tests.Any())
{
string lines = GetLines(descriptions);
if (!string.IsNullOrEmpty(lines))
{
long? subGroupId;
long breakAfter = dateTime.AddSeconds(_BreakAfterSeconds).Ticks;
long preWait = _FileConnectorConfiguration?.FileHandleWaitTime is null ? dateTime.AddMilliseconds(1234).Ticks : dateTime.AddMilliseconds(_FileConnectorConfiguration.FileHandleWaitTime.Value).Ticks;
if (string.IsNullOrEmpty(descriptions[0].Reactor) || string.IsNullOrEmpty(descriptions[0].PSN))
subGroupId = null;
else
(subGroupId, int? _, string _) = FromIQS.GetCommandText(_IqsConnectionString, _Logistics, descriptions[0], breakAfter, preWait);
if (subGroupId is null)
collection.Add(new(new ScopeInfo(tests[0], _OpenInsightFilePattern), lines));
else
collection.Add(new(new ScopeInfo(tests[0], $"{subGroupId.Value} {_OpenInsightFilePattern}"), lines));
string weekOfYear = _Calendar.GetWeekOfYear(_Logistics.DateTimeFromSequence, CalendarWeekRule.FirstDay, DayOfWeek.Sunday).ToString("00");
FromIQS.Save(_OpenInsightApiECDirectory, _OpenInsightApiIFXDirectory, _Logistics, reportFullPath, logistics, descriptions.First(), lines, subGroupId, weekOfYear);
}
}
if (_IsEAFHosted && _FileConnectorConfiguration.FileScanningIntervalInSeconds > 0)
WaitForFileConsumption(dateTime, descriptions, isDummyRun, successDirectory, duplicateDirectory, collection, duplicateFile);
}
private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime dateTime)
{
Tuple<string, Test[], JsonElement[], List<FileInfo>> results;
Tuple<string, string[], string[]> pdsf = ProcessDataStandardFormat.GetLogisticsColumnsAndBody(reportFullPath);
_Logistics = new Logistics(reportFullPath, pdsf.Item1);
SetFileParameterLotIDToLogisticsMID();
JsonElement[] jsonElements = ProcessDataStandardFormat.GetArray(pdsf);
// List<Shared.Properties.IDescription> descriptions = GetDuplicatorDescriptions(jsonElements);
List<json.Description> descriptions = json.ProcessData.GetDescriptions(jsonElements);
Test[] tests = (from l in descriptions select (Test)l.Test).ToArray();
if (_IsEAFHosted && _FileConnectorConfiguration.FileScanningIntervalInSeconds > 0)
SaveOpenInsightFile(reportFullPath, dateTime, pdsf.Item1, descriptions, tests);
results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(pdsf.Item1, tests, jsonElements, new List<FileInfo>());
return results;
}
}

View File

@ -0,0 +1,245 @@
using Adaptation.Shared;
using System;
using System.Data;
using System.Data.SqlClient;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading;
namespace Adaptation.FileHandlers.OpenInsight;
public class FromIQS
{
#nullable enable
private static string GetCommandText(Logistics logistics, json.Description description, string dateTime, long? subGroupId)
{
StringBuilder result = new();
_ = result
.AppendLine(" select iq.ev_count, iq.cl_count, iq.sl_count, iq.se_sgrp, iq.se_sgtm, iq.se_tsno, iq.td_test, iq.pr_name, iq.jd_name, iq.pl_name, iq.pd_name, iq.td_name, iq.se_val ")
.AppendLine(" from ( ")
.AppendLine(" select ")
.AppendLine(" se.f_sgrp se_sgrp, ")
.AppendLine(" se.f_sgtm se_sgtm, ")
.AppendLine(" se.f_tsno se_tsno, ")
.AppendLine(" se.f_val se_val, ")
.AppendLine(" pr.f_name pr_name, ")
.AppendLine(" jd.f_name jd_name, ")
.AppendLine(" pl.f_name pl_name, ")
.AppendLine(" pd.f_name pd_name, ")
.AppendLine(" td.f_test td_test, ")
.AppendLine(" td.f_name td_name, ")
.AppendLine(" (select count(cl.f_part) ")
.AppendLine(" from [spcepiworld].[dbo].[ctrl_lim] cl ")
.AppendLine(" where cl.f_part = pd.f_part ")
.AppendLine(" and cl.f_test = td.f_test ")
.AppendLine(" ) cl_count, ")
.AppendLine(" (select count(sl.f_part) ")
.AppendLine(" from [spcepiworld].[dbo].[spec_lim] sl ")
.AppendLine(" where sl.f_part = pd.f_part ")
.AppendLine(" and sl.f_test = td.f_test ")
.AppendLine(" ) sl_count, ")
.AppendLine(" (select count(ev.f_evnt) ")
.AppendLine(" from [spcepiworld].[dbo].[evnt_inf] ev ")
.AppendLine(" where ev.f_prcs = pr.f_prcs ")
.AppendLine(" and ev.f_part = pd.f_part ")
.AppendLine(" and ev.f_sgtm = se.f_sgtm ")
.AppendLine(" ) ev_count ")
.AppendLine(" from [spcepiworld].[dbo].[sgrp_ext] se ")
.AppendLine(" join [spcepiworld].[dbo].[prcs_dat] pr ")
.AppendLine(" on se.f_prcs = pr.f_prcs ")
.AppendLine(" join [spcepiworld].[dbo].[job_dat] jd ")
.AppendLine(" on se.f_job = jd.f_job ")
.AppendLine(" join [spcepiworld].[dbo].[part_lot] pl ")
.AppendLine(" on se.f_lot = pl.f_lot ")
.AppendLine(" join [spcepiworld].[dbo].[part_dat] pd ")
.AppendLine(" on se.f_part = pd.f_part ")
.AppendLine(" join [spcepiworld].[dbo].[test_dat] td ")
.AppendLine(" on se.f_test = td.f_test ")
.AppendLine(" where se.f_flag = 0 ");
if (subGroupId is not null)
_ = result.Append(" and se.f_sgrp = ").Append(subGroupId).AppendLine(" ");
if (!string.IsNullOrEmpty(description.RDS))
_ = result.Append(" and pl.f_name = '").Append(description.RDS).AppendLine("' ");
_ = result
.Append(" and pr.f_name = '").Append(description.Reactor).AppendLine("' ")
.Append(" and pd.f_name = '").Append(description.PSN).AppendLine("' ")
.AppendLine(" and jd.f_name in ('SRP2100') ")
.Append(" and jd.f_name = '").Append(logistics.MesEntity).AppendLine("' ")
.Append(" and dateadd(HH, -7, (dateadd(SS, convert(bigint, se.f_sgtm), '19700101'))) = '").Append(dateTime).AppendLine("' ")
.AppendLine(" ) as iq ")
.AppendLine(" order by iq.ev_count desc, iq.cl_count desc, iq.sl_count desc, iq.se_sgrp, iq.se_tsno, iq.td_test ")
.AppendLine(" for json path ");
return result.ToString();
}
private static StringBuilder GetForJsonPath(string connectionString, string commandText)
{
StringBuilder stringBuilder = new();
using SqlConnection sqlConnection = new(connectionString);
sqlConnection.Open();
using SqlCommand sqlCommand = new(commandText, sqlConnection);
SqlDataReader sqlDataReader = sqlCommand.ExecuteReader(CommandBehavior.SequentialAccess);
while (sqlDataReader.Read())
_ = stringBuilder.Append(sqlDataReader.GetString(0));
return stringBuilder;
}
internal static (long?, int?, string) GetCommandText(string connectionString, Logistics logistics, json.Description description, long breakAfter, long preWait)
{
string dateTime;
int? count = null;
string commandText;
long? result = null;
string dateFormat = json.Description.GetDateFormat();
if (DateTime.TryParseExact(description.Date, dateFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTime dateTimeParsed))
dateTime = dateTimeParsed.ToString("yyyy-MM-dd HH:mm:ss");
else if (DateTime.TryParse(description.Date, CultureInfo.InvariantCulture, DateTimeStyles.None, out dateTimeParsed))
dateTime = dateTimeParsed.ToString("yyyy-MM-dd HH:mm:ss");
else
dateTime = logistics.DateTimeFromSequence.ToString("yyyy-MM-dd HH:mm:ss");
commandText = GetCommandText(logistics, description, dateTime, subGroupId: null);
for (short i = 0; i < short.MaxValue; i++)
{
if (DateTime.Now.Ticks > preWait)
break;
Thread.Sleep(100);
}
StringBuilder stringBuilder = new();
for (short z = 0; z < short.MaxValue; z++)
{
stringBuilder = GetForJsonPath(connectionString, commandText);
if (stringBuilder.Length > 0)
{
long postBreakAfter = DateTime.Now.AddSeconds(5).Ticks;
for (short y = 0; y < short.MaxValue; y++)
{
if (DateTime.Now.Ticks > postBreakAfter)
break;
Thread.Sleep(250);
}
stringBuilder = GetForJsonPath(connectionString, commandText);
break;
}
if (DateTime.Now.Ticks > breakAfter)
// throw new Exception($"After {breakAfterSeconds} seconds, didn't find sub group id!");
break;
Thread.Sleep(250);
}
if (stringBuilder.Length == 0)
commandText = stringBuilder.ToString();
else
{
JsonElement[]? jsonElements = JsonSerializer.Deserialize<JsonElement[]>(stringBuilder.ToString());
if (jsonElements is null || !jsonElements.Any() || jsonElements[0].ValueKind != JsonValueKind.Object)
commandText = stringBuilder.ToString();
else
{
JsonProperty[] jsonProperties = jsonElements[0].EnumerateObject().ToArray();
if (!jsonProperties.Any() || jsonProperties[3].Name != "se_sgrp" || !long.TryParse(jsonProperties[3].Value.ToString(), out long subGroupId))
commandText = stringBuilder.ToString();
else
{
result = subGroupId;
if (jsonProperties.Any() && jsonProperties[0].Name == "ev_count" && int.TryParse(jsonProperties[0].Value.ToString(), out int evCount))
count = evCount;
}
}
}
return new(result, count, commandText);
}
private static string GetJson(Logistics logistics, string logisticLines, json.Description description)
{
string result;
StringBuilder stringBuilder = new();
var @object = new
{
description.MesEntity,
description.Employee,
description.Layer,
description.PSN,
description.RDS,
description.Reactor,
description.Recipe,
description.Zone,
logistics.DateTimeFromSequence.Ticks
};
string[] pair;
string safeValue;
string[] segments;
string serializerValue;
foreach (string line in logisticLines.Split(new string[] { Environment.NewLine }, StringSplitOptions.None))
{
segments = line.Split('\t');
if (segments.Length < 2)
continue;
segments = segments[1].Split(';');
_ = stringBuilder.Append('{');
foreach (string segment in segments)
{
pair = segment.Split('=');
if (pair.Length != 2 || pair[0].Length < 3)
continue;
serializerValue = JsonSerializer.Serialize(pair[1]);
safeValue = serializerValue.Substring(1, serializerValue.Length - 2);
_ = stringBuilder.Append('"').Append(pair[0].Substring(2)).Append('"').Append(':').Append('"').Append(safeValue).Append('"').Append(',');
}
if (stringBuilder.Length > 0)
_ = stringBuilder.Remove(stringBuilder.Length - 1, 1);
_ = stringBuilder.Append('}').Append(',');
}
if (stringBuilder.Length > 0)
_ = stringBuilder.Remove(stringBuilder.Length - 1, 1);
_ = stringBuilder.Append(']').Append('}');
_ = stringBuilder.Insert(0, ",\"Logistics\":[");
string json = JsonSerializer.Serialize(@object);
_ = stringBuilder.Insert(0, json.Substring(0, json.Length - 1));
JsonElement? jsonElement = JsonSerializer.Deserialize<JsonElement>(stringBuilder.ToString());
result = jsonElement is null ? "{}" : JsonSerializer.Serialize(jsonElement, new JsonSerializerOptions { WriteIndented = true });
return result;
}
internal static void Save(string openInsightApiECDirectory, string openInsightApiIFXDirectory, Logistics logistics, string reportFullPath, string logisticLines, json.Description description, string lines, long? subGroupId, string weekOfYear)
{
string checkFile;
string fileName = Path.GetFileName(reportFullPath);
string json = GetJson(logistics, logisticLines, description);
string? ecPathRoot = Path.GetPathRoot(openInsightApiECDirectory);
string? ifxPathRoot = Path.GetPathRoot(openInsightApiIFXDirectory);
bool ecExists = ecPathRoot is not null && Directory.Exists(ecPathRoot);
bool ifxExists = ifxPathRoot is not null && Directory.Exists(ifxPathRoot);
string weekYear = $"{logistics.DateTimeFromSequence:yyyy}_Week_{weekOfYear}";
string ecDirectory = Path.Combine(openInsightApiECDirectory, weekYear, $"-{description.PSN}", $"-{description.Reactor}", $"-{description.RDS}", $"-{subGroupId}");
string ifxDirectory = Path.Combine(openInsightApiIFXDirectory, weekYear, $"-{description.PSN}", $"-{description.Reactor}", $"-{description.RDS}", $"-{subGroupId}");
if (ecExists && !Directory.Exists(ecDirectory))
_ = Directory.CreateDirectory(ecDirectory);
if (ifxExists && !Directory.Exists(ifxDirectory))
_ = Directory.CreateDirectory(ifxDirectory);
checkFile = Path.Combine(ecDirectory, fileName);
if (ecExists && !File.Exists(checkFile))
File.Copy(reportFullPath, checkFile);
checkFile = Path.Combine(ifxDirectory, fileName);
if (ifxExists && !File.Exists(checkFile))
File.Copy(reportFullPath, checkFile);
checkFile = Path.Combine(ecDirectory, $"{logistics.DateTimeFromSequence.Ticks}.txt");
if (ecExists && !File.Exists(checkFile))
File.WriteAllText(checkFile, lines);
checkFile = Path.Combine(ifxDirectory, $"{logistics.DateTimeFromSequence.Ticks}.txt");
if (ifxExists && !File.Exists(checkFile))
File.WriteAllText(checkFile, lines);
checkFile = Path.Combine(ecDirectory, $"{logistics.DateTimeFromSequence.Ticks}.json");
if (ecExists && !File.Exists(checkFile))
File.WriteAllText(checkFile, json);
checkFile = Path.Combine(ifxDirectory, $"{logistics.DateTimeFromSequence.Ticks}.json");
if (ifxExists && !File.Exists(checkFile))
File.WriteAllText(checkFile, json);
}
#nullable disable
}