phares3757/oi-metrology
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PDSF (Process Data Standard Format) to use EAF for

Browse Source 
pushing to OI
This commit is contained in:
Mike Phares
2023-07-24 13:29:13 -07:00
parent 6668806432
commit 4cc5219409
28 changed files with 268 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
Server/ApiControllers/InboundController.cs
View File

@ -1,5 +1,4 @@
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json.Linq;
using OI.Metrology.Server.Models;
using OI.Metrology.Shared.Models;
using OI.Metrology.Shared.Models.Stateless;
@ -43,13 +42,6 @@ public partial class InboundController : ControllerBase, IInboundController<IAct
return result;
}
private static JToken GetJToken(Stream stream)
{
string? json = GetJson(stream);
JToken jsonbody = string.IsNullOrEmpty(json) ? JToken.Parse("{}") : JToken.Parse(json);
return jsonbody;
}
[HttpPost]
[Route("{tooltype}")]
public IActionResult Post(string tooltype)
@ -62,8 +54,8 @@ public partial class InboundController : ControllerBase, IInboundController<IAct
}
else
{
JToken jsonbody = GetJToken(Request.Body);
DataResponse dataResponse = _InboundRepository.Data(_MetrologyRepository, _InboundDataService, tooltype, jsonbody);
string? json = GetJson(Request.Body);
DataResponse dataResponse = _InboundRepository.Data(_MetrologyRepository, _InboundDataService, tooltype, json);
if (!dataResponse.Errors.Any())
return Ok(dataResponse);
else

6
Server/ApiControllers/ToolTypesController.cs
View File

@ -100,11 +100,11 @@ public class ToolTypesController : Controller, IToolTypesController<IActionResul
[Route("{toolTypeId}/headers/{headerid}/oiexport")]
public IActionResult OIExport(int toolTypeId, long headerid)
{
Exception? exception = _ToolTypesRepository.OIExport(_MetrologyRepo, _AppSettings.OIExportPath, toolTypeId, headerid);
if (exception is null)
string? message = _ToolTypesRepository.OIExport(_MetrologyRepo, _AttachmentsService, _AppSettings.AttachmentPath, _AppSettings.TableToPath, toolTypeId, headerid);
if (message is null)
return Ok(new { Message = "OK" });
else
return BadRequest(JsonConvert.SerializeObject(new { exception.Message }));
return BadRequest(JsonConvert.SerializeObject(new { message }));
}
}

2
Server/Models/AppSettings.cs
View File

@ -19,7 +19,7 @@ public record AppSettings(string ApiExportPath,
string MonAResource,
string MonASite,
string OI2SqlConnectionString,
string OIExportPath,
Dictionary<string, string> TableToPath,
string URLs,
string WorkingDirectoryName)
{

8
Server/Models/Binder/AppSettings.cs
View File

@ -25,7 +25,7 @@ public class AppSettings
[Display(Name = "MonA Resource"), Required] public string MonAResource { get; set; }
[Display(Name = "MonA Site"), Required] public string MonASite { get; set; }
[Display(Name = "Oi 2 Sql Connection String"), Required] public string Oi2SqlConnectionString { get; set; }
[Display(Name = "OI Export Path"), Required] public string OIExportPath { get; set; }
[Display(Name = "Table to Path"), Required] public Dictionary<string, string> TableToPath { get; set; }
[Display(Name = "URLs"), Required] public string URLs { get; set; }
[Display(Name = "Working Directory Name"), Required] public string WorkingDirectoryName { get; set; }
@ -76,10 +76,10 @@ public class AppSettings
throw new NullReferenceException(nameof(MonASite));
if (appSettings.Oi2SqlConnectionString is null)
throw new NullReferenceException(nameof(Oi2SqlConnectionString));
if (appSettings.OIExportPath is null)
throw new NullReferenceException(nameof(OIExportPath));
if (appSettings.URLs is null)
throw new NullReferenceException(nameof(URLs));
if (appSettings.TableToPath is null)
throw new NullReferenceException(nameof(TableToPath));
if (appSettings.WorkingDirectoryName is null)
throw new NullReferenceException(nameof(WorkingDirectoryName));
result = new(
@ -100,7 +100,7 @@ public class AppSettings
appSettings.MonAResource,
appSettings.MonASite,
appSettings.Oi2SqlConnectionString,
appSettings.OIExportPath,
appSettings.TableToPath,
appSettings.URLs,
appSettings.WorkingDirectoryName);
return result;

8
Server/OI.Metrology.Server.csproj
View File

@ -25,15 +25,15 @@
<Content Remove="compilerconfig.json" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Dapper" Version="2.0.123" />
<PackageReference Include="Dapper" Version="2.0.143" />
<PackageReference Include="EntityFramework" Version="6.4.4" />
<PackageReference Include="jQuery" Version="3.6.4" />
<PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly.Server" Version="7.0.5" />
<PackageReference Include="jQuery" Version="3.7.0" />
<PackageReference Include="Microsoft.AspNetCore.Components.WebAssembly.Server" Version="7.0.9" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.WindowsServices" Version="7.0.1" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="Serilog.AspNetCore.Ingestion" Version="1.0.0-dev-00032" />

2
Server/Properties/launchSettings.json
View File

@ -28,4 +28,4 @@
}
}
}
}
}

52
Server/Repositories/InboundRepository.cs
View File

@ -4,16 +4,13 @@ using OI.Metrology.Shared.Models;
using OI.Metrology.Shared.Models.Stateless;
using OI.Metrology.Shared.Services;
using System.Net;
using System.Text.Json;
namespace OI.Metrology.Server.Repository;
public class InboundRepository : IInboundRepository
{
private readonly Serilog.ILogger _Log;
public InboundRepository() => _Log = Serilog.Log.ForContext<InboundRepository>();
bool IInboundRepository.IsIPAddressAllowed(string inboundApiAllowedIPList, IPAddress? remoteIP)
{
if (string.IsNullOrWhiteSpace(inboundApiAllowedIPList))
@ -38,29 +35,50 @@ public class InboundRepository : IInboundRepository
// tooltype is the ToolTypeName column from the ToolType table
// JToken is how you can accept a JSON message without deserialization.
// Using "string" doesn't work because ASP.NET Core will expect a json encoded string, not give you the actual string.
DataResponse IInboundRepository.Data(IMetrologyRepository metrologyRepository, IInboundDataService inboundDataService, string tooltype, JToken jsonbody)
DataResponse IInboundRepository.Data(IMetrologyRepository metrologyRepository, IInboundDataService inboundDataService, string tooltype, string? json)
{
DataResponse result = new();
ToolType? toolType = metrologyRepository.GetToolTypeByName(tooltype);
if (toolType is null)
result.Errors.Add("Invalid tool type: " + tooltype);
result.Errors.Add($"Invalid tool type: {tooltype}");
else
{
List<ToolTypeMetadata> metaData = metrologyRepository.GetToolTypeMetadataByToolTypeID(toolType.ID).ToList();
if (metaData is null)
result.Errors.Add("Invalid metadata for tool type: " + tooltype);
else if (jsonbody is null)
result.Errors.Add("Invalid json");
InboundCommon? inboundCommon = string.IsNullOrEmpty(json) ? null : JsonSerializer.Deserialize<InboundCommon>(json);
if (inboundCommon is null || string.IsNullOrEmpty(inboundCommon.ProcessDataStandardFormat))
result.Errors.Add($"Invalid body: {json}");
else
inboundDataService.ValidateJSONFields(jsonbody, 0, metaData, result.Errors, result.Warnings);
if (metaData is not null && jsonbody is not null && !result.Errors.Any())
{
try
string? sourceDirectory = Path.GetDirectoryName(inboundCommon.ProcessDataStandardFormat);
string? parentDirectory = Path.GetDirectoryName(sourceDirectory);
if (string.IsNullOrEmpty(sourceDirectory) || string.IsNullOrEmpty(parentDirectory) || !Directory.Exists(parentDirectory))
result.Errors.Add($"Invalid body:path: <{inboundCommon.ProcessDataStandardFormat}>");
else
{
result.HeaderID = inboundDataService.DoSQLInsert(jsonbody, toolType, metaData);
result.Success = result.HeaderID > 0;
JToken jToken = string.IsNullOrEmpty(json) ? JToken.Parse("{}") : JToken.Parse(json);
if (jToken is null)
result.Errors.Add($"Invalid body: {json}");
else
{
List<ToolTypeMetadata> metaData = metrologyRepository.GetToolTypeMetadataByToolTypeID(toolType.ID).ToList();
if (metaData is null)
result.Errors.Add($"Invalid metadata for tool type: {tooltype}");
else
{
inboundDataService.ValidateJSONFields(jToken, 0, metaData, result.Errors, result.Warnings);
if (!result.Errors.Any())
{
try
{
result.HeaderID = inboundDataService.DoSQLInsert(jToken, toolType, metaData);
result.Success = result.HeaderID > 0;
string? destinationDirectory = Path.Combine(parentDirectory, result.HeaderID.ToString());
Directory.Move(sourceDirectory, destinationDirectory);
}
catch (Exception ex) { result.Errors.Add(ex.Message); }
}
}
}
}
catch (Exception ex) { result.Errors.Add(ex.Message); }
}
}
return result;

53
Server/Repositories/ToolTypesRepository.cs
View File

@ -204,50 +204,29 @@ public class ToolTypesRepository : IToolTypesRepository
return new(message, contentType, stream);
}
// This endpoint triggers writing of the OI Export file
Exception? IToolTypesRepository.OIExport(IMetrologyRepository metrologyRepository, string oiExportPath, int toolTypeId, long headerid)
string? IToolTypesRepository.OIExport(IMetrologyRepository metrologyRepository, IAttachmentsService attachmentsService, string attachmentPath, Dictionary<string, string> tableToPath, int toolTypeId, long headerid)
{
Exception? result = null;
// Call the export stored procedure
_Log.Debug($"Exporting to <{oiExportPath}>");
DataSet ds = metrologyRepository.GetOIExportData(toolTypeId, headerid);
try
string? result;
ToolType toolType = metrologyRepository.GetToolTypeByID(toolTypeId);
if (toolType?.HeaderTableName is null)
result = $"Invalid tool id [{toolTypeId}] [{headerid}]!";
else
{
// The SP must return 3 result tables
if (ds.Tables.Count != 3)
throw new Exception("Error exporting, invalid results");
// The first table has just one row, which is the export filename
if (ds.Tables[0].Rows.Count != 1)
throw new Exception("Error exporting, invalid filename");
string? filename = Convert.ToString(ds.Tables[0].Rows[0][0]);
// The second table has the header data
if (ds.Tables[1].Rows.Count != 1)
throw new Exception("Error exporting, invalid header data");
StringBuilder sb = new();
foreach (object? o in ds.Tables[1].Rows[0].ItemArray)
string? processDataStandardFormat = attachmentsService.GetProcessDataStandardFormat(metrologyRepository, attachmentPath, toolTypeId, headerid);
if (processDataStandardFormat is null)
result = $"Export file doesn't exist for [{toolTypeId}] [{headerid}] at <{attachmentPath}>!";
else if (!tableToPath.TryGetValue(toolType.HeaderTableName, out string? directly))
result = $"Export file doesn't exist for [{toolTypeId}] [{headerid}] at <{attachmentPath}>!";
else
{
if ((o is not null) && (!Convert.IsDBNull(o)))
_ = sb.Append(Convert.ToString(o));
_ = sb.Append('\t');
}
// The third table has the detail data
foreach (DataRow dr in ds.Tables[2].Rows)
{
foreach (object? o in dr.ItemArray)
try
{
if ((o is not null) && (!Convert.IsDBNull(o)))
_ = sb.Append(Convert.ToString(o));
_ = sb.Append('\t');
File.Copy(processDataStandardFormat, Path.Combine(directly, $"Viewer_{Path.GetFileName(processDataStandardFormat)}"));
result = null;
}
catch (Exception ex) { result = ex.Message; }
}
_ = sb.AppendLine();
// The output file will only have one line, the header columns are output first
// Then each detail rows has it's columns appended
// H1, H2, H3, D1.1, D1.2, D1.3, D2.1, D2.2, D2.3, etc
// Write the file
File.WriteAllText(Path.Join(oiExportPath, filename), sb.ToString());
}
catch (Exception ex) { result = ex; }
return result;
}

77
Server/Services/AttachmentsService.cs
View File

@ -27,14 +27,12 @@ public class AttachmentsService : IAttachmentsService
throw new NullReferenceException(nameof(tableName));
DateTime insertDate = Convert.ToDateTime(_MetrologyRepository.GetAttachmentInsertDateByGUID(tableName, attachmentId));
int year = insertDate.Year;
DateTime d = insertDate;
CultureInfo cul = CultureInfo.CurrentCulture;
int weekNum = cul.Calendar.GetWeekOfYear(d, CalendarWeekRule.FirstDay, DayOfWeek.Sunday);
string workWeek = "WW" + weekNum.ToString("00");
string dateDir = year + @"\" + workWeek;
string fullPath = Path.Combine(_AppSettings.AttachmentPath, tableName + "_", dateDir, attachmentId.ToString(), filename);
string year = insertDate.Year.ToString();
int weekNum = CultureInfo.CurrentCulture.Calendar.GetWeekOfYear(insertDate, CalendarWeekRule.FirstDay, DayOfWeek.Sunday);
string directory = Path.Combine(_AppSettings.AttachmentPath, tableName + "_", year, $"WW{weekNum:00}", attachmentId.ToString());
if (!Directory.Exists(directory))
_ = Directory.CreateDirectory(directory);
string fullPath = Path.Combine(directory, filename);
// Check to see if file exists in the "New" directory structure, if not change the path back to the old. and check there
if (!File.Exists(fullPath))
@ -48,7 +46,7 @@ public class AttachmentsService : IAttachmentsService
return new FileStream(fullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
}
public Stream GetAttachmentStreamByTitle(ToolType toolType, bool header, string title, string filename)
Stream IAttachmentsService.GetAttachmentStreamByTitle(ToolType toolType, bool header, string title, string filename)
{
if (toolType is null)
throw new Exception("Invalid tool type");
@ -67,7 +65,7 @@ public class AttachmentsService : IAttachmentsService
return GetAttachmentStream(tableName, attachmentId, filename);
}
public Stream GetAttachmentStreamByAttachmentId(ToolType toolType, bool header, Guid attachmentId, string filename)
Stream IAttachmentsService.GetAttachmentStreamByAttachmentId(ToolType toolType, bool header, Guid attachmentId, string filename)
{
if (toolType is null)
throw new Exception("Invalid tool type");
@ -94,7 +92,6 @@ public class AttachmentsService : IAttachmentsService
attachmentId = _MetrologyRepository.GetHeaderAttachmentID(toolType.ID, headerId);
insertDate = Convert.ToDateTime(_MetrologyRepository.GetHeaderInsertDate(toolType.ID, headerId));
tableName = toolType.HeaderTableName;
}
else
{
@ -102,37 +99,55 @@ public class AttachmentsService : IAttachmentsService
insertDate = Convert.ToDateTime(_MetrologyRepository.GetDataInsertDate(toolType.ID, headerId, dataUniqueId));
// Get Date for new directory name
tableName = toolType.DataTableName;
}
int year = insertDate.Year;
DateTime d = insertDate;
CultureInfo cul = CultureInfo.CurrentCulture;
int weekNum = cul.Calendar.GetWeekOfYear(d, CalendarWeekRule.FirstDay, DayOfWeek.Sunday);
string workWeek = "WW" + weekNum.ToString("00");
string dateDir = year + @"\" + workWeek;
if (Equals(attachmentId, Guid.Empty))
throw new Exception("Invalid attachment ID");
string directoryPathSecondary = Path.Combine(_AppSettings.AttachmentPath, tableName + "_", dateDir, attachmentId.ToString());
if (!Directory.Exists(directoryPathSecondary))
_ = Directory.CreateDirectory(directoryPathSecondary);
string fullPathSecondary = Path.Combine(directoryPathSecondary, filename);
string year = insertDate.Year.ToString();
int weekNum = CultureInfo.CurrentCulture.Calendar.GetWeekOfYear(insertDate, CalendarWeekRule.FirstDay, DayOfWeek.Sunday);
string directory = Path.Combine(_AppSettings.AttachmentPath, $"{tableName}_", year, $"WW{weekNum:00}", attachmentId.ToString());
if (!Directory.Exists(directory))
_ = Directory.CreateDirectory(directory);
using (FileStream s = new(fullPathSecondary, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
{
uploadedFile.CopyTo(s);
}
string fullPath = Path.Combine(directory, filename);
using (FileStream fileStream = new(fullPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None))
uploadedFile.CopyTo(fileStream);
trans.Complete();
}
public void SaveAttachment(ToolType toolType, long headerId, string dataUniqueId, string filename, object uploadedFile)
void IAttachmentsService.SaveAttachment(ToolType toolType, long headerId, string dataUniqueId, string filename, object uploadedFile)
{
IFormFile formFile = (IFormFile)uploadedFile;
SaveAttachment(toolType, headerId, dataUniqueId, filename, formFile);
}
string? IAttachmentsService.GetProcessDataStandardFormat(IMetrologyRepository metrologyRepository, string attachmentPath, int toolTypeId, long headerId)
{
string? result;
string year;
int weekNum;
string directory;
string checkDirectory;
List<string> files = new();
DateTime[] dateTimes = new DateTime[] { DateTime.Now, DateTime.Now.AddDays(-6.66) };
ToolType toolType = metrologyRepository.GetToolTypeByID(toolTypeId) ?? throw new Exception("Invalid tool type ID");
foreach (DateTime dateTime in dateTimes)
{
year = dateTime.Year.ToString();
weekNum = CultureInfo.CurrentCulture.Calendar.GetWeekOfYear(dateTime, CalendarWeekRule.FirstDay, DayOfWeek.Sunday);
directory = Path.Combine(_AppSettings.AttachmentPath, $"{toolType.HeaderTableName}_", year, $"WW{weekNum:00}");
if (!Directory.Exists(directory))
_ = Directory.CreateDirectory(directory);
checkDirectory = Path.Combine(directory, headerId.ToString());
if (!Directory.Exists(checkDirectory))
continue;
files.AddRange(Directory.GetFiles(checkDirectory));
if (files.Any())
break;
}
result = !files.Any() ? null : files.First();
return result;
}
}

13
Server/appsettings.json
View File

@ -6,7 +6,7 @@
"ApiUrl": "~/api",
"ApiLogPath": "D:\\Metrology\\MetrologyAPILogs",
"AttachmentPath": "\\\\messv02ecc1.ec.local\\EC_Metrology_Si\\MetrologyAttachments",
"BuildNumber": "1",
"BuildNumber": "1",
"Company": "Infineon Technologies Americas Corp.",
"ConnectionString": "Data Source=messv01ec.ec.local\\PROD1,53959;Integrated Security=True;Initial Catalog=Metrology;",
"GitCommitSeven": "1234567",
@ -25,7 +25,6 @@
"MonAResource": "OI_Metrology_Viewer_EC",
"MonASite": "auc",
"Oi2SqlConnectionString": "Data Source=messv01ec.ec.local\\PROD1,53959;Initial Catalog=LSL2SQL;Persist Security Info=True;User ID=srpadmin;Password=0okm9ijn;",
"OIExportPath": "\\\\openinsight-db-srv.na.infineon.com\\apps\\Metrology\\Data",
"Serilog": {
"Using": [
"Serilog.Sinks.Console",
@ -63,6 +62,16 @@
"Application": "Sample"
}
},
"TableToPath": {
"SPVRunHeader": "\\\\messa01ec.ec.local\\apps\\Metrology\\MET08ANLYSDIFAAST230\\Source",
"TencorRunHeader": "\\\\messa01ec.ec.local\\apps\\Metrology\\MET08DDUPSFS6420\\Source",
"SP1RunHeader": "\\\\messa01ec.ec.local\\apps\\Metrology\\MET08DDUPSP1TBI\\Source",
"MercuryProbeRunHeader": "\\\\messa01ec.ec.local\\apps\\Metrology\\MET08RESIHGCV\\Source",
"CDERunHeader": "\\\\messa01ec.ec.local\\apps\\Metrology\\MET08RESIMAPCDE\\Source",
"SRPRunHeader": "\\\\messa01ec.ec.local\\apps\\Metrology\\MET08RESISRP2100\\Source",
"BioRadRunHeader": "\\\\messa01ec.ec.local\\apps\\Metrology\\MET08THFTIRQS408M\\Source",
"StratusBioRadRunHeader": "\\\\messa01ec.ec.local\\apps\\Metrology\\MET08THFTIRSTRATUS\\Source"
},
"URLs": "http://localhost:5002;",
"WorkingDirectoryName": "IFXApps"
}