phares3757/dep08siasm
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

DEP08SIASM - Testing EDA

Browse Source
This commit is contained in:
Mike Phares 2022-03-10 17:29:39 -07:00
parent c6923ed84f
commit e476b14a54
5 changed files with 123 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
Adaptation/FileHandlers/jpeg/Description.cs
View File

@ -23,10 +23,16 @@ public class Description : IDescription
public string ProcessJobID { get; set; } public string ProcessJobID { get; set; }
public string MID { get; set; } public string MID { get; set; }
// //
public string Process { get; set; } //5 public string Date { get; set; }
public string Date { get; set; } //6 public string Employee { get; set; }
public string Part { get; set; } //7 public string Lot { get; set; }
public string Lot { get; set; } //8 public string PSN { get; set; }
public string Reactor { get; set; }
public string Recipe { get; set; }
//
public int Red { get; set; }
public int Green { get; set; }
public int TotalDelta { get; set; }
string IDescription.GetEventDescription() => "File Has been read and parsed"; string IDescription.GetEventDescription() => "File Has been read and parsed";
@ -51,8 +57,16 @@ public class Description : IDescription
List<string> IDescription.GetHeaderNames() List<string> IDescription.GetHeaderNames()
{ {
string[] results = new string[] { nameof(Process), nameof(Date), nameof(Part), nameof(Lot) }; List<string> results = new()
return results.ToList(); {
nameof(Date),
nameof(Employee),
nameof(Lot),
nameof(PSN),
nameof(Reactor),
nameof(Recipe)
};
return results;
} }
IDescription IDescription.GetDisplayNames() IDescription IDescription.GetDisplayNames()
@ -63,7 +77,12 @@ public class Description : IDescription
List<string> IDescription.GetParameterNames() List<string> IDescription.GetParameterNames()
{ {
List<string> results = new(); List<string> results = new()
{
nameof(Red),
nameof(Green),
nameof(TotalDelta),
};
return results; return results;
} }
@ -115,7 +134,7 @@ public class Description : IDescription
{ {
List<IDescription> results = new(); List<IDescription> results = new();
if (iProcessData is null || !iProcessData.Details.Any() || iProcessData is not ProcessData) if (iProcessData is null || !iProcessData.Details.Any() || iProcessData is not ProcessData processData)
results.Add(GetDefault(fileRead, logistics)); results.Add(GetDefault(fileRead, logistics));
else else
{ {
@ -128,7 +147,7 @@ public class Description : IDescription
nullData = configDataNullData.ToString(); nullData = configDataNullData.ToString();
for (int i = 0; i < iProcessData.Details.Count; i++) for (int i = 0; i < iProcessData.Details.Count; i++)
{ {
if (iProcessData.Details[i] is not string detail) if (iProcessData.Details[i] is not string _)
continue; continue;
description = new Description description = new Description
{ {
@ -145,10 +164,16 @@ public class Description : IDescription
ProcessJobID = logistics.ProcessJobID, ProcessJobID = logistics.ProcessJobID,
MID = logistics.MID, MID = logistics.MID,
// //
Process = logistics.ProcessJobID, Date = logistics.DateTimeFromSequence.ToUniversalTime().ToString("MM/dd/yyyy HH:mm:ss"),
Date = logistics.DateTimeFromSequence.Ticks.ToString(), Employee = string.Empty,
Part = string.Empty, Lot = processData.Text,
Lot = detail PSN = string.Empty,
Reactor = logistics.ProcessJobID,
Recipe = processData.Recipe,
//
Red = processData.Red,
Green = processData.Green,
TotalDelta = processData.TotalDelta,
}; };
results.Add(description); results.Add(description);
} }
@ -177,12 +202,19 @@ public class Description : IDescription
MesEntity = fileRead.MesEntity, MesEntity = fileRead.MesEntity,
ReportFullPath = logistics.ReportFullPath, ReportFullPath = logistics.ReportFullPath,
ProcessJobID = logistics.ProcessJobID, ProcessJobID = logistics.ProcessJobID,
MID = logistics.MID MID = logistics.MID,
//
Date = nameof(Date),
Employee = nameof(Employee),
Lot = nameof(Lot),
PSN = nameof(PSN),
Reactor = nameof(Reactor),
Recipe = nameof(Recipe),
//
Red = -1,
Green = -1,
TotalDelta = -1,
}; };
result.Process = nameof(Process);
result.Date = nameof(Date);
result.Part = nameof(Part);
result.Lot = nameof(Lot);
return result; return result;
} }

20
Adaptation/FileHandlers/jpeg/FileRead.cs
View File

@ -9,6 +9,7 @@ using System.IO;
using System.Linq; using System.Linq;
using System.Reflection; using System.Reflection;
using System.Text.Json; using System.Text.Json;
using System.Text.RegularExpressions;
namespace Adaptation.FileHandlers.jpeg; namespace Adaptation.FileHandlers.jpeg;
@ -23,6 +24,7 @@ public class FileRead : Shared.FileRead, IFileRead
protected long _LastChange; protected long _LastChange;
protected string _LastText; protected string _LastText;
protected string _TessDataDirectory; protected string _TessDataDirectory;
protected List<int> _PreviousTotalDeltaCollection;
protected readonly Dictionary<string, string> _Reactors; protected readonly Dictionary<string, string> _Reactors;
protected readonly List<(string, Color[])> _ColorCollections; protected readonly List<(string, Color[])> _ColorCollections;
@ -80,6 +82,7 @@ public class FileRead : Shared.FileRead, IFileRead
_StartY = int.Parse(startY); _StartY = int.Parse(startY);
_EndX = int.Parse(endX); _EndX = int.Parse(endX);
_EndY = int.Parse(endY); _EndY = int.Parse(endY);
_PreviousTotalDeltaCollection = new();
string masterImageDirectory = GetPropertyValue(cellInstanceConnectionName, modelObjectParameters, "Path.Memory.Master.Images"); string masterImageDirectory = GetPropertyValue(cellInstanceConnectionName, modelObjectParameters, "Path.Memory.Master.Images");
_ColorCollections = ProcessData.GetColorCollections(_StartX, _StartY, _EndX, _EndY, masterImageDirectory); _ColorCollections = ProcessData.GetColorCollections(_StartX, _StartY, _EndX, _EndY, masterImageDirectory);
} }
@ -160,22 +163,19 @@ public class FileRead : Shared.FileRead, IFileRead
results.Item4.Add(new FileInfo(reportFullPath)); results.Item4.Add(new FileInfo(reportFullPath));
else else
{ {
IProcessData iProcessData = new ProcessData(this, _Logistics, results.Item4, _TessDataDirectory, _StartX, _StartY, _EndX, _EndY, _ColorCollections); IProcessData iProcessData = new ProcessData(this, _Logistics, results.Item4, _StartX, _StartY, _EndX, _EndY, _ColorCollections, _PreviousTotalDeltaCollection, _LastText);
if (iProcessData is ProcessData _) if (iProcessData is not ProcessData processData)
{
if (!iProcessData.Details.Any())
throw new Exception(string.Concat("A) No Data - ", dateTime.Ticks)); throw new Exception(string.Concat("A) No Data - ", dateTime.Ticks));
if (iProcessData.Details[0] is not string mid || string.IsNullOrEmpty(mid)) string mid = processData.Text;
throw new Exception(string.Concat("B) No Data - ", dateTime.Ticks)); mid = Regex.Replace(mid, @"[\\,\/,\:,\*,\?,\"",\<,\>,\|]", "_").Split('\r')[0].Split('\n')[0];
_Logistics.MID = mid; _Logistics.MID = mid;
SetFileParameterLotID(mid); SetFileParameterLotID(mid);
_Logistics.ProcessJobID = iProcessData.GetCurrentReactor(this, _Logistics, _Reactors); _Logistics.ProcessJobID = iProcessData.GetCurrentReactor(this, _Logistics, _Reactors);
}
if (!iProcessData.Details.Any()) if (!iProcessData.Details.Any())
throw new Exception(string.Concat("C) No Data - ", dateTime.Ticks)); throw new Exception(string.Concat("B) No Data - ", dateTime.Ticks));
if (_LastText != _Logistics.MID || _LastChange < DateTime.Now.AddMinutes(-30).Ticks) if (_LastText != processData.Text || _LastChange < DateTime.Now.AddMinutes(-30).Ticks)
{ {
_LastText = _Logistics.MID; _LastText = processData.Text;
_LastChange = DateTime.Now.Ticks; _LastChange = DateTime.Now.Ticks;
results = iProcessData.GetResults(this, _Logistics, results.Item4); results = iProcessData.GetResults(this, _Logistics, results.Item4);
} }

198
Adaptation/FileHandlers/jpeg/ProcessData.cs
View File

@ -23,9 +23,15 @@ public class ProcessData : IProcessData
public string JobID { get; set; } public string JobID { get; set; }
public string MesEntity { get; set; } public string MesEntity { get; set; }
public int Red { get; set; }
public int Green { get; set; }
public string Text { get; set; }
public int TotalDelta { get; set; }
public string Recipe { get; set; }
List<object> Shared.Properties.IProcessData.Details => _Details; List<object> Shared.Properties.IProcessData.Details => _Details;
public ProcessData(IFileRead fileRead, Logistics logistics, List<FileInfo> fileInfoCollection, string tessDataDirectory, int startX, int startY, int endX, int endY, List<(string, Color[])> colorCollections) public ProcessData(IFileRead fileRead, Logistics logistics, List<FileInfo> fileInfoCollection, int startX, int startY, int endX, int endY, List<(string, Color[])> colorCollections, List<int> previousTotalDeltaCollection, string lastText)
{ {
if (logistics is null) if (logistics is null)
{ } { }
@ -34,7 +40,7 @@ public class ProcessData : IProcessData
_Details = new List<object>(); _Details = new List<object>();
MesEntity = logistics.MesEntity; MesEntity = logistics.MesEntity;
_Log = LogManager.GetLogger(typeof(ProcessData)); _Log = LogManager.GetLogger(typeof(ProcessData));
Parse(fileRead, fileInfoCollection, tessDataDirectory, startX, startY, endX, endY, colorCollections); Parse(fileRead, fileInfoCollection, startX, startY, endX, endY, colorCollections, previousTotalDeltaCollection, lastText);
} }
private static string Get(string value, bool useSplitForMID) private static string Get(string value, bool useSplitForMID)
@ -133,16 +139,17 @@ public class ProcessData : IProcessData
return imageFormat; return imageFormat;
} }
private static (Color[], int, int, MemoryStream) Get(string reportFullPath, string extension, int startX, int startY, int endX, int endY) private static (Color[], int, int) Get(IFileRead fileRead, List<FileInfo> fileInfoCollection, string extension, int startX, int startY, int endX, int endY)
{ {
Color color; Color color;
List<Color> colors = new(); List<Color> colors = new();
MemoryStream memoryStream = new(); MemoryStream memoryStream = new();
Bitmap selectedBitmap = new(endX - startX, endY - startY); Bitmap selectedBitmap = new(endX - startX, endY - startY);
System.Drawing.Imaging.ImageFormat imageFormat = Get(extension); System.Drawing.Imaging.ImageFormat imageFormat = Get(extension);
using Bitmap? bitmap = Image.FromFile(reportFullPath) as Bitmap; using Bitmap? bitmap = Image.FromFile(fileRead.ReportFullPath) as Bitmap;
string saveFileName = Path.ChangeExtension(fileRead.ReportFullPath, extension);
if (bitmap is null) if (bitmap is null)
throw new Exception($"Couldn't load image from <{reportFullPath}>"); throw new Exception($"Couldn't load image from <{fileRead.ReportFullPath}>");
for (int x = startX; x < endX; x++) for (int x = startX; x < endX; x++)
{ {
for (int y = startY; y < endY; y++) for (int y = startY; y < endY; y++)
@ -153,16 +160,21 @@ public class ProcessData : IProcessData
} }
} }
selectedBitmap.Save(memoryStream, imageFormat); selectedBitmap.Save(memoryStream, imageFormat);
return new(colors.ToArray(), endX - startX, endY - startY, memoryStream); if (!fileRead.IsEAFHosted)
{
fileInfoCollection.Add(new FileInfo(saveFileName));
SaveToFile(extension, saveFileName, memoryStream);
}
return new(colors.ToArray(), endX - startX, endY - startY);
} }
private static string Get(string saveFileName, string extension, string extra) private static string Get(IFileRead fileRead, string extension, string extra)
{ {
string result; string result;
string? directoryName = Path.GetDirectoryName(saveFileName); string? directoryName = Path.GetDirectoryName(fileRead.ReportFullPath);
if (string.IsNullOrEmpty(directoryName)) if (string.IsNullOrEmpty(directoryName))
throw new Exception("Couldn't get directoryName!"); throw new Exception("Couldn't get directoryName!");
string? fileNameWithoutExtension = Path.GetFileNameWithoutExtension(saveFileName); string? fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileRead.ReportFullPath);
if (string.IsNullOrEmpty(fileNameWithoutExtension)) if (string.IsNullOrEmpty(fileNameWithoutExtension))
throw new Exception("Couldn't get fileNameWithoutExtension!"); throw new Exception("Couldn't get fileNameWithoutExtension!");
result = Path.Combine(directoryName, $"{fileNameWithoutExtension} - {extra}{extension}"); result = Path.Combine(directoryName, $"{fileNameWithoutExtension} - {extra}{extension}");
@ -185,45 +197,17 @@ public class ProcessData : IProcessData
bitmap.Save(saveFileName, imageFormat); bitmap.Save(saveFileName, imageFormat);
} }
private static byte[] Get(bool development, string extension, string saveFileName, Color[] sourceColors, int width, int height, int thresHold, bool saveToFile)
{
int i = 0;
Color color;
MemoryStream memoryStream = new();
Bitmap selectedBitmap = new(width, height);
System.Drawing.Imaging.ImageFormat imageFormat = Get(extension);
string newSaveFileName = Get(saveFileName, extension, thresHold.ToString("000"));
for (int x = 0; x < width; x++)
{
for (int y = 0; y < height; y++)
{
color = sourceColors[i];
if (color.R > thresHold || color.G > thresHold || color.B > thresHold)
selectedBitmap.SetPixel(x, y, Color.Black);
i += 1;
}
}
selectedBitmap.Save(memoryStream, imageFormat);
if (development && saveToFile)
SaveToFile(extension, newSaveFileName, memoryStream);
byte[] bytes = memoryStream.GetBuffer();
return bytes;
}
#pragma warning restore CA1416 #pragma warning restore CA1416
private static void SaveText(int red, int green, string text, string textFileName, List<(string File, int TotalDelta)> totalDeltaCollection, int readAt, string reading, int score) private static void SaveText(int red, int green, string closestMatchFileNameEnding, string textFileName, List<(string File, int TotalDelta)> totalDeltaCollection)
{ {
string format = "00000"; string format = "00000";
List<string> lines = new() { red.ToString(format), green.ToString(format), text }; List<string> lines = new() { red.ToString(format), green.ToString(format), closestMatchFileNameEnding };
foreach ((string file, int totalDelta) in totalDeltaCollection) foreach ((string file, int totalDelta) in totalDeltaCollection)
{ {
lines.Add(file); lines.Add(file);
lines.Add(totalDelta.ToString(format)); lines.Add(totalDelta.ToString(format));
} }
lines.Add(readAt.ToString(format));
lines.Add(reading);
lines.Add(score.ToString(format));
File.WriteAllLines(textFileName, lines); File.WriteAllLines(textFileName, lines);
} }
@ -236,31 +220,19 @@ public class ProcessData : IProcessData
return results; return results;
} }
private void Parse(IFileRead fileRead, List<FileInfo> fileInfoCollection, string tessDataDirectory, int startX, int startY, int endX, int endY, List<(string, Color[])> colorCollections) private void Parse(IFileRead fileRead, List<FileInfo> fileInfoCollection, int startX, int startY, int endX, int endY, List<(string, Color[])> colorCollections, List<int> previousTotalDeltaCollection, string lastText)
{ {
Pix pix; Recipe = string.Empty;
_Log.Debug("TODO: Get recipe");
Red = 0;
Green = 0;
int delta; int delta;
Page page;
int readAt;
Color color; Color color;
int red = 0;
string text;
byte[] bytes;
int green = 0;
int totalDelta; int totalDelta;
string textFileName;
string closestMatchFile;
const int thresHold = 70; const int thresHold = 70;
string closestMatchFileName;
List<string> readings = new();
const int upperThresHold = 153;
Regex regex = new(@"[^a-zA-Z]");
const string extension = ".tiff"; const string extension = ".tiff";
string[] closestMatchFileNameSplit;
char[] closestMatchFileNameDistinct;
List<(string File, int TotalDelta)> totalDeltaCollection = new(); List<(string File, int TotalDelta)> totalDeltaCollection = new();
string saveFileName = Path.ChangeExtension(fileRead.ReportFullPath, extension); (Color[] sourceColors, int width, int height) = Get(fileRead, fileInfoCollection, extension, startX, startY, endX, endY);
(Color[] sourceColors, int width, int height, MemoryStream memoryStream) = Get(fileRead.ReportFullPath, extension, startX, startY, endX, endY);
foreach ((string file, Color[] colors) in colorCollections) foreach ((string file, Color[] colors) in colorCollections)
{ {
totalDelta = 0; totalDelta = 0;
@ -270,9 +242,9 @@ public class ProcessData : IProcessData
{ {
color = sourceColors[i]; color = sourceColors[i];
if (color.R > thresHold) if (color.R > thresHold)
red += 1; Red += 1;
if (color.G > thresHold) if (color.G > thresHold)
green += 1; Green += 1;
delta = color.R - colors[i].R; delta = color.R - colors[i].R;
if (delta > 0) if (delta > 0)
totalDelta += delta; totalDelta += delta;
@ -283,100 +255,40 @@ public class ProcessData : IProcessData
totalDelta += delta; totalDelta += delta;
else else
totalDelta += delta * -1; totalDelta += delta * -1;
delta = color.B - colors[i].B;
if (delta > 0)
totalDelta += delta;
else
totalDelta += delta * -1;
} }
totalDeltaCollection.Add(new(file, totalDelta)); totalDeltaCollection.Add(new(file, totalDelta));
} }
totalDeltaCollection = (from l in totalDeltaCollection orderby l.TotalDelta select l).ToList(); totalDeltaCollection = (from l in totalDeltaCollection orderby l.TotalDelta select l).ToList();
bytes = memoryStream.GetBuffer(); string closestMatchFile = totalDeltaCollection[0].File;
closestMatchFile = totalDeltaCollection[0].File; string[] closestMatchFileNameSplit = Path.GetFileNameWithoutExtension(closestMatchFile).Split('-');
closestMatchFileNameSplit = Path.GetFileNameWithoutExtension(closestMatchFile).Split('-'); Text = closestMatchFileNameSplit.Last().TrimStart();
closestMatchFileName = closestMatchFileNameSplit.Last().TrimStart(); TotalDelta = totalDeltaCollection[0].TotalDelta;
closestMatchFileNameDistinct = closestMatchFileName.Distinct().ToArray(); string textFileName = Get(fileRead, ".txt", $"{TotalDelta} - {Text}");
using TesseractEngine tesseractEngine = new(tessDataDirectory, "eng", EngineMode.Default);
pix = Pix.LoadTiffFromMemory(bytes);
page = tesseractEngine.Process(pix);
text = page.GetText().Trim();
pix.Dispose();
page.Dispose();
if (!fileRead.IsEAFHosted)
{
fileInfoCollection.Add(new FileInfo(saveFileName));
SaveToFile(extension, saveFileName, memoryStream);
}
if (!string.IsNullOrEmpty(text))
{
text = regex.Replace(text, string.Empty);
readings.Add(text);
}
if (text == closestMatchFileName)
{
readAt = thresHold;
_Log.Info(text);
textFileName = Get(saveFileName, ".txt", $"{thresHold:000} - {text}");
if (!fileRead.IsEAFHosted) if (!fileRead.IsEAFHosted)
{ {
fileInfoCollection.Add(new FileInfo(textFileName)); fileInfoCollection.Add(new FileInfo(textFileName));
SaveText(red, green, text, textFileName, totalDeltaCollection, readAt, text, int.MaxValue); SaveText(Red, Green, Text, textFileName, totalDeltaCollection);
} }
} if (Text != lastText && previousTotalDeltaCollection.Count > 50)
else
{ {
readAt = -1; double average = previousTotalDeltaCollection.Average();
memoryStream.Dispose(); double sum = previousTotalDeltaCollection.Sum(l => Math.Pow(l - average, 2));
for (int i = thresHold; i < upperThresHold; i += 10) double standardDeviation = Math.Sqrt(sum / previousTotalDeltaCollection.Count);
{ double deviation = standardDeviation * 3;
bytes = Get(!fileRead.IsEAFHosted, extension, saveFileName, sourceColors, width, height, i, i == thresHold); double upper = average + deviation;
pix = Pix.LoadTiffFromMemory(bytes); double lower = average - deviation;
page = tesseractEngine.Process(pix); if (TotalDelta > upper)
text = page.GetText().Trim(); throw new Exception();
pix.Dispose(); if (TotalDelta < lower)
page.Dispose(); throw new Exception();
if (!string.IsNullOrEmpty(text))
{
text = regex.Replace(text, string.Empty);
readings.Add(text);
} }
if (text == closestMatchFileName) previousTotalDeltaCollection.Add(TotalDelta);
{ _Details.Add(Text);
readAt = i;
_Log.Info(text);
textFileName = Get(saveFileName, ".txt", $"{i:000} - {text}");
if (!fileRead.IsEAFHosted)
{
fileInfoCollection.Add(new FileInfo(textFileName));
SaveText(red, green, text, textFileName, totalDeltaCollection, readAt, text, int.MaxValue);
}
break;
}
}
}
if (readAt < thresHold)
{
int score;
char[] readingDistinct;
text = closestMatchFileName;
List<(string Reading, int Score)> readingEvaluations = new();
foreach (string reading in readings)
{
score = 0;
readingDistinct = reading.Distinct().ToArray();
for (int i = 0; i < closestMatchFileNameDistinct.Length; i++)
{
if (!readingDistinct.Contains(closestMatchFileNameDistinct[i]))
continue;
score += 1;
}
readingEvaluations.Add(new(reading, score));
}
readingEvaluations = (from l in readingEvaluations orderby l.Score descending select l).ToList();
textFileName = Get(saveFileName, ".txt", $"{readAt:000} - {readingEvaluations[0].Reading} - {text}");
if (!fileRead.IsEAFHosted)
{
fileInfoCollection.Add(new FileInfo(textFileName));
SaveText(red, green, text, textFileName, totalDeltaCollection, readAt, readingEvaluations[0].Reading, readingEvaluations[0].Score);
}
}
_Details.Add(text);
} }
} }

2
DEP08SIASM.csproj
View File

@ -134,7 +134,7 @@
<Version>4.7.0</Version> <Version>4.7.0</Version>
</PackageReference> </PackageReference>
<PackageReference Include="Infineon.EAF.Runtime"> <PackageReference Include="Infineon.EAF.Runtime">
<Version>2.39.0</Version> <Version>2.39.2</Version>
</PackageReference> </PackageReference>
<PackageReference Include="Instances"> <PackageReference Include="Instances">
<Version>1.6.1</Version> <Version>1.6.1</Version>

4
Properties/AssemblyInfo.cs
View File

@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers // You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below: // by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")] // [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("2.39.0.0")] [assembly: AssemblyVersion("2.39.2.0")]
[assembly: AssemblyFileVersion("2.39.0.0")] [assembly: AssemblyFileVersion("2.39.2.0")]