From e476b14a54dd18edb8d5e578d9cd8d0ae4132623 Mon Sep 17 00:00:00 2001 From: Mike Phares Date: Thu, 10 Mar 2022 17:29:39 -0700 Subject: [PATCH] DEP08SIASM - Testing EDA --- Adaptation/FileHandlers/jpeg/Description.cs | 68 +++++-- Adaptation/FileHandlers/jpeg/FileRead.cs | 28 +-- Adaptation/FileHandlers/jpeg/ProcessData.cs | 200 ++++++-------------- DEP08SIASM.csproj | 2 +- Properties/AssemblyInfo.cs | 4 +- 5 files changed, 123 insertions(+), 179 deletions(-) diff --git a/Adaptation/FileHandlers/jpeg/Description.cs b/Adaptation/FileHandlers/jpeg/Description.cs index f7fc9de..80e93e1 100644 --- a/Adaptation/FileHandlers/jpeg/Description.cs +++ b/Adaptation/FileHandlers/jpeg/Description.cs @@ -23,10 +23,16 @@ public class Description : IDescription public string ProcessJobID { get; set; } public string MID { get; set; } // - public string Process { get; set; } //5 - public string Date { get; set; } //6 - public string Part { get; set; } //7 - public string Lot { get; set; } //8 + public string Date { get; set; } + public string Employee { get; set; } + public string Lot { get; set; } + public string PSN { get; set; } + public string Reactor { get; set; } + public string Recipe { get; set; } + // + public int Red { get; set; } + public int Green { get; set; } + public int TotalDelta { get; set; } string IDescription.GetEventDescription() => "File Has been read and parsed"; @@ -51,8 +57,16 @@ public class Description : IDescription List IDescription.GetHeaderNames() { - string[] results = new string[] { nameof(Process), nameof(Date), nameof(Part), nameof(Lot) }; - return results.ToList(); + List results = new() + { + nameof(Date), + nameof(Employee), + nameof(Lot), + nameof(PSN), + nameof(Reactor), + nameof(Recipe) + }; + return results; } IDescription IDescription.GetDisplayNames() @@ -63,7 +77,12 @@ public class Description : IDescription List IDescription.GetParameterNames() { - List results = new(); + List results = new() + { + nameof(Red), + nameof(Green), + nameof(TotalDelta), + }; return results; } @@ -115,7 +134,7 @@ public class Description : IDescription { List results = new(); - if (iProcessData is null || !iProcessData.Details.Any() || iProcessData is not ProcessData) + if (iProcessData is null || !iProcessData.Details.Any() || iProcessData is not ProcessData processData) results.Add(GetDefault(fileRead, logistics)); else { @@ -128,7 +147,7 @@ public class Description : IDescription nullData = configDataNullData.ToString(); for (int i = 0; i < iProcessData.Details.Count; i++) { - if (iProcessData.Details[i] is not string detail) + if (iProcessData.Details[i] is not string _) continue; description = new Description { @@ -145,10 +164,16 @@ public class Description : IDescription ProcessJobID = logistics.ProcessJobID, MID = logistics.MID, // - Process = logistics.ProcessJobID, - Date = logistics.DateTimeFromSequence.Ticks.ToString(), - Part = string.Empty, - Lot = detail + Date = logistics.DateTimeFromSequence.ToUniversalTime().ToString("MM/dd/yyyy HH:mm:ss"), + Employee = string.Empty, + Lot = processData.Text, + PSN = string.Empty, + Reactor = logistics.ProcessJobID, + Recipe = processData.Recipe, + // + Red = processData.Red, + Green = processData.Green, + TotalDelta = processData.TotalDelta, }; results.Add(description); } @@ -177,12 +202,19 @@ public class Description : IDescription MesEntity = fileRead.MesEntity, ReportFullPath = logistics.ReportFullPath, ProcessJobID = logistics.ProcessJobID, - MID = logistics.MID + MID = logistics.MID, + // + Date = nameof(Date), + Employee = nameof(Employee), + Lot = nameof(Lot), + PSN = nameof(PSN), + Reactor = nameof(Reactor), + Recipe = nameof(Recipe), + // + Red = -1, + Green = -1, + TotalDelta = -1, }; - result.Process = nameof(Process); - result.Date = nameof(Date); - result.Part = nameof(Part); - result.Lot = nameof(Lot); return result; } diff --git a/Adaptation/FileHandlers/jpeg/FileRead.cs b/Adaptation/FileHandlers/jpeg/FileRead.cs index 994a6be..71ddde6 100644 --- a/Adaptation/FileHandlers/jpeg/FileRead.cs +++ b/Adaptation/FileHandlers/jpeg/FileRead.cs @@ -9,6 +9,7 @@ using System.IO; using System.Linq; using System.Reflection; using System.Text.Json; +using System.Text.RegularExpressions; namespace Adaptation.FileHandlers.jpeg; @@ -23,6 +24,7 @@ public class FileRead : Shared.FileRead, IFileRead protected long _LastChange; protected string _LastText; protected string _TessDataDirectory; + protected List _PreviousTotalDeltaCollection; protected readonly Dictionary _Reactors; protected readonly List<(string, Color[])> _ColorCollections; @@ -80,6 +82,7 @@ public class FileRead : Shared.FileRead, IFileRead _StartY = int.Parse(startY); _EndX = int.Parse(endX); _EndY = int.Parse(endY); + _PreviousTotalDeltaCollection = new(); string masterImageDirectory = GetPropertyValue(cellInstanceConnectionName, modelObjectParameters, "Path.Memory.Master.Images"); _ColorCollections = ProcessData.GetColorCollections(_StartX, _StartY, _EndX, _EndY, masterImageDirectory); } @@ -160,22 +163,19 @@ public class FileRead : Shared.FileRead, IFileRead results.Item4.Add(new FileInfo(reportFullPath)); else { - IProcessData iProcessData = new ProcessData(this, _Logistics, results.Item4, _TessDataDirectory, _StartX, _StartY, _EndX, _EndY, _ColorCollections); - if (iProcessData is ProcessData _) - { - if (!iProcessData.Details.Any()) - throw new Exception(string.Concat("A) No Data - ", dateTime.Ticks)); - if (iProcessData.Details[0] is not string mid || string.IsNullOrEmpty(mid)) - throw new Exception(string.Concat("B) No Data - ", dateTime.Ticks)); - _Logistics.MID = mid; - SetFileParameterLotID(mid); - _Logistics.ProcessJobID = iProcessData.GetCurrentReactor(this, _Logistics, _Reactors); - } + IProcessData iProcessData = new ProcessData(this, _Logistics, results.Item4, _StartX, _StartY, _EndX, _EndY, _ColorCollections, _PreviousTotalDeltaCollection, _LastText); + if (iProcessData is not ProcessData processData) + throw new Exception(string.Concat("A) No Data - ", dateTime.Ticks)); + string mid = processData.Text; + mid = Regex.Replace(mid, @"[\\,\/,\:,\*,\?,\"",\<,\>,\|]", "_").Split('\r')[0].Split('\n')[0]; + _Logistics.MID = mid; + SetFileParameterLotID(mid); + _Logistics.ProcessJobID = iProcessData.GetCurrentReactor(this, _Logistics, _Reactors); if (!iProcessData.Details.Any()) - throw new Exception(string.Concat("C) No Data - ", dateTime.Ticks)); - if (_LastText != _Logistics.MID || _LastChange < DateTime.Now.AddMinutes(-30).Ticks) + throw new Exception(string.Concat("B) No Data - ", dateTime.Ticks)); + if (_LastText != processData.Text || _LastChange < DateTime.Now.AddMinutes(-30).Ticks) { - _LastText = _Logistics.MID; + _LastText = processData.Text; _LastChange = DateTime.Now.Ticks; results = iProcessData.GetResults(this, _Logistics, results.Item4); } diff --git a/Adaptation/FileHandlers/jpeg/ProcessData.cs b/Adaptation/FileHandlers/jpeg/ProcessData.cs index 45e1472..4df8df6 100644 --- a/Adaptation/FileHandlers/jpeg/ProcessData.cs +++ b/Adaptation/FileHandlers/jpeg/ProcessData.cs @@ -23,9 +23,15 @@ public class ProcessData : IProcessData public string JobID { get; set; } public string MesEntity { get; set; } + public int Red { get; set; } + public int Green { get; set; } + public string Text { get; set; } + public int TotalDelta { get; set; } + public string Recipe { get; set; } + List Shared.Properties.IProcessData.Details => _Details; - public ProcessData(IFileRead fileRead, Logistics logistics, List fileInfoCollection, string tessDataDirectory, int startX, int startY, int endX, int endY, List<(string, Color[])> colorCollections) + public ProcessData(IFileRead fileRead, Logistics logistics, List fileInfoCollection, int startX, int startY, int endX, int endY, List<(string, Color[])> colorCollections, List previousTotalDeltaCollection, string lastText) { if (logistics is null) { } @@ -34,7 +40,7 @@ public class ProcessData : IProcessData _Details = new List(); MesEntity = logistics.MesEntity; _Log = LogManager.GetLogger(typeof(ProcessData)); - Parse(fileRead, fileInfoCollection, tessDataDirectory, startX, startY, endX, endY, colorCollections); + Parse(fileRead, fileInfoCollection, startX, startY, endX, endY, colorCollections, previousTotalDeltaCollection, lastText); } private static string Get(string value, bool useSplitForMID) @@ -133,16 +139,17 @@ public class ProcessData : IProcessData return imageFormat; } - private static (Color[], int, int, MemoryStream) Get(string reportFullPath, string extension, int startX, int startY, int endX, int endY) + private static (Color[], int, int) Get(IFileRead fileRead, List fileInfoCollection, string extension, int startX, int startY, int endX, int endY) { Color color; List colors = new(); MemoryStream memoryStream = new(); Bitmap selectedBitmap = new(endX - startX, endY - startY); System.Drawing.Imaging.ImageFormat imageFormat = Get(extension); - using Bitmap? bitmap = Image.FromFile(reportFullPath) as Bitmap; + using Bitmap? bitmap = Image.FromFile(fileRead.ReportFullPath) as Bitmap; + string saveFileName = Path.ChangeExtension(fileRead.ReportFullPath, extension); if (bitmap is null) - throw new Exception($"Couldn't load image from <{reportFullPath}>"); + throw new Exception($"Couldn't load image from <{fileRead.ReportFullPath}>"); for (int x = startX; x < endX; x++) { for (int y = startY; y < endY; y++) @@ -153,16 +160,21 @@ public class ProcessData : IProcessData } } selectedBitmap.Save(memoryStream, imageFormat); - return new(colors.ToArray(), endX - startX, endY - startY, memoryStream); + if (!fileRead.IsEAFHosted) + { + fileInfoCollection.Add(new FileInfo(saveFileName)); + SaveToFile(extension, saveFileName, memoryStream); + } + return new(colors.ToArray(), endX - startX, endY - startY); } - private static string Get(string saveFileName, string extension, string extra) + private static string Get(IFileRead fileRead, string extension, string extra) { string result; - string? directoryName = Path.GetDirectoryName(saveFileName); + string? directoryName = Path.GetDirectoryName(fileRead.ReportFullPath); if (string.IsNullOrEmpty(directoryName)) throw new Exception("Couldn't get directoryName!"); - string? fileNameWithoutExtension = Path.GetFileNameWithoutExtension(saveFileName); + string? fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileRead.ReportFullPath); if (string.IsNullOrEmpty(fileNameWithoutExtension)) throw new Exception("Couldn't get fileNameWithoutExtension!"); result = Path.Combine(directoryName, $"{fileNameWithoutExtension} - {extra}{extension}"); @@ -185,45 +197,17 @@ public class ProcessData : IProcessData bitmap.Save(saveFileName, imageFormat); } - private static byte[] Get(bool development, string extension, string saveFileName, Color[] sourceColors, int width, int height, int thresHold, bool saveToFile) - { - int i = 0; - Color color; - MemoryStream memoryStream = new(); - Bitmap selectedBitmap = new(width, height); - System.Drawing.Imaging.ImageFormat imageFormat = Get(extension); - string newSaveFileName = Get(saveFileName, extension, thresHold.ToString("000")); - for (int x = 0; x < width; x++) - { - for (int y = 0; y < height; y++) - { - color = sourceColors[i]; - if (color.R > thresHold || color.G > thresHold || color.B > thresHold) - selectedBitmap.SetPixel(x, y, Color.Black); - i += 1; - } - } - selectedBitmap.Save(memoryStream, imageFormat); - if (development && saveToFile) - SaveToFile(extension, newSaveFileName, memoryStream); - byte[] bytes = memoryStream.GetBuffer(); - return bytes; - } - #pragma warning restore CA1416 - private static void SaveText(int red, int green, string text, string textFileName, List<(string File, int TotalDelta)> totalDeltaCollection, int readAt, string reading, int score) + private static void SaveText(int red, int green, string closestMatchFileNameEnding, string textFileName, List<(string File, int TotalDelta)> totalDeltaCollection) { string format = "00000"; - List lines = new() { red.ToString(format), green.ToString(format), text }; + List lines = new() { red.ToString(format), green.ToString(format), closestMatchFileNameEnding }; foreach ((string file, int totalDelta) in totalDeltaCollection) { lines.Add(file); lines.Add(totalDelta.ToString(format)); } - lines.Add(readAt.ToString(format)); - lines.Add(reading); - lines.Add(score.ToString(format)); File.WriteAllLines(textFileName, lines); } @@ -236,31 +220,19 @@ public class ProcessData : IProcessData return results; } - private void Parse(IFileRead fileRead, List fileInfoCollection, string tessDataDirectory, int startX, int startY, int endX, int endY, List<(string, Color[])> colorCollections) + private void Parse(IFileRead fileRead, List fileInfoCollection, int startX, int startY, int endX, int endY, List<(string, Color[])> colorCollections, List previousTotalDeltaCollection, string lastText) { - Pix pix; + Recipe = string.Empty; + _Log.Debug("TODO: Get recipe"); + Red = 0; + Green = 0; int delta; - Page page; - int readAt; Color color; - int red = 0; - string text; - byte[] bytes; - int green = 0; int totalDelta; - string textFileName; - string closestMatchFile; const int thresHold = 70; - string closestMatchFileName; - List readings = new(); - const int upperThresHold = 153; - Regex regex = new(@"[^a-zA-Z]"); const string extension = ".tiff"; - string[] closestMatchFileNameSplit; - char[] closestMatchFileNameDistinct; List<(string File, int TotalDelta)> totalDeltaCollection = new(); - string saveFileName = Path.ChangeExtension(fileRead.ReportFullPath, extension); - (Color[] sourceColors, int width, int height, MemoryStream memoryStream) = Get(fileRead.ReportFullPath, extension, startX, startY, endX, endY); + (Color[] sourceColors, int width, int height) = Get(fileRead, fileInfoCollection, extension, startX, startY, endX, endY); foreach ((string file, Color[] colors) in colorCollections) { totalDelta = 0; @@ -270,9 +242,9 @@ public class ProcessData : IProcessData { color = sourceColors[i]; if (color.R > thresHold) - red += 1; + Red += 1; if (color.G > thresHold) - green += 1; + Green += 1; delta = color.R - colors[i].R; if (delta > 0) totalDelta += delta; @@ -283,100 +255,40 @@ public class ProcessData : IProcessData totalDelta += delta; else totalDelta += delta * -1; + delta = color.B - colors[i].B; + if (delta > 0) + totalDelta += delta; + else + totalDelta += delta * -1; } totalDeltaCollection.Add(new(file, totalDelta)); } totalDeltaCollection = (from l in totalDeltaCollection orderby l.TotalDelta select l).ToList(); - bytes = memoryStream.GetBuffer(); - closestMatchFile = totalDeltaCollection[0].File; - closestMatchFileNameSplit = Path.GetFileNameWithoutExtension(closestMatchFile).Split('-'); - closestMatchFileName = closestMatchFileNameSplit.Last().TrimStart(); - closestMatchFileNameDistinct = closestMatchFileName.Distinct().ToArray(); - using TesseractEngine tesseractEngine = new(tessDataDirectory, "eng", EngineMode.Default); - pix = Pix.LoadTiffFromMemory(bytes); - page = tesseractEngine.Process(pix); - text = page.GetText().Trim(); - pix.Dispose(); - page.Dispose(); + string closestMatchFile = totalDeltaCollection[0].File; + string[] closestMatchFileNameSplit = Path.GetFileNameWithoutExtension(closestMatchFile).Split('-'); + Text = closestMatchFileNameSplit.Last().TrimStart(); + TotalDelta = totalDeltaCollection[0].TotalDelta; + string textFileName = Get(fileRead, ".txt", $"{TotalDelta} - {Text}"); if (!fileRead.IsEAFHosted) { - fileInfoCollection.Add(new FileInfo(saveFileName)); - SaveToFile(extension, saveFileName, memoryStream); + fileInfoCollection.Add(new FileInfo(textFileName)); + SaveText(Red, Green, Text, textFileName, totalDeltaCollection); } - if (!string.IsNullOrEmpty(text)) + if (Text != lastText && previousTotalDeltaCollection.Count > 50) { - text = regex.Replace(text, string.Empty); - readings.Add(text); + double average = previousTotalDeltaCollection.Average(); + double sum = previousTotalDeltaCollection.Sum(l => Math.Pow(l - average, 2)); + double standardDeviation = Math.Sqrt(sum / previousTotalDeltaCollection.Count); + double deviation = standardDeviation * 3; + double upper = average + deviation; + double lower = average - deviation; + if (TotalDelta > upper) + throw new Exception(); + if (TotalDelta < lower) + throw new Exception(); } - if (text == closestMatchFileName) - { - readAt = thresHold; - _Log.Info(text); - textFileName = Get(saveFileName, ".txt", $"{thresHold:000} - {text}"); - if (!fileRead.IsEAFHosted) - { - fileInfoCollection.Add(new FileInfo(textFileName)); - SaveText(red, green, text, textFileName, totalDeltaCollection, readAt, text, int.MaxValue); - } - } - else - { - readAt = -1; - memoryStream.Dispose(); - for (int i = thresHold; i < upperThresHold; i += 10) - { - bytes = Get(!fileRead.IsEAFHosted, extension, saveFileName, sourceColors, width, height, i, i == thresHold); - pix = Pix.LoadTiffFromMemory(bytes); - page = tesseractEngine.Process(pix); - text = page.GetText().Trim(); - pix.Dispose(); - page.Dispose(); - if (!string.IsNullOrEmpty(text)) - { - text = regex.Replace(text, string.Empty); - readings.Add(text); - } - if (text == closestMatchFileName) - { - readAt = i; - _Log.Info(text); - textFileName = Get(saveFileName, ".txt", $"{i:000} - {text}"); - if (!fileRead.IsEAFHosted) - { - fileInfoCollection.Add(new FileInfo(textFileName)); - SaveText(red, green, text, textFileName, totalDeltaCollection, readAt, text, int.MaxValue); - } - break; - } - } - } - if (readAt < thresHold) - { - int score; - char[] readingDistinct; - text = closestMatchFileName; - List<(string Reading, int Score)> readingEvaluations = new(); - foreach (string reading in readings) - { - score = 0; - readingDistinct = reading.Distinct().ToArray(); - for (int i = 0; i < closestMatchFileNameDistinct.Length; i++) - { - if (!readingDistinct.Contains(closestMatchFileNameDistinct[i])) - continue; - score += 1; - } - readingEvaluations.Add(new(reading, score)); - } - readingEvaluations = (from l in readingEvaluations orderby l.Score descending select l).ToList(); - textFileName = Get(saveFileName, ".txt", $"{readAt:000} - {readingEvaluations[0].Reading} - {text}"); - if (!fileRead.IsEAFHosted) - { - fileInfoCollection.Add(new FileInfo(textFileName)); - SaveText(red, green, text, textFileName, totalDeltaCollection, readAt, readingEvaluations[0].Reading, readingEvaluations[0].Score); - } - } - _Details.Add(text); + previousTotalDeltaCollection.Add(TotalDelta); + _Details.Add(Text); } } \ No newline at end of file diff --git a/DEP08SIASM.csproj b/DEP08SIASM.csproj index 516a66e..95d55e2 100644 --- a/DEP08SIASM.csproj +++ b/DEP08SIASM.csproj @@ -134,7 +134,7 @@ 4.7.0 - 2.39.0 + 2.39.2 1.6.1 diff --git a/Properties/AssemblyInfo.cs b/Properties/AssemblyInfo.cs index dc65d63..2d398ef 100644 --- a/Properties/AssemblyInfo.cs +++ b/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("2.39.0.0")] -[assembly: AssemblyFileVersion("2.39.0.0")] +[assembly: AssemblyVersion("2.39.2.0")] +[assembly: AssemblyFileVersion("2.39.2.0")]