using Dapper; using Microsoft.Extensions.Caching.Memory; using Newtonsoft.Json.Linq; using OI.Metrology.Shared.DataModels; using OI.Metrology.Shared.Repositories; using System; using System.Collections.Generic; using System.Data; using System.Data.Common; using System.Linq; using System.Transactions; namespace OI.Metrology.Viewer.Repositories; public class MetrologyRepo : IMetrologyRepo { private readonly IMemoryCache _Cache; private readonly IDbConnectionFactory _DBConnectionFactory; public MetrologyRepo(IDbConnectionFactory dbConnectionFactory, IMemoryCache memoryCache) { _Cache = memoryCache; _DBConnectionFactory = dbConnectionFactory; } private DbConnection GetDbConnection() => _DBConnectionFactory.GetDbConnection(); protected DbProviderFactory GetDbProviderFactory(IDbConnection conn) => DbProviderFactories.GetFactory(conn.GetType().Namespace); public bool IsTestDatabase() { int c = 0; using (DbConnection conn = GetDbConnection()) { c = conn.Query( "SELECT COUNT(*) " + "FROM Configuration " + "WHERE KeyName = 'TestDatabase' " + "AND ValueString = '1'").FirstOrDefault(); } return c > 0; } public TransactionScope StartTransaction() => new(); protected void CacheItem(string key, object v) { System.Diagnostics.Debug.WriteLine("CacheItem: " + key); _ = _Cache.Set(key, v, new MemoryCacheEntryOptions().SetSlidingExpiration(TimeSpan.FromHours(1))); } public IEnumerable GetToolTypes() { IEnumerable cached; string cacheKey = "GetToolTypes"; if (_Cache.TryGetValue(cacheKey, out cached)) return cached; using DbConnection conn = GetDbConnection(); IEnumerable r = conn.Query("SELECT * FROM ToolType"); CacheItem(cacheKey, r); return r; } public ToolType GetToolTypeByName(string name) { ToolType cached; string cacheKey = "GetToolTypeByName_" + name; if (_Cache.TryGetValue(cacheKey, out cached)) return cached; using DbConnection conn = GetDbConnection(); ToolType r = conn.QueryFirstOrDefault( "SELECT * FROM ToolType WHERE ToolTypeName = @name", new { name }); CacheItem(cacheKey, r); return r; } public ToolType GetToolTypeByID(int id) { ToolType cached; string cacheKey = "GetToolTypeByID_" + id.ToString(); if (_Cache.TryGetValue(cacheKey, out cached)) return cached; using DbConnection conn = GetDbConnection(); ToolType r = conn.QueryFirstOrDefault( "SELECT * FROM ToolType WHERE ID = @id", new { id }); CacheItem(cacheKey, r); return r; } public IEnumerable GetToolTypeMetadataByToolTypeID(int id) { IEnumerable cached; string cacheKey = "GetToolTypeMetadataByToolTypeID_" + id.ToString(); if (_Cache.TryGetValue(cacheKey, out cached)) return cached; using DbConnection conn = GetDbConnection(); IEnumerable r = conn.Query( "SELECT * FROM ToolTypeMetadata WHERE ToolTypeID = @id", new { id }); CacheItem(cacheKey, r); return r; } public long InsertToolDataJSON(JToken jsonrow, long headerId, List metaData, string tableName) { long r = -1; using (DbConnection conn = GetDbConnection()) { bool isHeader = headerId <= 0; // get fields from metadata List fields = metaData.Where(md => md.Header == isHeader).ToList(); // maps ApiName to ColumnName Dictionary fieldmap = new(); // store property name of container field string containerField = null; // maps container ApiName to ColumnName Dictionary containerFieldMap = new(); // build field map foreach (ToolTypeMetadata f in fields) { if ((f.ApiName != null) && f.ApiName.Contains('\\')) { string n = f.ApiName.Split('\\')[0].Trim().ToUpper(); if (containerField == null) containerField = n; else if (!string.Equals(containerField, n)) throw new Exception("Only one container field is allowed"); string pn = f.ApiName.Split('\\')[1].Trim().ToUpper(); containerFieldMap.Add(pn, f.ColumnName.Trim()); } else if (!string.IsNullOrWhiteSpace(f.ApiName) && !string.IsNullOrWhiteSpace(f.ColumnName)) { fieldmap.Add(f.ApiName.Trim().ToUpper(), f.ColumnName.Trim()); } } if (containerField == null) { // No container field, just insert a single row r = InsertRowFromJSON(conn, tableName, jsonrow, fieldmap, headerId, null, null); } else { // Find the container field in the json JProperty contJP = jsonrow.Children().Where(c => string.Equals(c.Name.Trim(), containerField, StringComparison.OrdinalIgnoreCase)).SingleOrDefault(); if ((contJP != null) && (contJP.Value is JArray array)) { JArray contRows = array; // Insert a row for each row in the container field foreach (JToken contRow in contRows) { r = InsertRowFromJSON(conn, tableName, jsonrow, fieldmap, headerId, contRow, containerFieldMap); } } else throw new Exception("Invalid container field type"); } } return r; } protected void AddParameter(IDbCommand cmd, string name, object value) { IDbDataParameter p = cmd.CreateParameter(); p.ParameterName = name; p.Value = value; _ = cmd.Parameters.Add(p); } protected long InsertRowFromJSON( IDbConnection conn, string tableName, JToken jsonrow, Dictionary fieldmap, long headerId, JToken containerrow, Dictionary containerFieldmap) { // Translate the json into a SQL INSERT using the field map IDbCommand cmd = conn.CreateCommand(); string columns = "INSERT INTO [" + tableName + "]("; string parms = ") SELECT "; int parmnumber = 1; if (headerId > 0) { columns += "HeaderID,"; parms += "@HeaderID,"; AddParameter(cmd, "@HeaderID", headerId); } foreach (JProperty jp in jsonrow.Children()) { string apifield = jp.Name.Trim().ToUpper(); if (fieldmap.ContainsKey(apifield)) { string parmname = string.Format("@p{0}", parmnumber); columns += string.Format("[{0}],", fieldmap[apifield]); parms += parmname; parms += ","; parmnumber += 1; object sqlValue = ((JValue)jp.Value).Value; if (sqlValue == null) sqlValue = DBNull.Value; AddParameter(cmd, parmname, sqlValue); } } if ((containerrow != null) && (containerFieldmap != null)) { foreach (JProperty jp in containerrow.Children()) { string apifield = jp.Name.Trim().ToUpper(); if (containerFieldmap.ContainsKey(apifield)) { string parmname = string.Format("@p{0}", parmnumber); columns += string.Format("[{0}],", containerFieldmap[apifield]); parms += parmname; parms += ","; parmnumber += 1; object sqlValue = ((JValue)jp.Value).Value; if (sqlValue == null) sqlValue = DBNull.Value; AddParameter(cmd, parmname, sqlValue); } } } if (parmnumber == 1) throw new Exception("JSON had no fields"); cmd.CommandText = columns.TrimEnd(',') + parms.TrimEnd(',') + ";SELECT SCOPE_IDENTITY();"; object o = cmd.ExecuteScalar(); if ((o == null) || Convert.IsDBNull(o)) throw new Exception("Unexpected query result"); return Convert.ToInt64(o); } public DataTable ExportData(string spName, DateTime startTime, DateTime endTime) { DataTable dt = new(); DateTime endTimeLocal = endTime.ToLocalTime(); DateTime startTimeLocal = startTime.ToLocalTime(); using (DbConnection conn = GetDbConnection()) { DbProviderFactory factory = GetDbProviderFactory(conn); DbCommand cmd = factory.CreateCommand(); cmd.Connection = conn; cmd.CommandText = spName; cmd.CommandType = CommandType.StoredProcedure; AddParameter(cmd, "@StartTime", startTimeLocal); AddParameter(cmd, "@EndTime", endTimeLocal); DbDataAdapter da = factory.CreateDataAdapter(); da.SelectCommand = cmd; _ = da.Fill(dt); } return dt; } protected string FormDynamicSelectQuery(IEnumerable fields, string tableName) { System.Text.StringBuilder sb = new(); _ = sb.Append("SELECT "); bool firstField = true; foreach (ToolTypeMetadata f in fields) { if (!string.IsNullOrWhiteSpace(f.ColumnName)) { if (!firstField) _ = sb.Append(','); if (f.GridAttributes != null && f.GridAttributes.Contains("isNull")) { _ = sb.AppendFormat("{0}", "ISNULL(" + f.ColumnName + ", '')[" + f.ColumnName + "]"); } else { _ = sb.AppendFormat("[{0}]", f.ColumnName); } firstField = false; } } _ = sb.AppendFormat(" FROM [{0}] ", tableName); return sb.ToString(); } public DataTable GetHeaders(int toolTypeId, DateTime? startTime, DateTime? endTime, int? pageNo, int? pageSize, long? headerId, out long totalRecords) { ToolType tt = GetToolTypeByID(toolTypeId); if (tt == null) throw new Exception("Invalid tool type ID"); IEnumerable md = GetToolTypeMetadataByToolTypeID(toolTypeId); if (md == null) throw new Exception("Invalid tool type metadata"); DataTable dt = new(); using (DbConnection conn = GetDbConnection()) { System.Text.StringBuilder sb = new(); _ = sb.Append( FormDynamicSelectQuery( md.Where(m => m.Header == true).ToList(), tt.HeaderTableName) ); DbProviderFactory factory = GetDbProviderFactory(conn); DbCommand cmd = factory.CreateCommand(); string whereClause = ""; if (headerId.HasValue && headerId.Value > 0) { whereClause = "ID = @HeaderID "; AddParameter(cmd, "@HeaderID", headerId.Value); } else { if (startTime.HasValue) { whereClause = "[Date] >= @StartTime "; DateTime startTimeLocal = startTime.Value.ToLocalTime(); AddParameter(cmd, "@StartTime", startTimeLocal); } if (endTime.HasValue) { DateTime endTimeLocal = endTime.Value.ToLocalTime(); TimeSpan timeSpan = new(DateTime.Now.Ticks - endTimeLocal.Ticks); if (timeSpan.TotalMinutes > 5) { if (whereClause.Length > 0) whereClause += "AND "; whereClause += "[Date] <= @EndTime "; AddParameter(cmd, "@EndTime", endTimeLocal); } } } if (whereClause.Length > 0) { _ = sb.Append("WHERE "); _ = sb.Append(whereClause); } if (pageNo.HasValue && pageSize.HasValue) { _ = sb.Append("ORDER BY [Date] DESC OFFSET @PageNum * @PageSize ROWS FETCH NEXT @PageSize ROWS ONLY"); AddParameter(cmd, "@PageNum", pageNo.Value); AddParameter(cmd, "@PageSize", pageSize.Value); } else { _ = sb.Append("ORDER BY [Date] DESC"); } cmd.Connection = conn; cmd.CommandText = sb.ToString(); cmd.CommandType = CommandType.Text; DbDataAdapter da = factory.CreateDataAdapter(); da.SelectCommand = cmd; _ = da.Fill(dt); cmd.CommandText = "SELECT COUNT(*) FROM [" + tt.HeaderTableName + "] "; if (whereClause.Length > 0) { cmd.CommandText += "WHERE "; cmd.CommandText += whereClause; } totalRecords = Convert.ToInt64(cmd.ExecuteScalar()); } return dt; } public DataTable GetData(int toolTypeId, long headerid) { ToolType tt = GetToolTypeByID(toolTypeId); if (tt == null) throw new Exception("Invalid tool type ID"); IEnumerable md = GetToolTypeMetadataByToolTypeID(toolTypeId); if (md == null) throw new Exception("Invalid tool type metadata"); DataTable dt = new(); using (DbConnection conn = GetDbConnection()) { System.Text.StringBuilder sb = new(); _ = sb.Append( FormDynamicSelectQuery( md.Where(m => m.Header == false).OrderBy(m => m.GridDisplayOrder).ToList(), tt.DataTableName) ); DbProviderFactory factory = GetDbProviderFactory(conn); DbCommand cmd = factory.CreateCommand(); _ = sb.Append("WHERE [HeaderID] = @HeaderID "); if (!string.IsNullOrWhiteSpace(tt.DataGridSortBy)) { _ = sb.AppendFormat("ORDER BY {0} ", tt.DataGridSortBy); } AddParameter(cmd, "@HeaderID", headerid); cmd.Connection = conn; cmd.CommandText = sb.ToString(); cmd.CommandType = CommandType.Text; DbDataAdapter da = factory.CreateDataAdapter(); da.SelectCommand = cmd; _ = da.Fill(dt); } // this code will add a couple of rows with stats calculations if (!string.IsNullOrWhiteSpace(tt.DataGridStatsColumn)) { if (dt.Columns.Contains(tt.DataGridStatsColumn)) { double sumAll = 0; double sumAllQ = 0; foreach (DataRow dr in dt.Rows) { try { object v = dr[tt.DataGridStatsColumn]; if (!Convert.IsDBNull(v)) { double d = Convert.ToDouble(v); sumAll += d; sumAllQ += d * d; } } catch { } } double length = Convert.ToDouble(dt.Rows.Count); double meanAverage = Math.Round(sumAll / length, 4); double stdDev = Math.Sqrt((sumAllQ - sumAll * sumAll / length) * (1.0d / (length - 1))); string stdDevStr = ""; if (tt.DataGridStatsStdDevType == "%") stdDevStr = Math.Round(stdDev / meanAverage * 100.0d, 2).ToString("0.####") + "%"; else stdDevStr = Math.Round(stdDev, 4).ToString("0.####"); int labelIndex = dt.Columns[tt.DataGridStatsColumn].Ordinal - 1; DataRow newrow = dt.NewRow(); newrow["ID"] = -1; newrow[labelIndex] = "Average"; newrow[tt.DataGridStatsColumn] = meanAverage; dt.Rows.Add(newrow); newrow = dt.NewRow(); newrow["ID"] = -2; newrow[labelIndex] = "Std Dev"; newrow[tt.DataGridStatsColumn] = stdDevStr; dt.Rows.Add(newrow); } } return dt; } public Guid GetHeaderAttachmentID(int toolTypeId, long headerId) { ToolType tt = GetToolTypeByID(toolTypeId); if (tt == null) throw new Exception("Invalid tool type ID"); using DbConnection conn = GetDbConnection(); string sql = $"UPDATE [{tt.HeaderTableName}] SET AttachmentID = NEWID() WHERE ID = @HeaderID AND AttachmentID IS NULL; " + $"SELECT AttachmentID FROM [{tt.HeaderTableName}] WHERE ID = @HeaderID"; return conn.ExecuteScalar(sql, param: new { HeaderID = headerId }); } public string GetHeaderInsertDate(int toolTypeId, long headerId) { ToolType tt = GetToolTypeByID(toolTypeId); if (tt == null) throw new Exception("Invalid tool type ID"); using DbConnection conn = GetDbConnection(); string sql = $"SELECT CONVERT(varchar, case when [InsertDate] < [Date] or [Date] is null then [InsertDate] else [Date] end, 120) d FROM[{tt.HeaderTableName}] where ID = @HeaderID"; return conn.ExecuteScalar(sql, param: new { HeaderID = headerId }); } public string GetAttachmentInsertDateByGUID(string tableName, Guid attachmentId) { using DbConnection conn = GetDbConnection(); string sql = ""; if (tableName is "SP1RunData" or "TencorRunData") { sql = $"SELECT [InsertDate] FROM[{tableName}] where AttachmentID = @AttachmentID"; } else { sql = $"SELECT [InsertDate] FROM[{tableName}] where AttachmentID = @AttachmentID"; } return conn.ExecuteScalar(sql, param: new { AttachmentID = attachmentId }); } public void SetHeaderDirName(string tableName, long headerId, string dateDir) { using DbConnection conn = GetDbConnection(); _ = conn.Execute($"UPDATE [{tableName}] SET AttachDirName = @AttachDirName WHERE ID = @HeaderID;", new { HeaderID = headerId, AttachDirName = dateDir }); } public Guid GetDataAttachmentID(int toolTypeId, long headerId, string title) { ToolType tt = GetToolTypeByID(toolTypeId); if (tt == null) throw new Exception("Invalid tool type ID"); using DbConnection conn = GetDbConnection(); string sql = $"UPDATE [{tt.DataTableName}] SET AttachmentID = NEWID() WHERE HeaderID = @HeaderID AND Title = @Title AND AttachmentID IS NULL; " + $"SELECT AttachmentID FROM [{tt.DataTableName}] WHERE HeaderID = @HeaderID AND Title = @Title"; return conn.ExecuteScalar(sql, param: new { HeaderID = headerId, Title = title }); } // J Ouellette Added public string GetDataInsertDate(int toolTypeId, long headerId, string title) { ToolType tt = GetToolTypeByID(toolTypeId); if (tt == null) throw new Exception("Invalid tool type ID"); using DbConnection conn = GetDbConnection(); string sql = ""; if (tt.DataTableName is "" or "") { sql = $"SELECT InsertDate FROM [{tt.DataTableName}] WHERE HeaderID = @HeaderID AND Title = @Title"; } else { sql = $"SELECT [InsertDate] FROM[{tt.DataTableName}] WHERE HeaderID = @HeaderID AND Title = @Title"; } return conn.ExecuteScalar(sql, param: new { HeaderID = headerId, Title = title }); } public void SetDataDirName(string tableName, long headerId, string title, string dateDir) { using DbConnection conn = GetDbConnection(); string sql = $"UPDATE [{tableName}] SET AttachDirName = @AttachDirName WHERE HeaderID = @HeaderID AND Title = @Title;"; _ = conn.Execute(sql, param: new { HeaderID = headerId, Title = title, AttachDirName = dateDir }); } public void PurgeExistingData(int toolTypeId, string title) { using DbConnection conn = GetDbConnection(); _ = conn.Execute("PurgeExistingData", param: new { ToolTypeID = toolTypeId, Title = title }, commandType: CommandType.StoredProcedure); } public DataSet GetOIExportData(int toolTypeId, long headerid) { ToolType tt = GetToolTypeByID(toolTypeId); if (tt == null) throw new Exception("Invalid tool type ID"); if (string.IsNullOrWhiteSpace(tt.OIExportSPName)) throw new Exception("OpenInsight export not available for " + tt.ToolTypeName); DataSet ds = new(); using (DbConnection conn = GetDbConnection()) { DbProviderFactory factory = GetDbProviderFactory(conn); DbCommand cmd = factory.CreateCommand(); cmd.Connection = conn; cmd.CommandText = tt.OIExportSPName; cmd.CommandType = CommandType.StoredProcedure; AddParameter(cmd, "@ID", headerid); DbDataAdapter da = factory.CreateDataAdapter(); da.SelectCommand = cmd; _ = da.Fill(ds); } return ds; } public IEnumerable GetHeaderTitles(int toolTypeId, int? pageNo, int? pageSize, out long totalRecords) { ToolType tt = GetToolTypeByID(toolTypeId); if (tt == null) throw new Exception("Invalid tool type ID"); using DbConnection conn = GetDbConnection(); string sql = $"SELECT ID, InsertDate, AttachmentID, Title, [Date] FROM {tt.HeaderTableName} ORDER BY [Date] DESC "; IEnumerable headers; if (pageNo.HasValue && pageSize.HasValue) { sql += "OFFSET @PageNum * @PageSize ROWS FETCH NEXT @PageSize ROWS ONLY"; headers = conn.Query(sql, param: new { PageNum = pageNo.Value, PageSize = pageSize.Value }).ToList(); } else { headers = conn.Query(sql).ToList(); } sql = $"SELECT COUNT(*) FROM [{tt.HeaderTableName}] "; totalRecords = Convert.ToInt64(conn.ExecuteScalar(sql)); return headers; } public IEnumerable> GetHeaderFields(int toolTypeId, long headerid) { ToolType tt = GetToolTypeByID(toolTypeId); if (tt == null) throw new Exception("Invalid tool type ID"); IEnumerable md = GetToolTypeMetadataByToolTypeID(toolTypeId); if (md == null) throw new Exception("Invalid tool type metadata"); List> r = new(); using (DbConnection conn = GetDbConnection()) { DbProviderFactory factory = GetDbProviderFactory(conn); DbCommand cmd = factory.CreateCommand(); cmd.Connection = conn; cmd.CommandText = $"SELECT * FROM [{tt.HeaderTableName}] WHERE ID = @HeaderID"; AddParameter(cmd, "@HeaderID", headerid); DataTable dt = new(); DbDataAdapter da = factory.CreateDataAdapter(); da.SelectCommand = cmd; _ = da.Fill(dt); DataRow dr = null; if (dt.Rows.Count > 0) dr = dt.Rows[0]; foreach (ToolTypeMetadata m in md.Where(m => m.Header == true && m.TableDisplayOrder > 0).OrderBy(m => m.TableDisplayOrder)) { string v = ""; if (dr != null) { object o = dr[m.ColumnName]; if (o != null && !Convert.IsDBNull(o)) v = Convert.ToString(o); } KeyValuePair kvp = new(m.DisplayTitle, v); r.Add(kvp); } } return r; } public IEnumerable GetAwaitingDispo() { using DbConnection conn = GetDbConnection(); return conn.Query("GetAwaitingDispo", commandType: CommandType.StoredProcedure); } public int UpdateReviewDate(int toolTypeId, long headerId, bool clearDate) { ToolType tt = GetToolTypeByID(toolTypeId); if (tt == null) throw new Exception("Invalid tool type ID"); using DbConnection conn = GetDbConnection(); if (clearDate) { // if it's already past the 6 hour window, then it won't show in queue anyway, so need to return value so we can show that string sql = $"SELECT DATEDIFF(HH, INSERTDATE, GETDATE()) FROM [{tt.HeaderTableName}] WHERE ID = @HeaderID"; int hrs = conn.ExecuteScalar(sql, param: new { HeaderId = headerId }); _ = conn.Execute($"UPDATE [{tt.HeaderTableName}] SET ReviewDate = NULL WHERE ID = @HeaderID", new { HeaderID = headerId }); return hrs; } else { _ = conn.Execute($"UPDATE [{tt.HeaderTableName}] SET ReviewDate = GETDATE() WHERE ID = @HeaderID", new { HeaderID = headerId }); return 1; } } public Guid GetHeaderAttachmentIDByTitle(int toolTypeId, string title) { ToolType tt = GetToolTypeByID(toolTypeId); if (tt == null) throw new Exception("Invalid tool type ID"); using DbConnection conn = GetDbConnection(); string sql = $"SELECT TOP 1 AttachmentID FROM [{tt.HeaderTableName}] WHERE Title = @Title ORDER BY InsertDate DESC"; return conn.ExecuteScalar(sql, param: new { Title = title }); } public Guid GetDataAttachmentIDByTitle(int toolTypeId, string title) { ToolType tt = GetToolTypeByID(toolTypeId); if (tt == null) throw new Exception("Invalid tool type ID"); using DbConnection conn = GetDbConnection(); string sql = $"SELECT TOP 1 AttachmentID FROM [{tt.DataTableName}] WHERE Title = @Title ORDER BY InsertDate DESC"; return conn.ExecuteScalar(sql, param: new { Title = title }); } DataTable IMetrologyRepo.GetDataSharePoint(int toolTypeId, string headerId) => throw new NotImplementedException(); }