using Adaptation.Eaf.Management.ConfigurationData.CellAutomation; using Adaptation.FileHandlers.json.WorkItems; using Adaptation.Ifx.Eaf.EquipmentConnector.File.Configuration; using Adaptation.Shared; using Adaptation.Shared.Duplicator; using Adaptation.Shared.Methods; using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; using System.Text; using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; namespace Adaptation.FileHandlers.json; public class FileRead : Shared.FileRead, IFileRead { private readonly Timer _Timer; public FileRead(ISMTP smtp, Dictionary fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList modelObjectParameters, string equipmentDictionaryName, Dictionary> dummyRuns, Dictionary> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) : base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null) { _MinFileLength = 10; _NullData = string.Empty; _Logistics = new(this); if (_FileParameter is null) throw new Exception(cellInstanceConnectionName); if (_ModelObjectParameterDefinitions is null) throw new Exception(cellInstanceConnectionName); if (_IsDuplicator) throw new Exception(cellInstanceConnectionName); if (_IsEAFHosted) NestExistingFiles(_FileConnectorConfiguration); if (!Debugger.IsAttached && fileConnectorConfiguration.PreProcessingMode != FileConnectorConfiguration.PreProcessingModeEnum.Process) _Timer = new Timer(Callback, null, (int)(fileConnectorConfiguration.FileScanningIntervalInSeconds * 1000), Timeout.Infinite); else { _Timer = new Timer(Callback, null, Timeout.Infinite, Timeout.Infinite); Callback(null); } } void IFileRead.Move(Tuple> extractResults, Exception exception) => Move(extractResults); void IFileRead.WaitForThread() => WaitForThread(thread: null, threadExceptions: null); string IFileRead.GetEventDescription() { string result = _Description.GetEventDescription(); return result; } List IFileRead.GetHeaderNames() { List results = _Description.GetHeaderNames(); return results; } string[] IFileRead.Move(Tuple> extractResults, string to, string from, string resolvedFileLocation, Exception exception) { string[] results = Move(extractResults, to, from, resolvedFileLocation, exception); return results; } JsonProperty[] IFileRead.GetDefault() { JsonProperty[] results = _Description.GetDefault(this, _Logistics); return results; } Dictionary IFileRead.GetDisplayNamesJsonElement() { Dictionary results = _Description.GetDisplayNamesJsonElement(this); return results; } List IFileRead.GetDescriptions(IFileRead fileRead, List tests, IProcessData processData) { List results = _Description.GetDescriptions(fileRead, _Logistics, tests, processData); return results; } Tuple> IFileRead.GetExtractResult(string reportFullPath, string eventName) { Tuple> results; if (string.IsNullOrEmpty(eventName)) throw new Exception(); _ReportFullPath = reportFullPath; DateTime dateTime = DateTime.Now; results = GetExtractResult(reportFullPath, dateTime); if (results.Item3 is null) results = new Tuple>(results.Item1, Array.Empty(), JsonSerializer.Deserialize("[]"), results.Item4); if (results.Item3.Length > 0 && _IsEAFHosted) WritePDSF(this, results.Item3); UpdateLastTicksDuration(DateTime.Now.Ticks - dateTime.Ticks); return results; } Tuple> IFileRead.ReExtract() { Tuple> results; List headerNames = _Description.GetHeaderNames(); Dictionary keyValuePairs = _Description.GetDisplayNamesJsonElement(this); results = ReExtract(this, headerNames, keyValuePairs); return results; } #nullable enable private static string GetParamCase(string value) { string result; StringBuilder stringBuilder = new(value); List matches = new(); MatchCollection matchCollection = Regex.Matches(value, "([A-Z]+(.))"); foreach (object? item in matchCollection) { if (item is not Match match) continue; matches.Add(match); } for (int i = matches.Count - 1; i > -1; i--) _ = stringBuilder.Insert(matches[i].Index, '-'); string[] segments = Regex.Split(stringBuilder.ToString().ToLower(), "[\\s!?.,@:;|\\\\/\"'`£$%\\^&*{}[\\]()<>~#+\\-=_¬]+"); result = string.Join("-", segments).Trim('-'); return result; } private static ReadOnlyDictionary GetKeyValuePairs(ReadOnlyCollection valueWithReqCollection) { Dictionary results = new(); Fields fields; WorkItem workItem; Relation relation; string[] segments; List relations; WorkItems.Attribute attributes; foreach (Value value in valueWithReqCollection) { relations = new(); fields = value.Fields; if (value.Relations is not null) { foreach (WIQL.Relation r in value.Relations) { segments = r?.Attributes is null ? Array.Empty() : r.URL.Split('/'); if (segments.Length < 2) continue; if (r?.Attributes is null) continue; if (!int.TryParse(segments[segments.Length - 1], out int id)) continue; attributes = new(r.Attributes.IsLocked, r.Attributes.Name, null, null, null); relation = new(attributes, id, r.Rel); relations.Add(relation); } } workItem = new(activatedDate: fields.MicrosoftVSTSCommonActivatedDate == DateTime.MinValue ? null : fields.MicrosoftVSTSCommonActivatedDate, areaPath: fields.SystemAreaPath, assignedTo: fields.SystemAssignedTo?.DisplayName, businessValue: fields.MicrosoftVSTSCommonBusinessValue is null or 0 ? null : (long)fields.MicrosoftVSTSCommonBusinessValue, changedDate: fields.SystemChangedDate, closedDate: fields.MicrosoftVSTSCommonClosedDate == DateTime.MinValue ? null : fields.MicrosoftVSTSCommonClosedDate, commentCount: fields.SystemCommentCount, createdDate: fields.SystemCreatedDate, description: fields.SystemDescription, effort: fields.MicrosoftVSTSSchedulingEffort is null or 0 ? null : (long)fields.MicrosoftVSTSSchedulingEffort, id: value.Id, iterationPath: fields.SystemIterationPath, parent: fields.SystemParent == 0 ? null : fields.SystemParent, priority: fields.MicrosoftVSTSCommonPriority == 0 ? null : fields.MicrosoftVSTSCommonPriority, relations: relations.ToArray(), remainingWork: fields.MicrosoftVSTSSchedulingRemainingWork is null ? null : (long)fields.MicrosoftVSTSSchedulingRemainingWork, requester: fields.CustomRequester?.DisplayName, resolvedDate: fields.MicrosoftVSTSCommonResolvedDate == DateTime.MinValue ? null : fields.MicrosoftVSTSCommonResolvedDate, revision: value.Rev, riskReductionMinusOpportunityEnablement: fields.CustomRRminusOE is null or 0 ? null : (long)fields.CustomRRminusOE, startDate: fields.MicrosoftVSTSSchedulingStartDate == DateTime.MinValue ? null : fields.MicrosoftVSTSSchedulingStartDate, state: fields.SystemState, storyPoints: fields.MicrosoftVSTSSchedulingStoryPoints is null ? null : (long)fields.MicrosoftVSTSSchedulingStoryPoints, tags: fields.SystemTags, targetDate: fields.MicrosoftVSTSSchedulingTargetDate == DateTime.MinValue ? null : fields.MicrosoftVSTSSchedulingTargetDate, timeCriticality: fields.MicrosoftVSTSCommonTimeCriticality is null or 0 ? null : (long)fields.MicrosoftVSTSCommonTimeCriticality, title: fields.SystemTitle.Trim(), violation: null, weightedShortestJobFirst: fields.CustomWSJF is null or 0 ? null : (long)fields.CustomWSJF, workItemType: fields.SystemWorkItemType); results.Add(workItem.Id, workItem); } return new(results); } private static ReadOnlyCollection GetWorkItems(ReadOnlyCollection valueWithReqCollection) { List results = new(); WorkItem workItem; Relation relation; string parameterTitle; List relations; WorkItem? relationWorkItem; WorkItems.Attribute attributes; ReadOnlyDictionary keyValuePairs = GetKeyValuePairs(valueWithReqCollection); foreach (KeyValuePair keyValuePair in keyValuePairs) { relations = new(); if (keyValuePair.Value.Relations is not null) { foreach (Relation r in keyValuePair.Value.Relations) { if (!keyValuePairs.TryGetValue(r.Id, out relationWorkItem)) continue; parameterTitle = GetParamCase(relationWorkItem.Title); attributes = new(r.Attributes.IsLocked, r.Attributes.Name, parameterTitle, relationWorkItem.State, relationWorkItem.WorkItemType); relation = new(attributes, r.Id, r.Rel); relations.Add(relation); } } workItem = WorkItem.Get(keyValuePair.Value, relations.ToArray()); results.Add(workItem); } return new(results); } private static ReadOnlyCollection GetWorkItems(string[] files) { List results = new(); string json; Value? value; foreach (string file in from l in files orderby l.Length, l select l) { json = File.ReadAllText(file); value = JsonSerializer.Deserialize(json); if (value is null) continue; results.Add(value); } return new(results); } private static void WriteCollectionJson(FileConnectorConfiguration fileConnectorConfiguration, string[] alternateTargetFolders, ReadOnlyCollection workItems) { string old; string checkFile; string? pathRoot; string json = JsonSerializer.Serialize(workItems.ToArray(), WorkItemCollectionSourceGenerationContext.Default.WorkItemArray); foreach (string alternateTargetFolder in alternateTargetFolders) { if (alternateTargetFolder == fileConnectorConfiguration.TargetFileLocation) continue; pathRoot = Path.GetPathRoot(alternateTargetFolder); if (string.IsNullOrEmpty(pathRoot)) continue; try { if (!Directory.Exists(pathRoot)) continue; } catch (Exception) { continue; } if (!Directory.Exists(alternateTargetFolder)) _ = Directory.CreateDirectory(alternateTargetFolder); checkFile = Path.Combine(alternateTargetFolder, fileConnectorConfiguration.TargetFileName); old = !File.Exists(checkFile) ? string.Empty : File.ReadAllText(checkFile); if (old == json) continue; File.WriteAllText(checkFile, json); } } private static void ParseWorkItemsAsync(FileConnectorConfiguration fileConnectorConfiguration, Calendar calendar, string[] alternateTargetFolders) { if (!Directory.Exists(fileConnectorConfiguration.TargetFileLocation)) _ = Directory.CreateDirectory(fileConnectorConfiguration.TargetFileLocation); DateTime dateTime = DateTime.Now; string weekOfYear = calendar.GetWeekOfYear(dateTime, CalendarWeekRule.FirstDay, DayOfWeek.Sunday).ToString("00"); string weekDirectory = Path.Combine(fileConnectorConfiguration.TargetFileLocation, $"{dateTime:yyyy}_Week_{weekOfYear}"); string[] files = Directory.GetFiles(fileConnectorConfiguration.TargetFileLocation, fileConnectorConfiguration.SourceFileFilter, SearchOption.TopDirectoryOnly); if (!Directory.Exists(weekDirectory)) { _ = Directory.CreateDirectory(weekDirectory); foreach (string file in files) File.Copy(file, Path.Combine(weekDirectory, Path.GetFileName(file))); } ReadOnlyCollection collection = GetWorkItems(files); ReadOnlyCollection workItems = GetWorkItems(collection); if (workItems.Count > 0) WriteCollectionJson(fileConnectorConfiguration, alternateTargetFolders, workItems); } private static void ParseWorkItemsAsync(FileConnectorConfiguration fileConnectorConfiguration, Calendar calendar) { string[] alternateTargetFolders = fileConnectorConfiguration.AlternateTargetFolder.Split('|'); if (alternateTargetFolders.Length > 0) ParseWorkItemsAsync(fileConnectorConfiguration, calendar, alternateTargetFolders); } private void Callback(object state) { try { if (_IsEAFHosted) ParseWorkItemsAsync(_FileConnectorConfiguration, _Calendar); } catch (Exception exception) { string subject = string.Concat("Exception:", _CellInstanceConnectionName); string body = string.Concat(exception.Message, Environment.NewLine, Environment.NewLine, exception.StackTrace); try { _SMTP.SendHighPriorityEmailMessage(subject, body); File.WriteAllText(".email", body); } catch (Exception) { } } try { if (_FileConnectorConfiguration?.FileScanningIntervalInSeconds is null) throw new Exception(); TimeSpan timeSpan = new(DateTime.Now.AddSeconds(_FileConnectorConfiguration.FileScanningIntervalInSeconds.Value).Ticks - DateTime.Now.Ticks); _ = _Timer.Change((long)timeSpan.TotalMilliseconds, Timeout.Infinite); } catch (Exception exception) { string subject = string.Concat("Exception:", _CellInstanceConnectionName); string body = string.Concat(exception.Message, Environment.NewLine, Environment.NewLine, exception.StackTrace); try { _SMTP.SendHighPriorityEmailMessage(subject, body); File.WriteAllText(".email", body); } catch (Exception) { } } } private void MoveJson(string reportFullPath, DateTime dateTime) { if (dateTime == DateTime.MinValue) throw new ArgumentNullException(nameof(dateTime)); string json = File.ReadAllText(reportFullPath); Value? value = JsonSerializer.Deserialize(json); if (value is null) throw new Exception(nameof(value)); string checkFile = Path.Combine(_FileConnectorConfiguration.TargetFileLocation, $"{value.Id}.json"); string old = !File.Exists(checkFile) ? string.Empty : File.ReadAllText(checkFile); if (json != old) { File.Delete(checkFile); File.Move(reportFullPath, checkFile); } } private Tuple> GetExtractResult(string reportFullPath, DateTime dateTime) { Tuple> results; string[] lines = new string[] { string.Empty, "NUM_DATA_ROWS", $"LOGISTICS_1{'\t'}A_JOBID={"BACKLOG"};A_MES_ENTITY={"BACKLOG"};" }; ProcessDataStandardFormat processDataStandardFormat = ProcessDataStandardFormat.GetProcessDataStandardFormat(reportFullPath, lines); _Logistics = new Logistics(reportFullPath, processDataStandardFormat); if (_IsEAFHosted && _FileConnectorConfiguration.FileScanningIntervalInSeconds > 0) MoveJson(reportFullPath, dateTime); results = new(_Logistics.Logistics1[0], Array.Empty(), Array.Empty(), new List()); return results; } }