json in process-data-standard-format

process-data-standard-format with HeaderId and SubgroupId

process-data-standard-format with pipes

UniqueId replacement for attachments

Write input PDSF in output after EOF

ProcessDataStandardFormat over Tuple

MoveMatchingFiles to use ProcessDataStandardFormatMapping
This commit is contained in:
Mike Phares 2025-05-22 09:28:44 -07:00
parent 36d06684f8
commit f4d00bdfeb
16 changed files with 313 additions and 235 deletions

View File

@ -1,92 +1 @@
[
{
"DocumentId": {
"ProjectId": {
"Id": "09d3713c-65ba-4edc-825c-25bdd8365c7b"
},
"Id": "6581d230-abcd-4361-be56-35654f1f73d9"
},
"FileName": "DEP08CEPIEPSILON.cs",
"FilePath": "L:\\DevOps\\EAF-Mesa-Integration\\dep08cepiepsilon\\Adaptation\\_Tests\\Static\\DEP08CEPIEPSILON.cs",
"FileChanges": [
{
"LineNumber": 54,
"CharNumber": 9,
"DiagnosticId": "MSTEST0037",
"FormatDescription": "error MSTEST0037: Use \u0027Assert.AreEqual\u0027 instead of \u0027Assert.IsTrue\u0027"
}
]
},
{
"DocumentId": {
"ProjectId": {
"Id": "09d3713c-65ba-4edc-825c-25bdd8365c7b"
},
"Id": "63aef27c-c033-4bfc-86bc-3b9c034490c1"
},
"FileName": "AdaptationTesting.cs",
"FilePath": "L:\\DevOps\\EAF-Mesa-Integration\\dep08cepiepsilon\\Adaptation\\_Tests\\Shared\\AdaptationTesting.cs",
"FileChanges": [
{
"LineNumber": 1114,
"CharNumber": 13,
"DiagnosticId": "MSTEST0037",
"FormatDescription": "error MSTEST0037: Use \u0027Assert.AreNotEqual\u0027 instead of \u0027Assert.IsTrue\u0027"
}
]
},
{
"DocumentId": {
"ProjectId": {
"Id": "09d3713c-65ba-4edc-825c-25bdd8365c7b"
},
"Id": "63aef27c-c033-4bfc-86bc-3b9c034490c1"
},
"FileName": "AdaptationTesting.cs",
"FilePath": "L:\\DevOps\\EAF-Mesa-Integration\\dep08cepiepsilon\\Adaptation\\_Tests\\Shared\\AdaptationTesting.cs",
"FileChanges": [
{
"LineNumber": 1262,
"CharNumber": 17,
"DiagnosticId": "MSTEST0037",
"FormatDescription": "error MSTEST0037: Use \u0027Assert.AreEqual\u0027 instead of \u0027Assert.IsTrue\u0027"
}
]
},
{
"DocumentId": {
"ProjectId": {
"Id": "09d3713c-65ba-4edc-825c-25bdd8365c7b"
},
"Id": "63aef27c-c033-4bfc-86bc-3b9c034490c1"
},
"FileName": "AdaptationTesting.cs",
"FilePath": "L:\\DevOps\\EAF-Mesa-Integration\\dep08cepiepsilon\\Adaptation\\_Tests\\Shared\\AdaptationTesting.cs",
"FileChanges": [
{
"LineNumber": 1265,
"CharNumber": 17,
"DiagnosticId": "MSTEST0037",
"FormatDescription": "error MSTEST0037: Use \u0027Assert.AreEqual\u0027 instead of \u0027Assert.IsTrue\u0027"
}
]
},
{
"DocumentId": {
"ProjectId": {
"Id": "09d3713c-65ba-4edc-825c-25bdd8365c7b"
},
"Id": "63aef27c-c033-4bfc-86bc-3b9c034490c1"
},
"FileName": "AdaptationTesting.cs",
"FilePath": "L:\\DevOps\\EAF-Mesa-Integration\\dep08cepiepsilon\\Adaptation\\_Tests\\Shared\\AdaptationTesting.cs",
"FileChanges": [
{
"LineNumber": 1268,
"CharNumber": 17,
"DiagnosticId": "MSTEST0037",
"FormatDescription": "error MSTEST0037: Use \u0027Assert.AreEqual\u0027 instead of \u0027Assert.IsTrue\u0027"
}
]
}
]
[]

View File

@ -9,7 +9,7 @@ namespace Adaptation.FileHandlers;
public class CellInstanceConnectionName
{
internal static IFileRead Get(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, int? connectionCount)
internal static IFileRead Get(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, int? connectionCount)
{
IFileRead result = cellInstanceConnectionName switch
{

View File

@ -23,7 +23,7 @@ public class FileRead : Shared.FileRead, IFileRead
private int _LastDummyRunIndex;
private readonly string[] _CellNames;
public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
{
_MinFileLength = 10;

View File

@ -20,7 +20,7 @@ public class FileRead : Shared.FileRead, IFileRead
private readonly string _TimestampFormat;
private readonly ReadOnlyDictionary<string, string> _SystemStateToNames;
public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
{
_MinFileLength = 10;

View File

@ -20,7 +20,7 @@ public class FileRead : Shared.FileRead, IFileRead
private readonly string _TimestampFormat;
private readonly ReadOnlyDictionary<string, string> _SystemStateToNames;
public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
{
_MinFileLength = 10;

View File

@ -20,7 +20,7 @@ public class FileRead : Shared.FileRead, IFileRead
private readonly string _TimestampFormat;
private readonly ReadOnlyDictionary<string, string> _SystemStateToNames;
public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
{
_MinFileLength = 10;

View File

@ -20,7 +20,7 @@ public class FileRead : Shared.FileRead, IFileRead
private readonly string _TimestampFormat;
private readonly ReadOnlyDictionary<string, string> _SystemStateToNames;
public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
{
_MinFileLength = 10;

View File

@ -20,7 +20,7 @@ public class FileRead : Shared.FileRead, IFileRead
private readonly string _TimestampFormat;
private readonly ReadOnlyDictionary<string, string> _SystemStateToNames;
public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
{
_MinFileLength = 10;

View File

@ -20,7 +20,7 @@ public class FileRead : Shared.FileRead, IFileRead
private readonly string _TimestampFormat;
private readonly ReadOnlyDictionary<string, string> _SystemStateToNames;
public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
public FileRead(ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Shared.Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted) :
base(new Description(), false, smtp, fileParameter, cellInstanceName, connectionCount, cellInstanceConnectionName, fileConnectorConfiguration, equipmentTypeName, parameterizedModelObjectDefinitionType, modelObjectParameters, equipmentDictionaryName, dummyRuns, staticRuns, useCyclicalForDescription, isEAFHosted: connectionCount is null)
{
_MinFileLength = 10;

View File

@ -44,9 +44,9 @@ public class FileRead : Properties.IFileRead
protected readonly string _CellInstanceConnectionNameBase;
protected readonly Dictionary<string, List<long>> _DummyRuns;
protected readonly Dictionary<string, string> _FileParameter;
protected readonly Dictionary<long, List<string>> _StaticRuns;
protected readonly string _ParameterizedModelObjectDefinitionType;
protected readonly FileConnectorConfiguration _FileConnectorConfiguration;
protected readonly Dictionary<long, List<Metrology.WS.Results>> _StaticRuns;
protected readonly IList<ModelObjectParameterDefinition> _ModelObjectParameterDefinitions;
bool Properties.IFileRead.IsEvent => _IsEvent;
@ -203,7 +203,7 @@ public class FileRead : Properties.IFileRead
}
}
public FileRead(IDescription description, bool isEvent, ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<string>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted)
public FileRead(IDescription description, bool isEvent, ISMTP smtp, Dictionary<string, string> fileParameter, string cellInstanceName, int? connectionCount, string cellInstanceConnectionName, FileConnectorConfiguration fileConnectorConfiguration, string equipmentTypeName, string parameterizedModelObjectDefinitionType, IList<ModelObjectParameterDefinition> modelObjectParameters, string equipmentDictionaryName, Dictionary<string, List<long>> dummyRuns, Dictionary<long, List<Metrology.WS.Results>> staticRuns, bool useCyclicalForDescription, bool isEAFHosted)
{
_SMTP = smtp;
_IsEvent = isEvent;
@ -377,17 +377,25 @@ public class FileRead : Properties.IFileRead
internal string[] GetInProcessDirectory(string jobIdDirectory)
{
string[] results;
List<string> results = new();
if (!_IsEAFHosted)
results = new string[] { jobIdDirectory };
results = new string[] { jobIdDirectory }.ToList();
else
{
string[] files;
string logisticsSequence = _Logistics.Sequence.ToString();
results = Directory.GetDirectories(jobIdDirectory, string.Concat(_Logistics.MID, '*', logisticsSequence, '*'), SearchOption.TopDirectoryOnly);
string[] directories = Directory.GetDirectories(jobIdDirectory, $"*{logisticsSequence}*", SearchOption.TopDirectoryOnly);
foreach (string directory in directories)
{
files = Directory.GetFiles(directory, "*", SearchOption.TopDirectoryOnly);
if (files.Length == 0)
continue;
results.Add(directory);
}
if ((results is null) || results.Length != 1)
}
if ((results is null) || results.Count != 1)
throw new Exception("Didn't find directory by logistics sequence");
return results;
return results.ToArray();
}
protected static string[] GetMatches(FileConnectorConfiguration fileConnectorConfiguration)
@ -616,6 +624,9 @@ public class FileRead : Properties.IFileRead
case FileConnectorConfiguration.PostProcessingModeEnum.Delete:
File.Delete(sourceFile.FullName);
break;
case FileConnectorConfiguration.PostProcessingModeEnum.None:
File.Move(sourceFile.FullName, itemFile);
break;
default:
throw new Exception();
}

View File

@ -6,23 +6,25 @@ public partial class WS
public class Attachment
{
public string SubGroupId { get; set; }
public long HeaderId { get; set; }
public string HeaderIdDirectory { get; set; }
public string UniqueId { get; set; }
public string DestinationFileName { get; set; }
public string SourceFileName { get; set; }
public string AttachmentId { get; set; }
#nullable enable
public Attachment(string subGroupId, long headerId, string headerIdDirectory, string uniqueId, string destinationFileName, string sourceFileName)
public long HeaderId { get; set; }
public string UniqueId { get; set; }
public string SubGroupId { get; set; }
public string AttachmentId { get; set; }
public string SourceFileName { get; set; }
public string HeaderIdDirectory { get; set; }
public string DestinationFileName { get; set; }
public Attachment(Results? results, string headerIdDirectory, string uniqueId, string destinationFileName, string sourceFileName)
{
SubGroupId = subGroupId;
HeaderId = headerId;
HeaderIdDirectory = headerIdDirectory;
UniqueId = uniqueId;
DestinationFileName = destinationFileName;
SourceFileName = sourceFileName;
HeaderIdDirectory = headerIdDirectory;
DestinationFileName = destinationFileName;
AttachmentId = System.Guid.NewGuid().ToString();
HeaderId = results?.HeaderId is null ? -1 : results.HeaderId.Value;
SubGroupId = results?.SubgroupId is null ? string.Empty : results.SubgroupId.Value.ToString();
}
}

View File

@ -1,27 +1,75 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Adaptation.Shared.Metrology;
public partial class WS
{
// this class represents the response from the Inbound API endpoint
public class Results
{
// true or false if data was written to the database
public bool Success { get; set; }
// if true, contains ID of the Header record in the database
public long HeaderID { get; set; }
#nullable enable
// if false, this collection will contain a list of errors
public List<string> Errors { get; set; }
[JsonConstructor]
public Results(List<string>? errors,
long? headerId,
long? subgroupId,
bool? success,
List<string>? warnings)
{
Errors = errors;
Success = success;
HeaderId = headerId;
Warnings = warnings;
SubgroupId = subgroupId;
}
// this collection will contain a list of warnings, they will not prevent data from being saved
public List<string> Warnings { get; set; }
[JsonPropertyName("errors")] public List<string>? Errors { get; set; }
[JsonPropertyName("headerID")] public long? HeaderId { get; set; }
[JsonPropertyName("subgroupId")] public long? SubgroupId { get; set; }
[JsonPropertyName("success")] public bool? Success { get; set; }
[JsonPropertyName("warnings")] public List<string>? Warnings { get; set; }
public override string ToString()
{
string result = JsonSerializer.Serialize(this, ResultsSourceGenerationContext.Default.Results);
return result;
}
internal static Results Get(Results results, long? subgroupId) =>
new(results.Errors, results.HeaderId, subgroupId, results.Success, results.Warnings);
internal static Results Get(string resultsJson, Exception e)
{
Results results;
Exception? exception = e;
List<string> errors = new();
StringBuilder stringBuilder = new();
while (exception is not null)
{
_ = stringBuilder.AppendLine(exception.Message);
exception = exception.InnerException;
}
errors.Add(resultsJson);
errors.Add(stringBuilder.ToString());
results = new(errors: errors,
headerId: null,
subgroupId: null,
success: false,
warnings: new());
return results;
}
// this is just a helper function to make displaying the results easier
public override string ToString() => JsonSerializer.Serialize(this, GetType());
}
}
[JsonSourceGenerationOptions(WriteIndented = true)]
[JsonSerializable(typeof(WS.Results))]
internal partial class ResultsSourceGenerationContext : JsonSerializerContext
{
}

View File

@ -10,9 +10,11 @@ namespace Adaptation.Shared.Metrology;
public partial class WS
{
#nullable enable
public static (string, Results) SendData(string url, long sequence, string directory, object payload, int timeoutSeconds = 120)
{
Results results = new();
Results? wsResults = null;
string resultsJson = string.Empty;
try
{
@ -30,29 +32,20 @@ public partial class WS
};
HttpResponseMessage httpResponseMessage = httpClient.SendAsync(httpRequestMessage, HttpCompletionOption.ResponseContentRead).Result;
resultsJson = httpResponseMessage.Content.ReadAsStringAsync().Result;
results = JsonSerializer.Deserialize<Results>(resultsJson, new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
string checkDirectory = Path.Combine(directory, $"-{results.HeaderID}");
wsResults = JsonSerializer.Deserialize(resultsJson, ResultsSourceGenerationContext.Default.Results);
if (wsResults is null)
throw new NullReferenceException(nameof(wsResults));
string checkDirectory = Path.Combine(directory, $"-{wsResults.HeaderId}");
if (!Directory.Exists(checkDirectory))
_ = Directory.CreateDirectory(checkDirectory);
File.WriteAllText(Path.Combine(checkDirectory, $"{sequence}.json"), json);
}
if (!results.Success)
results.Errors.Add(results.ToString());
if (wsResults.Success is null || !wsResults.Success.Value)
wsResults.Errors?.Add(wsResults.ToString());
}
catch (Exception e)
{
Exception exception = e;
StringBuilder stringBuilder = new();
while (exception is not null)
{
_ = stringBuilder.AppendLine(exception.Message);
exception = exception.InnerException;
}
results.Errors ??= new List<string>();
results.Errors.Add(resultsJson);
results.Errors.Add(stringBuilder.ToString());
}
return new(resultsJson, results);
{ wsResults ??= Results.Get(resultsJson, e); }
return new(resultsJson, wsResults);
}
public static void AttachFile(string url, Attachment attachment, int timeoutSeconds = 60)
@ -69,16 +62,20 @@ public partial class WS
}
}
public static void AttachFiles(string url, List<Attachment> headerAttachments = null, List<Attachment> dataAttachments = null)
public static void AttachFiles(string url, List<Attachment>? headerAttachments = null, List<Attachment>? dataAttachments = null)
{
string directory;
try
{
string? directoryName;
if (headerAttachments is not null)
{
foreach (Attachment attachment in headerAttachments)
{
directory = Path.Combine(Path.GetDirectoryName(attachment.HeaderIdDirectory), attachment.AttachmentId) ?? throw new Exception();
directoryName = Path.GetDirectoryName(attachment.HeaderIdDirectory);
if (string.IsNullOrEmpty(directoryName))
continue;
directory = Path.Combine(directoryName, attachment.AttachmentId) ?? throw new Exception();
if (!Directory.Exists(directory))
_ = Directory.CreateDirectory(directory);
File.Copy(attachment.SourceFileName, Path.Combine(directory, attachment.DestinationFileName), overwrite: true);
@ -88,7 +85,10 @@ public partial class WS
{
foreach (Attachment attachment in dataAttachments)
{
directory = Path.Combine(Path.GetDirectoryName(attachment.HeaderIdDirectory.Replace("Header", "Data")), attachment.AttachmentId) ?? throw new Exception();
directoryName = Path.GetDirectoryName(attachment.HeaderIdDirectory.Replace("Header", "Data"));
if (string.IsNullOrEmpty(directoryName))
continue;
directory = Path.Combine(directoryName, attachment.AttachmentId) ?? throw new Exception();
if (!Directory.Exists(directory))
_ = Directory.CreateDirectory(directory);
File.Copy(attachment.SourceFileName, Path.Combine(directory, attachment.DestinationFileName), overwrite: true);
@ -108,7 +108,7 @@ public partial class WS
}
catch (Exception e)
{
Exception exception = e;
Exception? exception = e;
StringBuilder stringBuilder = new();
while (exception is not null)
{

View File

@ -26,16 +26,25 @@ internal class ProcessDataStandardFormat
internal long? Sequence { get; private set; }
internal ReadOnlyCollection<string> Body { get; private set; }
internal ReadOnlyCollection<string> Footer { get; private set; }
internal ReadOnlyCollection<string> Header { get; private set; }
internal ReadOnlyCollection<string> Columns { get; private set; }
internal ProcessDataStandardFormat? InputPDSF { get; private set; }
internal ReadOnlyCollection<string> Logistics { get; private set; }
internal ProcessDataStandardFormat(ReadOnlyCollection<string> body,
ReadOnlyCollection<string> columns,
ReadOnlyCollection<string> footer,
ReadOnlyCollection<string> header,
ProcessDataStandardFormat? inputPDSF,
ReadOnlyCollection<string> logistics,
long? sequence)
{
Body = body;
Columns = columns;
Footer = footer;
Header = header;
InputPDSF = inputPDSF;
Logistics = logistics;
Sequence = sequence;
}
@ -52,8 +61,8 @@ internal class ProcessDataStandardFormat
internal static string Archive(bool addSpaces = true, char separator = ' ') =>
GetString(SearchFor.Archive, addSpaces, separator);
internal static ProcessDataStandardFormat GetEmpty() =>
new(new(Array.Empty<string>()), new(Array.Empty<string>()), new(Array.Empty<string>()), null);
internal static ProcessDataStandardFormat GetEmpty(Logistics logistics) =>
new(new(Array.Empty<string>()), new(Array.Empty<string>()), new(Array.Empty<string>()), new(Array.Empty<string>()), null, new(logistics.Logistics1), null);
internal static List<string> PDSFToFixedWidth(string reportFullPath)
{
@ -124,19 +133,26 @@ internal class ProcessDataStandardFormat
return results;
}
internal static ProcessDataStandardFormat GetProcessDataStandardFormat(string reportFullPath, string[]? lines = null)
internal static ProcessDataStandardFormat GetProcessDataStandardFormat(string reportFullPath, string[]? lines = null, int columnsLine = 6)
{
ProcessDataStandardFormat result;
string segment;
List<string> body = new();
List<string> logistics = new();
lines ??= File.ReadAllLines(reportFullPath);
string[] segments;
if (lines.Length < 7)
bool addToFooter = false;
List<string> body = new();
List<string> header = new();
List<string> footer = new();
List<string> columns = new();
ReadOnlyCollection<string> logistics;
lines ??= File.ReadAllLines(reportFullPath);
if (lines.Length < columnsLine + 1)
segments = Array.Empty<string>();
else
segments = lines[6].Trim().Split('\t');
List<string> columns = new();
{
segments = lines[columnsLine].Trim().Split('\t');
for (int i = 0; i < columnsLine; i++)
header.Add(lines[i]);
}
for (int c = 0; c < segments.Length; c++)
{
segment = segments[c].Substring(1, segments[c].Length - 2);
@ -155,87 +171,102 @@ internal class ProcessDataStandardFormat
}
}
}
bool lookForLogistics = false;
for (int r = 7; r < lines.Length; r++)
for (int r = columnsLine + 1; r < lines.Length; r++)
{
if (lines[r].StartsWith("NUM_DATA_ROWS"))
lookForLogistics = true;
if (!lookForLogistics)
{
addToFooter = true;
if (!addToFooter)
body.Add(lines[r]);
continue;
}
if (lines[r].StartsWith("LOGISTICS_1"))
{
for (int i = r; i < lines.Length; i++)
else
{
footer.Add(lines[r]);
if (lines[r].StartsWith("END_HEADER"))
break;
logistics.Add(lines[i]);
}
break;
}
}
result = new(logistics.AsReadOnly(), columns.AsReadOnly(), body.AsReadOnly(), null);
string? linesOne = lines.Length > 0 && body.Count == 0 && columns.Count == 0 ? lines[1] : null;
logistics = GetLogistics(footer, linesOne: linesOne);
result = new(body: body.AsReadOnly(),
columns: columns.AsReadOnly(),
footer: footer.AsReadOnly(),
header: header.AsReadOnly(),
inputPDSF: null,
logistics: logistics,
sequence: null);
return result;
}
internal static ProcessDataStandardFormat? GetProcessDataStandardFormat(string reportFullPath, ProcessDataStandardFormatMapping pdsfMapping)
private static ReadOnlyCollection<string> GetLogistics(List<string> footer, string? linesOne)
{
ProcessDataStandardFormat? result;
List<string> results = new();
bool foundLogistics1 = false;
foreach (string line in footer)
{
if (line.StartsWith("END_HEADER"))
break;
if (line.StartsWith("LOGISTICS_1"))
foundLogistics1 = true;
if (foundLogistics1 && line.StartsWith("LOGISTICS_"))
results.Add(line);
}
if (!string.IsNullOrEmpty(linesOne) && results.Count == 0)
results.Add(linesOne);
return results.AsReadOnly();
}
internal static ProcessDataStandardFormat GetProcessDataStandardFormat(string reportFullPath, ProcessDataStandardFormatMapping pdsfMapping)
{
ProcessDataStandardFormat result;
const int columnsLine = 6;
FileInfo fileInfo = new(reportFullPath);
ProcessDataStandardFormat processDataStandardFormat = GetProcessDataStandardFormat(fileInfo.LastWriteTime, pdsfMapping.NewColumnNames.Count, columnsLine, fileInfo.FullName, lines: null);
JsonElement[]? jsonElements = GetArray(pdsfMapping.NewColumnNames.Count, processDataStandardFormat, lookForNumbers: false);
if (jsonElements is null || pdsfMapping.OldColumnNames.Count != pdsfMapping.ColumnIndices.Count)
result = null;
ProcessDataStandardFormat processDataStandardFormat = GetProcessDataStandardFormat(fileInfo.LastWriteTime, columnsLine, fileInfo.FullName, lines: null);
JsonElement[]? jsonElements = pdsfMapping.OldColumnNames.Count != pdsfMapping.ColumnIndices.Count ? null : GetFullArray(processDataStandardFormat);
JsonProperty[]? jsonProperties = jsonElements is null || jsonElements.Length == 0 ? null : jsonElements[0].EnumerateObject().ToArray();
if (jsonElements is null || jsonProperties is null || jsonProperties.Length != pdsfMapping.NewColumnNames.Count)
result = processDataStandardFormat;
else
{
result = GetProcessDataStandardFormat(pdsfMapping, jsonElements, processDataStandardFormat);
if (result.Sequence is null || result.Columns.Count == 0 || result.Body.Count == 0 || result.Logistics.Count == 0)
result = null;
result = processDataStandardFormat;
}
return result;
}
private static ProcessDataStandardFormat GetProcessDataStandardFormat(DateTime lastWriteTime, int expectedColumns, int columnsLine, string path, string[]? lines)
private static ProcessDataStandardFormat GetProcessDataStandardFormat(DateTime lastWriteTime, int columnsLine, string path, string[]? lines)
{
ProcessDataStandardFormat result;
long sequence;
string[] segments;
bool addToFooter = false;
List<string> body = new();
bool lookForLogistics = false;
List<string> logistics = new();
List<string> header = new();
List<string> footer = new();
ReadOnlyCollection<string> logistics;
lines ??= File.ReadAllLines(path);
if (lines.Length <= columnsLine)
segments = Array.Empty<string>();
else
{
segments = lines[columnsLine].Split('\t');
if (segments.Length != expectedColumns)
segments = Array.Empty<string>();
for (int i = 0; i < columnsLine; i++)
header.Add(lines[i]);
}
string[] columns = segments.Select(l => l.Trim('"')).ToArray();
for (int r = columnsLine + 1; r < lines.Length; r++)
{
if (lines[r].StartsWith("NUM_DATA_ROWS"))
lookForLogistics = true;
if (!lookForLogistics)
{
addToFooter = true;
if (!addToFooter)
body.Add(lines[r]);
continue;
}
if (lines[r].StartsWith("LOGISTICS_1"))
{
for (int i = r; i < lines.Length; i++)
else
{
footer.Add(lines[r]);
if (lines[r].StartsWith("END_HEADER"))
break;
logistics.Add(lines[i]);
}
break;
}
}
logistics = GetLogistics(footer, linesOne: null);
if (logistics.Count == 0)
sequence = lastWriteTime.Ticks;
else
@ -245,12 +276,15 @@ internal class ProcessDataStandardFormat
}
result = new(body: body.AsReadOnly(),
columns: new(columns),
logistics: logistics.AsReadOnly(),
footer: footer.AsReadOnly(),
header: header.AsReadOnly(),
inputPDSF: null,
logistics: logistics,
sequence: sequence);
return result;
}
private static JsonElement[]? GetArray(int expectedColumns, ProcessDataStandardFormat processDataStandardFormat, bool lookForNumbers)
private static JsonElement[]? GetFullArray(ProcessDataStandardFormat processDataStandardFormat)
{
JsonElement[]? results;
if (processDataStandardFormat.Body.Count == 0 || !processDataStandardFormat.Body[0].Contains('\t'))
@ -258,37 +292,19 @@ internal class ProcessDataStandardFormat
else
{
string value;
string[] segments;
List<string> segments;
List<string> lines = new();
StringBuilder stringBuilder = new();
foreach (string bodyLine in processDataStandardFormat.Body)
{
_ = stringBuilder.Clear();
_ = stringBuilder.Append('{');
segments = bodyLine.Split('\t');
if (segments.Length != expectedColumns)
continue;
if (!lookForNumbers)
{
for (int c = 0; c < segments.Length; c++)
segments = bodyLine.Split('\t').ToList();
for (int c = 0; c < segments.Count; c++)
{
value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\");
_ = stringBuilder.Append('"').Append(processDataStandardFormat.Columns[c]).Append("\":\"").Append(value).Append("\",");
}
}
else
{
for (int c = 0; c < segments.Length; c++)
{
value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\");
if (string.IsNullOrEmpty(value))
_ = stringBuilder.Append('"').Append(processDataStandardFormat.Columns[c]).Append("\":").Append(value).Append("null,");
else if (value.All(char.IsDigit))
_ = stringBuilder.Append('"').Append(processDataStandardFormat.Columns[c]).Append("\":").Append(value).Append(',');
else
_ = stringBuilder.Append('"').Append(processDataStandardFormat.Columns[c]).Append("\":\"").Append(value).Append("\",");
}
}
_ = stringBuilder.Remove(stringBuilder.Length - 1, 1);
_ = stringBuilder.AppendLine("}");
lines.Add(stringBuilder.ToString());
@ -335,13 +351,73 @@ internal class ProcessDataStandardFormat
results.Add(string.Join("\t", values));
}
result = new(body: new(results),
columns: processDataStandardFormat.Columns,
columns: processDataStandardFormatMapping.OldColumnNames,
footer: processDataStandardFormat.Footer,
header: processDataStandardFormat.Header,
inputPDSF: processDataStandardFormat,
logistics: processDataStandardFormat.Logistics,
sequence: processDataStandardFormat.Sequence);
return result;
}
internal static void Write(string path, ProcessDataStandardFormat processDataStandardFormat)
private static string GetJson(ProcessDataStandardFormat processDataStandardFormat)
{
if (processDataStandardFormat.InputPDSF is null)
throw new NullReferenceException(nameof(processDataStandardFormat.InputPDSF));
#pragma warning disable CA1845, IDE0057
string result;
string line;
string value;
string[] segments;
List<string> lines = new();
for (int i = 0; i < processDataStandardFormat.InputPDSF.Body.Count; i++)
{
line = "{";
segments = processDataStandardFormat.InputPDSF.Body[i].Trim().Split('\t');
if (segments.Length != processDataStandardFormat.InputPDSF.Columns.Count)
break;
for (int c = 0; c < segments.Length; c++)
{
value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\");
line += string.Concat('"', processDataStandardFormat.InputPDSF.Columns[c].Trim('"'), '"', ':', '"', value, '"', ',');
}
line = string.Concat(line.Substring(0, line.Length - 1), '}');
lines.Add(line);
}
result = string.Concat(
'{',
Environment.NewLine,
'"',
"Count",
'"',
": ",
processDataStandardFormat.Body.Count,
',',
Environment.NewLine,
'"',
"Records",
'"',
": ",
Environment.NewLine,
'[',
Environment.NewLine,
string.Join($",{Environment.NewLine}", lines),
Environment.NewLine,
']',
',',
Environment.NewLine,
'"',
"Sequence",
'"',
": ",
processDataStandardFormat.Sequence,
Environment.NewLine,
'}');
return result;
#pragma warning restore CA1845, IDE0057
}
internal static void Write(string path, ProcessDataStandardFormat processDataStandardFormat, List<Metrology.WS.Results>? wsResults)
{
List<string> results = new();
if (processDataStandardFormat.Sequence is null)
@ -357,7 +433,7 @@ internal class ProcessDataStandardFormat
results.Add($"HEADER_OFFSET\t{headerOffset}");
results.Add($"DATA_OFFSET\t{dataOffset}");
results.Add($"END_OFFSET\t{endOffset}");
results.Add($"\"{string.Join("\",\t\"", processDataStandardFormat.Columns)}\"");
results.Add($"\"{string.Join("\"\t\"", processDataStandardFormat.Columns)}\"");
results.AddRange(processDataStandardFormat.Body);
results.Add($"NUM_DATA_ROWS\t{processDataStandardFormat.Body.Count.ToString().PadLeft(9, '0')}");
results.Add($"NUM_DATA_COLUMNS\t{processDataStandardFormat.Columns.Count.ToString().PadLeft(9, '0')}");
@ -366,7 +442,39 @@ internal class ProcessDataStandardFormat
results.Add($"START_TIME\t{startTime}");
results.Add("LOGISTICS_COLUMN\tA_LOGISTICS");
results.Add("LOGISTICS_COLUMN\tB_LOGISTICS");
if (wsResults is null || wsResults.Count != 1)
results.AddRange(processDataStandardFormat.Logistics);
else
{
string[] segments;
foreach (string logistics in processDataStandardFormat.Logistics)
{
segments = logistics.Split(new string[] { "\t" }, StringSplitOptions.None);
if (segments.Length != 2 || string.IsNullOrEmpty(segments[1]))
results.Add(logistics);
else
results.Add($"{segments[0]}\t{segments[1][0]}_HeaderId={wsResults[0].HeaderId};{segments[1][0]}_SubgroupId={wsResults[0].SubgroupId};{segments[1]}");
}
}
results.Add("END_HEADER");
if (processDataStandardFormat.InputPDSF is not null)
{
results.Add(string.Empty);
List<char> hyphens = new();
results.AddRange(processDataStandardFormat.InputPDSF.Header.Select(l => l.Replace('\t', '|')));
results.Add($"|{string.Join("|", processDataStandardFormat.InputPDSF.Columns)}|");
for (int i = 0; i < processDataStandardFormat.InputPDSF.Columns.Count; i++)
hyphens.Add('-');
results.Add($"|{string.Join("|", hyphens)}|");
results.AddRange(processDataStandardFormat.InputPDSF.Body.Select(l => l.Replace('\t', '|')));
results.Add(string.Empty);
results.AddRange(processDataStandardFormat.InputPDSF.Footer.Select(l => l.Replace('\t', '|')));
results.Add(string.Empty);
results.Add("EOF");
results.Add(string.Empty);
string json = GetJson(processDataStandardFormat);
results.Add(json);
}
File.WriteAllText(path, string.Join(Environment.NewLine, results));
}

View File

@ -977,7 +977,7 @@ public class AdaptationTesting : ISMTP
if (!string.IsNullOrEmpty(mbn.CellInstanceConnectionName) && !Directory.Exists(fileInfo.DirectoryName))
_ = Directory.CreateDirectory(fileInfo.Directory.FullName);
Dictionary<string, List<long>> dummyRuns = new();
Dictionary<long, List<string>> staticRuns = new();
Dictionary<long, List<Adaptation.Shared.Metrology.WS.Results>> staticRuns = new();
Tuple<string, CellInstanceVersion> cellInstanceVersionTuple = GetCellInstanceVersionTuple(mbn.CellInstanceName, mbn.CellInstanceVersionName);
Tuple<string, FileConnectorConfiguration> fileConnectorConfigurationTuple = GetFileConnectorConfigurationTuple(cellInstanceVersionTuple, mbn.CellInstanceConnectionName);
Tuple<string, string, string, EquipmentTypeVersion> equipmentTypeVersionTuple = GetEquipmentTypeVersionTuple(cellInstanceVersionTuple.Item2, mbn.CellInstanceConnectionName);
@ -1182,7 +1182,7 @@ public class AdaptationTesting : ISMTP
Assert.IsNotNull(extractResult.Item3);
Assert.IsNotNull(extractResult.Item4);
if (!validatePDSF)
_ = GetProcessDataStandardFormat(fileRead, logistics, extractResult, ProcessDataStandardFormat.GetEmpty());
_ = GetProcessDataStandardFormat(fileRead, logistics, extractResult, ProcessDataStandardFormat.GetEmpty(logistics));
else
{
Assert.IsTrue(extractResult.Item3.Length > 0, "extractResult Array Length check!");

View File

@ -37,7 +37,7 @@ public partial class FileRead : FileReaderHandler, ISMTP
private FilePathGenerator _FilePathGeneratorForTarget;
private readonly List<EquipmentParameter> _EquipmentParameters;
private static readonly Dictionary<string, List<long>> _DummyRuns;
private static readonly Dictionary<long, List<string>> _StaticRuns;
private static readonly Dictionary<long, List<Adaptation.Shared.Metrology.WS.Results>> _StaticRuns;
static FileRead()
{