427 lines
19 KiB
C#
427 lines
19 KiB
C#
|
|
using System.Collections.ObjectModel;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Serialization;
|
|
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace File_Folder_Helper.ADO2025.PI6;
|
|
|
|
internal static partial class Helper20250701 {
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
|
|
[JsonSerializable(typeof(JsonElement))]
|
|
internal partial class JsonElementSourceGenerationContext : JsonSerializerContext {
|
|
}
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)]
|
|
[JsonSerializable(typeof(ReadOnlyDictionary<string, ReadOnlyDictionary<string, string>>))]
|
|
internal partial class ReadOnlyDictionaryStringReadOnlyDictionaryStringStringSourceGenerationContext : JsonSerializerContext {
|
|
}
|
|
|
|
internal static void ProcessDataStandardFormatTo(ILogger<Worker> logger, List<string> args) {
|
|
logger.LogInformation(args[0]);
|
|
logger.LogInformation(args[1]);
|
|
logger.LogInformation(args[2]);
|
|
logger.LogInformation(args[3]);
|
|
logger.LogInformation(args[4]);
|
|
logger.LogInformation(args[5]);
|
|
logger.LogInformation(args[6]);
|
|
logger.LogInformation(args[7]);
|
|
string[] segments;
|
|
string extension = args[5];
|
|
string timeColumn = args[4];
|
|
string searchPattern = args[2];
|
|
int sizeFilter = int.Parse(args[3]);
|
|
string[] columns = args[6].Split(',');
|
|
Dictionary<string, string> columnMapping = [];
|
|
string sourceDirectory = Path.GetFullPath(args[0].Split('~')[0]);
|
|
string destinationDirectory = Path.GetFullPath(args[7].Split('~')[0]);
|
|
foreach (string column in columns) {
|
|
segments = column.Split('~');
|
|
columnMapping.Add(segments[0], segments[1]);
|
|
}
|
|
string[] directories = Directory.GetDirectories(sourceDirectory, "*", SearchOption.TopDirectoryOnly);
|
|
ProcessDataStandardFormatTo(logger, sourceDirectory, searchPattern, sizeFilter, timeColumn, extension, columnMapping, destinationDirectory, directories);
|
|
Helpers.HelperDeleteEmptyDirectories.DeleteEmptyDirectories(logger, sourceDirectory);
|
|
}
|
|
|
|
private static void ProcessDataStandardFormatTo(ILogger<Worker> logger, string sourceDirectory, string searchPattern, int sizeFilter, string timeColumn, string extension, Dictionary<string, string> columnMapping, string destinationDirectory, string[] directories) {
|
|
string text;
|
|
string? json;
|
|
string[] files;
|
|
string markdown;
|
|
string checkFile;
|
|
string[] matches;
|
|
FileInfo fileInfo;
|
|
string? pipeTable;
|
|
string? collections;
|
|
string directoryName;
|
|
string checkDirectory;
|
|
foreach (string directory in directories) {
|
|
if (sizeFilter < 987654321 && Path.GetFileName(directory).Contains('-')) {
|
|
continue;
|
|
}
|
|
files = Directory.GetFiles(directory, searchPattern, SearchOption.TopDirectoryOnly);
|
|
foreach (string file in files) {
|
|
fileInfo = new(file);
|
|
if (fileInfo.LastWriteTime > DateTime.Now.AddSeconds(-5)) {
|
|
continue;
|
|
}
|
|
directoryName = Path.GetFileName(fileInfo.DirectoryName);
|
|
if (fileInfo.Length > sizeFilter && !directoryName.StartsWith('Z')) {
|
|
checkDirectory = Path.Combine(sourceDirectory, $"Z{directoryName}");
|
|
if (!Directory.Exists(checkDirectory)) {
|
|
_ = Directory.CreateDirectory(checkDirectory);
|
|
}
|
|
checkFile = Path.Combine(checkDirectory, fileInfo.Name);
|
|
if (File.Exists(checkFile)) {
|
|
continue;
|
|
}
|
|
File.Move(file, checkFile);
|
|
continue;
|
|
}
|
|
checkDirectory = Path.Combine(destinationDirectory, directoryName);
|
|
if (!Directory.Exists(checkDirectory)) {
|
|
_ = Directory.CreateDirectory(checkDirectory);
|
|
}
|
|
checkFile = Path.Combine(checkDirectory, $"{fileInfo.Name}{extension}");
|
|
if (File.Exists(checkFile)) {
|
|
continue;
|
|
}
|
|
if (extension == ".md") {
|
|
collections = GetMarkdown(logger, timeColumn, columnMapping, file, fileInfo.Name);
|
|
if (string.IsNullOrEmpty(collections)) {
|
|
logger.LogWarning("collections is null");
|
|
continue;
|
|
}
|
|
text = collections;
|
|
} else if (extension == ".pipe.md") {
|
|
json = GetJavaScriptObjectNotation(logger, file);
|
|
if (string.IsNullOrEmpty(json)) {
|
|
logger.LogWarning("json is null");
|
|
continue;
|
|
}
|
|
pipeTable = GetPipeTable(logger, json);
|
|
if (string.IsNullOrEmpty(pipeTable)) {
|
|
logger.LogWarning("pipeTable is null");
|
|
continue;
|
|
}
|
|
markdown = $"# {fileInfo.Name}{Environment.NewLine}{Environment.NewLine}{pipeTable}{Environment.NewLine}";
|
|
text = markdown;
|
|
} else if (extension == ".json") {
|
|
json = GetJavaScriptObjectNotation(logger, file);
|
|
if (string.IsNullOrEmpty(json)) {
|
|
logger.LogWarning("json is null");
|
|
continue;
|
|
}
|
|
text = json;
|
|
} else {
|
|
logger.LogWarning("{extension} is not mapped!", extension);
|
|
continue;
|
|
}
|
|
File.WriteAllText(checkFile, text);
|
|
File.SetLastWriteTime(checkFile, fileInfo.LastAccessTime);
|
|
logger.LogInformation("<{checkFile}> was written", checkFile);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static string? GetMarkdown(ILogger<Worker> logger, string timeColumn, Dictionary<string, string> columnMapping, string file, string name) {
|
|
string? result = null;
|
|
string[] lines = File.ReadAllLines(file);
|
|
int? columnTitlesLine = GetProcessDataStandardFormatColumnTitlesLine(lines);
|
|
if (columnTitlesLine is null) {
|
|
logger.LogWarning("<{columnTitlesLine}> is null", nameof(columnTitlesLine));
|
|
} else {
|
|
if (lines.Length < columnTitlesLine.Value + 1) {
|
|
logger.LogWarning("<{lines}>(s)", lines.Length);
|
|
} else {
|
|
result = GetMarkdown(timeColumn, columnMapping, name, lines, columnTitlesLine);
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
private static int? GetProcessDataStandardFormatColumnTitlesLine(string[] lines) {
|
|
int? result = null;
|
|
for (int i = 0; i < lines.Length; i++) {
|
|
if (lines[i].StartsWith("END_OFFSET") && i + 3 < lines.Length) {
|
|
result = i + 1;
|
|
break;
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
private static ReadOnlyDictionary<string, ReadOnlyCollection<string>> GetKeyValuePairs(int columnTitlesLine, string[] lines) {
|
|
Dictionary<string, ReadOnlyCollection<string>> results = [];
|
|
string value;
|
|
string[] segments;
|
|
List<List<string>> collections = [];
|
|
string[] columns = lines[columnTitlesLine].Split('\t');
|
|
foreach (string column in columns) {
|
|
collections.Add([]);
|
|
}
|
|
for (int i = columnTitlesLine + 1; i < lines.Length; i++) {
|
|
if (lines[i].StartsWith("NUM_DATA_ROWS")) {
|
|
break;
|
|
}
|
|
segments = lines[i].Split('\t');
|
|
if (segments.Length > columns.Length) {
|
|
continue;
|
|
}
|
|
for (int c = 0; c < segments.Length; c++) {
|
|
collections[c].Add(segments[c]);
|
|
}
|
|
}
|
|
for (int i = 0; i < collections.Count; i++) {
|
|
if (collections[i].Count > 1) {
|
|
if (string.IsNullOrEmpty(collections[i][0])) {
|
|
collections[i][0] = collections[i][1];
|
|
}
|
|
if (string.IsNullOrEmpty(collections[i][^1])) {
|
|
collections[i][^1] = collections[i][^2];
|
|
}
|
|
}
|
|
results.Add(columns[i].Trim('"'), collections[i].AsReadOnly());
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static string? GetMarkdown(string timeColumn, Dictionary<string, string> columnMapping, string name, string[] lines, int? columnTitlesLine) {
|
|
string? result;
|
|
List<string> charts = [];
|
|
List<string> results = [];
|
|
ReadOnlyDictionary<string, ReadOnlyCollection<string>> keyValuePairs = GetKeyValuePairs(columnTitlesLine.Value, lines);
|
|
string[] columns = keyValuePairs.Keys.OrderBy(l => l).ToArray();
|
|
if (!columns.Contains(timeColumn)) {
|
|
result = null;
|
|
} else {
|
|
string? alias;
|
|
string[] mappedColumns = columnMapping.Keys.ToArray();
|
|
string labels = string.Join(',', keyValuePairs[timeColumn].Select(l => l.Replace(',', '_')));
|
|
foreach (string column in columns) {
|
|
if (column == timeColumn) {
|
|
continue;
|
|
}
|
|
if (!columnMapping.TryGetValue(column, out alias) || keyValuePairs[column].Count == 0 || string.IsNullOrEmpty(keyValuePairs[column][0])) {
|
|
results.Add(string.Concat("## ", column, Environment.NewLine, Environment.NewLine,
|
|
"- ", string.Join(',', keyValuePairs[column].Select(l => l.Replace(',', '_'))), Environment.NewLine
|
|
));
|
|
} else {
|
|
charts.Add(string.Concat(
|
|
"## ", column, " - ", alias, Environment.NewLine, Environment.NewLine,
|
|
"```chart", Environment.NewLine,
|
|
"{\"type\": \"line\", \"data\": {\"datasets\": [{", Environment.NewLine,
|
|
"\"label\": \"", column, " - ", alias, "\",", Environment.NewLine,
|
|
"\"data\": [", string.Join(',', keyValuePairs[column].Select(l => l.Replace(',', '_'))), "],", Environment.NewLine,
|
|
"\"borderColor\": \"rgb(75, 192, 192)\"", Environment.NewLine,
|
|
"}],", Environment.NewLine,
|
|
"\"labels\": [", labels, "]", Environment.NewLine,
|
|
"}}", Environment.NewLine,
|
|
"```", Environment.NewLine
|
|
));
|
|
}
|
|
}
|
|
if (results.Count == 0 && charts.Count == 0) {
|
|
result = null;
|
|
} else {
|
|
string[] segments;
|
|
results.Add($"## Footer{Environment.NewLine}");
|
|
results.Insert(0, $"# {name}{Environment.NewLine}");
|
|
for (int i = columnTitlesLine.Value + 1; i < lines.Length; i++) {
|
|
if (lines[i].StartsWith("NUM_DATA_ROWS")) {
|
|
for (int j = i; j < lines.Length; j++) {
|
|
results.Add($"- {lines[j]}");
|
|
}
|
|
}
|
|
}
|
|
results.Add(string.Empty);
|
|
results.Add(string.Empty);
|
|
result = $"{string.Join(Environment.NewLine, results)}{string.Join(Environment.NewLine, charts)}";
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
private static string? GetJavaScriptObjectNotation(ILogger<Worker> logger, string file) {
|
|
string? result = null;
|
|
string[] lines = File.ReadAllLines(file);
|
|
int? columnTitlesLine = GetProcessDataStandardFormatColumnTitlesLine(lines);
|
|
if (columnTitlesLine is null) {
|
|
logger.LogWarning("<{columnTitlesLine}> is null", nameof(columnTitlesLine));
|
|
} else {
|
|
if (lines.Length < columnTitlesLine.Value + 1) {
|
|
logger.LogWarning("<{lines}>(s)", lines.Length);
|
|
} else {
|
|
result = GetJavaScriptObjectNotation(columnTitlesLine.Value, lines);
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
private static string GetJavaScriptObjectNotation(int columnTitlesLine, string[] lines) {
|
|
#pragma warning disable CA1845, IDE0057
|
|
string result;
|
|
string record;
|
|
string value;
|
|
string[] segments;
|
|
string? json = null;
|
|
List<string> records = [];
|
|
ReadOnlyCollection<string>? footerLines = null;
|
|
string[] columns = lines[columnTitlesLine].Split('\t');
|
|
for (int i = columnTitlesLine + 1; i < lines.Length; i++) {
|
|
if (lines[i].StartsWith("NUM_DATA_ROWS")) {
|
|
footerLines = GetFooterLines(lines, i);
|
|
break;
|
|
}
|
|
record = "{";
|
|
segments = lines[i].Split('\t');
|
|
if (segments.Length > columns.Length) {
|
|
continue;
|
|
}
|
|
for (int c = 0; c < segments.Length; c++) {
|
|
value = segments[c].Replace("\\", "\\\\").Replace("\"", "\\\"");
|
|
record += string.Concat('"', columns[c].Trim('"'), '"', ':', '"', value, '"', ',');
|
|
}
|
|
record = string.Concat(record.Substring(0, record.Length - 1), '}');
|
|
records.Add(record);
|
|
}
|
|
if (footerLines is not null && footerLines.Count > 0) {
|
|
ReadOnlyDictionary<string, string> footerKeyValuePairs = GetFooterKeyValuePairs(footerLines);
|
|
ReadOnlyDictionary<string, ReadOnlyDictionary<string, string>> logisticKeyValuePairs = GetLogisticKeyValuePairs(footerLines, footerKeyValuePairs);
|
|
json = JsonSerializer.Serialize(logisticKeyValuePairs, ReadOnlyDictionaryStringReadOnlyDictionaryStringStringSourceGenerationContext.Default.ReadOnlyDictionaryStringReadOnlyDictionaryStringString);
|
|
}
|
|
string footerText = string.IsNullOrEmpty(json) || json == "{}" ? string.Empty : $",{Environment.NewLine}\"PDSF\":{Environment.NewLine}{json}";
|
|
result = string.Concat(
|
|
'{',
|
|
Environment.NewLine,
|
|
'"',
|
|
"Count",
|
|
'"',
|
|
": ",
|
|
records.Count,
|
|
',',
|
|
Environment.NewLine,
|
|
'"',
|
|
"Records",
|
|
'"',
|
|
": ",
|
|
Environment.NewLine,
|
|
'[',
|
|
Environment.NewLine,
|
|
string.Join($",{Environment.NewLine}", records),
|
|
Environment.NewLine,
|
|
']',
|
|
footerText,
|
|
'}');
|
|
return result;
|
|
#pragma warning restore CA1845, IDE0057
|
|
}
|
|
|
|
private static ReadOnlyCollection<string> GetFooterLines(string[] lines, int i) {
|
|
List<string> results = [];
|
|
for (int j = i; j < lines.Length; j++) {
|
|
results.Add(lines[j]);
|
|
if (lines[j].StartsWith("END_HEADER"))
|
|
break;
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static ReadOnlyDictionary<string, string> GetFooterKeyValuePairs(ReadOnlyCollection<string> footerLines) {
|
|
Dictionary<string, string> results = [];
|
|
string[] segments;
|
|
foreach (string footerLine in footerLines) {
|
|
segments = footerLine.Split('\t');
|
|
if (segments.Length != 2 || string.IsNullOrEmpty(segments[1].Trim())) {
|
|
continue;
|
|
}
|
|
if (segments[1].Contains(';')) {
|
|
continue;
|
|
} else {
|
|
results.Add(segments[0], segments[1]);
|
|
}
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static ReadOnlyDictionary<string, ReadOnlyDictionary<string, string>> GetLogisticKeyValuePairs(ReadOnlyCollection<string> footerLines, ReadOnlyDictionary<string, string> footerKeyValuePairs) {
|
|
Dictionary<string, ReadOnlyDictionary<string, string>> results = [];
|
|
string[] segments;
|
|
string[] subSegments;
|
|
string[] subSubSegments;
|
|
Dictionary<string, string>? keyValue;
|
|
results.Add("Footer", footerKeyValuePairs);
|
|
Dictionary<string, Dictionary<string, string>> keyValuePairs = [];
|
|
foreach (string footerLine in footerLines) {
|
|
segments = footerLine.Split('\t');
|
|
if (segments.Length != 2 || string.IsNullOrEmpty(segments[1].Trim())) {
|
|
continue;
|
|
}
|
|
if (!segments[1].Contains(';') || !segments[1].Contains('=')) {
|
|
continue;
|
|
} else {
|
|
subSegments = segments[1].Split(';');
|
|
if (subSegments.Length < 1) {
|
|
continue;
|
|
}
|
|
if (!keyValuePairs.TryGetValue(segments[0], out keyValue)) {
|
|
keyValuePairs.Add(segments[0], []);
|
|
if (!keyValuePairs.TryGetValue(segments[0], out keyValue)) {
|
|
throw new Exception();
|
|
}
|
|
}
|
|
foreach (string segment in subSegments) {
|
|
subSubSegments = segment.Split('=');
|
|
if (subSubSegments.Length != 2) {
|
|
continue;
|
|
}
|
|
keyValue.Add(subSubSegments[0], subSubSegments[1]);
|
|
}
|
|
}
|
|
}
|
|
foreach (KeyValuePair<string, Dictionary<string, string>> keyValuePair in keyValuePairs) {
|
|
results.Add(keyValuePair.Key, keyValuePair.Value.AsReadOnly());
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static string? GetPipeTable(ILogger<Worker> logger, string json) {
|
|
string? result = null;
|
|
string? value;
|
|
string[]? columns = null;
|
|
List<string> values = [];
|
|
List<string> results = [];
|
|
Dictionary<string, string> keyValuePairs = [];
|
|
JsonElement jsonElement = JsonSerializer.Deserialize(json, JsonElementSourceGenerationContext.Default.JsonElement);
|
|
JsonElement[] jsonElements = jsonElement.EnumerateArray().ToArray();
|
|
foreach (JsonElement j in jsonElements) {
|
|
values.Clear();
|
|
keyValuePairs.Clear();
|
|
foreach (JsonProperty jsonProperty in j.EnumerateObject().ToArray()) {
|
|
if (columns is null) {
|
|
JsonProperty[] jsonProperties = j.EnumerateObject().OrderBy(l => l.Name).ToArray();
|
|
columns = jsonProperties.Select(l => l.Name).ToArray();
|
|
results.Add($"|{string.Join('|', columns)}|");
|
|
results.Add($"|{string.Join('|', columns.Select(l => '-'))}|");
|
|
}
|
|
keyValuePairs.Add(jsonProperty.Name, jsonProperty.Value.ToString());
|
|
}
|
|
foreach (string column in columns) {
|
|
if (!keyValuePairs.TryGetValue(column, out value)) {
|
|
values.Add(new string(' ', column.Length));
|
|
} else {
|
|
values.Add(value.PadLeft(column.Length, ' '));
|
|
}
|
|
}
|
|
results.Add($"|{string.Join('|', values)}|");
|
|
}
|
|
if (results.Count > 0) {
|
|
result = string.Join(Environment.NewLine, results);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
} |