using System.Collections.ObjectModel; using System.Text.Json; using System.Text.Json.Serialization; using Microsoft.Extensions.Logging; namespace File_Folder_Helper.ADO2025.PI6; internal static partial class Helper20250701 { [JsonSourceGenerationOptions(WriteIndented = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] [JsonSerializable(typeof(JsonElement))] internal partial class JsonElementSourceGenerationContext : JsonSerializerContext { } [JsonSourceGenerationOptions(WriteIndented = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] [JsonSerializable(typeof(ReadOnlyDictionary>))] internal partial class ReadOnlyDictionaryStringReadOnlyDictionaryStringStringSourceGenerationContext : JsonSerializerContext { } internal static void ProcessDataStandardFormatTo(ILogger logger, List args) { logger.LogInformation(args[0]); logger.LogInformation(args[1]); logger.LogInformation(args[2]); logger.LogInformation(args[3]); logger.LogInformation(args[4]); logger.LogInformation(args[5]); logger.LogInformation(args[6]); logger.LogInformation(args[7]); string[] segments; string extension = args[5]; string timeColumn = args[4]; string searchPattern = args[2]; int sizeFilter = int.Parse(args[3]); string[] columns = args[6].Split(','); Dictionary columnMapping = []; string sourceDirectory = Path.GetFullPath(args[0].Split('~')[0]); string destinationDirectory = Path.GetFullPath(args[7].Split('~')[0]); foreach (string column in columns) { segments = column.Split('~'); columnMapping.Add(segments[0], segments[1]); } string[] directories = Directory.GetDirectories(sourceDirectory, "*", SearchOption.TopDirectoryOnly); ProcessDataStandardFormatTo(logger, sourceDirectory, searchPattern, sizeFilter, timeColumn, extension, columnMapping.AsReadOnly(), destinationDirectory, directories.AsReadOnly()); Helpers.HelperDeleteEmptyDirectories.DeleteEmptyDirectories(logger, sourceDirectory); } private static void ProcessDataStandardFormatTo(ILogger logger, string sourceDirectory, string searchPattern, int sizeFilter, string timeColumn, string extension, ReadOnlyDictionary columnMapping, string destinationDirectory, ReadOnlyCollection directories) { string text; string? json; string[] files; string markdown; string checkFile; string[] matches; FileInfo fileInfo; string? pipeTable; string? collections; string directoryName; string checkDirectory; foreach (string directory in directories) { if (sizeFilter < 987654321 && Path.GetFileName(directory).Contains('-')) { continue; } files = Directory.GetFiles(directory, searchPattern, SearchOption.TopDirectoryOnly); foreach (string file in files) { fileInfo = new(file); if (fileInfo.LastWriteTime > DateTime.Now.AddSeconds(-5)) { continue; } directoryName = Path.GetFileName(fileInfo.DirectoryName); if (fileInfo.Length > sizeFilter && !directoryName.StartsWith('Z')) { checkDirectory = Path.Combine(sourceDirectory, $"Z{directoryName}"); if (!Directory.Exists(checkDirectory)) { _ = Directory.CreateDirectory(checkDirectory); } checkFile = Path.Combine(checkDirectory, fileInfo.Name); if (File.Exists(checkFile)) { continue; } File.Move(file, checkFile); continue; } checkDirectory = Path.Combine(destinationDirectory, directoryName); if (!Directory.Exists(checkDirectory)) { _ = Directory.CreateDirectory(checkDirectory); } checkFile = Path.Combine(checkDirectory, $"{fileInfo.Name}{extension}"); if (File.Exists(checkFile)) { continue; } if (extension == ".md") { collections = GetMarkdown(logger, timeColumn, columnMapping, file, fileInfo.Name); if (string.IsNullOrEmpty(collections)) { logger.LogWarning("collections is null"); continue; } text = collections; } else if (extension == ".pipe.md") { json = GetJavaScriptObjectNotation(logger, file); if (string.IsNullOrEmpty(json)) { logger.LogWarning("json is null"); continue; } pipeTable = GetPipeTable(logger, json); if (string.IsNullOrEmpty(pipeTable)) { logger.LogWarning("pipeTable is null"); continue; } markdown = $"# {fileInfo.Name}{Environment.NewLine}{Environment.NewLine}{pipeTable}{Environment.NewLine}"; text = markdown; } else if (extension == ".json") { json = GetJavaScriptObjectNotation(logger, file); if (string.IsNullOrEmpty(json)) { logger.LogWarning("json is null"); continue; } text = json; } else { logger.LogWarning("{extension} is not mapped!", extension); continue; } File.WriteAllText(checkFile, text); File.SetLastWriteTime(checkFile, fileInfo.LastWriteTime); logger.LogInformation("<{checkFile}> was written", checkFile); } } } private static string? GetMarkdown(ILogger logger, string timeColumn, ReadOnlyDictionary columnMapping, string file, string name) { string? result = null; string[] lines = File.ReadAllLines(file); int? columnTitlesLine = GetProcessDataStandardFormatColumnTitlesLine(lines); if (columnTitlesLine is null) { logger.LogWarning("<{columnTitlesLine}> is null", nameof(columnTitlesLine)); } else { if (lines.Length < columnTitlesLine.Value + 1) { logger.LogWarning("<{lines}>(s)", lines.Length); } else { result = GetMarkdown(timeColumn, columnMapping, name, lines, columnTitlesLine); } } return result; } private static int? GetProcessDataStandardFormatColumnTitlesLine(string[] lines) { int? result = null; for (int i = 0; i < lines.Length; i++) { if (lines[i].StartsWith("END_OFFSET") && i + 3 < lines.Length) { result = i + 1; break; } } return result; } private static ReadOnlyDictionary> GetKeyValuePairs(int columnTitlesLine, string[] lines) { Dictionary> results = []; string value; string[] segments; List> collections = []; string[] columns = lines[columnTitlesLine].Split('\t'); foreach (string column in columns) { collections.Add([]); } for (int i = columnTitlesLine + 1; i < lines.Length; i++) { if (lines[i].StartsWith("NUM_DATA_ROWS")) { break; } segments = lines[i].Split('\t'); if (segments.Length > columns.Length) { continue; } for (int c = 0; c < segments.Length; c++) { collections[c].Add(segments[c]); } } for (int i = 0; i < collections.Count; i++) { if (collections[i].Count > 1) { if (string.IsNullOrEmpty(collections[i][0])) { collections[i][0] = collections[i][1]; } if (string.IsNullOrEmpty(collections[i][^1])) { collections[i][^1] = collections[i][^2]; } } results.Add(columns[i].Trim('"'), collections[i].AsReadOnly()); } return results.AsReadOnly(); } private static string? GetMarkdown(string timeColumn, ReadOnlyDictionary columnMapping, string name, string[] lines, int? columnTitlesLine) { string? result; List charts = []; List results = []; ReadOnlyDictionary> keyValuePairs = GetKeyValuePairs(columnTitlesLine.Value, lines); string[] columns = keyValuePairs.Keys.OrderBy(l => l).ToArray(); if (!columns.Contains(timeColumn)) { result = null; } else { string? alias; string[] mappedColumns = columnMapping.Keys.ToArray(); string labels = string.Join(',', keyValuePairs[timeColumn].Select(l => l.Replace(',', '_'))); foreach (string column in columns) { if (column == timeColumn) { continue; } if (!columnMapping.TryGetValue(column, out alias) || keyValuePairs[column].Count == 0 || string.IsNullOrEmpty(keyValuePairs[column][0])) { results.Add(string.Concat("## ", column, Environment.NewLine, Environment.NewLine, "- ", string.Join(',', keyValuePairs[column].Select(l => l.Replace(',', '_'))), Environment.NewLine )); } else { charts.Add(string.Concat( "## ", column, " - ", alias, Environment.NewLine, Environment.NewLine, "```chart", Environment.NewLine, "{\"type\": \"line\", \"data\": {\"datasets\": [{", Environment.NewLine, "\"label\": \"", column, " - ", alias, "\",", Environment.NewLine, "\"data\": [", string.Join(',', keyValuePairs[column].Select(l => l.Replace(',', '_'))), "],", Environment.NewLine, "\"borderColor\": \"rgb(75, 192, 192)\"", Environment.NewLine, "}],", Environment.NewLine, "\"labels\": [", labels, "]", Environment.NewLine, "}}", Environment.NewLine, "```", Environment.NewLine )); } } if (results.Count == 0 && charts.Count == 0) { result = null; } else { string[] segments; results.Add($"## Footer{Environment.NewLine}"); results.Insert(0, $"# {name}{Environment.NewLine}"); for (int i = columnTitlesLine.Value + 1; i < lines.Length; i++) { if (lines[i].StartsWith("NUM_DATA_ROWS")) { for (int j = i; j < lines.Length; j++) { results.Add($"- {lines[j]}"); } } } results.Add(string.Empty); results.Add(string.Empty); result = $"{string.Join(Environment.NewLine, results)}{string.Join(Environment.NewLine, charts)}"; } } return result; } private static string? GetJavaScriptObjectNotation(ILogger logger, string file) { string? result = null; string[] lines = File.ReadAllLines(file); int? columnTitlesLine = GetProcessDataStandardFormatColumnTitlesLine(lines); if (columnTitlesLine is null) { logger.LogWarning("<{columnTitlesLine}> is null", nameof(columnTitlesLine)); } else { if (lines.Length < columnTitlesLine.Value + 1) { logger.LogWarning("<{lines}>(s)", lines.Length); } else { result = GetJavaScriptObjectNotation(columnTitlesLine.Value, lines); } } return result; } private static string GetJavaScriptObjectNotation(int columnTitlesLine, string[] lines) { #pragma warning disable CA1845, IDE0057 string result; string record; string value; string[] segments; string? json = null; List records = []; ReadOnlyCollection? footerLines = null; string[] columns = lines[columnTitlesLine].Split('\t'); for (int i = columnTitlesLine + 1; i < lines.Length; i++) { if (lines[i].StartsWith("NUM_DATA_ROWS")) { footerLines = GetFooterLines(lines, i); break; } record = "{"; segments = lines[i].Split('\t'); if (segments.Length > columns.Length) { continue; } for (int c = 0; c < segments.Length; c++) { value = segments[c].Replace("\\", "\\\\").Replace("\"", "\\\""); record += string.Concat('"', columns[c].Trim('"'), '"', ':', '"', value, '"', ','); } record = string.Concat(record.Substring(0, record.Length - 1), '}'); records.Add(record); } if (footerLines is not null && footerLines.Count > 0) { ReadOnlyDictionary footerKeyValuePairs = GetFooterKeyValuePairs(footerLines); ReadOnlyDictionary> logisticKeyValuePairs = GetLogisticKeyValuePairs(footerLines, footerKeyValuePairs); json = JsonSerializer.Serialize(logisticKeyValuePairs, ReadOnlyDictionaryStringReadOnlyDictionaryStringStringSourceGenerationContext.Default.ReadOnlyDictionaryStringReadOnlyDictionaryStringString); } string footerText = string.IsNullOrEmpty(json) || json == "{}" ? string.Empty : $",{Environment.NewLine}\"ProcessDataStandardFormat\":{Environment.NewLine}{json}"; result = string.Concat( '{', Environment.NewLine, '"', "Count", '"', ": ", records.Count, ',', Environment.NewLine, '"', "Records", '"', ": ", Environment.NewLine, '[', Environment.NewLine, string.Join($",{Environment.NewLine}", records), Environment.NewLine, ']', footerText, '}'); return result; #pragma warning restore CA1845, IDE0057 } private static ReadOnlyCollection GetFooterLines(string[] lines, int i) { List results = []; for (int j = i; j < lines.Length; j++) { results.Add(lines[j]); if (lines[j].StartsWith("END_HEADER")) break; } return results.AsReadOnly(); } private static ReadOnlyDictionary GetFooterKeyValuePairs(ReadOnlyCollection footerLines) { Dictionary results = []; string[] segments; foreach (string footerLine in footerLines) { segments = footerLine.Split('\t'); if (segments.Length != 2 || string.IsNullOrEmpty(segments[1].Trim())) { continue; } if (segments[1].Contains(';')) { continue; } else { results.Add(segments[0], segments[1]); } } return results.AsReadOnly(); } private static ReadOnlyDictionary> GetLogisticKeyValuePairs(ReadOnlyCollection footerLines, ReadOnlyDictionary footerKeyValuePairs) { Dictionary> results = []; string[] segments; string[] subSegments; string[] subSubSegments; Dictionary? keyValue; results.Add("Footer", footerKeyValuePairs); Dictionary> keyValuePairs = []; foreach (string footerLine in footerLines) { segments = footerLine.Split('\t'); if (segments.Length != 2 || string.IsNullOrEmpty(segments[1].Trim())) { continue; } if (!segments[1].Contains(';') || !segments[1].Contains('=')) { continue; } else { subSegments = segments[1].Split(';'); if (subSegments.Length < 1) { continue; } if (!keyValuePairs.TryGetValue(segments[0], out keyValue)) { keyValuePairs.Add(segments[0], []); if (!keyValuePairs.TryGetValue(segments[0], out keyValue)) { throw new Exception(); } } foreach (string segment in subSegments) { subSubSegments = segment.Split('='); if (subSubSegments.Length != 2) { continue; } keyValue.Add(subSubSegments[0], subSubSegments[1]); } } } foreach (KeyValuePair> keyValuePair in keyValuePairs) { results.Add(keyValuePair.Key, keyValuePair.Value.AsReadOnly()); } return results.AsReadOnly(); } private static string? GetPipeTable(ILogger logger, string json) { string? result = null; string? value; string[]? columns = null; List values = []; List results = []; Dictionary keyValuePairs = []; JsonElement jsonElement = JsonSerializer.Deserialize(json, JsonElementSourceGenerationContext.Default.JsonElement); JsonElement[] jsonElements = jsonElement.EnumerateArray().ToArray(); foreach (JsonElement j in jsonElements) { values.Clear(); keyValuePairs.Clear(); foreach (JsonProperty jsonProperty in j.EnumerateObject().ToArray()) { if (columns is null) { JsonProperty[] jsonProperties = j.EnumerateObject().OrderBy(l => l.Name).ToArray(); columns = jsonProperties.Select(l => l.Name).ToArray(); results.Add($"|{string.Join('|', columns)}|"); results.Add($"|{string.Join('|', columns.Select(l => '-'))}|"); } keyValuePairs.Add(jsonProperty.Name, jsonProperty.Value.ToString()); } foreach (string column in columns) { if (!keyValuePairs.TryGetValue(column, out value)) { values.Add(new string(' ', column.Length)); } else { values.Add(value.PadLeft(column.Length, ' ')); } } results.Add($"|{string.Join('|', values)}|"); } if (results.Count > 0) { result = string.Join(Environment.NewLine, results); } return result; } }