using System.Collections.ObjectModel; using System.Text.Json; using System.Text.Json.Serialization; using Microsoft.Extensions.Logging; namespace File_Folder_Helper.ADO2025.PI6; internal static partial class Helper20250701 { [JsonSourceGenerationOptions(WriteIndented = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] [JsonSerializable(typeof(JsonElement))] internal partial class JsonElementSourceGenerationContext : JsonSerializerContext { } internal static void ProcessDataStandardFormatTo(ILogger logger, List args) { logger.LogInformation(args[0]); logger.LogInformation(args[1]); logger.LogInformation(args[2]); logger.LogInformation(args[3]); logger.LogInformation(args[4]); logger.LogInformation(args[5]); logger.LogInformation(args[6]); string[] segments; string timeColumn = args[4]; string searchPattern = args[2]; int sizeFilter = int.Parse(args[3]); string[] columns = args[5].Split(','); Dictionary columnMapping = []; string sourceDirectory = Path.GetFullPath(args[0].Split('~')[0]); string destinationDirectory = Path.GetFullPath(args[6].Split('~')[0]); foreach (string column in columns) { segments = column.Split('~'); columnMapping.Add(segments[0], segments[1]); } string[] directories = Directory.GetDirectories(sourceDirectory, "*", SearchOption.TopDirectoryOnly); ProcessDataStandardFormatTo(logger, sourceDirectory, searchPattern, sizeFilter, timeColumn, columnMapping, destinationDirectory, directories); Helpers.HelperDeleteEmptyDirectories.DeleteEmptyDirectories(logger, sourceDirectory); } private static void ProcessDataStandardFormatTo(ILogger logger, string sourceDirectory, string searchPattern, int sizeFilter, string timeColumn, Dictionary columnMapping, string destinationDirectory, string[] directories) { string? json; string[] files; string markdown; string checkFile; string[] matches; FileInfo fileInfo; string? pipeTable; string? collections; string directoryName; string checkDirectory; foreach (string directory in directories) { if (Path.GetFileName(directory).Contains('-')) { continue; } files = Directory.GetFiles(directory, searchPattern, SearchOption.TopDirectoryOnly); foreach (string file in files) { fileInfo = new(file); if (fileInfo.LastWriteTime > DateTime.Now.AddSeconds(-5)) { continue; } directoryName = Path.GetFileName(fileInfo.DirectoryName); if (fileInfo.Length > sizeFilter && !directoryName.StartsWith('Z')) { checkDirectory = Path.Combine(sourceDirectory, $"Z{directoryName}"); if (!Directory.Exists(checkDirectory)) { _ = Directory.CreateDirectory(checkDirectory); } checkFile = Path.Combine(checkDirectory, fileInfo.Name); if (File.Exists(checkFile)) { continue; } File.Move(file, checkFile); continue; } checkDirectory = Path.Combine(destinationDirectory, directoryName); if (!Directory.Exists(checkDirectory)) { _ = Directory.CreateDirectory(checkDirectory); } checkFile = Path.Combine(checkDirectory, $"{fileInfo.Name}.md"); if (File.Exists(checkFile)) { continue; } collections = GetMarkdown(logger, timeColumn, columnMapping, file, fileInfo.Name); if (string.IsNullOrEmpty(collections)) { logger.LogWarning("collections is null"); continue; } File.WriteAllText(checkFile, collections); File.SetLastWriteTime(checkFile, fileInfo.LastAccessTime); if (!string.IsNullOrEmpty(destinationDirectory)) { continue; } json = GetJavaScriptObjectNotation(logger, file); if (string.IsNullOrEmpty(json)) { logger.LogWarning("json is null"); continue; } File.WriteAllText($"{checkFile}.md", json); File.SetLastWriteTime($"{checkFile}.md", fileInfo.LastAccessTime); pipeTable = GetPipeTable(logger, json); if (string.IsNullOrEmpty(pipeTable)) { logger.LogWarning("pipeTable is null"); continue; } markdown = $"# {fileInfo.Name}{Environment.NewLine}{Environment.NewLine}{pipeTable}{Environment.NewLine}"; File.WriteAllText($"{checkFile}.md", markdown); File.SetLastWriteTime($"{checkFile}.md", fileInfo.LastAccessTime); logger.LogInformation("<{checkFile}> was written", checkFile); } } } private static string? GetMarkdown(ILogger logger, string timeColumn, Dictionary columnMapping, string file, string name) { string? result = null; string[] lines = File.ReadAllLines(file); int? columnTitlesLine = GetProcessDataStandardFormatColumnTitlesLine(lines); if (columnTitlesLine is null) { logger.LogWarning("<{columnTitlesLine}> is null", nameof(columnTitlesLine)); } else { if (lines.Length < columnTitlesLine.Value + 1) { logger.LogWarning("<{lines}>(s)", lines.Length); } else { result = GetMarkdown(timeColumn, columnMapping, name, lines, columnTitlesLine); } } return result; } private static int? GetProcessDataStandardFormatColumnTitlesLine(string[] lines) { int? result = null; for (int i = 0; i < lines.Length; i++) { if (lines[i].StartsWith("END_OFFSET") && i + 3 < lines.Length) { result = i + 1; break; } } return result; } private static ReadOnlyDictionary> GetKeyValuePairs(int columnTitlesLine, string[] lines) { Dictionary> results = []; string value; string[] segments; List> collections = []; string[] columns = lines[columnTitlesLine].Split('\t'); foreach (string column in columns) { collections.Add([]); } for (int i = columnTitlesLine + 1; i < lines.Length; i++) { if (lines[i].StartsWith("NUM_DATA_ROWS")) { break; } segments = lines[i].Split('\t'); if (segments.Length > columns.Length) { continue; } for (int c = 0; c < segments.Length; c++) { collections[c].Add(segments[c]); } } for (int i = 0; i < collections.Count; i++) { if (collections[i].Count > 1) { if (string.IsNullOrEmpty(collections[i][0])) { collections[i][0] = collections[i][1]; } if (string.IsNullOrEmpty(collections[i][^1])) { collections[i][^1] = collections[i][^2]; } } results.Add(columns[i].Trim('"'), collections[i].AsReadOnly()); } return results.AsReadOnly(); } private static string? GetMarkdown(string timeColumn, Dictionary columnMapping, string name, string[] lines, int? columnTitlesLine) { string? result; List charts = []; List results = []; ReadOnlyDictionary> keyValuePairs = GetKeyValuePairs(columnTitlesLine.Value, lines); string[] columns = keyValuePairs.Keys.OrderBy(l => l).ToArray(); if (!columns.Contains(timeColumn)) { result = null; } else { string? alias; string[] mappedColumns = columnMapping.Keys.ToArray(); string labels = string.Join(',', keyValuePairs[timeColumn].Select(l => l.Replace(',', '_'))); foreach (string column in columns) { if (column == timeColumn) { continue; } if (!columnMapping.TryGetValue(column, out alias) || keyValuePairs[column].Count == 0 || string.IsNullOrEmpty(keyValuePairs[column][0])) { results.Add(string.Concat("## ", column, Environment.NewLine, Environment.NewLine, "- ", string.Join(',', keyValuePairs[column].Select(l => l.Replace(',', '_'))), Environment.NewLine )); } else { charts.Add(string.Concat( "## ", column, " - ", alias, Environment.NewLine, Environment.NewLine, "```chart", Environment.NewLine, "{\"type\": \"line\", \"data\": {\"datasets\": [{", Environment.NewLine, "\"label\": \"", column, " - ", alias, "\",", Environment.NewLine, "\"data\": [", string.Join(',', keyValuePairs[column].Select(l => l.Replace(',', '_'))), "],", Environment.NewLine, "\"borderColor\": \"rgb(75, 192, 192)\"", Environment.NewLine, "}],", Environment.NewLine, "\"labels\": [", labels, "]", Environment.NewLine, "}}", Environment.NewLine, "```", Environment.NewLine )); } } if (results.Count == 0 && charts.Count == 0) { result = null; } else { string[] segments; results.Add($"## Footer{Environment.NewLine}"); results.Insert(0, $"# {name}{Environment.NewLine}"); for (int i = columnTitlesLine.Value + 1; i < lines.Length; i++) { if (lines[i].StartsWith("NUM_DATA_ROWS")) { for (int j = i; j < lines.Length; j++) { results.Add($"- {lines[j]}"); } } } results.Add(string.Empty); results.Add(string.Empty); result = $"{string.Join(Environment.NewLine, results)}{string.Join(Environment.NewLine, charts)}"; } } return result; } private static string? GetJavaScriptObjectNotation(ILogger logger, string file) { string? result = null; string[] lines = File.ReadAllLines(file); int? columnTitlesLine = GetProcessDataStandardFormatColumnTitlesLine(lines); if (columnTitlesLine is null) { logger.LogWarning("<{columnTitlesLine}> is null", nameof(columnTitlesLine)); } else { if (lines.Length < columnTitlesLine.Value + 1) { logger.LogWarning("<{lines}>(s)", lines.Length); } else { result = GetJavaScriptObjectNotation(columnTitlesLine.Value, [], lines); } } return result; } private static string GetJavaScriptObjectNotation(int columnTitlesLine, string[] columns, string[] lines) { #pragma warning disable CA1845, IDE0057 string result = "[\n"; string line; string value; string[] segments; if (columns.Length == 0) { columns = lines[columnTitlesLine].Split('\t'); } for (int i = columnTitlesLine + 1; i < lines.Length; i++) { if (lines[i].StartsWith("NUM_DATA_ROWS")) { break; } line = "{"; segments = lines[i].Split('\t'); if (segments.Length > columns.Length) { continue; } for (int c = 0; c < segments.Length; c++) { value = segments[c].Replace("\\", "\\\\").Replace("\"", "\\\""); line += '"' + columns[c].Trim('"') + '"' + ':' + '"' + value + '"' + ','; } line = line.Substring(0, line.Length - 1) + '}' + ',' + '\n'; result += line; } result = result.Substring(0, result.Length - 2) + ']'; return result; #pragma warning restore CA1845, IDE0057 } private static string? GetPipeTable(ILogger logger, string json) { string? result = null; string? value; string[]? columns = null; List values = []; List results = []; Dictionary keyValuePairs = []; JsonElement jsonElement = JsonSerializer.Deserialize(json, JsonElementSourceGenerationContext.Default.JsonElement); JsonElement[] jsonElements = jsonElement.EnumerateArray().ToArray(); foreach (JsonElement j in jsonElements) { values.Clear(); keyValuePairs.Clear(); foreach (JsonProperty jsonProperty in j.EnumerateObject().ToArray()) { if (columns is null) { JsonProperty[] jsonProperties = j.EnumerateObject().OrderBy(l => l.Name).ToArray(); columns = jsonProperties.Select(l => l.Name).ToArray(); results.Add($"|{string.Join('|', columns)}|"); results.Add($"|{string.Join('|', columns.Select(l => '-'))}|"); } keyValuePairs.Add(jsonProperty.Name, jsonProperty.Value.ToString()); } foreach (string column in columns) { if (!keyValuePairs.TryGetValue(column, out value)) { values.Add(new string(' ', column.Length)); } else { values.Add(value.PadLeft(column.Length, ' ')); } } results.Add($"|{string.Join('|', values)}|"); } if (results.Count > 0) { result = string.Join(Environment.NewLine, results); } return result; } }