using File_Folder_Helper.Models; using Microsoft.Extensions.Logging; using System.Collections.ObjectModel; using System.Globalization; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using System.Text.RegularExpressions; using YamlDotNet.Serialization; namespace File_Folder_Helper.Helpers; internal static partial class HelperMarkdown { private record Input(string? Destination, string Source, string? StartAt); private record Record(string Directory, string File, string[] Lines); private record MarkdownFile(DateTime CreationDateTime, string Directory, string Extension, string File, string FileName, string FileNameWithoutExtension, ReadOnlyDictionary FrontMatterYaml, string H1, bool IsKanbanIndex, bool IsKanbanMarkdown, DateTime LastWriteDateTime, LineNumber LineNumber, string Type); private record MarkdownFileAndLines(MarkdownFile MarkdownFile, string[] Lines); private record MarkdownExtra(ReadOnlyCollection? Assignees, string? Effort, ReadOnlyCollection? H2HexColorCollection, ReadOnlyCollection? H2NoCheckboxesCollection, ReadOnlyCollection? H2WithCheckboxesCollection, string? RequestedDateTime); private record MarkdownFileH1AndRelativePath(MarkdownFile? MarkdownFile, string[]? Lines, string? H1, string? RelativePath); [JsonSourceGenerationOptions(WriteIndented = true, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull)] [JsonSerializable(typeof(Dictionary))] internal partial class DictionaryStringAndJsonElementSourceGenerationContext : JsonSerializerContext { } private static void SetRecursiveLines(AppSettings appSettings, ILogger logger, ReadOnlyDictionary> keyValuePairs, string linkTitle, MarkdownFile markdownFile, string[] lines, List indentations, List recursiveLines) { if (recursiveLines is null) throw new Exception(); string file; string[] segmentsA; string[] segmentsB; string segmentsALast; bool fencedCodeBlock = false; string indentation = new(indentations.ToArray()); MarkdownFileH1AndRelativePath markdownFileH1AndRelativePath; for (int i = 0; i < lines.Length; i++) { if (indentations.Count > 15) { recursiveLines.Add("```Error```"); break; } if (lines[i].Length < 1) continue; if (lines[i].Length > 4 && lines[i][..3] == "```") fencedCodeBlock = !fencedCodeBlock; if (fencedCodeBlock) continue; if (lines[i][0] == '#') { if (lines[i] == $"# {linkTitle}") continue; recursiveLines.Add($"{indentation}{lines[i]}"); continue; } segmentsA = lines[i].Split("]("); if (segmentsA.Length != 2) continue; segmentsALast = segmentsA[^1]; if (appSettings.ExcludeSchemes.Any(l => segmentsALast.StartsWith(l))) continue; segmentsB = segmentsALast.Split(")"); if (segmentsB.Length != 2) continue; file = Path.GetFullPath(Path.Combine(markdownFile.Directory, segmentsB[0])); markdownFileH1AndRelativePath = GetRelativePath(keyValuePairs, markdownFile, file); if (markdownFileH1AndRelativePath.MarkdownFile is null || markdownFileH1AndRelativePath.H1 is null || markdownFileH1AndRelativePath.RelativePath is null) { recursiveLines.Add($"???{indentation}{lines[i]}"); logger.LogInformation("Didn't find {line} in <{file}>", lines[i], markdownFile.FileNameWithoutExtension); continue; } if (markdownFileH1AndRelativePath.Lines is null) continue; indentations.Add('\t'); recursiveLines.Add($"{indentation}{lines[i]}"); SetRecursiveLines(appSettings, logger, keyValuePairs, segmentsA[0].Split('[')[^1], markdownFileH1AndRelativePath.MarkdownFile, markdownFileH1AndRelativePath.Lines, indentations, recursiveLines); } if (indentations.Count > 0) indentations.RemoveAt(0); } private static List GetFrontMatterLines(string[] parsedLines) { List results = []; string afterTrim; string[] segments; StringBuilder stringBuilder = new(); for (int i = 0; i < parsedLines.Length; i++) { afterTrim = parsedLines[i].Trim(); if (string.IsNullOrEmpty(afterTrim) || afterTrim[0] is '{' or '}') continue; segments = afterTrim.Split(": "); if (segments.Length != 2) { if (results[^1][^1] == '[') { _ = stringBuilder.Clear(); _ = stringBuilder.Append(results[^1]); results.RemoveAt(results.Count - 1); for (int j = i; j < parsedLines.Length; j++) { i = j; afterTrim = parsedLines[j].Trim(); if (afterTrim == "],") _ = stringBuilder.Append(afterTrim[..^1]); else if (afterTrim[^1] == ',') _ = stringBuilder.Append(afterTrim).Append(' '); else _ = stringBuilder.Append(afterTrim); if (afterTrim is "]" or "],") { results.Add(stringBuilder.ToString()); break; } } continue; } results.Clear(); break; } if (afterTrim[^1] != ',') results.Add(afterTrim[1..].Replace("\": ", ": ")); else results.Add(afterTrim[1..^1].Replace("\": ", ": ")); } return results; } /// /// Determines files text file's encoding by analyzing its byte order mark (BOM). /// Defaults to ASCII when detection of the text file's endianness fails. /// /// The text file to analyze. /// The detected encoding. private static Encoding? GetEncoding(string filename) { Encoding? result; byte[] bom = new byte[4]; using FileStream file = new(filename, FileMode.Open, FileAccess.Read); _ = file.Read(bom, 0, 4); if (bom[0] == 0x2b && bom[1] == 0x2f && bom[2] == 0x76) #pragma warning disable SYSLIB0001 result = Encoding.UTF7; #pragma warning restore SYSLIB0001 if (bom[0] == 0xef && bom[1] == 0xbb && bom[2] == 0xbf) result = Encoding.UTF8; if (bom[0] == 0xff && bom[1] == 0xfe && bom[2] == 0 && bom[3] == 0) result = Encoding.UTF32; //UTF-32LE if (bom[0] == 0xff && bom[1] == 0xfe) result = Encoding.Unicode; //UTF-16LE if (bom[0] == 0xfe && bom[1] == 0xff) result = Encoding.BigEndianUnicode; //UTF-16BE if (bom[0] == 0 && bom[1] == 0 && bom[2] == 0xfe && bom[3] == 0xff) result = new UTF32Encoding(true, true); //UTF-32BE else result = null; return result; } [GeneratedRegex("(~~)?(#)([a-zA-Z0-9]{6})(~~)?( )")] private static partial Regex HtmlColor(); private static List GetMarkdownFileAndLines(string file, List markdownFiles) { List results = []; List distinct = []; string? directory = Path.GetDirectoryName(file); foreach (MarkdownFileAndLines markdownFileAndLines in markdownFiles) { if (string.IsNullOrEmpty(directory) || markdownFileAndLines.MarkdownFile.Directory != directory) continue; if (distinct.Contains(markdownFileAndLines.MarkdownFile.File)) continue; distinct.Add(markdownFileAndLines.MarkdownFile.File); results.Add(markdownFileAndLines); } return results; } private static (string?, Dictionary?, List) Get(List jsonLines) { string? result; List results; string jsonLinesLast = jsonLines[^1]; jsonLines.RemoveAt(jsonLines.Count - 1); jsonLines.Add(jsonLinesLast[..^1]); jsonLines.Insert(0, "{"); jsonLines.Add("}"); Dictionary? keyValuePairs; result = string.Join(Environment.NewLine, jsonLines); keyValuePairs = JsonSerializer.Deserialize(result, DictionaryStringAndJsonElementSourceGenerationContext.Default.DictionaryStringJsonElement); if (keyValuePairs is null) throw new NullReferenceException(nameof(keyValuePairs)); result = JsonSerializer.Serialize(keyValuePairs, DictionaryStringAndJsonElementSourceGenerationContext.Default.DictionaryStringJsonElement); string[] parsedLines = result.Split(Environment.NewLine).ToArray(); results = GetFrontMatterLines(parsedLines); if (results.Count == 0) { result = null; keyValuePairs = null; } return (result, keyValuePairs, results); } internal static (List, LineNumber) GetStatusAndFrontMatterYamlEndLineNumbers(FileInfo fileInfo) { string line; int? h1LineNumber = null; int? typeLineNumber = null; int? statusLineNumber = null; int? createdLineNumber = null; int? updatedLineNumber = null; int? frontMatterYamlEndLineNumber = null; Encoding? encoding = GetEncoding(fileInfo.FullName) ?? Encoding.Default; string[] lines = File.ReadAllLines(fileInfo.FullName, encoding); for (int i = 0; i < lines.Length; i++) { line = lines[i]; if (line.Length < 3) continue; if (i == 0 && line[..3] == "---") continue; if (h1LineNumber is null && line[..3] == "---") { frontMatterYamlEndLineNumber = i; continue; } if (line.Length > 6 && line[..6] == "type: ") { typeLineNumber = i; continue; } if (line.Length > 8 && line[..8] == "status: ") { statusLineNumber = i; continue; } if (line.Length > 9 && line[..9] == "created: ") { createdLineNumber = i; continue; } if (line.Length > 9 && line[..9] == "updated: ") { updatedLineNumber = i; continue; } if (h1LineNumber is null && line.Length > 2 && line[0] == '#' && line[1] == ' ') { h1LineNumber = i; continue; } } LineNumber lineNumber = new(createdLineNumber, h1LineNumber, frontMatterYamlEndLineNumber, statusLineNumber, typeLineNumber, updatedLineNumber); return (lines.ToList(), lineNumber); } private static MarkdownExtra GetMarkdownExtra(MarkdownFileAndLines markdownFileAndLines) { MarkdownExtra result; int skip; Match match; string line; int completed; int notCompleted; List lines; string? effort = null; List assignees = []; string? requestedDateTime = null; ReadOnlyCollection groups; List h2HexColors = []; List h2NoCheckboxes = []; List h2WithCheckboxes = []; if (markdownFileAndLines.MarkdownFile.LineNumber.FrontMatterYamlEnd is not null) { for (int i = 1; i < markdownFileAndLines.Lines.Length; i++) { line = markdownFileAndLines.Lines[i]; if (line.Length < 3) continue; if (line.Length > 8 && line[..8] == "effort: ") { effort = line[7..].Trim().Trim('"'); continue; } if (line.Length > 10 && line[..10] == "assigned: ") { foreach (string item in line[10..].Split(',', StringSplitOptions.RemoveEmptyEntries)) assignees.Add(item.Trim().Trim('"')); continue; } if (line.Length > 11 && line[..11] == "requested: ") { requestedDateTime = line[10..].Trim().Trim('"'); continue; } if (line.Length > 3 && line[0] == '#' && line[1] == '#' && line[2] == ' ') { completed = 0; notCompleted = 0; match = HtmlColor().Match(line[3..]); if (line.Length > 3 && match.Success) { groups = match.Groups.AsReadOnly(); skip = 3 + groups.Skip(1).Sum(l => l.Length); h2HexColors.Add(new(line[skip..], $"#{groups.First(l => l.Value.Length == 6)}")); continue; } lines = []; if (i + 1 == markdownFileAndLines.Lines.Length) continue; for (int j = i + 1; j < markdownFileAndLines.Lines.Length; j++) { line = markdownFileAndLines.Lines[j]; if (line.Length == 0) continue; if (line.Length > 2 && line[0] == '#') break; lines.Add(line); if (line.Length < 5 || line[0] != '-' || line[1] != ' ' || line[2] != '[') continue; if (line[3] == ' ' && line[4] == ']') notCompleted++; else if (line[3] is 'x' or 'X' && line[4] == ']') completed++; } if (completed != 0 || notCompleted != 0) h2WithCheckboxes.Add(new(completed, markdownFileAndLines.Lines[i][3..], notCompleted, notCompleted + completed)); else if (lines.Count > 0) h2NoCheckboxes.Add(new(markdownFileAndLines.Lines[i][3..], new(lines))); continue; } } } result = new(new(assignees), effort, new(h2HexColors), new(h2NoCheckboxes), new(h2WithCheckboxes), requestedDateTime); return result; } private static List Distinct(IEnumerable? markdownFileAndLinesCollection) { List results = []; if (markdownFileAndLinesCollection is not null) { List distinct = []; foreach (MarkdownFileAndLines markdownFileAndLines in markdownFileAndLinesCollection) { if (distinct.Contains(markdownFileAndLines.MarkdownFile.File)) continue; distinct.Add(markdownFileAndLines.MarkdownFile.File); results.Add(markdownFileAndLines); } } return results; } private static List GetMarkdownFileAndLines(ReadOnlyDictionary> keyValuePairs) { List results = []; foreach (KeyValuePair> keyValuePair in keyValuePairs) { foreach (MarkdownFileAndLines markdownFileAndLines in keyValuePair.Value) results.Add(markdownFileAndLines); } return results; } private static ReadOnlyDictionary> GetKeyValuePairs(ReadOnlyDictionary relativeToCollection) { Dictionary> results = []; MarkdownFile markdownFile; string fileNameWithoutExtension; string fileNameWithoutExtensionB; List? markdownFiles; foreach (KeyValuePair relativeTo in relativeToCollection) { markdownFile = relativeTo.Value.MarkdownFile; if (!results.TryGetValue(relativeTo.Key, out markdownFiles)) { results.Add(relativeTo.Key, []); if (!results.TryGetValue(relativeTo.Key, out markdownFiles)) throw new NotSupportedException(); } markdownFiles.Add(relativeTo.Value); } foreach (KeyValuePair relativeTo in relativeToCollection) { markdownFile = relativeTo.Value.MarkdownFile; fileNameWithoutExtension = markdownFile.FileNameWithoutExtension.ToLower(); fileNameWithoutExtensionB = fileNameWithoutExtension.Replace("%20", "-").Replace(' ', '-'); if (!results.TryGetValue(markdownFile.FileNameWithoutExtension, out markdownFiles)) { results.Add(markdownFile.FileNameWithoutExtension, []); if (!results.TryGetValue(markdownFile.FileNameWithoutExtension, out markdownFiles)) throw new NotSupportedException(); } markdownFiles.Add(relativeTo.Value); if (fileNameWithoutExtension == markdownFile.FileNameWithoutExtension) continue; if (!results.TryGetValue(fileNameWithoutExtension, out markdownFiles)) { results.Add(fileNameWithoutExtension, []); if (!results.TryGetValue(fileNameWithoutExtension, out markdownFiles)) throw new NotSupportedException(); } if (fileNameWithoutExtensionB == markdownFile.FileNameWithoutExtension) continue; if (!results.TryGetValue(fileNameWithoutExtensionB, out markdownFiles)) { results.Add(fileNameWithoutExtensionB, []); if (!results.TryGetValue(fileNameWithoutExtensionB, out markdownFiles)) throw new NotSupportedException(); } markdownFiles.Add(relativeTo.Value); } foreach (KeyValuePair relativeTo in relativeToCollection) { markdownFile = relativeTo.Value.MarkdownFile; if (!results.TryGetValue(markdownFile.H1, out markdownFiles)) { results.Add(markdownFile.H1, []); if (!results.TryGetValue(markdownFile.H1, out markdownFiles)) throw new NotSupportedException(); } markdownFiles.Add(relativeTo.Value); } return new(results); } private static int ConvertFileToSlugName(ReadOnlyDictionary relativeToCollection, ReadOnlyCollection gitOthersModifiedAndDeletedExcludingStandardFiles) { int result = 0; string h1; bool gitCheck; string h1Check; string[] lines; string checkName; string checkFileName; MarkdownFile markdownFile; foreach (KeyValuePair relativeTo in relativeToCollection) { if (relativeTo.Value.Lines.Length == 0) continue; lines = relativeTo.Value.Lines; markdownFile = relativeTo.Value.MarkdownFile; gitCheck = gitOthersModifiedAndDeletedExcludingStandardFiles.Contains(markdownFile.File); if (markdownFile.LineNumber.H1 is not null) { h1 = lines[markdownFile.LineNumber.H1.Value]; if (h1.Length > 2) { h1Check = $"# {h1[2..]}"; if (h1Check.Length == h1.Length && h1Check != h1) { if (!gitCheck) continue; lines[markdownFile.LineNumber.H1.Value] = h1Check; File.WriteAllLines(markdownFile.File, lines); result += 1; } } } checkFileName = markdownFile.FileName.ToLower().Replace("%20", "-").Replace(' ', '-'); if (checkFileName == markdownFile.FileName) continue; if (!File.Exists(markdownFile.File)) continue; checkName = Path.Combine(markdownFile.Directory, checkFileName); if (checkName == markdownFile.File) continue; if (!gitCheck) continue; File.Move(markdownFile.File, checkName); result += 1; } return result; } private static string[] GetFiles(AppSettings appSettings, string directory) { string[] results = Directory.GetFiles(directory, "*.md", SearchOption.AllDirectories). Where(l => !appSettings.ExcludeDirectoryNames.Any(m => l.Contains(m))).ToArray(); return results; } private static ReadOnlyCollection GetFromMatterYamlLines(List lines, LineNumber lineNumber) { List results = []; if (lineNumber.FrontMatterYamlEnd is not null && lines.Count >= lineNumber.FrontMatterYamlEnd.Value) { for (int i = 1; i < lineNumber.FrontMatterYamlEnd.Value; i++) results.Add(lines[i]); } return new(results); } private static ReadOnlyDictionary GetFromMatterYaml(List lines, LineNumber lineNumber) { Dictionary results = []; IDeserializer deserializer = new DeserializerBuilder().Build(); ReadOnlyCollection frontMatterYamlLines = GetFromMatterYamlLines(lines, lineNumber); string frontMatterYaml = string.Join(Environment.NewLine, frontMatterYamlLines); Dictionary? keyValuePairs = deserializer.Deserialize>(frontMatterYaml); if (keyValuePairs is not null) { foreach (string key in keyValuePairs.Keys.OrderBy(l => l)) results.Add(key, keyValuePairs[key]); } return new(results); } private static ReadOnlyDictionary GetRelativeToCollection(AppSettings appSettings, Input input, string[] files, ReadOnlyCollection gitOthersModifiedAndDeletedExcludingStandardFiles, bool force) { Dictionary results = []; string h1; string key; string type; bool gitCheck; FileInfo fileInfo; bool isKanbanIndex; List lines; bool isKanbanMarkdown; LineNumber lineNumber; MarkdownFile markdownFile; string fileNameWithoutExtension; ReadOnlyDictionary frontMatterYaml; foreach (string file in files) { // cSpell:disable fileInfo = new(file); if (fileInfo.DirectoryName is null) continue; key = Path.GetRelativePath(input.Source, file); (lines, lineNumber) = GetStatusAndFrontMatterYamlEndLineNumbers(fileInfo); fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileInfo.FullName); h1 = fileNameWithoutExtension.ToLower().Replace("%20", "-").Replace(' ', '-'); frontMatterYaml = GetFromMatterYaml(lines, lineNumber); if (lines.Count > 0) (type, h1) = GetTypeAndH1(appSettings, h1, lines, lineNumber); else { gitCheck = gitOthersModifiedAndDeletedExcludingStandardFiles.Contains(file); if (!gitCheck) continue; type = appSettings.DefaultNoteType; File.WriteAllLines(file, ["---", $"type: \"{type}\"", "---", string.Empty, $"# {h1}"]); lines = File.ReadAllLines(file).ToList(); } isKanbanMarkdown = fileInfo.Name.EndsWith(".knb.md"); isKanbanIndex = fileNameWithoutExtension == "index" && type.StartsWith("kanb", StringComparison.OrdinalIgnoreCase); markdownFile = new(fileInfo.CreationTime, fileInfo.DirectoryName, fileInfo.Extension, file, fileInfo.Name, fileNameWithoutExtension, frontMatterYaml, h1, isKanbanIndex, isKanbanMarkdown, fileInfo.LastWriteTime, lineNumber, type); if (force || input.StartAt is null || file.StartsWith(input.StartAt)) results.Add(key, new(markdownFile, lines.ToArray())); else results.Add(key, new(markdownFile, [])); } // cSpell:restore return new(results); } private static List GetKeys(ReadOnlyDictionary relativeToCollection) { List results = []; MarkdownFile markdownFile; foreach (KeyValuePair relativeTo in relativeToCollection) { markdownFile = relativeTo.Value.MarkdownFile; if (markdownFile.IsKanbanMarkdown) continue; results.Add(relativeTo.Key); } return results; } private static MarkdownFileAndLines? GetKanbanIndexMarkdownFileAndLines(ReadOnlyDictionary keyValuePairs) { MarkdownFile markdownFile; MarkdownFileAndLines? result = null; foreach (KeyValuePair keyValuePair in keyValuePairs) { markdownFile = keyValuePair.Value.MarkdownFile; if (markdownFile.IsKanbanIndex) { if (result is not null) { result = null; break; } result = keyValuePair.Value; } } return result; } private static void SetCards(Input input, ReadOnlyDictionary relativeToCollection, List notLinkedKey, MarkdownFile markdownFile, string[] lines, List cards, List allKeys, int i) { Card card; string key; string[] segmentsA; MarkdownExtra markdownExtra; MarkdownFileAndLines? markdownFileAndLines; for (int j = i + 1; j < lines.Length; j++) { if (lines[j].Length < 5) continue; if (lines[j].Length >= 4 && lines[j][0] == '#' && lines[j][1] == '#' && lines[j][2] == ' ') break; segmentsA = lines[j].Split("]("); if (segmentsA.Length != 2 || segmentsA[1][^1] != ')') continue; key = Path.GetRelativePath(input.Source, Path.Combine(markdownFile.Directory, segmentsA[1][..^1])); if (!relativeToCollection.TryGetValue(key, out markdownFileAndLines)) continue; markdownExtra = GetMarkdownExtra(markdownFileAndLines); card = new(markdownExtra.Assignees, markdownFileAndLines.MarkdownFile.CreationDateTime, markdownFileAndLines.MarkdownFile.Directory, markdownExtra.Effort, markdownFileAndLines.MarkdownFile.Extension, markdownFileAndLines.MarkdownFile.File, markdownFileAndLines.MarkdownFile.FileName, markdownFileAndLines.MarkdownFile.FileNameWithoutExtension, markdownFileAndLines.MarkdownFile.H1, markdownExtra.H2HexColorCollection, markdownExtra.H2NoCheckboxesCollection, markdownExtra.H2WithCheckboxesCollection, markdownFileAndLines.MarkdownFile.LastWriteDateTime, markdownFileAndLines.MarkdownFile.LineNumber, markdownExtra.RequestedDateTime, markdownFileAndLines.MarkdownFile.Type); if (allKeys.Remove(key)) cards.Add(card); else notLinkedKey.Add(card); } } private static ReadOnlyDictionary> GetColumnsToCards(Input input, ReadOnlyDictionary relativeToCollection, MarkdownFileAndLines markdownFileAndLines) { Dictionary> results = []; string? column; string[] lines; List cards = []; List notLinkedKey = []; lines = markdownFileAndLines.Lines; List allKeys = GetKeys(relativeToCollection); for (int i = 0; i < lines.Length; i++) { if (lines[i].Length < 4 || lines[i][0] != '#' || lines[i][1] != '#' || lines[i][2] != ' ') continue; column = lines[i][3..].TrimEnd(); if (lines.Length == i + 1) continue; SetCards(input, relativeToCollection, notLinkedKey, markdownFileAndLines.MarkdownFile, lines, cards, allKeys, i); results.Add(column, cards); cards = []; } if (notLinkedKey.Count > 1) results.Add("Not Linked", notLinkedKey); return new(results); } private static MarkdownFileAndLines? GetMarkdownFile(ReadOnlyDictionary> keyValuePairs, MarkdownFile markdownFile, string file) { MarkdownFileAndLines? result; List? markdownFileAndLinesCollection; string fileNameWithoutExtension = Path.GetFileNameWithoutExtension(file); if (!keyValuePairs.TryGetValue(fileNameWithoutExtension, out markdownFileAndLinesCollection)) _ = keyValuePairs.TryGetValue(fileNameWithoutExtension.ToLower(), out markdownFileAndLinesCollection); markdownFileAndLinesCollection = Distinct(markdownFileAndLinesCollection); if (markdownFileAndLinesCollection is not null && markdownFileAndLinesCollection.Count == 1) result = markdownFileAndLinesCollection[0]; else { List matches; matches = markdownFileAndLinesCollection is null ? [] : GetMarkdownFileAndLines(file, markdownFileAndLinesCollection); if (matches.Count == 1) result = matches[0]; else { markdownFileAndLinesCollection = GetMarkdownFileAndLines(keyValuePairs); matches = Distinct(markdownFileAndLinesCollection.Where(l => l.MarkdownFile.FileNameWithoutExtension.Length == fileNameWithoutExtension.Length && l.MarkdownFile.FileNameWithoutExtension.Contains(fileNameWithoutExtension, StringComparison.OrdinalIgnoreCase))); if (matches.Count == 1) result = matches[0]; else { string checkName = fileNameWithoutExtension.ToLower().Replace("%20", "-").Replace(' ', '-'); matches = Distinct(markdownFileAndLinesCollection.Where(l => l.MarkdownFile.FileNameWithoutExtension.Length == checkName.Length && l.MarkdownFile.FileNameWithoutExtension.Contains(checkName, StringComparison.OrdinalIgnoreCase))); if (matches.Count == 1) result = matches[0]; else if (matches.Count == 0) result = null; else { checkName = matches[0].MarkdownFile.FileNameWithoutExtension; matches = Distinct(markdownFileAndLinesCollection.Where(l => l.MarkdownFile.File.Contains(checkName, StringComparison.OrdinalIgnoreCase))); if (matches.Count == 1) result = matches[0]; else { checkName = $"{checkName}{markdownFile.Extension}"; matches = Distinct(markdownFileAndLinesCollection.Where(l => l.MarkdownFile.File.EndsWith(checkName, StringComparison.OrdinalIgnoreCase))); if (matches.Count == 1) result = matches[0]; else { checkName = $"\\{checkName}"; matches = Distinct(markdownFileAndLinesCollection.Where(l => l.MarkdownFile.File.EndsWith(checkName, StringComparison.OrdinalIgnoreCase))); if (matches.Count == 1) result = matches[0]; else result = null; } } } } } } return result; } private static Input GetInput(List args) { Input result; string? startAt = null; string? destination = null; string source = Path.GetFullPath(args[0]); for (int i = 1; i < args.Count; i++) { if (args[i].Length == 2 && i + 1 < args.Count) { if (args[i][1] == 's') startAt = Path.GetFullPath(args[i + 1]); else if (args[i][1] == 'd') destination = Path.GetFullPath(args[i + 1]); i++; } } if (startAt is not null && !Directory.Exists(startAt)) throw new Exception($"Start at directory <{startAt}> doesn't exist!"); if (startAt is not null && startAt.Length < source.Length) throw new Exception($"Start at directory <{startAt}> must be a subdirectory!"); if (destination is not null) { string? root = Path.GetPathRoot(destination); if (root is null || !Directory.Exists(root)) throw new NotSupportedException($"This method requires frontMatterYamlLines valid -d path <{root}>!"); if (!Directory.Exists(destination)) _ = Directory.CreateDirectory(destination); } result = new(destination, source, startAt); return result; } private static int CircularReference(ILogger logger, ReadOnlyDictionary relativeToCollection, ReadOnlyCollection gitOthersModifiedAndDeletedExcludingStandardFiles) { int result = 0; string line; string check; bool gitCheck; string[] lines; bool circularReference; MarkdownFile markdownFile; foreach (KeyValuePair relativeTo in relativeToCollection) { if (relativeTo.Value.Lines.Length == 0) continue; circularReference = false; lines = relativeTo.Value.Lines; markdownFile = relativeTo.Value.MarkdownFile; for (int i = 0; i < lines.Length; i++) { check = $"[[{markdownFile.FileNameWithoutExtension}]]"; if (!lines[i].Contains(check)) continue; line = lines[i].Replace(check, $"~~{markdownFile.FileName}~~"); if (lines[i] == line) continue; lines[i] = line; if (!circularReference) circularReference = true; } for (int i = 0; i < lines.Length; i++) { check = $"{markdownFile.FileNameWithoutExtension}|{markdownFile.FileNameWithoutExtension}]]"; if (!lines[i].Contains(check)) continue; line = lines[i].Replace(check, $"~~{markdownFile.FileName}~~"); if (lines[i] == line) continue; lines[i] = line; if (!circularReference) circularReference = true; } for (int i = 0; i < lines.Length; i++) { check = $"[{markdownFile.FileNameWithoutExtension}]({markdownFile.FileName})"; if (!lines[i].Contains(check)) continue; line = lines[i].Replace(check, $"~~{markdownFile.FileName}~~"); if (lines[i] == line) continue; lines[i] = line; logger.LogInformation("circular reference for <{file}>", markdownFile.FileName); if (!circularReference) circularReference = true; } if (!circularReference) continue; gitCheck = gitOthersModifiedAndDeletedExcludingStandardFiles.Contains(markdownFile.File); if (!gitCheck) continue; File.WriteAllLines(markdownFile.File, lines); result += 1; } return result; } private static int FindReplace(ReadOnlyDictionary relativeToCollection, ReadOnlyCollection gitOthersModifiedAndDeletedExcludingStandardFiles) { int result = 0; bool found; string line; string check; bool gitCheck; string[] lines; MarkdownFile markdownFile; foreach (KeyValuePair relativeTo in relativeToCollection) { if (relativeTo.Value.Lines.Length == 0) continue; found = false; lines = relativeTo.Value.Lines; markdownFile = relativeTo.Value.MarkdownFile; for (int i = 0; i < lines.Length; i++) { check = $"[[K-A/"; if (!lines[i].Contains(check)) continue; line = lines[i].Replace(check, "[[.kanbn/Archive/"); if (lines[i] == line) continue; lines[i] = line; if (!found) found = true; } for (int i = 0; i < lines.Length; i++) { check = $"[[K-T/"; if (!lines[i].Contains(check)) continue; line = lines[i].Replace(check, "[[.kanbn/Tasks/"); if (lines[i] == line) continue; lines[i] = line; if (!found) found = true; } if (!found) continue; gitCheck = gitOthersModifiedAndDeletedExcludingStandardFiles.Contains(markdownFile.File); if (!gitCheck) continue; File.WriteAllLines(markdownFile.File, lines); result += 1; } return result; } private static int ConvertToRelativePath(ILogger logger, ReadOnlyDictionary relativeToCollection, ReadOnlyCollection gitOthersModifiedAndDeletedExcludingStandardFiles) { int result = 0; bool write; string line; bool gitCheck; string[] lines; string[] segmentsA; string[] segmentsB; string[] segmentsC; MarkdownFile markdownFile; MarkdownFileH1AndRelativePath markdownFileH1AndRelativePath; ReadOnlyDictionary> keyValuePairs = GetKeyValuePairs(relativeToCollection); foreach (KeyValuePair relativeTo in relativeToCollection) { if (relativeTo.Value.Lines.Length == 0) continue; write = false; lines = relativeTo.Value.Lines; markdownFile = relativeTo.Value.MarkdownFile; for (int i = 0; i < lines.Length; i++) { segmentsA = lines[i].Split("]]"); if (segmentsA.Length is not 2 or 3) continue; segmentsB = segmentsA[0].Split("[["); if (segmentsB.Length is not 2 or 3) continue; segmentsC = segmentsB[^1].Split('|'); markdownFileH1AndRelativePath = GetRelativePath(keyValuePairs, markdownFile, segmentsC[0]); if (markdownFileH1AndRelativePath.MarkdownFile is null || markdownFileH1AndRelativePath.H1 is null || markdownFileH1AndRelativePath.RelativePath is null) { logger.LogInformation("Didn't find {line} in <{file}>", lines[i], markdownFile.FileNameWithoutExtension); continue; } line = $"{segmentsB[0]}[{markdownFileH1AndRelativePath.H1}]({markdownFileH1AndRelativePath.RelativePath.Replace('\\', '/')}){segmentsA[^1]}"; if (lines[i] == line) continue; lines[i] = line; if (!write) write = true; } if (!write) continue; gitCheck = gitOthersModifiedAndDeletedExcludingStandardFiles.Contains(markdownFile.File); if (!gitCheck) continue; File.WriteAllLines(markdownFile.File, lines); result += 1; } return result; } private static int ConvertFileToSlugName(AppSettings appSettings, ILogger logger, ReadOnlyDictionary relativeToCollection, ReadOnlyCollection gitOthersModifiedAndDeletedExcludingStandardFiles) { int result = 0; bool write; string file; string line; bool gitCheck; string[] lines; string fileName; string checkName; string? directory; string[] segmentsA; string[] segmentsB; string[] segmentsC; string checkFileName; string segmentsALast; string segmentsBFirst; MarkdownFile markdownFile; MarkdownFileH1AndRelativePath markdownFileH1AndRelativePath; ReadOnlyDictionary> keyValuePairs = GetKeyValuePairs(relativeToCollection); foreach (KeyValuePair relativeTo in relativeToCollection) { if (relativeTo.Value.Lines.Length == 0) continue; lines = relativeTo.Value.Lines; markdownFile = relativeTo.Value.MarkdownFile; if (markdownFile.IsKanbanIndex) continue; if (!File.Exists(markdownFile.File)) continue; write = false; for (int i = 0; i < lines.Length; i++) { segmentsA = lines[i].Split("]("); if (segmentsA.Length != 2) continue; segmentsALast = segmentsA[^1]; if (appSettings.ExcludeSchemes.Any(l => segmentsALast.StartsWith(l))) continue; segmentsB = segmentsALast.Split(")"); if (segmentsB.Length != 2) continue; segmentsBFirst = segmentsB[0]; file = Path.GetFullPath(Path.Combine(markdownFile.Directory, segmentsBFirst)); fileName = Path.GetFileName(file); directory = Path.GetDirectoryName(file); if (string.IsNullOrEmpty(directory)) continue; checkFileName = fileName.ToLower().Replace("%20", "-").Replace(' ', '-'); checkName = Path.Combine(directory, checkFileName); segmentsC = segmentsA[0].Split('['); markdownFileH1AndRelativePath = GetRelativePath(keyValuePairs, markdownFile, file); if (markdownFileH1AndRelativePath.MarkdownFile is null || markdownFileH1AndRelativePath.H1 is null || markdownFileH1AndRelativePath.RelativePath is null) { logger.LogInformation("Didn't find {line} in <{file}>", lines[i], markdownFile.FileNameWithoutExtension); continue; } line = $"{string.Join('[', segmentsC, 0, segmentsC.Length - 1)}[{markdownFileH1AndRelativePath.H1}]({markdownFileH1AndRelativePath.RelativePath.Replace('\\', '/')}){segmentsB[^1]}"; if (lines[i] == line) continue; if (fileName.Contains(' ') || fileName.Contains("%20")) { if (!File.Exists(file)) { logger.LogInformation("Didn't find <{file}>", file); continue; } if (File.Exists(checkName)) continue; File.Move(file, checkName); } else if (!fileName.Equals(fileName, StringComparison.CurrentCultureIgnoreCase)) { if (file != checkName) { if (!File.Exists(file)) { logger.LogInformation("Didn't find <{file}>", file); continue; } File.Move(file, checkName); } } lines[i] = line; if (!write) write = true; } if (!write) continue; gitCheck = gitOthersModifiedAndDeletedExcludingStandardFiles.Contains(markdownFile.File); if (!gitCheck) continue; File.WriteAllLines(markdownFile.File, lines); result += 1; } if (result == 0) result = ConvertFileToSlugName(relativeToCollection, gitOthersModifiedAndDeletedExcludingStandardFiles); return result; } private static ReadOnlyDictionary GetRelativeToCollection(AppSettings appSettings, Input input, ReadOnlyCollection gitOthersModifiedAndDeletedExcludingStandardFiles, bool force = false) { ReadOnlyDictionary results; string[] files = GetFiles(appSettings, input.Source); results = GetRelativeToCollection(appSettings, input, files, gitOthersModifiedAndDeletedExcludingStandardFiles, force); return new(results); } private static List GetRecursiveLines(AppSettings appSettings, Input input, ILogger logger, ReadOnlyDictionary relativeToCollection) { List results = []; string[] lines; List indentations; MarkdownFile markdownFile; List recursiveLines; ReadOnlyDictionary> keyValuePairs = GetKeyValuePairs(relativeToCollection); foreach (KeyValuePair relativeTo in relativeToCollection) { if (relativeTo.Value.Lines.Length == 0) continue; if (input.StartAt is null || !relativeTo.Value.MarkdownFile.File.StartsWith(input.StartAt) || Path.GetFileName(relativeTo.Value.MarkdownFile.Directory) != Path.GetFileName(input.StartAt)) continue; indentations = []; recursiveLines = []; lines = relativeTo.Value.Lines; markdownFile = relativeTo.Value.MarkdownFile; SetRecursiveLines(appSettings, logger, keyValuePairs, markdownFile.FileNameWithoutExtension, markdownFile, lines, indentations, recursiveLines); results.Add(new(relativeTo.Value.MarkdownFile, recursiveLines.ToArray())); } return results; } private static void Write(Input input, List markdownFileAndLinesCollection, ReadOnlyCollection gitOthersModifiedAndDeletedExcludingStandardFiles) { bool gitCheck; foreach (MarkdownFileAndLines markdownFileAndLines in markdownFileAndLinesCollection) { if (input.Destination is null) continue; gitCheck = gitOthersModifiedAndDeletedExcludingStandardFiles.Contains(markdownFileAndLines.MarkdownFile.File); if (!gitCheck) continue; File.WriteAllLines(Path.Combine(input.Destination, markdownFileAndLines.MarkdownFile.FileName), markdownFileAndLines.Lines); } } private static void SaveColumnToCards(Input input, ReadOnlyDictionary relativeToCollection) { if (string.IsNullOrEmpty(input.StartAt) || string.IsNullOrEmpty(input.Destination)) throw new NotSupportedException(); MarkdownFileAndLines? markdownFileAndLines = GetKanbanIndexMarkdownFileAndLines(relativeToCollection); if (markdownFileAndLines is not null && File.Exists(markdownFileAndLines.MarkdownFile.File)) { ReadOnlyDictionary> columnsToCards; string jsonFile = Path.Combine(input.Destination, $"{nameof(columnsToCards)}.json"); if (File.Exists(jsonFile)) File.Delete(jsonFile); columnsToCards = GetColumnsToCards(input, relativeToCollection, markdownFileAndLines); if (columnsToCards.Count == 0) File.WriteAllText(jsonFile, "{}"); else { string json = JsonSerializer.Serialize(columnsToCards, ColumnsAndCardsSourceGenerationContext.Default.ReadOnlyDictionaryStringListCard); File.WriteAllText(jsonFile, json); } } } private static (string type, string h1) GetTypeAndH1(AppSettings appSettings, string h1, List lines, LineNumber lineNumber) { string type = lineNumber.Type is null ? appSettings.DefaultNoteType : lines[lineNumber.Type.Value][5..].Trim().Trim('"'); string h1FromFile = lineNumber.H1 is null ? h1 : lines[lineNumber.H1.Value][2..]; return (type, h1FromFile); } private static int SetFrontMatterAndH1(AppSettings appSettings, ILogger logger, Input input, ReadOnlyDictionary relativeToCollection, ReadOnlyCollection gitOthersModifiedAndDeletedExcludingStandardFiles) { int result = 0; List results = []; bool gitCheck; string h1Line; string[] lines; string typeLine; TimeSpan timeSpan; string createdLine; string updatedLine; string lineDateTime; DateTime checkDateTime; DateTime creationDateTime; MarkdownFile markdownFile; string lineCreationFormat = "yyyy-MM-ddTHH:mm:ss.fffZ"; foreach (KeyValuePair relativeTo in relativeToCollection) { if (relativeTo.Value.Lines.Length == 0) continue; results.Clear(); lines = relativeTo.Value.Lines; markdownFile = relativeTo.Value.MarkdownFile; if (markdownFile.IsKanbanMarkdown) continue; results.AddRange(lines); typeLine = $"type: \"{appSettings.DefaultNoteType}\""; h1Line = $"# {markdownFile.FileNameWithoutExtension}"; creationDateTime = markdownFile.CreationDateTime > markdownFile.LastWriteDateTime ? markdownFile.LastWriteDateTime : markdownFile.CreationDateTime; gitCheck = gitOthersModifiedAndDeletedExcludingStandardFiles.Contains(markdownFile.File); createdLine = $"created: \"{creationDateTime.ToUniversalTime():yyyy-MM-ddTHH:mm:ss.fffZ}\""; updatedLine = $"updated: \"{markdownFile.LastWriteDateTime.ToUniversalTime():yyyy-MM-ddTHH:mm:ss.fffZ}\""; if (markdownFile.IsKanbanIndex) HelperKanbanMetadata.SetMetadata(markdownFile.Directory, new(lines), markdownFile.LineNumber, gitOthersModifiedAndDeletedExcludingStandardFiles); if (markdownFile.LineNumber.FrontMatterYamlEnd is null) { if (markdownFile.LineNumber.H1 is not null) results.Insert(0, string.Empty); else { results.Insert(0, string.Empty); results.Insert(0, h1Line); results.Insert(0, string.Empty); } results.Insert(0, "---"); if (gitCheck) { results.Insert(0, updatedLine); results.Insert(0, createdLine); } results.Insert(0, typeLine); results.Insert(0, "---"); } else { if (markdownFile.LineNumber.H1 is null) { results.Insert(markdownFile.LineNumber.FrontMatterYamlEnd.Value + 1, string.Empty); results.Insert(markdownFile.LineNumber.FrontMatterYamlEnd.Value + 1, h1Line); results.Insert(markdownFile.LineNumber.FrontMatterYamlEnd.Value + 1, string.Empty); } if (markdownFile.LineNumber.Type is null) results.Insert(markdownFile.LineNumber.FrontMatterYamlEnd.Value, typeLine); if (gitCheck) { if (markdownFile.LineNumber.Updated is null) results.Insert(markdownFile.LineNumber.FrontMatterYamlEnd.Value, updatedLine); else { lineDateTime = results[markdownFile.LineNumber.Updated.Value].Split(": ")[1].Trim('"'); if (!DateTime.TryParseExact(lineDateTime, lineCreationFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) results[markdownFile.LineNumber.Updated.Value] = updatedLine; else { timeSpan = new(checkDateTime.Ticks - markdownFile.LastWriteDateTime.Ticks); if (timeSpan.TotalDays is > 1 or < -1) results[markdownFile.LineNumber.Updated.Value] = updatedLine; } } if (markdownFile.LineNumber.Created is null) results.Insert(markdownFile.LineNumber.FrontMatterYamlEnd.Value, createdLine); else { lineDateTime = results[markdownFile.LineNumber.Created.Value].Split(": ")[1].Trim('"'); if (!DateTime.TryParseExact(lineDateTime, lineCreationFormat, CultureInfo.InvariantCulture, DateTimeStyles.None, out checkDateTime)) results[markdownFile.LineNumber.Created.Value] = createdLine; else { timeSpan = new(checkDateTime.Ticks - creationDateTime.Ticks); if (timeSpan.TotalDays > 1) results[markdownFile.LineNumber.Created.Value] = createdLine; if (timeSpan.TotalDays < -1) File.SetCreationTime(markdownFile.File, checkDateTime); } } } } if (results.Count == lines.Length && string.Join('\r', lines) == string.Join('\r', results)) continue; if (!gitCheck) continue; File.WriteAllLines(markdownFile.File, results); File.SetLastWriteTime(markdownFile.File, markdownFile.LastWriteDateTime); result += 1; } return result; } private static MarkdownFileH1AndRelativePath GetRelativePath(ReadOnlyDictionary> keyValuePairs, MarkdownFile markdownFile, string file) { MarkdownFileAndLines? result = GetMarkdownFile(keyValuePairs, markdownFile, file); return new(result?.MarkdownFile, result?.Lines, result?.MarkdownFile.H1, result is null ? null : Path.GetRelativePath(markdownFile.Directory, Path.GetFullPath(result.MarkdownFile.File))); } private static int SortFrontMatter(AppSettings appSettings, ILogger logger, Input input, ReadOnlyDictionary relativeToCollection, ReadOnlyCollection gitOthersModifiedAndDeletedExcludingStandardFiles) { int result = 0; List results = []; bool gitCheck; string[] lines; string frontMatterYaml; MarkdownFile markdownFile; string[] frontMatterYamlLines; ISerializer serializer = new SerializerBuilder().WithIndentedSequences().Build(); foreach (KeyValuePair relativeTo in relativeToCollection) { results.Clear(); if (relativeTo.Value.Lines.Length < 2) continue; lines = relativeTo.Value.Lines; markdownFile = relativeTo.Value.MarkdownFile; if (markdownFile.IsKanbanMarkdown) continue; gitCheck = gitOthersModifiedAndDeletedExcludingStandardFiles.Contains(markdownFile.File); if (!gitCheck) continue; if (markdownFile.LineNumber.FrontMatterYamlEnd is null) continue; frontMatterYaml = serializer.Serialize(markdownFile.FrontMatterYaml).Trim(); frontMatterYamlLines = frontMatterYaml.Split(Environment.NewLine); results.Add("---"); results.AddRange(frontMatterYamlLines); for (int i = markdownFile.LineNumber.FrontMatterYamlEnd.Value; i < lines.Length; i++) results.Add(lines[i]); if (results.Count == lines.Length && string.Join('\r', lines) == string.Join('\r', results)) continue; File.WriteAllLines(markdownFile.File, results); File.SetLastWriteTime(markdownFile.File, markdownFile.LastWriteDateTime); result += 1; } return result; } internal static void MarkdownWikiLinkVerification(AppSettings appSettings, ILogger logger, List args, CancellationToken cancellationToken) { int updated; bool usePathCombine = true; Input input = GetInput(args); ReadOnlyCollection gitOthersModifiedAndDeletedExcludingStandardFiles = HelperGit.GetOthersModifiedAndDeletedExcludingStandardFiles(input.Source, usePathCombine, cancellationToken); ReadOnlyDictionary relativeToCollection = GetRelativeToCollection(appSettings, input, gitOthersModifiedAndDeletedExcludingStandardFiles); updated = SetFrontMatterAndH1(appSettings, logger, input, relativeToCollection, gitOthersModifiedAndDeletedExcludingStandardFiles); if (updated != 0) { relativeToCollection = GetRelativeToCollection(appSettings, input, gitOthersModifiedAndDeletedExcludingStandardFiles); logger.LogInformation("{updated} Markdown file(s) were updated", updated); } updated = SortFrontMatter(appSettings, logger, input, relativeToCollection, gitOthersModifiedAndDeletedExcludingStandardFiles); if (updated != 0) { relativeToCollection = GetRelativeToCollection(appSettings, input, gitOthersModifiedAndDeletedExcludingStandardFiles); logger.LogInformation("{updated} Markdown file(s) were updated", updated); } updated = CircularReference(logger, relativeToCollection, gitOthersModifiedAndDeletedExcludingStandardFiles); if (updated != 0) { relativeToCollection = GetRelativeToCollection(appSettings, input, gitOthersModifiedAndDeletedExcludingStandardFiles); logger.LogInformation("{updated} Markdown file(s) were updated", updated); } updated = FindReplace(relativeToCollection, gitOthersModifiedAndDeletedExcludingStandardFiles); if (updated != 0) { relativeToCollection = GetRelativeToCollection(appSettings, input, gitOthersModifiedAndDeletedExcludingStandardFiles); logger.LogInformation("{updated} Markdown file(s) were updated", updated); } updated = ConvertToRelativePath(logger, relativeToCollection, gitOthersModifiedAndDeletedExcludingStandardFiles); if (updated != 0) { relativeToCollection = GetRelativeToCollection(appSettings, input, gitOthersModifiedAndDeletedExcludingStandardFiles); logger.LogInformation("{updated} Markdown file(s) were updated", updated); } updated = ConvertFileToSlugName(appSettings, logger, relativeToCollection, gitOthersModifiedAndDeletedExcludingStandardFiles); if (updated != 0) { relativeToCollection = GetRelativeToCollection(appSettings, input, gitOthersModifiedAndDeletedExcludingStandardFiles); logger.LogInformation("{updated} Markdown file(s) were updated", updated); } if (!string.IsNullOrEmpty(input.StartAt) && !string.IsNullOrEmpty(input.Destination)) { relativeToCollection = GetRelativeToCollection(appSettings, input, gitOthersModifiedAndDeletedExcludingStandardFiles, force: true); List markdownFileAndLinesCollection = GetRecursiveLines(appSettings, input, logger, relativeToCollection); Write(input, markdownFileAndLinesCollection, gitOthersModifiedAndDeletedExcludingStandardFiles); } if (!string.IsNullOrEmpty(input.StartAt) && !string.IsNullOrEmpty(input.Destination)) SaveColumnToCards(input, relativeToCollection); } }