using System.Collections.ObjectModel; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using System.Web; using Microsoft.Extensions.FileSystemGlobbing; using Microsoft.Extensions.Logging; namespace File_Folder_Helper.ADO2025.PI6; internal static partial class Helper20250519 { private record RelativePath(string LeftDirectory, string? RightDirectory, Record[] Records); [JsonSourceGenerationOptions(WriteIndented = true)] [JsonSerializable(typeof(RelativePath))] private partial class RelativePathSourceGenerationContext : JsonSerializerContext { } private record Review(Segment[]? AreEqual, Segment[]? LeftSideIsNewer, Segment[]? LeftSideOnly, Segment[]? NotEqualBut, Record[]? Records, Segment[]? RightSideIsNewer, Segment[]? RightSideOnly); [JsonSourceGenerationOptions(WriteIndented = true)] [JsonSerializable(typeof(Review))] private partial class ReviewSourceGenerationContext : JsonSerializerContext { } private record Record(string RelativePath, long Size, long Ticks); private record Segment(Record? Left, Record? Right); private record Verb(string Directory, string Display, string File, string Multipart, string RelativePath, long Size, long Ticks, string UrlEncodedFile); private record Logic(string Comment, char GreaterThan, bool? LeftSideIsNewer, int LeftSideIsNewerIndex, bool? LeftSideOnly, int LeftSideOnlyIndex, char LessThan, char Minus, bool? NotEqualBut, int NotEqualButIndex, char Plus, string[] Raw, bool? RightSideIsNewer, int RightSideIsNewerIndex, bool? RightSideOnly, int RightSideOnlyIndex) { internal static Logic? Get(string[] segments) { Logic? result; bool check = true; bool? notEqualBut; bool? leftSideOnly; bool? rightSideOnly; bool? leftSideIsNewer; const char plus = '+'; bool? rightSideIsNewer; const char minus = '-'; const char lessThan = 'L'; const int commentIndex = 5; const char greaterThan = 'G'; const int notEqualButIndex = 2; const int leftSideOnlyIndex = 0; const int rightSideOnlyIndex = 4; const int leftSideIsNewerIndex = 1; const int rightSideIsNewerIndex = 3; string comment = segments[commentIndex]; if (string.IsNullOrEmpty(segments[leftSideOnlyIndex])) leftSideOnly = null; else if (segments[leftSideOnlyIndex][0] == plus) leftSideOnly = true; else if (segments[leftSideOnlyIndex][0] == minus) leftSideOnly = false; else { check = false; leftSideOnly = null; } if (string.IsNullOrEmpty(segments[leftSideIsNewerIndex])) leftSideIsNewer = null; else if (segments[leftSideIsNewerIndex][0] == greaterThan) leftSideIsNewer = true; else if (segments[leftSideIsNewerIndex][0] == lessThan) leftSideIsNewer = false; else { check = false; leftSideIsNewer = null; } if (string.IsNullOrEmpty(segments[notEqualButIndex])) notEqualBut = null; else if (segments[notEqualButIndex][0] == greaterThan) notEqualBut = true; else if (segments[notEqualButIndex][0] == lessThan) notEqualBut = false; else { check = false; notEqualBut = null; } if (string.IsNullOrEmpty(segments[rightSideIsNewerIndex])) rightSideIsNewer = null; else if (segments[rightSideIsNewerIndex][0] == greaterThan) rightSideIsNewer = true; else if (segments[rightSideIsNewerIndex][0] == lessThan) rightSideIsNewer = false; else { check = false; rightSideIsNewer = null; } if (string.IsNullOrEmpty(segments[rightSideOnlyIndex])) rightSideOnly = null; else if (segments[rightSideOnlyIndex][0] == plus) rightSideOnly = true; else if (segments[rightSideOnlyIndex][0] == minus) rightSideOnly = false; else { check = false; rightSideOnly = null; } result = !check ? null : new(Comment: comment, GreaterThan: greaterThan, LeftSideIsNewerIndex: leftSideIsNewerIndex, LeftSideIsNewer: leftSideIsNewer, LeftSideOnly: leftSideOnly, LeftSideOnlyIndex: leftSideOnlyIndex, LessThan: lessThan, Minus: minus, NotEqualBut: notEqualBut, NotEqualButIndex: notEqualButIndex, Plus: plus, RightSideIsNewer: rightSideIsNewer, RightSideIsNewerIndex: rightSideIsNewerIndex, RightSideOnly: rightSideOnly, Raw: segments, RightSideOnlyIndex: rightSideOnlyIndex); return result; } } internal static void LiveSync(ILogger logger, List args) { string[] segments = args[9].Split('~'); Logic? logic = segments.Length != 6 ? null : Logic.Get(segments); string[] baseAddresses = args.Count < 5 ? [] : args[5].Split('~'); if (logic is null || baseAddresses.Length == 0) logger.LogInformation("Invalid input!"); else { string rightDirectory = Path.GetFullPath(args[0].Split('~')[0]); string excludePatternsFile = Path.Combine(rightDirectory, args[4]); string includePatternsFile = Path.Combine(rightDirectory, args[3]); Matcher matcher = GetMatcher(excludePatternsFile, includePatternsFile); ReadOnlyCollection records = GetRecords(rightDirectory, matcher); if (records.Count == 0) logger.LogInformation("No source records"); else { string page = args[6]; string leftDirectory = Path.GetFullPath(args[2].Split('~')[0]); RelativePath relativePath = new(LeftDirectory: leftDirectory, RightDirectory: rightDirectory, Records: records.ToArray()); LiveSync180(logger, logic, baseAddresses, page, relativePath); } } } private static Matcher GetMatcher(string excludePatternsFile, string includePatternsFile) { Matcher result = new(); result.AddIncludePatterns(!File.Exists(includePatternsFile) ? ["*"] : File.ReadAllLines(includePatternsFile)); result.AddExcludePatterns(!File.Exists(excludePatternsFile) ? ["System Volume Information"] : File.ReadAllLines(excludePatternsFile)); return result; } private static ReadOnlyCollection GetRecords(string directory, Matcher matcher) { List results = [ new(RelativePath: directory, Size: 0, Ticks: 0)]; Record record; FileInfo fileInfo; string relativePath; ReadOnlyCollection> collection = Helpers.HelperDirectory.GetFilesCollection(directory, "*", "*"); foreach (ReadOnlyCollection c in collection) { foreach (string f in c) { if (!matcher.Match(directory, f).HasMatches) continue; fileInfo = new(f); if (fileInfo.Length == 0) continue; relativePath = Path.GetRelativePath(directory, fileInfo.FullName); record = new(RelativePath: relativePath, Size: fileInfo.Length, Ticks: fileInfo.LastWriteTime.ToUniversalTime().Ticks); results.Add(record); } } return results.AsReadOnly(); } private static void LiveSync180(ILogger logger, Logic logic, string[] baseAddresses, string page, RelativePath relativePath) { Review? review; Task response; Task httpResponseMessage; StringContent stringContent = new(JsonSerializer.Serialize(relativePath, RelativePathSourceGenerationContext.Default.RelativePath), Encoding.UTF8, "application/json"); foreach (string baseAddress in baseAddresses) { if (!baseAddress.StartsWith("http:")) { logger.LogInformation("Not supported URL <{url}>", baseAddress); } else { HttpClient httpClient = new(); httpClient.BaseAddress = new(baseAddress); httpResponseMessage = httpClient.PostAsync(page, stringContent); httpResponseMessage.Wait(); if (!httpResponseMessage.Result.IsSuccessStatusCode) { logger.LogInformation("Failed to download: <{uniformResourceLocator}>;", httpClient.BaseAddress); } else { response = httpResponseMessage.Result.Content.ReadAsStringAsync(); response.Wait(); review = JsonSerializer.Deserialize(response.Result, ReviewSourceGenerationContext.Default.Review); if (review is null) { logger.LogInformation("Failed to download: <{uniformResourceLocator}>;", httpClient.BaseAddress); continue; } LiveSync(logger, logic, page, relativePath, httpClient, review); } } } } private static void LiveSync(ILogger logger, Logic l, string page, RelativePath relativePath, HttpClient httpClient, Review review) { if (review.NotEqualBut.Length > 0 && l is not null && l.NotEqualBut is not null && l.Raw[l.NotEqualButIndex][0] == l.Minus && !l.NotEqualBut.Value) logger.LogDebug("Doing nothing with {name}", nameof(Logic.NotEqualBut)); if (review.LeftSideOnly.Length > 0 && l is not null && l.LeftSideOnly is not null && l.Raw[l.LeftSideOnlyIndex][0] == l.Minus && !l.LeftSideOnly.Value) LiveSync(logger, page, relativePath, httpClient, relativePath.LeftDirectory, (from x in review.LeftSideOnly select x.Left).ToArray().AsReadOnly(), HttpMethod.Delete, delete: false); if (review.LeftSideIsNewer.Length > 0 && l is not null && l.LeftSideIsNewer is not null && l.Raw[l.LeftSideIsNewerIndex][0] == l.LessThan && !l.LeftSideIsNewer.Value) throw new Exception(); // LiveSync(logger, page, relativePath, httpClient, relativePath.LeftDirectory, (from x in review.LeftSideIsNewer select x.Left).ToArray().AsReadOnly(), HttpMethod.Patch, delete: true); if (review.RightSideIsNewer.Length > 0 && l is not null && l.RightSideIsNewer is not null && l.Raw[l.RightSideIsNewerIndex][0] == l.LessThan && !l.RightSideIsNewer.Value) throw new Exception(); // LiveSync(logger, page, relativePath, httpClient, relativePath.RightDirectory, (from x in review.RightSideIsNewer select x.Right).ToArray().AsReadOnly(), HttpMethod.Patch, delete: true); if (review.RightSideOnly.Length > 0 && l is not null && l.RightSideOnly is not null && l.Raw[l.RightSideOnlyIndex][0] == l.Plus && l.RightSideOnly.Value) throw new Exception(); // LiveSync(logger, page, relativePath, httpClient, relativePath.RightDirectory, (from x in review.RightSideOnly select x.Right).ToArray().AsReadOnly(), HttpMethod.Put, delete: false); if (review.RightSideOnly.Length > 0 && l is not null && l.RightSideOnly is not null && l.Raw[l.RightSideOnlyIndex][0] == l.Minus && !l.RightSideOnly.Value) LiveSync(logger, page, relativePath, httpClient, relativePath.RightDirectory, (from x in review.RightSideOnly select x.Right).ToArray().AsReadOnly(), httpMethod: null, delete: true); if (review.LeftSideOnly.Length > 0 && l is not null && l.LeftSideOnly is not null && l.Raw[l.LeftSideOnlyIndex][0] == l.Plus && l.LeftSideOnly.Value) LiveSync(logger, page, relativePath, httpClient, relativePath.LeftDirectory, (from x in review.LeftSideOnly select x.Left).ToArray().AsReadOnly(), HttpMethod.Get, delete: false); if (review.LeftSideIsNewer.Length > 0 && l is not null && l.LeftSideIsNewer is not null && l.Raw[l.LeftSideIsNewerIndex][0] == l.GreaterThan && l.LeftSideIsNewer.Value) LiveSync(logger, page, relativePath, httpClient, relativePath.LeftDirectory, (from x in review.LeftSideIsNewer select x.Left).ToArray().AsReadOnly(), HttpMethod.Get, delete: true); if (review.NotEqualBut.Length > 0 && l is not null && l.NotEqualBut is not null && l.Raw[l.NotEqualButIndex][0] == l.Plus && l.NotEqualBut.Value) LiveSync(logger, page, relativePath, httpClient, relativePath.LeftDirectory, (from x in review.NotEqualBut select x.Left).ToArray().AsReadOnly(), HttpMethod.Get, delete: true); if (review.RightSideIsNewer.Length > 0 && l is not null && l.RightSideIsNewer is not null && l.Raw[l.RightSideIsNewerIndex][0] == l.GreaterThan && l.RightSideIsNewer.Value) LiveSync(logger, page, relativePath, httpClient, relativePath.RightDirectory, (from x in review.RightSideIsNewer select x.Right).ToArray().AsReadOnly(), HttpMethod.Get, delete: true); } private static void LiveSync(ILogger logger, string page, RelativePath relativePath, HttpClient httpClient, string directory, ReadOnlyCollection records, HttpMethod? httpMethod, bool delete) { long sum; try { sum = records.Sum(l => l.Size); } catch (Exception) { sum = 0; } string size = GetSizeWithSuffix(sum); if (delete) { logger.LogInformation("Starting to delete {count} file(s) [{sum}]", records.Count, size); PreformDeletes(logger, relativePath.RightDirectory, records); logger.LogInformation("Deleted {count} file(s) [{sum}]", records.Count, size); } if (httpMethod is not null) { logger.LogInformation("Starting to {httpMethod} {count} file(s) [{sum}]", httpMethod.ToString().ToLower(), records.Count, size); Preform(logger, page, directory, records, httpClient, httpMethod); logger.LogInformation("{httpMethod}'ed {count} file(s) [{sum}]", httpMethod.ToString(), records.Count, size); } } private static string GetSizeWithSuffix(long value) { string result; int i = 0; string[] SizeSuffixes = ["bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"]; if (value < 0) { result = "-" + GetSizeWithSuffix(-value); } else { while (Math.Round(value / 1024f) >= 1) { value /= 1024; i++; } result = string.Format("{0:n1} {1}", value, SizeSuffixes[i]); } return result; } private static void PreformDeletes(ILogger logger, string directory, ReadOnlyCollection records) { string size; Record? record; string count = records.Count.ToString("000000"); #if ShellProgressBar ProgressBar progressBar = new(records.Count, $"Deleting: {count};", new ProgressBarOptions() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }); #endif for (int i = 0; i < records.Count; i++) { #if ShellProgressBar progressBar.Tick(); #endif record = records[i]; if (record is null) continue; size = GetSizeWithSuffix(record.Size); try { File.Delete(Path.Combine(directory, record.RelativePath)); logger.LogInformation("{i} of {count} - Deleted: <{RelativePath}> - {size};", i.ToString("000000"), count, record.RelativePath, size); } catch (Exception) { logger.LogInformation("Failed to delete: <{RelativePath}> - {size};", record.RelativePath, size); } } #if ShellProgressBar progressBar.Dispose(); #endif } private static void Preform(ILogger logger, string page, string directory, ReadOnlyCollection records, HttpClient httpClient, HttpMethod httpMethod) { Verb verb; long ticks; string size; string iValue; string duration; DateTime dateTime; Task response; HttpRequestMessage httpRequestMessage; Task httpResponseMessage; string count = records.Count.ToString("000000"); MultipartFormDataContent multipartFormDataContent; ReadOnlyCollection collection = GetVerbCollection(directory, records); #if ShellProgressBar ProgressBar progressBar = new(downloads.Count, $"{httpMethod}ing: {count};", new ProgressBarOptions() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true }); #endif for (int i = 0; i < collection.Count; i++) { verb = collection[i]; #if ShellProgressBar progressBar.Tick(); #endif ticks = DateTime.Now.Ticks; iValue = (i + 1).ToString("000000"); size = GetSizeWithSuffix(verb.Size); if (httpMethod == HttpMethod.Get || httpMethod == HttpMethod.Delete) { httpRequestMessage = new(httpMethod, $"{page}size={verb.Size}&ticks={verb.Ticks}&path={verb.UrlEncodedFile}"); } else if (httpMethod == HttpMethod.Patch || httpMethod == HttpMethod.Put) { httpRequestMessage = new(httpMethod, $"{page}path={verb.Directory}"); multipartFormDataContent = new(); multipartFormDataContent.Add(new ByteArrayContent(File.ReadAllBytes(verb.File)), "formFiles", verb.Multipart); multipartFormDataContent.Add(new StringContent(verb.Directory), "path", iValue); httpRequestMessage.Content = multipartFormDataContent; } else throw new NotImplementedException(); httpResponseMessage = httpClient.SendAsync(httpRequestMessage); httpResponseMessage.Wait(-1); if (!httpResponseMessage.Result.IsSuccessStatusCode) logger.LogInformation("Failed to {httpMethod}: <{display}> - {size};", httpMethod, verb.Display, size); else { try { if (httpMethod != HttpMethod.Get) { duration = GetDurationWithSuffix(ticks); } else { response = httpResponseMessage.Result.Content.ReadAsStringAsync(); response.Wait(); File.WriteAllText(verb.File, response.Result); duration = GetDurationWithSuffix(ticks); dateTime = new DateTime(verb.Ticks).ToLocalTime(); File.SetLastWriteTime(verb.File, dateTime); } logger.LogInformation("{i} of {count} - {httpMethod}'ed: <{display}> - {size} - {timeSpan};", iValue, count, httpMethod, verb.Display, size, duration); } catch (Exception) { logger.LogInformation("Failed to {httpMethod}: <{display}> - {size};", httpMethod, verb.Display, size); } } } #if ShellProgressBar progressBar.Dispose(); #endif } private static ReadOnlyCollection GetVerbCollection(string directory, ReadOnlyCollection records) { List results = []; Verb verb; string checkFile; string checkFileName; string? checkDirectory; List collection = []; foreach (Record record in records) { checkFile = Path.Combine(directory, record.RelativePath); checkFileName = Path.GetFileName(checkFile); checkDirectory = Path.GetDirectoryName(checkFile); if (string.IsNullOrEmpty(checkDirectory)) continue; if (!Directory.Exists(checkDirectory)) _ = Directory.CreateDirectory(checkDirectory); if (File.Exists(checkFile) && new FileInfo(checkFile).Length == 0) File.Delete(checkFile); verb = new(Directory: checkDirectory, Display: $"{checkFileName}{Environment.NewLine}{checkDirectory}", File: checkFile, Multipart: $"RelativePath:{record.RelativePath}|Size:{record.Size}|Ticks:{record.Ticks};", RelativePath: record.RelativePath, Size: record.Size, Ticks: record.Ticks, UrlEncodedFile: HttpUtility.UrlEncode(checkFile)); collection.Add(verb); } Verb[] sorted = (from l in collection orderby l.Size select l).ToArray(); int stop = sorted.Length < 100 ? sorted.Length : 100; for (int i = 0; i < stop; i++) results.Add(sorted[i]); for (int i = sorted.Length - 1; i > stop - 1; i--) results.Add(sorted[i]); if (collection.Count != results.Count) throw new Exception(); return results.AsReadOnly(); } private static string GetDurationWithSuffix(long ticks) { string result; TimeSpan timeSpan = new(DateTime.Now.Ticks - ticks); if (timeSpan.TotalMilliseconds < 1000) result = $"{timeSpan.Milliseconds} ms"; else if (timeSpan.TotalMilliseconds < 60000) result = $"{Math.Floor(timeSpan.TotalSeconds)} s"; else if (timeSpan.TotalMilliseconds < 3600000) result = $"{Math.Floor(timeSpan.TotalMinutes)} m"; else result = $"{Math.Floor(timeSpan.TotalHours)} h"; return result; } }