445 lines
23 KiB
C#
445 lines
23 KiB
C#
using System.Collections.ObjectModel;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Serialization;
|
|
using System.Web;
|
|
|
|
using Microsoft.Extensions.FileSystemGlobbing;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
namespace File_Folder_Helper.ADO2025.PI6;
|
|
|
|
internal static partial class Helper20250519 {
|
|
|
|
private record RelativePath(string LeftDirectory,
|
|
string? RightDirectory,
|
|
Record[] Records);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(RelativePath))]
|
|
private partial class RelativePathSourceGenerationContext : JsonSerializerContext {
|
|
}
|
|
|
|
private record Review(Segment[]? AreEqual,
|
|
Segment[]? LeftSideIsNewer,
|
|
Segment[]? LeftSideOnly,
|
|
Segment[]? NotEqualBut,
|
|
Record[]? Records,
|
|
Segment[]? RightSideIsNewer,
|
|
Segment[]? RightSideOnly);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(Review))]
|
|
private partial class ReviewSourceGenerationContext : JsonSerializerContext {
|
|
}
|
|
|
|
private record Record(string RelativePath,
|
|
long Size,
|
|
long Ticks);
|
|
|
|
private record Segment(Record? Left,
|
|
Record? Right);
|
|
|
|
private record Verb(string Directory,
|
|
string Display,
|
|
string File,
|
|
string Multipart,
|
|
string RelativePath,
|
|
long Size,
|
|
long Ticks,
|
|
string UrlEncodedFile);
|
|
|
|
private record Logic(string Comment,
|
|
char GreaterThan,
|
|
bool? LeftSideIsNewer,
|
|
int LeftSideIsNewerIndex,
|
|
bool? LeftSideOnly,
|
|
int LeftSideOnlyIndex,
|
|
char LessThan,
|
|
char Minus,
|
|
bool? NotEqualBut,
|
|
int NotEqualButIndex,
|
|
char Plus,
|
|
string[] Raw,
|
|
bool? RightSideIsNewer,
|
|
int RightSideIsNewerIndex,
|
|
bool? RightSideOnly,
|
|
int RightSideOnlyIndex) {
|
|
|
|
internal static Logic? Get(string[] segments) {
|
|
Logic? result;
|
|
bool check = true;
|
|
bool? notEqualBut;
|
|
bool? leftSideOnly;
|
|
bool? rightSideOnly;
|
|
bool? leftSideIsNewer;
|
|
const char plus = '+';
|
|
bool? rightSideIsNewer;
|
|
const char minus = '-';
|
|
const char lessThan = 'L';
|
|
const int commentIndex = 5;
|
|
const char greaterThan = 'G';
|
|
const int notEqualButIndex = 2;
|
|
const int leftSideOnlyIndex = 0;
|
|
const int rightSideOnlyIndex = 4;
|
|
const int leftSideIsNewerIndex = 1;
|
|
const int rightSideIsNewerIndex = 3;
|
|
string comment = segments[commentIndex];
|
|
if (string.IsNullOrEmpty(segments[leftSideOnlyIndex]))
|
|
leftSideOnly = null;
|
|
else if (segments[leftSideOnlyIndex][0] == plus)
|
|
leftSideOnly = true;
|
|
else if (segments[leftSideOnlyIndex][0] == minus)
|
|
leftSideOnly = false;
|
|
else {
|
|
check = false;
|
|
leftSideOnly = null;
|
|
}
|
|
if (string.IsNullOrEmpty(segments[leftSideIsNewerIndex]))
|
|
leftSideIsNewer = null;
|
|
else if (segments[leftSideIsNewerIndex][0] == greaterThan)
|
|
leftSideIsNewer = true;
|
|
else if (segments[leftSideIsNewerIndex][0] == lessThan)
|
|
leftSideIsNewer = false;
|
|
else {
|
|
check = false;
|
|
leftSideIsNewer = null;
|
|
}
|
|
if (string.IsNullOrEmpty(segments[notEqualButIndex]))
|
|
notEqualBut = null;
|
|
else if (segments[notEqualButIndex][0] == greaterThan)
|
|
notEqualBut = true;
|
|
else if (segments[notEqualButIndex][0] == lessThan)
|
|
notEqualBut = false;
|
|
else {
|
|
check = false;
|
|
notEqualBut = null;
|
|
}
|
|
if (string.IsNullOrEmpty(segments[rightSideIsNewerIndex]))
|
|
rightSideIsNewer = null;
|
|
else if (segments[rightSideIsNewerIndex][0] == greaterThan)
|
|
rightSideIsNewer = true;
|
|
else if (segments[rightSideIsNewerIndex][0] == lessThan)
|
|
rightSideIsNewer = false;
|
|
else {
|
|
check = false;
|
|
rightSideIsNewer = null;
|
|
}
|
|
if (string.IsNullOrEmpty(segments[rightSideOnlyIndex]))
|
|
rightSideOnly = null;
|
|
else if (segments[rightSideOnlyIndex][0] == plus)
|
|
rightSideOnly = true;
|
|
else if (segments[rightSideOnlyIndex][0] == minus)
|
|
rightSideOnly = false;
|
|
else {
|
|
check = false;
|
|
rightSideOnly = null;
|
|
}
|
|
result = !check ? null : new(Comment: comment,
|
|
GreaterThan: greaterThan,
|
|
LeftSideIsNewerIndex: leftSideIsNewerIndex,
|
|
LeftSideIsNewer: leftSideIsNewer,
|
|
LeftSideOnly: leftSideOnly,
|
|
LeftSideOnlyIndex: leftSideOnlyIndex,
|
|
LessThan: lessThan,
|
|
Minus: minus,
|
|
NotEqualBut: notEqualBut,
|
|
NotEqualButIndex: notEqualButIndex,
|
|
Plus: plus,
|
|
RightSideIsNewer: rightSideIsNewer,
|
|
RightSideIsNewerIndex: rightSideIsNewerIndex,
|
|
RightSideOnly: rightSideOnly,
|
|
Raw: segments,
|
|
RightSideOnlyIndex: rightSideOnlyIndex);
|
|
return result;
|
|
}
|
|
|
|
}
|
|
|
|
internal static void LiveSync(ILogger<Worker> logger, List<string> args) {
|
|
string[] segments = args[9].Split('~');
|
|
Logic? logic = segments.Length != 6 ? null : Logic.Get(segments);
|
|
string[] baseAddresses = args.Count < 5 ? [] : args[5].Split('~');
|
|
if (logic is null || baseAddresses.Length == 0)
|
|
logger.LogInformation("Invalid input!");
|
|
else {
|
|
string rightDirectory = Path.GetFullPath(args[0].Split('~')[0]);
|
|
string excludePatternsFile = Path.Combine(rightDirectory, args[4]);
|
|
string includePatternsFile = Path.Combine(rightDirectory, args[3]);
|
|
Matcher matcher = GetMatcher(excludePatternsFile, includePatternsFile);
|
|
ReadOnlyCollection<Record> records = GetRecords(rightDirectory, matcher);
|
|
if (records.Count == 0)
|
|
logger.LogInformation("No source records");
|
|
else {
|
|
string page = args[6];
|
|
string leftDirectory = Path.GetFullPath(args[2].Split('~')[0]);
|
|
RelativePath relativePath = new(LeftDirectory: leftDirectory, RightDirectory: rightDirectory, Records: records.ToArray());
|
|
LiveSync180(logger, logic, baseAddresses, page, relativePath);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static Matcher GetMatcher(string excludePatternsFile, string includePatternsFile) {
|
|
Matcher result = new();
|
|
result.AddIncludePatterns(!File.Exists(includePatternsFile) ? ["*"] : File.ReadAllLines(includePatternsFile));
|
|
result.AddExcludePatterns(!File.Exists(excludePatternsFile) ? ["System Volume Information"] : File.ReadAllLines(excludePatternsFile));
|
|
return result;
|
|
}
|
|
|
|
private static ReadOnlyCollection<Record> GetRecords(string directory, Matcher matcher) {
|
|
List<Record> results = [
|
|
new(RelativePath: directory,
|
|
Size: 0,
|
|
Ticks: 0)];
|
|
Record record;
|
|
FileInfo fileInfo;
|
|
string relativePath;
|
|
ReadOnlyCollection<ReadOnlyCollection<string>> collection = Helpers.HelperDirectory.GetFilesCollection(directory, "*", "*");
|
|
foreach (ReadOnlyCollection<string> c in collection) {
|
|
foreach (string f in c) {
|
|
if (!matcher.Match(directory, f).HasMatches)
|
|
continue;
|
|
fileInfo = new(f);
|
|
if (fileInfo.Length == 0)
|
|
continue;
|
|
relativePath = Path.GetRelativePath(directory, fileInfo.FullName);
|
|
record = new(RelativePath: relativePath,
|
|
Size: fileInfo.Length,
|
|
Ticks: fileInfo.LastWriteTime.ToUniversalTime().Ticks);
|
|
results.Add(record);
|
|
}
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static void LiveSync180(ILogger<Worker> logger, Logic logic, string[] baseAddresses, string page, RelativePath relativePath) {
|
|
Review? review;
|
|
Task<string> response;
|
|
Task<HttpResponseMessage> httpResponseMessage;
|
|
StringContent stringContent = new(JsonSerializer.Serialize(relativePath, RelativePathSourceGenerationContext.Default.RelativePath), Encoding.UTF8, "application/json");
|
|
foreach (string baseAddress in baseAddresses) {
|
|
if (!baseAddress.StartsWith("http:")) {
|
|
logger.LogInformation("Not supported URL <{url}>", baseAddress);
|
|
} else {
|
|
HttpClient httpClient = new();
|
|
httpClient.BaseAddress = new(baseAddress);
|
|
httpResponseMessage = httpClient.PostAsync(page, stringContent);
|
|
httpResponseMessage.Wait();
|
|
if (!httpResponseMessage.Result.IsSuccessStatusCode) {
|
|
logger.LogInformation("Failed to download: <{uniformResourceLocator}>;", httpClient.BaseAddress);
|
|
} else {
|
|
response = httpResponseMessage.Result.Content.ReadAsStringAsync();
|
|
response.Wait();
|
|
review = JsonSerializer.Deserialize(response.Result, ReviewSourceGenerationContext.Default.Review);
|
|
if (review is null) {
|
|
logger.LogInformation("Failed to download: <{uniformResourceLocator}>;", httpClient.BaseAddress);
|
|
continue;
|
|
}
|
|
LiveSync(logger, logic, page, relativePath, httpClient, review);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private static void LiveSync(ILogger<Worker> logger, Logic l, string page, RelativePath relativePath, HttpClient httpClient, Review review) {
|
|
if (review.NotEqualBut.Length > 0 && l is not null && l.NotEqualBut is not null && l.Raw[l.NotEqualButIndex][0] == l.Minus && !l.NotEqualBut.Value)
|
|
logger.LogDebug("Doing nothing with {name}", nameof(Logic.NotEqualBut));
|
|
if (review.LeftSideOnly.Length > 0 && l is not null && l.LeftSideOnly is not null && l.Raw[l.LeftSideOnlyIndex][0] == l.Minus && !l.LeftSideOnly.Value)
|
|
LiveSync(logger, page, relativePath, httpClient, relativePath.LeftDirectory, (from x in review.LeftSideOnly select x.Left).ToArray().AsReadOnly(), HttpMethod.Delete, delete: false);
|
|
if (review.LeftSideIsNewer.Length > 0 && l is not null && l.LeftSideIsNewer is not null && l.Raw[l.LeftSideIsNewerIndex][0] == l.LessThan && !l.LeftSideIsNewer.Value)
|
|
throw new Exception(); // LiveSync(logger, page, relativePath, httpClient, relativePath.LeftDirectory, (from x in review.LeftSideIsNewer select x.Left).ToArray().AsReadOnly(), HttpMethod.Patch, delete: true);
|
|
if (review.RightSideIsNewer.Length > 0 && l is not null && l.RightSideIsNewer is not null && l.Raw[l.RightSideIsNewerIndex][0] == l.LessThan && !l.RightSideIsNewer.Value)
|
|
throw new Exception(); // LiveSync(logger, page, relativePath, httpClient, relativePath.RightDirectory, (from x in review.RightSideIsNewer select x.Right).ToArray().AsReadOnly(), HttpMethod.Patch, delete: true);
|
|
if (review.RightSideOnly.Length > 0 && l is not null && l.RightSideOnly is not null && l.Raw[l.RightSideOnlyIndex][0] == l.Plus && l.RightSideOnly.Value)
|
|
throw new Exception(); // LiveSync(logger, page, relativePath, httpClient, relativePath.RightDirectory, (from x in review.RightSideOnly select x.Right).ToArray().AsReadOnly(), HttpMethod.Put, delete: false);
|
|
if (review.RightSideOnly.Length > 0 && l is not null && l.RightSideOnly is not null && l.Raw[l.RightSideOnlyIndex][0] == l.Minus && !l.RightSideOnly.Value)
|
|
LiveSync(logger, page, relativePath, httpClient, relativePath.RightDirectory, (from x in review.RightSideOnly select x.Right).ToArray().AsReadOnly(), httpMethod: null, delete: true);
|
|
if (review.LeftSideOnly.Length > 0 && l is not null && l.LeftSideOnly is not null && l.Raw[l.LeftSideOnlyIndex][0] == l.Plus && l.LeftSideOnly.Value)
|
|
LiveSync(logger, page, relativePath, httpClient, relativePath.LeftDirectory, (from x in review.LeftSideOnly select x.Left).ToArray().AsReadOnly(), HttpMethod.Get, delete: false);
|
|
if (review.LeftSideIsNewer.Length > 0 && l is not null && l.LeftSideIsNewer is not null && l.Raw[l.LeftSideIsNewerIndex][0] == l.GreaterThan && l.LeftSideIsNewer.Value)
|
|
LiveSync(logger, page, relativePath, httpClient, relativePath.LeftDirectory, (from x in review.LeftSideIsNewer select x.Left).ToArray().AsReadOnly(), HttpMethod.Get, delete: true);
|
|
if (review.NotEqualBut.Length > 0 && l is not null && l.NotEqualBut is not null && l.Raw[l.NotEqualButIndex][0] == l.Plus && l.NotEqualBut.Value)
|
|
LiveSync(logger, page, relativePath, httpClient, relativePath.LeftDirectory, (from x in review.NotEqualBut select x.Left).ToArray().AsReadOnly(), HttpMethod.Get, delete: true);
|
|
if (review.RightSideIsNewer.Length > 0 && l is not null && l.RightSideIsNewer is not null && l.Raw[l.RightSideIsNewerIndex][0] == l.GreaterThan && l.RightSideIsNewer.Value)
|
|
LiveSync(logger, page, relativePath, httpClient, relativePath.RightDirectory, (from x in review.RightSideIsNewer select x.Right).ToArray().AsReadOnly(), HttpMethod.Get, delete: true);
|
|
}
|
|
|
|
private static void LiveSync(ILogger<Worker> logger, string page, RelativePath relativePath, HttpClient httpClient, string directory, ReadOnlyCollection<Record> records, HttpMethod? httpMethod, bool delete) {
|
|
long sum;
|
|
try { sum = records.Sum(l => l.Size); } catch (Exception) { sum = 0; }
|
|
string size = GetSizeWithSuffix(sum);
|
|
if (delete) {
|
|
logger.LogInformation("Starting to delete {count} file(s) [{sum}]", records.Count, size);
|
|
PreformDeletes(logger, relativePath.RightDirectory, records);
|
|
logger.LogInformation("Deleted {count} file(s) [{sum}]", records.Count, size);
|
|
}
|
|
if (httpMethod is not null) {
|
|
logger.LogInformation("Starting to {httpMethod} {count} file(s) [{sum}]", httpMethod.ToString().ToLower(), records.Count, size);
|
|
Preform(logger, page, directory, records, httpClient, httpMethod);
|
|
logger.LogInformation("{httpMethod}'ed {count} file(s) [{sum}]", httpMethod.ToString(), records.Count, size);
|
|
}
|
|
}
|
|
|
|
private static string GetSizeWithSuffix(long value) {
|
|
string result;
|
|
int i = 0;
|
|
string[] SizeSuffixes = ["bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
|
|
if (value < 0) {
|
|
result = "-" + GetSizeWithSuffix(-value);
|
|
} else {
|
|
while (Math.Round(value / 1024f) >= 1) {
|
|
value /= 1024;
|
|
i++;
|
|
}
|
|
result = string.Format("{0:n1} {1}", value, SizeSuffixes[i]);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
private static void PreformDeletes(ILogger<Worker> logger, string directory, ReadOnlyCollection<Record> records) {
|
|
string size;
|
|
Record? record;
|
|
string count = records.Count.ToString("000000");
|
|
#if ShellProgressBar
|
|
ProgressBar progressBar = new(records.Count, $"Deleting: {count};", new ProgressBarOptions() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true });
|
|
#endif
|
|
for (int i = 0; i < records.Count; i++) {
|
|
#if ShellProgressBar
|
|
progressBar.Tick();
|
|
#endif
|
|
record = records[i];
|
|
if (record is null)
|
|
continue;
|
|
size = GetSizeWithSuffix(record.Size);
|
|
try {
|
|
File.Delete(Path.Combine(directory, record.RelativePath));
|
|
logger.LogInformation("{i} of {count} - Deleted: <{RelativePath}> - {size};", i.ToString("000000"), count, record.RelativePath, size);
|
|
} catch (Exception) {
|
|
logger.LogInformation("Failed to delete: <{RelativePath}> - {size};", record.RelativePath, size);
|
|
}
|
|
}
|
|
#if ShellProgressBar
|
|
progressBar.Dispose();
|
|
#endif
|
|
}
|
|
|
|
private static void Preform(ILogger<Worker> logger, string page, string directory, ReadOnlyCollection<Record> records, HttpClient httpClient, HttpMethod httpMethod) {
|
|
Verb verb;
|
|
long ticks;
|
|
string size;
|
|
string iValue;
|
|
string duration;
|
|
DateTime dateTime;
|
|
Task<string> response;
|
|
HttpRequestMessage httpRequestMessage;
|
|
Task<HttpResponseMessage> httpResponseMessage;
|
|
string count = records.Count.ToString("000000");
|
|
MultipartFormDataContent multipartFormDataContent;
|
|
ReadOnlyCollection<Verb> collection = GetVerbCollection(directory, records);
|
|
#if ShellProgressBar
|
|
ProgressBar progressBar = new(downloads.Count, $"{httpMethod}ing: {count};", new ProgressBarOptions() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true });
|
|
#endif
|
|
for (int i = 0; i < collection.Count; i++) {
|
|
verb = collection[i];
|
|
#if ShellProgressBar
|
|
progressBar.Tick();
|
|
#endif
|
|
ticks = DateTime.Now.Ticks;
|
|
iValue = (i + 1).ToString("000000");
|
|
size = GetSizeWithSuffix(verb.Size);
|
|
if (httpMethod == HttpMethod.Get || httpMethod == HttpMethod.Delete) {
|
|
httpRequestMessage = new(httpMethod, $"{page}size={verb.Size}&ticks={verb.Ticks}&path={verb.UrlEncodedFile}");
|
|
} else if (httpMethod == HttpMethod.Patch || httpMethod == HttpMethod.Put) {
|
|
httpRequestMessage = new(httpMethod, $"{page}path={verb.Directory}");
|
|
multipartFormDataContent = new();
|
|
multipartFormDataContent.Add(new ByteArrayContent(File.ReadAllBytes(verb.File)), "formFiles", verb.Multipart);
|
|
multipartFormDataContent.Add(new StringContent(verb.Directory), "path", iValue);
|
|
httpRequestMessage.Content = multipartFormDataContent;
|
|
} else
|
|
throw new NotImplementedException();
|
|
httpResponseMessage = httpClient.SendAsync(httpRequestMessage);
|
|
httpResponseMessage.Wait(-1);
|
|
if (!httpResponseMessage.Result.IsSuccessStatusCode)
|
|
logger.LogInformation("Failed to {httpMethod}: <{display}> - {size};", httpMethod, verb.Display, size);
|
|
else {
|
|
try {
|
|
if (httpMethod != HttpMethod.Get) {
|
|
duration = GetDurationWithSuffix(ticks);
|
|
} else {
|
|
response = httpResponseMessage.Result.Content.ReadAsStringAsync();
|
|
response.Wait();
|
|
File.WriteAllText(verb.File, response.Result);
|
|
duration = GetDurationWithSuffix(ticks);
|
|
dateTime = new DateTime(verb.Ticks).ToLocalTime();
|
|
File.SetLastWriteTime(verb.File, dateTime);
|
|
}
|
|
logger.LogInformation("{i} of {count} - {httpMethod}'ed: <{display}> - {size} - {timeSpan};",
|
|
iValue,
|
|
count,
|
|
httpMethod,
|
|
verb.Display,
|
|
size,
|
|
duration);
|
|
} catch (Exception) {
|
|
logger.LogInformation("Failed to {httpMethod}: <{display}> - {size};", httpMethod, verb.Display, size);
|
|
}
|
|
}
|
|
}
|
|
#if ShellProgressBar
|
|
progressBar.Dispose();
|
|
#endif
|
|
}
|
|
|
|
private static ReadOnlyCollection<Verb> GetVerbCollection(string directory, ReadOnlyCollection<Record> records) {
|
|
List<Verb> results = [];
|
|
Verb verb;
|
|
string checkFile;
|
|
string checkFileName;
|
|
string? checkDirectory;
|
|
List<Verb> collection = [];
|
|
foreach (Record record in records) {
|
|
checkFile = Path.Combine(directory, record.RelativePath);
|
|
checkFileName = Path.GetFileName(checkFile);
|
|
checkDirectory = Path.GetDirectoryName(checkFile);
|
|
if (string.IsNullOrEmpty(checkDirectory))
|
|
continue;
|
|
if (!Directory.Exists(checkDirectory))
|
|
_ = Directory.CreateDirectory(checkDirectory);
|
|
if (File.Exists(checkFile) && new FileInfo(checkFile).Length == 0)
|
|
File.Delete(checkFile);
|
|
verb = new(Directory: checkDirectory,
|
|
Display: $"{checkFileName}{Environment.NewLine}{checkDirectory}",
|
|
File: checkFile,
|
|
Multipart: $"RelativePath:{record.RelativePath}|Size:{record.Size}|Ticks:{record.Ticks};",
|
|
RelativePath: record.RelativePath,
|
|
Size: record.Size,
|
|
Ticks: record.Ticks,
|
|
UrlEncodedFile: HttpUtility.UrlEncode(checkFile));
|
|
collection.Add(verb);
|
|
}
|
|
Verb[] sorted = (from l in collection orderby l.Size select l).ToArray();
|
|
int stop = sorted.Length < 100 ? sorted.Length : 100;
|
|
for (int i = 0; i < stop; i++)
|
|
results.Add(sorted[i]);
|
|
for (int i = sorted.Length - 1; i > stop - 1; i--)
|
|
results.Add(sorted[i]);
|
|
if (collection.Count != results.Count)
|
|
throw new Exception();
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static string GetDurationWithSuffix(long ticks) {
|
|
string result;
|
|
TimeSpan timeSpan = new(DateTime.Now.Ticks - ticks);
|
|
if (timeSpan.TotalMilliseconds < 1000)
|
|
result = $"{timeSpan.Milliseconds} ms";
|
|
else if (timeSpan.TotalMilliseconds < 60000)
|
|
result = $"{Math.Floor(timeSpan.TotalSeconds)} s";
|
|
else if (timeSpan.TotalMilliseconds < 3600000)
|
|
result = $"{Math.Floor(timeSpan.TotalMinutes)} m";
|
|
else
|
|
result = $"{Math.Floor(timeSpan.TotalHours)} h";
|
|
return result;
|
|
}
|
|
|
|
} |