693 lines
34 KiB
C#
693 lines
34 KiB
C#
using System.Collections.ObjectModel;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Serialization;
|
|
|
|
using File_Folder_Helper.Models;
|
|
|
|
using Microsoft.Extensions.FileSystemGlobbing;
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
#if ShellProgressBar
|
|
using ShellProgressBar;
|
|
#endif
|
|
|
|
namespace File_Folder_Helper.ADO2025.PI5;
|
|
|
|
internal static partial class Helper20250407 {
|
|
|
|
private record Record(string RelativePath,
|
|
long Size,
|
|
long Ticks);
|
|
|
|
private record Download(string Directory,
|
|
string Display,
|
|
string File,
|
|
long Size,
|
|
long Ticks,
|
|
string UniformResourceLocator);
|
|
|
|
private record Segment(Record? Left,
|
|
string? LeftDirectory,
|
|
Record? Right,
|
|
string RightDirectory,
|
|
string RootUniformResourceLocator);
|
|
|
|
private record Logic(char GreaterThan,
|
|
bool? LeftSideIsNewer,
|
|
int LeftSideIsNewerIndex,
|
|
bool? LeftSideOnly,
|
|
int LeftSideOnlyIndex,
|
|
char LessThan,
|
|
char Minus,
|
|
bool? NotEqualBut,
|
|
int NotEqualButIndex,
|
|
char Plus,
|
|
string[] Raw,
|
|
bool? RightSideIsNewer,
|
|
int RightSideIsNewerIndex,
|
|
bool? RightSideOnly,
|
|
int RightSideOnlyIndex) {
|
|
|
|
internal static Logic? Get(string[] segments) {
|
|
Logic? result;
|
|
bool check = true;
|
|
bool? notEqualBut;
|
|
bool? leftSideOnly;
|
|
bool? rightSideOnly;
|
|
bool? leftSideIsNewer;
|
|
const char plus = '+';
|
|
bool? rightSideIsNewer;
|
|
const char minus = '-';
|
|
const char lessThan = 'L';
|
|
const char greaterThan = 'G';
|
|
const int notEqualButIndex = 2;
|
|
const int leftSideOnlyIndex = 0;
|
|
const int rightSideOnlyIndex = 4;
|
|
const int leftSideIsNewerIndex = 1;
|
|
const int rightSideIsNewerIndex = 3;
|
|
if (string.IsNullOrEmpty(segments[leftSideOnlyIndex]))
|
|
leftSideOnly = null;
|
|
else if (segments[leftSideOnlyIndex][0] == plus)
|
|
leftSideOnly = true;
|
|
else if (segments[leftSideOnlyIndex][0] == minus)
|
|
leftSideOnly = false;
|
|
else {
|
|
check = false;
|
|
leftSideOnly = null;
|
|
}
|
|
if (string.IsNullOrEmpty(segments[leftSideIsNewerIndex]))
|
|
leftSideIsNewer = null;
|
|
else if (segments[leftSideIsNewerIndex][0] == greaterThan)
|
|
leftSideIsNewer = true;
|
|
else if (segments[leftSideIsNewerIndex][0] == lessThan)
|
|
leftSideIsNewer = false;
|
|
else {
|
|
check = false;
|
|
leftSideIsNewer = null;
|
|
}
|
|
if (string.IsNullOrEmpty(segments[notEqualButIndex]))
|
|
notEqualBut = null;
|
|
else if (segments[notEqualButIndex][0] == greaterThan)
|
|
notEqualBut = true;
|
|
else if (segments[notEqualButIndex][0] == lessThan)
|
|
notEqualBut = false;
|
|
else {
|
|
check = false;
|
|
notEqualBut = null;
|
|
}
|
|
if (string.IsNullOrEmpty(segments[rightSideIsNewerIndex]))
|
|
rightSideIsNewer = null;
|
|
else if (segments[rightSideIsNewerIndex][0] == greaterThan)
|
|
rightSideIsNewer = true;
|
|
else if (segments[rightSideIsNewerIndex][0] == lessThan)
|
|
rightSideIsNewer = false;
|
|
else {
|
|
check = false;
|
|
rightSideIsNewer = null;
|
|
}
|
|
if (string.IsNullOrEmpty(segments[rightSideOnlyIndex]))
|
|
rightSideOnly = null;
|
|
else if (segments[rightSideOnlyIndex][0] == plus)
|
|
rightSideOnly = true;
|
|
else if (segments[rightSideOnlyIndex][0] == minus)
|
|
rightSideOnly = false;
|
|
else {
|
|
check = false;
|
|
rightSideOnly = null;
|
|
}
|
|
result = !check ? null : new(GreaterThan: greaterThan,
|
|
LeftSideIsNewerIndex: leftSideIsNewerIndex,
|
|
LeftSideIsNewer: leftSideIsNewer,
|
|
LeftSideOnly: leftSideOnly,
|
|
LeftSideOnlyIndex: leftSideOnlyIndex,
|
|
LessThan: lessThan,
|
|
Minus: minus,
|
|
NotEqualBut: notEqualBut,
|
|
NotEqualButIndex: notEqualButIndex,
|
|
Plus: plus,
|
|
RightSideIsNewer: rightSideIsNewer,
|
|
RightSideIsNewerIndex: rightSideIsNewerIndex,
|
|
RightSideOnly: rightSideOnly,
|
|
Raw: segments,
|
|
RightSideOnlyIndex: rightSideOnlyIndex);
|
|
return result;
|
|
}
|
|
|
|
}
|
|
|
|
private record Review(Segment[]? AreEqual,
|
|
Segment[]? LeftSideIsNewer,
|
|
Segment[]? LeftSideOnly,
|
|
Segment[]? NotEqualBut,
|
|
Record[]? Records,
|
|
Segment[]? RightSideIsNewer,
|
|
Segment[]? RightSideOnly);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(Review))]
|
|
private partial class ReviewCommonSourceGenerationContext : JsonSerializerContext {
|
|
}
|
|
|
|
internal static void Sync(ILogger<Worker> logger, List<string> args) {
|
|
Matcher matcher = new();
|
|
string fileName = $"{args[1]}.json";
|
|
string[] segments = args[5].Split('|');
|
|
string rightDirectory = Path.GetFullPath(args[0].Split('|')[0]);
|
|
Logic? logic = segments.Length != 5 ? null : Logic.Get(segments);
|
|
string includePatternsFile = Path.Combine(rightDirectory, args[2]);
|
|
string excludePatternsFile = Path.Combine(rightDirectory, args[3]);
|
|
string[] rootUniformResourceLocators = args.Count < 5 ? [] : args[4].Split('|');
|
|
matcher.AddIncludePatterns(!File.Exists(includePatternsFile) ? ["*"] : File.ReadAllLines(includePatternsFile));
|
|
matcher.AddExcludePatterns(!File.Exists(excludePatternsFile) ? ["System Volume Information"] : File.ReadAllLines(excludePatternsFile));
|
|
ReadOnlyCollection<Record> rightRecords = GetRecords(rightDirectory, matcher);
|
|
if (rightRecords.Count == 0)
|
|
logger.LogInformation("No source records");
|
|
else {
|
|
string checkFile = Path.Combine(rightDirectory, fileName);
|
|
Review review = new(AreEqual: null,
|
|
LeftSideIsNewer: null,
|
|
LeftSideOnly: null,
|
|
NotEqualBut: null,
|
|
Records: rightRecords.ToArray(),
|
|
RightSideIsNewer: null,
|
|
RightSideOnly: null);
|
|
string json = JsonSerializer.Serialize(review, ReviewCommonSourceGenerationContext.Default.Review);
|
|
WriteAllText(checkFile, json);
|
|
if (rootUniformResourceLocators.Length == 0)
|
|
logger.LogInformation("No urls");
|
|
else {
|
|
string format = NginxFileSystem.GetFormat();
|
|
TimeZoneInfo timeZoneInfo = TimeZoneInfo.Local;
|
|
Sync(logger, rightDirectory, fileName, logic, rootUniformResourceLocators, rightRecords, format, timeZoneInfo);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static ReadOnlyCollection<Record> GetRecords(string rightDirectory, Matcher matcher) {
|
|
List<Record> results = [
|
|
new(RelativePath: rightDirectory,
|
|
Size: 0,
|
|
Ticks: 0)];
|
|
Record record;
|
|
FileInfo fileInfo;
|
|
string relativePath;
|
|
ReadOnlyCollection<ReadOnlyCollection<string>> collection = Helpers.HelperDirectory.GetFilesCollection(rightDirectory, "*", "*");
|
|
foreach (ReadOnlyCollection<string> c in collection) {
|
|
foreach (string f in c) {
|
|
if (!matcher.Match(rightDirectory, f).HasMatches)
|
|
continue;
|
|
fileInfo = new(f);
|
|
if (fileInfo.Length == 0)
|
|
continue;
|
|
relativePath = Path.GetRelativePath(rightDirectory, fileInfo.FullName);
|
|
record = new(RelativePath: relativePath,
|
|
Size: fileInfo.Length,
|
|
Ticks: fileInfo.LastWriteTime.ToUniversalTime().Ticks);
|
|
results.Add(record);
|
|
}
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static void WriteAllText(string path, string text) {
|
|
string check = !File.Exists(path) ? string.Empty : File.ReadAllText(path);
|
|
if (check != text)
|
|
File.WriteAllText(path, text);
|
|
}
|
|
|
|
private static void Sync(ILogger<Worker> logger, string rightDirectory, string fileName, Logic? logic, string[] rootUniformResourceLocators, ReadOnlyCollection<Record> rightRecords, string format, TimeZoneInfo timeZoneInfo) {
|
|
Review? review;
|
|
foreach (string rootUniformResourceLocator in rootUniformResourceLocators) {
|
|
if (!rootUniformResourceLocator.StartsWith("https:"))
|
|
logger.LogInformation("Not supported URL <{url}>", rootUniformResourceLocator);
|
|
else {
|
|
review = GetJsonResponse(logger, fileName, rootUniformResourceLocator, format, timeZoneInfo);
|
|
if (review?.Records is null || review.Records.Length == 0)
|
|
logger.LogInformation("No response records");
|
|
else {
|
|
ReadOnlyCollection<Record> leftRecords = review.Records.AsReadOnly();
|
|
Sync(logger, rightDirectory, fileName, logic, rightRecords, rootUniformResourceLocator, leftRecords);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private static Review? GetJsonResponse(ILogger<Worker> logger, string fileName, string rootUniformResourceLocator, string format, TimeZoneInfo timeZoneInfo) {
|
|
Review? result;
|
|
Task<string> response;
|
|
HttpClient httpClient = new();
|
|
Task<HttpResponseMessage> httpResponseMessage;
|
|
string url = new(rootUniformResourceLocator.EndsWith('/') ?
|
|
$"{rootUniformResourceLocator[..^1]}/{fileName}" :
|
|
$"{rootUniformResourceLocator}/{fileName}");
|
|
httpResponseMessage = httpClient.GetAsync(rootUniformResourceLocator);
|
|
httpResponseMessage.Wait();
|
|
if (!httpResponseMessage.Result.IsSuccessStatusCode) {
|
|
logger.LogInformation("Failed to download: <{rootUniformResourceLocator}>;", rootUniformResourceLocator);
|
|
result = null;
|
|
} else {
|
|
response = httpResponseMessage.Result.Content.ReadAsStringAsync();
|
|
response.Wait();
|
|
NginxFileSystem[]? nginxFileSystems = JsonSerializer.Deserialize(response.Result, NginxFileSystemCollectionSourceGenerationContext.Default.NginxFileSystemArray);
|
|
bool isNewest = nginxFileSystems is not null && IsNewest(fileName, format, timeZoneInfo, new(rootUniformResourceLocator), nginxFileSystems);
|
|
if (nginxFileSystems is null) {
|
|
logger.LogInformation("Failed to parse: <{rootUniformResourceLocator}>;", rootUniformResourceLocator);
|
|
result = null;
|
|
} else if (!isNewest) {
|
|
logger.LogInformation("Outdated remote file: <{rootUniformResourceLocator}>;", rootUniformResourceLocator);
|
|
result = null;
|
|
} else {
|
|
httpResponseMessage = httpClient.GetAsync(url);
|
|
httpResponseMessage.Wait();
|
|
if (!httpResponseMessage.Result.IsSuccessStatusCode) {
|
|
logger.LogInformation("Failed to download: <{url}>;", url);
|
|
result = null;
|
|
} else {
|
|
response = httpResponseMessage.Result.Content.ReadAsStringAsync();
|
|
response.Wait();
|
|
result = string.IsNullOrEmpty(response.Result) ?
|
|
null :
|
|
JsonSerializer.Deserialize(response.Result, ReviewCommonSourceGenerationContext.Default.Review);
|
|
}
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
private static bool IsNewest(string fileName, string format, TimeZoneInfo timeZoneInfo, Uri uri, NginxFileSystem[] nginxFileSystems) {
|
|
bool result;
|
|
DateTime? match = null;
|
|
NginxFileSystem nginxFileSystem;
|
|
DateTime dateTime = DateTime.MinValue;
|
|
for (int i = 0; i < nginxFileSystems.Length; i++) {
|
|
nginxFileSystem = NginxFileSystem.Get(format, timeZoneInfo, uri, nginxFileSystems[i]);
|
|
if (nginxFileSystem.LastModified is not null && nginxFileSystem.Name == fileName) {
|
|
match = nginxFileSystem.LastModified.Value;
|
|
continue;
|
|
}
|
|
if (nginxFileSystem.LastModified is null || nginxFileSystem.LastModified <= dateTime)
|
|
continue;
|
|
dateTime = nginxFileSystem.LastModified.Value;
|
|
}
|
|
result = match is not null && match.Value > dateTime;
|
|
return result;
|
|
}
|
|
|
|
private static void Sync(ILogger<Worker> logger, string rightDirectory, string fileName, Logic? l, ReadOnlyCollection<Record> rightRecords, string rootUniformResourceLocators, ReadOnlyCollection<Record> leftRecords) {
|
|
string json;
|
|
string checkFile;
|
|
HttpClient httpClient = new();
|
|
checkFile = Path.Combine(rightDirectory, fileName);
|
|
if (File.Exists(checkFile))
|
|
File.Delete(checkFile);
|
|
ReadOnlyCollection<Segment> areEqual = GetAreEqual(rightDirectory, fileName, rightRecords, rootUniformResourceLocators, leftRecords);
|
|
ReadOnlyCollection<Segment> notEqualBut = GetNotEqualBut(rightDirectory, fileName, rightRecords, rootUniformResourceLocators, leftRecords);
|
|
ReadOnlyCollection<Segment> leftSideOnly = GetLeftSideOnly(rightDirectory, fileName, rightRecords, rootUniformResourceLocators, leftRecords);
|
|
ReadOnlyCollection<Segment> rightSideOnly = GetRightSideOnly(rightDirectory, fileName, rightRecords, rootUniformResourceLocators, leftRecords);
|
|
ReadOnlyCollection<Segment> leftSideIsNewer = GetLeftSideIsNewer(rightDirectory, fileName, rightRecords, rootUniformResourceLocators, leftRecords);
|
|
ReadOnlyCollection<Segment> rightSideIsNewer = GetRightSideIsNewer(rightDirectory, fileName, rightRecords, rootUniformResourceLocators, leftRecords);
|
|
Review review = new(AreEqual: areEqual.ToArray(),
|
|
LeftSideIsNewer: leftSideIsNewer.ToArray(),
|
|
LeftSideOnly: leftSideOnly.ToArray(),
|
|
NotEqualBut: notEqualBut.ToArray(),
|
|
Records: null,
|
|
RightSideIsNewer: rightSideIsNewer.ToArray(),
|
|
RightSideOnly: rightSideOnly.ToArray());
|
|
json = JsonSerializer.Serialize(review, ReviewCommonSourceGenerationContext.Default.Review);
|
|
checkFile = Path.Combine(rightDirectory, fileName);
|
|
WriteAllText(checkFile, json);
|
|
if (notEqualBut.Count > 0 && l is not null && l.NotEqualBut is not null && l.Raw[l.NotEqualButIndex][0] == l.Minus && !l.NotEqualBut.Value)
|
|
logger.LogDebug("Doing nothing with {name}", nameof(Logic.NotEqualBut));
|
|
if (leftSideOnly.Count > 0 && l is not null && l.LeftSideOnly is not null && l.Raw[l.LeftSideOnlyIndex][0] == l.Minus && !l.LeftSideOnly.Value)
|
|
throw new NotImplementedException("Not possible with https!");
|
|
if (leftSideIsNewer.Count > 0 && l is not null && l.LeftSideIsNewer is not null && l.Raw[l.LeftSideIsNewerIndex][0] == l.LessThan && !l.LeftSideIsNewer.Value)
|
|
throw new NotImplementedException("Not possible with https!");
|
|
if (rightSideIsNewer.Count > 0 && l is not null && l.RightSideIsNewer is not null && l.Raw[l.RightSideIsNewerIndex][0] == l.LessThan && !l.RightSideIsNewer.Value)
|
|
throw new NotImplementedException("Not possible with https!");
|
|
if (rightSideOnly.Count > 0 && l is not null && l.RightSideOnly is not null && l.Raw[l.RightSideOnlyIndex][0] == l.Plus && l.RightSideOnly.Value)
|
|
throw new NotImplementedException("Not possible with https!");
|
|
if (rightSideOnly.Count > 0 && l is not null && l.RightSideOnly is not null && l.Raw[l.RightSideOnlyIndex][0] == l.Minus && !l.RightSideOnly.Value)
|
|
DoWork(logger, rightDirectory, httpClient, rightSideOnly, delete: true, download: false);
|
|
if (leftSideOnly.Count > 0 && l is not null && l.LeftSideOnly is not null && l.Raw[l.LeftSideOnlyIndex][0] == l.Plus && l.LeftSideOnly.Value)
|
|
DoWork(logger, rightDirectory, httpClient, leftSideOnly, delete: false, download: true);
|
|
if (leftSideIsNewer.Count > 0 && l is not null && l.LeftSideIsNewer is not null && l.Raw[l.LeftSideIsNewerIndex][0] == l.GreaterThan && l.LeftSideIsNewer.Value)
|
|
DoWork(logger, rightDirectory, httpClient, leftSideIsNewer, delete: true, download: true);
|
|
if (notEqualBut.Count > 0 && l is not null && l.NotEqualBut is not null && l.Raw[l.NotEqualButIndex][0] == l.Plus && l.NotEqualBut.Value)
|
|
DoWork(logger, rightDirectory, httpClient, notEqualBut, delete: true, download: true);
|
|
if (rightSideIsNewer.Count > 0 && l is not null && l.RightSideIsNewer is not null && l.Raw[l.RightSideIsNewerIndex][0] == l.GreaterThan && l.RightSideIsNewer.Value)
|
|
DoWork(logger, rightDirectory, httpClient, rightSideIsNewer, delete: true, download: true);
|
|
}
|
|
|
|
private static ReadOnlyCollection<Segment> GetAreEqual(string rightDirectory, string fileName, ReadOnlyCollection<Record> rightRecords, string rootUniformResourceLocators, ReadOnlyCollection<Record> leftRecords) {
|
|
List<Segment> results = [];
|
|
Record? record;
|
|
Segment segment;
|
|
double totalSeconds;
|
|
string? checkDirectory = null;
|
|
ReadOnlyDictionary<string, Record> keyValuePairs = GetKeyValuePairs(rightRecords);
|
|
foreach (Record r in leftRecords) {
|
|
if (checkDirectory is null && r.Size == 0 && r.Ticks == 0) {
|
|
checkDirectory = r.RelativePath;
|
|
continue;
|
|
}
|
|
if (r.RelativePath == rightDirectory || r.RelativePath == fileName)
|
|
continue;
|
|
if (!keyValuePairs.TryGetValue(r.RelativePath, out record))
|
|
continue;
|
|
totalSeconds = new TimeSpan(record.Ticks - r.Ticks).TotalSeconds;
|
|
if (record.Size != r.Size || totalSeconds is > 2 or < -2)
|
|
continue;
|
|
segment = new(Left: r,
|
|
LeftDirectory: checkDirectory,
|
|
Right: record,
|
|
RightDirectory: rightDirectory,
|
|
RootUniformResourceLocator: rootUniformResourceLocators);
|
|
results.Add(segment);
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static ReadOnlyDictionary<string, Record> GetKeyValuePairs(ReadOnlyCollection<Record> records) {
|
|
Dictionary<string, Record> results = [];
|
|
foreach (Record record in records)
|
|
results.Add(record.RelativePath, record);
|
|
return new(results);
|
|
}
|
|
|
|
private static ReadOnlyCollection<Segment> GetNotEqualBut(string rightDirectory, string fileName, ReadOnlyCollection<Record> rightRecords, string rootUniformResourceLocators, ReadOnlyCollection<Record> leftRecords) {
|
|
List<Segment> results = [];
|
|
Record? record;
|
|
Segment segment;
|
|
double totalSeconds;
|
|
string? checkDirectory = null;
|
|
ReadOnlyDictionary<string, Record> keyValuePairs = GetKeyValuePairs(rightRecords);
|
|
foreach (Record r in leftRecords) {
|
|
if (checkDirectory is null && r.Size == 0 && r.Ticks == 0) {
|
|
checkDirectory = r.RelativePath;
|
|
continue;
|
|
}
|
|
if (r.RelativePath == rightDirectory || r.RelativePath == fileName)
|
|
continue;
|
|
if (!keyValuePairs.TryGetValue(r.RelativePath, out record))
|
|
continue;
|
|
if (record.Size == r.Size)
|
|
continue;
|
|
totalSeconds = new TimeSpan(record.Ticks - r.Ticks).TotalSeconds;
|
|
if (totalSeconds is >= 2 or <= -2)
|
|
continue;
|
|
segment = new(Left: r,
|
|
LeftDirectory: checkDirectory,
|
|
Right: record,
|
|
RightDirectory: rightDirectory,
|
|
RootUniformResourceLocator: rootUniformResourceLocators);
|
|
results.Add(segment);
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static ReadOnlyCollection<Segment> GetLeftSideOnly(string rightDirectory, string fileName, ReadOnlyCollection<Record> rightRecords, string rootUniformResourceLocators, ReadOnlyCollection<Record> leftRecords) {
|
|
List<Segment> results = [];
|
|
Record? record;
|
|
Segment segment;
|
|
string? checkDirectory = null;
|
|
ReadOnlyDictionary<string, Record> keyValuePairs = GetKeyValuePairs(rightRecords);
|
|
foreach (Record r in leftRecords) {
|
|
if (checkDirectory is null && r.Size == 0 && r.Ticks == 0) {
|
|
checkDirectory = r.RelativePath;
|
|
continue;
|
|
}
|
|
if (r.RelativePath == rightDirectory || r.RelativePath == fileName)
|
|
continue;
|
|
if (keyValuePairs.TryGetValue(r.RelativePath, out record))
|
|
continue;
|
|
segment = new(Left: r,
|
|
LeftDirectory: checkDirectory,
|
|
Right: record,
|
|
RightDirectory: rightDirectory,
|
|
RootUniformResourceLocator: rootUniformResourceLocators);
|
|
results.Add(segment);
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static ReadOnlyCollection<Segment> GetRightSideOnly(string rightDirectory, string fileName, ReadOnlyCollection<Record> rightRecords, string rootUniformResourceLocators, ReadOnlyCollection<Record> leftRecords) {
|
|
List<Segment> results = [];
|
|
Record? record;
|
|
Segment segment;
|
|
string? checkDirectory = null;
|
|
ReadOnlyDictionary<string, Record> keyValuePairs = GetKeyValuePairs(leftRecords);
|
|
foreach (Record r in rightRecords) {
|
|
if (checkDirectory is null && r.Size == 0 && r.Ticks == 0) {
|
|
checkDirectory = r.RelativePath;
|
|
continue;
|
|
}
|
|
if (r.RelativePath == rightDirectory || r.RelativePath == fileName)
|
|
continue;
|
|
if (keyValuePairs.TryGetValue(r.RelativePath, out record))
|
|
continue;
|
|
segment = new(Left: record,
|
|
LeftDirectory: null,
|
|
Right: r,
|
|
RightDirectory: rightDirectory,
|
|
RootUniformResourceLocator: rootUniformResourceLocators);
|
|
results.Add(segment);
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static ReadOnlyCollection<Segment> GetLeftSideIsNewer(string rightDirectory, string fileName, ReadOnlyCollection<Record> rightRecords, string rootUniformResourceLocators, ReadOnlyCollection<Record> leftRecords) {
|
|
List<Segment> results = [];
|
|
Record? record;
|
|
Segment segment;
|
|
double totalSeconds;
|
|
string? checkDirectory = null;
|
|
ReadOnlyDictionary<string, Record> keyValuePairs = GetKeyValuePairs(rightRecords);
|
|
foreach (Record r in leftRecords) {
|
|
if (checkDirectory is null && r.Size == 0 && r.Ticks == 0) {
|
|
checkDirectory = r.RelativePath;
|
|
continue;
|
|
}
|
|
if (r.RelativePath == rightDirectory || r.RelativePath == fileName)
|
|
continue;
|
|
if (!keyValuePairs.TryGetValue(r.RelativePath, out record))
|
|
continue;
|
|
totalSeconds = new TimeSpan(record.Ticks - r.Ticks).TotalSeconds;
|
|
if (totalSeconds is > -2)
|
|
continue;
|
|
segment = new(Left: r,
|
|
LeftDirectory: checkDirectory,
|
|
Right: record,
|
|
RightDirectory: rightDirectory,
|
|
RootUniformResourceLocator: rootUniformResourceLocators);
|
|
results.Add(segment);
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static ReadOnlyCollection<Segment> GetRightSideIsNewer(string rightDirectory, string fileName, ReadOnlyCollection<Record> rightRecords, string rootUniformResourceLocators, ReadOnlyCollection<Record> leftRecords) {
|
|
List<Segment> results = [];
|
|
Record? record;
|
|
Segment segment;
|
|
double totalSeconds;
|
|
string? checkDirectory = null;
|
|
ReadOnlyDictionary<string, Record> keyValuePairs = GetKeyValuePairs(leftRecords);
|
|
foreach (Record r in rightRecords) {
|
|
if (checkDirectory is null && r.Size == 0 && r.Ticks == 0) {
|
|
checkDirectory = r.RelativePath;
|
|
continue;
|
|
}
|
|
if (r.RelativePath == rightDirectory || r.RelativePath == fileName)
|
|
continue;
|
|
if (!keyValuePairs.TryGetValue(r.RelativePath, out record))
|
|
continue;
|
|
totalSeconds = new TimeSpan(record.Ticks - r.Ticks).TotalSeconds;
|
|
if (totalSeconds is > -2)
|
|
continue;
|
|
segment = new(Left: record,
|
|
LeftDirectory: null,
|
|
Right: r,
|
|
RightDirectory: rightDirectory,
|
|
RootUniformResourceLocator: rootUniformResourceLocators);
|
|
results.Add(segment);
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static void DoWork(ILogger<Worker> logger, string rightDirectory, HttpClient httpClient, ReadOnlyCollection<Segment> segments, bool delete, bool download) {
|
|
long sum;
|
|
Record[] records = (from l in segments where l.Left is not null select l.Left).ToArray();
|
|
try { sum = records.Sum(l => l.Size); } catch (Exception) { sum = 0; }
|
|
string size = GetSizeWithSuffix(sum);
|
|
if (delete) {
|
|
logger.LogInformation("Starting to delete {count} file(s) [{sum}]", segments.Count, size);
|
|
DoDeletes(logger, rightDirectory, segments);
|
|
logger.LogInformation("Deleted {count} file(s) [{sum}]", segments.Count, size);
|
|
}
|
|
if (download) {
|
|
logger.LogInformation("Starting to download {count} file(s) [{sum}]", segments.Count, size);
|
|
DoDownloads(logger, rightDirectory, segments, httpClient);
|
|
logger.LogInformation("Downloaded {count} file(s) [{sum}]", segments.Count, size);
|
|
}
|
|
}
|
|
|
|
private static string GetSizeWithSuffix(long value) {
|
|
string result;
|
|
int i = 0;
|
|
string[] SizeSuffixes = ["bytes", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
|
|
if (value < 0) {
|
|
result = "-" + GetSizeWithSuffix(-value);
|
|
} else {
|
|
while (Math.Round(value / 1024f) >= 1) {
|
|
value /= 1024;
|
|
i++;
|
|
}
|
|
result = string.Format("{0:n1} {1}", value, SizeSuffixes[i]);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
private static string GetDurationWithSuffix(long ticks) {
|
|
string result;
|
|
TimeSpan timeSpan = new(DateTime.Now.Ticks - ticks);
|
|
if (timeSpan.TotalMilliseconds < 1000)
|
|
result = $"{timeSpan.Milliseconds} ms";
|
|
else if (timeSpan.TotalMilliseconds < 60000)
|
|
result = $"{Math.Floor(timeSpan.TotalSeconds)} s";
|
|
else if (timeSpan.TotalMilliseconds < 3600000)
|
|
result = $"{Math.Floor(timeSpan.TotalMinutes)} m";
|
|
else
|
|
result = $"{Math.Floor(timeSpan.TotalHours)} h";
|
|
return result;
|
|
}
|
|
|
|
private static void DoDeletes(ILogger<Worker> logger, string rightDirectory, ReadOnlyCollection<Segment> segments) {
|
|
Record? record;
|
|
string size;
|
|
string count = segments.Count.ToString("000000");
|
|
#if ShellProgressBar
|
|
ProgressBar progressBar = new(segments.Count, $"Deleting: {count};", new ProgressBarOptions() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true });
|
|
#endif
|
|
for (int i = 0; i < segments.Count; i++) {
|
|
#if ShellProgressBar
|
|
progressBar.Tick();
|
|
#endif
|
|
record = segments[i].Right;
|
|
if (record is null)
|
|
continue;
|
|
size = GetSizeWithSuffix(record.Size);
|
|
try {
|
|
File.Delete(Path.Combine(rightDirectory, record.RelativePath));
|
|
logger.LogInformation("{i} of {count} - Deleted: <{RelativePath}> - {size};", i.ToString("000000"), count, record.RelativePath, size);
|
|
} catch (Exception) {
|
|
logger.LogInformation("Failed to delete: <{RelativePath}> - {size};", record.RelativePath, size);
|
|
}
|
|
}
|
|
#if ShellProgressBar
|
|
progressBar.Dispose();
|
|
#endif
|
|
}
|
|
|
|
private static void DoDownloads(ILogger<Worker> logger, string rightDirectory, ReadOnlyCollection<Segment> segments, HttpClient httpClient) {
|
|
int i = 0;
|
|
long ticks;
|
|
string size;
|
|
string duration;
|
|
DateTime dateTime;
|
|
Task<string> response;
|
|
string count = segments.Count.ToString("000000");
|
|
ReadOnlyCollection<Download> downloads = GetDownloads(rightDirectory, segments);
|
|
Task<HttpResponseMessage> httpResponseMessage;
|
|
#if ShellProgressBar
|
|
ProgressBar progressBar = new(downloads.Count, $"Downloading: {count};", new ProgressBarOptions() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true });
|
|
#endif
|
|
foreach (Download download in downloads) {
|
|
#if ShellProgressBar
|
|
progressBar.Tick();
|
|
#endif
|
|
i += 1;
|
|
ticks = DateTime.Now.Ticks;
|
|
size = GetSizeWithSuffix(download.Size);
|
|
httpResponseMessage = httpClient.GetAsync(download.UniformResourceLocator);
|
|
httpResponseMessage.Wait(-1);
|
|
if (!httpResponseMessage.Result.IsSuccessStatusCode)
|
|
logger.LogInformation("Failed to download: <{checkURL}> - {size};", download.UniformResourceLocator, size);
|
|
else {
|
|
response = httpResponseMessage.Result.Content.ReadAsStringAsync();
|
|
response.Wait();
|
|
try {
|
|
File.WriteAllText(download.File, response.Result);
|
|
duration = GetDurationWithSuffix(ticks);
|
|
dateTime = new DateTime(download.Ticks).ToLocalTime();
|
|
File.SetLastWriteTime(download.File, dateTime);
|
|
logger.LogInformation("{i} of {count} - Downloaded: <{checkURL}> - {size} - {timeSpan};",
|
|
i.ToString("000000"),
|
|
count,
|
|
download.Display,
|
|
size,
|
|
duration);
|
|
} catch (Exception) {
|
|
logger.LogInformation("Failed to download: <{checkURL}> - {size};", download.UniformResourceLocator, size);
|
|
}
|
|
}
|
|
}
|
|
#if ShellProgressBar
|
|
progressBar.Dispose();
|
|
#endif
|
|
}
|
|
|
|
private static ReadOnlyCollection<Download> GetDownloads(string rightDirectory, ReadOnlyCollection<Segment> segments) {
|
|
List<Download> results = [];
|
|
string checkFile;
|
|
Download download;
|
|
string? checkDirectory;
|
|
List<Download> collection = [];
|
|
string? checkUniformResourceLocator;
|
|
foreach (Segment segment in segments) {
|
|
if (segment.Left is null)
|
|
continue;
|
|
checkFile = Path.Combine(rightDirectory, segment.Left.RelativePath);
|
|
checkDirectory = Path.GetDirectoryName(checkFile);
|
|
if (string.IsNullOrEmpty(checkDirectory))
|
|
continue;
|
|
if (!Directory.Exists(checkDirectory))
|
|
_ = Directory.CreateDirectory(checkDirectory);
|
|
if (File.Exists(checkFile) && new FileInfo(checkFile).Length == 0)
|
|
File.Delete(checkFile);
|
|
checkUniformResourceLocator = ConvertTo(segment.RootUniformResourceLocator, segment.Left.RelativePath);
|
|
if (string.IsNullOrEmpty(checkUniformResourceLocator))
|
|
continue;
|
|
download = new(Directory: checkDirectory,
|
|
Display: checkUniformResourceLocator[segment.RootUniformResourceLocator.Length..],
|
|
File: checkFile,
|
|
Size: segment.Left.Size,
|
|
Ticks: segment.Left.Ticks,
|
|
UniformResourceLocator: checkUniformResourceLocator);
|
|
collection.Add(download);
|
|
}
|
|
Download[] sorted = (from l in collection orderby l.Size select l).ToArray();
|
|
int stop = sorted.Length < 100 ? sorted.Length : 100;
|
|
for (int i = 0; i < stop; i++)
|
|
results.Add(sorted[i]);
|
|
for (int i = sorted.Length - 1; i > stop - 1; i--)
|
|
results.Add(sorted[i]);
|
|
if (collection.Count != results.Count)
|
|
throw new Exception();
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static string? ConvertTo(string rootURL, string relativePath) {
|
|
string? result = rootURL.EndsWith('/') ? rootURL[..^1] : rootURL;
|
|
string windowsRoot = "c:\\";
|
|
string windowsMock = $"{windowsRoot}{relativePath}";
|
|
string fileName = Path.GetFileName(windowsMock);
|
|
ReadOnlyCollection<string> directoryNames = Helpers.HelperDirectory.GetDirectoryNames(windowsMock);
|
|
foreach (string directoryName in directoryNames) {
|
|
if (directoryName == windowsRoot || directoryName == fileName)
|
|
continue;
|
|
result = $"{result}/{directoryName}";
|
|
}
|
|
result = result == rootURL ? null : $"{result}/{fileName}";
|
|
return result;
|
|
}
|
|
|
|
} |