phares3757/file-folder-helper
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updates to Backup method

Browse Source
This commit is contained in:
Mike Phares 2025-03-23 15:57:07 -07:00
parent b74a6a5267
commit 0621d0f07e
2 changed files with 205 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
.vscode/tasks.json vendored
View File

@ -228,6 +228,20 @@
"type": "shell",
"command": "npx jest",
"problemMatcher": []
},
{
"label": "File-Folder-Helper AOT s X Day-Helper-2025-03-20",
"type": "shell",
"command": "L:/DevOps/Mesa_FI/File-Folder-Helper/bin/Release/net8.0/win-x64/publish/File-Folder-Helper.exe",
"args": [
"s",
"X",
"L:/DevOps/Mesa_FI/File-Folder-Helper",
"Day-Helper-2025-03-20",
"false",
"4"
],
"problemMatcher": []
}
]
}

409
ADO2024/PI4/Helper-2024-12-17.cs
View File

@ -11,12 +11,31 @@ namespace File_Folder_Helper.ADO2024.PI4;
internal static partial class Helper20241217
{
private record Record(string Directory, Job? Job, string Path);
private record Job(string AlternatePath, string Directory, string Extension, File[] Files, int FilesCount, double FilesTotalLength, int Keep, Target[] Targets);
private record SecureShell();
private record ServerMessageBlock(string Path, bool Required);
private record Target(SecureShell? SecureShell, ServerMessageBlock? ServerMessageBlock);
private record File(long LastWriteTicks, long Length, string RelativePath);
private record SecureShell(
);
private record ServerMessageBlock(string Path,
bool Required);
private record Target(SecureShell? SecureShell,
ServerMessageBlock? ServerMessageBlock);
private record File(long LastWriteTicks,
long Length,
string RelativePath);
private record Record(string Directory,
Job Job,
string Path);
private record Job(string? AlternatePath,
string Directory,
string Extension,
File[] Files,
int FilesCount,
double FilesTotalLength,
int Keep,
Target[] Targets);
[JsonSourceGenerationOptions(WriteIndented = true)]
[JsonSerializable(typeof(Job))]
@ -30,6 +49,95 @@ internal static partial class Helper20241217
{
}
internal static void Backup(ILogger<Worker> logger, List<string> args)
{
Job jobNew;
string path;
string? json;
bool areTheyTheSame;
string directoryName;
ReadOnlyCollection<File> files;
string searchPattern = args[2];
string[] ignoreFileNames = args[3].Split('|');
string sourceDirectory = Path.GetFullPath(args[0]);
char destinationDriveLetter = args[4].Split(':')[0][0];
logger.LogInformation("Searching <{sourceDirectory}> with search pattern {searchPattern}", args[0], searchPattern);
if (Debugger.IsAttached)
Verify(searchPattern, ignoreFileNames);
IEnumerable<Record> records = GetRecords(sourceDirectory, searchPattern);
foreach (Record record in records)
{
if (record.Job is null || string.IsNullOrEmpty(record.Job.Extension))
continue;
logger.LogInformation("Searching <{directory}>", record.Directory);
files = GetFiles(searchPattern, ignoreFileNames, record);
jobNew = GetJob(searchPattern, ignoreFileNames, record, files);
json = JsonSerializer.Serialize(jobNew, JobSourceGenerationContext.Default.Job);
areTheyTheSame = GetAreTheyTheSame(logger, searchPattern, ignoreFileNames, record, jobNew);
if (areTheyTheSame)
{
WriteAllText(record.Path, json);
continue;
}
directoryName = Path.GetFileName(record.Directory);
path = Path.Combine(record.Directory, $"{directoryName}-{DateTime.Now:yyyy-MM-dd-HH-mm-ss-fff}{record.Job.Extension}");
logger.LogInformation("Writing <{directory}> extension", record.Directory);
WritePassedExtension(destinationDriveLetter, record, files, path);
WriteAllText(record.Path, json);
}
if (Debugger.IsAttached && records.Count() == 0)
{
files = GetFiles(sourceDirectory, searchPattern, ignoreFileNames);
json = JsonSerializer.Serialize(files.ToArray(), FilesSourceGenerationContext.Default.FileArray);
WriteAllText(Path.Combine(Environment.CurrentDirectory, ".vscode", "helper", ".json"), json);
}
}
private static void Verify(string searchPattern, string[] ignoreFileNames)
{
List<Target> targets = [
new(new SecureShell(), null),
new(null, new ServerMessageBlock("\\\\mesfs.infineon.com\\EC_APC\\DEV", true))
];
string directory = Path.Combine(Environment.CurrentDirectory, ".vscode", "helper");
if (!Directory.Exists(directory))
_ = Directory.CreateDirectory(directory);
string path = Path.Combine(directory, "verify.json");
ReadOnlyCollection<File> files = GetFiles(directory, searchPattern, ignoreFileNames);
ReadOnlyCollection<File> collection = GetFilteredFiles(searchPattern, ignoreFileNames, files);
double filesTotalLength = collection.Select(l => l.Length).Sum();
Job job = new(AlternatePath: "C:/Users/phares",
Directory: directory,
Extension: ".iso",
Files: collection.ToArray(),
FilesCount: files.Count,
FilesTotalLength: filesTotalLength,
Keep: 3,
Targets: targets.ToArray());
string json = JsonSerializer.Serialize(job, JobSourceGenerationContext.Default.Job);
WriteAllText(path, json);
}
private static ReadOnlyCollection<File> GetFilteredFiles(string searchPattern, string[] ignoreFileNames, ReadOnlyCollection<File> files)
{
List<File> results = [];
string fileName;
foreach (File file in files)
{
if (file.RelativePath == searchPattern)
continue;
fileName = Path.GetFileName(file.RelativePath);
if (fileName == searchPattern)
throw new Exception("Found nested file!");
if (ignoreFileNames.Any(l => l == fileName))
continue;
if (file.Length == 0)
continue;
results.Add(file);
}
return results.AsReadOnly();
}
private static IEnumerable<Record> GetRecords(string directory, string searchPattern)
{
Job? job;
@ -48,8 +156,19 @@ internal static partial class Helper20241217
continue;
}
json = System.IO.File.ReadAllText(file);
job = JsonSerializer.Deserialize(json, JobSourceGenerationContext.Default.Job);
record = new(directoryName, job, file);
if (string.IsNullOrEmpty(json) || json is "{}" or "[]")
job = null;
else
job = JsonSerializer.Deserialize(json, JobSourceGenerationContext.Default.Job);
job ??= new(AlternatePath: null,
Directory: directory,
Extension: ".iso",
Files: [],
FilesCount: 0,
FilesTotalLength: 0,
Keep: 3,
Targets: []);
record = new(Directory: directoryName, Job: job, Path: file);
yield return record;
}
}
@ -72,7 +191,7 @@ internal static partial class Helper20241217
relativePath = Path.GetRelativePath(directory, fileInfo.FullName).Replace(';', '_');
if (relativePath.StartsWith(".."))
relativePath = relativePath[3..];
file = new(fileInfo.LastWriteTime.Ticks, fileInfo.Length, relativePath);
file = new(LastWriteTicks: fileInfo.LastWriteTime.Ticks, Length: fileInfo.Length, RelativePath: relativePath);
results.Add(file);
}
return results.AsReadOnly();
@ -81,61 +200,78 @@ internal static partial class Helper20241217
private static ReadOnlyCollection<File> GetFiles(string searchPattern, string[] ignoreFileNames, Record record) =>
GetFiles(record.Directory, searchPattern, ignoreFileNames);
private static string? GetJsonIfNotEqual(string searchPattern, string[] ignoreFileNames, Record record, Job job, ReadOnlyCollection<File> files)
private static Job GetJob(string searchPattern, string[] ignoreFileNames, Record record, ReadOnlyCollection<File> files)
{
string? result;
string? jsonNew;
string? jsonOld;
string fileName;
int ignoreCount = 0;
double filesTotalLengthNew = 0;
File[] filesArray = files.ToArray();
double filesTotalLengthOld = job.FilesTotalLength;
foreach (File file in files)
filesTotalLengthNew += file.Length;
Job jobNew = new(job.AlternatePath,
record.Directory,
job.Extension,
filesArray,
files.Count,
filesTotalLengthNew,
job.Keep,
job.Targets);
result = JsonSerializer.Serialize(jobNew, JobSourceGenerationContext.Default.Job);
if (filesTotalLengthNew != filesTotalLengthOld)
Job result;
ReadOnlyCollection<File> collection = GetFilteredFiles(searchPattern, ignoreFileNames, files);
double filesTotalLengthNew = collection.Select(l => l.Length).Sum();
result = new(AlternatePath: record.Job.AlternatePath,
Directory: record.Directory,
Extension: record.Job.Extension,
Files: collection.ToArray(),
FilesCount: collection.Count,
FilesTotalLength: filesTotalLengthNew,
Keep: record.Job.Keep,
Targets: record.Job.Targets);
return result;
}
private static bool GetAreTheyTheSame(ILogger<Worker> logger, string searchPattern, string[] ignoreFileNames, Record record, Job jobNew)
{
bool result;
ReadOnlyCollection<File> collection = GetFilteredFiles(searchPattern, ignoreFileNames, record.Job.Files.AsReadOnly());
int filesCountOld = collection.Count;
int filesCountNew = jobNew.Files.Length;
if (filesCountNew != filesCountOld)
{
filesTotalLengthOld = 0;
foreach (File file in job.Files)
{
fileName = Path.GetFileName(file.RelativePath);
if (fileName == searchPattern || ignoreFileNames.Any(l => l == fileName))
{
ignoreCount += 1;
continue;
}
if (file.Length == 0)
{
ignoreCount += 1;
continue;
}
filesTotalLengthOld += file.Length;
}
}
if (filesTotalLengthNew != filesTotalLengthOld || files.Count != (job.Files.Length - ignoreCount))
{
jsonNew = null;
jsonOld = null;
result = false;
logger.LogInformation("<{directory}> file count has changed {filesCountNew} != {filesCountOld}", record.Directory, filesCountNew, filesCountOld);
}
else
{
jsonNew = JsonSerializer.Serialize((from l in filesArray orderby l.RelativePath.Length, l.RelativePath select l).ToArray(), FilesSourceGenerationContext.Default.FileArray);
jsonOld = JsonSerializer.Serialize((from l in job.Files orderby l.RelativePath.Length, l.RelativePath where l.RelativePath != searchPattern select l).ToArray(), FilesSourceGenerationContext.Default.FileArray);
double filesTotalLengthOld = collection.Select(l => l.Length).Sum();
double filesTotalLengthNew = jobNew.Files.Select(l => l.Length).Sum();
if (filesTotalLengthNew != filesTotalLengthOld)
{
result = false;
logger.LogInformation("<{directory}> file length has changed {filesTotalLengthNew} != {filesTotalLengthOld}", record.Directory, filesTotalLengthNew, filesTotalLengthOld);
}
else
{
string jsonNew = JsonSerializer.Serialize(jobNew.Files, FilesSourceGenerationContext.Default.FileArray);
string jsonOld = JsonSerializer.Serialize(collection.ToArray(), FilesSourceGenerationContext.Default.FileArray);
if (jsonNew == jsonOld)
result = true;
else
{
result = false;
WriteAllText(Path.Combine(Environment.CurrentDirectory, ".vscode", "helper", "old.json"), jsonOld);
WriteAllText(Path.Combine(Environment.CurrentDirectory, ".vscode", "helper", "new.json"), jsonNew);
logger.LogInformation("<{directory}> file serialized are different {filesTotalLengthNew} != {filesTotalLengthOld}", record.Directory, filesTotalLengthNew, filesTotalLengthOld);
}
}
}
if (!string.IsNullOrEmpty(jsonNew) && !string.IsNullOrEmpty(jsonOld) && jsonNew == jsonOld)
result = null;
return result;
}
private static void WriteAllText(string path, string text)
{
string check = !System.IO.File.Exists(path) ? string.Empty : System.IO.File.ReadAllText(path);
if (check != text)
System.IO.File.WriteAllText(path, text);
}
private static void WritePassedExtension(char destinationDriveLetter, Record record, ReadOnlyCollection<File> files, string path)
{
string directoryName = Path.GetFileName(record.Directory);
if (record.Job.Extension.Equals(".iso", StringComparison.OrdinalIgnoreCase))
WriteISO(destinationDriveLetter, record, files, path, directoryName);
else if (record.Job.Extension.Equals(".zip", StringComparison.OrdinalIgnoreCase))
WriteZIP(destinationDriveLetter, record, files, path);
else
throw new NotImplementedException();
}
private static void WriteISO(char destinationDriveLetter, Record record, ReadOnlyCollection<File> files, string path, string directoryName)
{
string checkFile = $"{destinationDriveLetter}{path[1..]}";
@ -169,167 +305,4 @@ internal static partial class Helper20241217
_ = zip.CreateEntryFromFile(Path.Combine(record.Directory, file.RelativePath), file.RelativePath);
}
private static void WriteExtension(char destinationDriveLetter, Record record, Job job, ReadOnlyCollection<File> files, string path)
{
string directoryName = Path.GetFileName(record.Directory);
if (job.Extension.Equals(".iso", StringComparison.OrdinalIgnoreCase))
WriteISO(destinationDriveLetter, record, files, path, directoryName);
else if (job.Extension.Equals(".zip", StringComparison.OrdinalIgnoreCase))
WriteZIP(destinationDriveLetter, record, files, path);
else
throw new NotImplementedException();
}
private static void PushTo(ServerMessageBlock serverMessageBlock, string path)
{
string remotePath = Path.Combine(serverMessageBlock.Path, Path.GetFileName(path));
System.IO.File.Copy(path, remotePath);
}
private static void PushTo(string directory, string path)
{
string remotePath = Path.Combine(directory, Path.GetFileName(path));
System.IO.File.Copy(path, remotePath);
}
private static ReadOnlyCollection<Exception> PushTo(Job job, string path)
{
List<Exception> results = [];
foreach (Target target in job.Targets)
{
if (target.SecureShell is not null)
continue;
else if (target.ServerMessageBlock is not null)
{
try
{ PushTo(target.ServerMessageBlock, path); }
catch (Exception ex)
{
if (target.ServerMessageBlock.Required)
results.Add(ex);
}
}
else
throw new NotImplementedException();
}
return results.AsReadOnly();
}
private static void DeleteOld(Job job, ServerMessageBlock serverMessageBlock, string path)
{
List<string> results = [];
string[] files = Directory.GetFiles(serverMessageBlock.Path, $"*{job.Extension}", SearchOption.TopDirectoryOnly);
foreach (string file in files)
{
if (file == path)
continue;
results.Add(file);
}
for (int i = job.Keep - 1; i < results.Count; i++)
System.IO.File.Delete(results[i]);
}
private static ReadOnlyCollection<Exception> DeleteOld(Job job, string path)
{
List<Exception> results = [];
foreach (Target target in job.Targets)
{
if (target.SecureShell is not null)
continue;
else if (target.ServerMessageBlock is not null)
{
try
{ DeleteOld(job, target.ServerMessageBlock, path); }
catch (Exception ex)
{
if (target.ServerMessageBlock.Required)
results.Add(ex);
}
}
else
throw new NotImplementedException();
}
return results.AsReadOnly();
}
private static void Verify(string searchPattern, string[] ignoreFileNames)
{
List<Target> targets = [
new(new SecureShell(), null),
new(null, new ServerMessageBlock("\\\\mesfs.infineon.com\\EC_APC\\DEV", true))
];
string directory = Path.Combine(Environment.CurrentDirectory, ".vscode", "helper");
if (!Directory.Exists(directory))
_ = Directory.CreateDirectory(directory);
ReadOnlyCollection<File> files = GetFiles(directory, searchPattern, ignoreFileNames);
double filesTotalLength = 0;
foreach (File file in files)
filesTotalLength += file.Length;
Job job = new(
"C:/Users/phares",
directory,
"*.iso",
files.ToArray(),
files.Count,
filesTotalLength,
3,
targets.ToArray());
string json = JsonSerializer.Serialize(job, JobSourceGenerationContext.Default.Job);
System.IO.File.WriteAllText(Path.Combine(directory, "verify.json"), json);
}
internal static void Backup(ILogger<Worker> logger, List<string> args)
{
string path;
string? json;
string directoryName;
ReadOnlyCollection<File> files;
string searchPattern = args[2];
ReadOnlyCollection<Exception> exceptions;
string[] ignoreFileNames = args[3].Split('|');
string sourceDirectory = Path.GetFullPath(args[0]);
char destinationDriveLetter = args[4].Split(':')[0][0];
logger.LogInformation("Searching <{sourceDirectory}> with search pattern {searchPattern}", args[0], searchPattern);
if (Debugger.IsAttached)
Verify(searchPattern, ignoreFileNames);
IEnumerable<Record> records = GetRecords(sourceDirectory, searchPattern);
foreach (Record record in records)
{
if (record.Job is null || record.Job.Targets.Length == 0 || string.IsNullOrEmpty(record.Job.Extension))
continue;
logger.LogInformation("Searching <{directory}>", record.Directory);
files = GetFiles(searchPattern, ignoreFileNames, record);
json = GetJsonIfNotEqual(searchPattern, ignoreFileNames, record, record.Job, files);
if (string.IsNullOrEmpty(json))
continue;
directoryName = Path.GetFileName(record.Directory);
path = Path.Combine(record.Directory, $"{directoryName}-{DateTime.Now:yyyy-MM-dd-HH-mm-ss-fff}{record.Job.Extension}");
logger.LogInformation("Writing <{directory}> extension", record.Directory);
WriteExtension(destinationDriveLetter, record, record.Job, files, path);
logger.LogInformation("Pushing <{directory}> extension", record.Directory);
exceptions = PushTo(record.Job, path);
if (exceptions.Count != 0)
{
foreach (Exception exception in exceptions)
logger.LogError(exception, exception.Message);
PushTo(record.Job.AlternatePath, path);
}
System.IO.File.WriteAllText(record.Path, json);
System.IO.File.Delete(path);
logger.LogInformation("Deleting old <{directory}> extension", record.Directory);
exceptions = DeleteOld(record.Job, path);
if (exceptions.Count != 0)
{
foreach (Exception exception in exceptions)
logger.LogError(exception, exception.Message);
}
}
if (Debugger.IsAttached && records.Count() == 0)
{
files = GetFiles(sourceDirectory, searchPattern, ignoreFileNames);
json = JsonSerializer.Serialize(files.ToArray(), FilesSourceGenerationContext.Default.FileArray);
System.IO.File.WriteAllText(Path.Combine(Environment.CurrentDirectory, ".vscode", "helper", ".json"), json);
}
}
}