From 0621d0f07eb2e1d59dc616735dcacd41d410b510 Mon Sep 17 00:00:00 2001 From: Mike Phares Date: Sun, 23 Mar 2025 15:57:07 -0700 Subject: [PATCH] Updates to Backup method --- .vscode/tasks.json | 14 ++ ADO2024/PI4/Helper-2024-12-17.cs | 409 +++++++++++++++---------------- 2 files changed, 205 insertions(+), 218 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 875c099..842e9d3 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -228,6 +228,20 @@ "type": "shell", "command": "npx jest", "problemMatcher": [] + }, + { + "label": "File-Folder-Helper AOT s X Day-Helper-2025-03-20", + "type": "shell", + "command": "L:/DevOps/Mesa_FI/File-Folder-Helper/bin/Release/net8.0/win-x64/publish/File-Folder-Helper.exe", + "args": [ + "s", + "X", + "L:/DevOps/Mesa_FI/File-Folder-Helper", + "Day-Helper-2025-03-20", + "false", + "4" + ], + "problemMatcher": [] } ] } \ No newline at end of file diff --git a/ADO2024/PI4/Helper-2024-12-17.cs b/ADO2024/PI4/Helper-2024-12-17.cs index 2a82ad4..a3e1c19 100644 --- a/ADO2024/PI4/Helper-2024-12-17.cs +++ b/ADO2024/PI4/Helper-2024-12-17.cs @@ -11,12 +11,31 @@ namespace File_Folder_Helper.ADO2024.PI4; internal static partial class Helper20241217 { - private record Record(string Directory, Job? Job, string Path); - private record Job(string AlternatePath, string Directory, string Extension, File[] Files, int FilesCount, double FilesTotalLength, int Keep, Target[] Targets); - private record SecureShell(); - private record ServerMessageBlock(string Path, bool Required); - private record Target(SecureShell? SecureShell, ServerMessageBlock? ServerMessageBlock); - private record File(long LastWriteTicks, long Length, string RelativePath); + private record SecureShell( + ); + + private record ServerMessageBlock(string Path, + bool Required); + + private record Target(SecureShell? SecureShell, + ServerMessageBlock? ServerMessageBlock); + + private record File(long LastWriteTicks, + long Length, + string RelativePath); + + private record Record(string Directory, + Job Job, + string Path); + + private record Job(string? AlternatePath, + string Directory, + string Extension, + File[] Files, + int FilesCount, + double FilesTotalLength, + int Keep, + Target[] Targets); [JsonSourceGenerationOptions(WriteIndented = true)] [JsonSerializable(typeof(Job))] @@ -30,6 +49,95 @@ internal static partial class Helper20241217 { } + internal static void Backup(ILogger logger, List args) + { + Job jobNew; + string path; + string? json; + bool areTheyTheSame; + string directoryName; + ReadOnlyCollection files; + string searchPattern = args[2]; + string[] ignoreFileNames = args[3].Split('|'); + string sourceDirectory = Path.GetFullPath(args[0]); + char destinationDriveLetter = args[4].Split(':')[0][0]; + logger.LogInformation("Searching <{sourceDirectory}> with search pattern {searchPattern}", args[0], searchPattern); + if (Debugger.IsAttached) + Verify(searchPattern, ignoreFileNames); + IEnumerable records = GetRecords(sourceDirectory, searchPattern); + foreach (Record record in records) + { + if (record.Job is null || string.IsNullOrEmpty(record.Job.Extension)) + continue; + logger.LogInformation("Searching <{directory}>", record.Directory); + files = GetFiles(searchPattern, ignoreFileNames, record); + jobNew = GetJob(searchPattern, ignoreFileNames, record, files); + json = JsonSerializer.Serialize(jobNew, JobSourceGenerationContext.Default.Job); + areTheyTheSame = GetAreTheyTheSame(logger, searchPattern, ignoreFileNames, record, jobNew); + if (areTheyTheSame) + { + WriteAllText(record.Path, json); + continue; + } + directoryName = Path.GetFileName(record.Directory); + path = Path.Combine(record.Directory, $"{directoryName}-{DateTime.Now:yyyy-MM-dd-HH-mm-ss-fff}{record.Job.Extension}"); + logger.LogInformation("Writing <{directory}> extension", record.Directory); + WritePassedExtension(destinationDriveLetter, record, files, path); + WriteAllText(record.Path, json); + } + if (Debugger.IsAttached && records.Count() == 0) + { + files = GetFiles(sourceDirectory, searchPattern, ignoreFileNames); + json = JsonSerializer.Serialize(files.ToArray(), FilesSourceGenerationContext.Default.FileArray); + WriteAllText(Path.Combine(Environment.CurrentDirectory, ".vscode", "helper", ".json"), json); + } + } + + private static void Verify(string searchPattern, string[] ignoreFileNames) + { + List targets = [ + new(new SecureShell(), null), + new(null, new ServerMessageBlock("\\\\mesfs.infineon.com\\EC_APC\\DEV", true)) + ]; + string directory = Path.Combine(Environment.CurrentDirectory, ".vscode", "helper"); + if (!Directory.Exists(directory)) + _ = Directory.CreateDirectory(directory); + string path = Path.Combine(directory, "verify.json"); + ReadOnlyCollection files = GetFiles(directory, searchPattern, ignoreFileNames); + ReadOnlyCollection collection = GetFilteredFiles(searchPattern, ignoreFileNames, files); + double filesTotalLength = collection.Select(l => l.Length).Sum(); + Job job = new(AlternatePath: "C:/Users/phares", + Directory: directory, + Extension: ".iso", + Files: collection.ToArray(), + FilesCount: files.Count, + FilesTotalLength: filesTotalLength, + Keep: 3, + Targets: targets.ToArray()); + string json = JsonSerializer.Serialize(job, JobSourceGenerationContext.Default.Job); + WriteAllText(path, json); + } + + private static ReadOnlyCollection GetFilteredFiles(string searchPattern, string[] ignoreFileNames, ReadOnlyCollection files) + { + List results = []; + string fileName; + foreach (File file in files) + { + if (file.RelativePath == searchPattern) + continue; + fileName = Path.GetFileName(file.RelativePath); + if (fileName == searchPattern) + throw new Exception("Found nested file!"); + if (ignoreFileNames.Any(l => l == fileName)) + continue; + if (file.Length == 0) + continue; + results.Add(file); + } + return results.AsReadOnly(); + } + private static IEnumerable GetRecords(string directory, string searchPattern) { Job? job; @@ -48,8 +156,19 @@ internal static partial class Helper20241217 continue; } json = System.IO.File.ReadAllText(file); - job = JsonSerializer.Deserialize(json, JobSourceGenerationContext.Default.Job); - record = new(directoryName, job, file); + if (string.IsNullOrEmpty(json) || json is "{}" or "[]") + job = null; + else + job = JsonSerializer.Deserialize(json, JobSourceGenerationContext.Default.Job); + job ??= new(AlternatePath: null, + Directory: directory, + Extension: ".iso", + Files: [], + FilesCount: 0, + FilesTotalLength: 0, + Keep: 3, + Targets: []); + record = new(Directory: directoryName, Job: job, Path: file); yield return record; } } @@ -72,7 +191,7 @@ internal static partial class Helper20241217 relativePath = Path.GetRelativePath(directory, fileInfo.FullName).Replace(';', '_'); if (relativePath.StartsWith("..")) relativePath = relativePath[3..]; - file = new(fileInfo.LastWriteTime.Ticks, fileInfo.Length, relativePath); + file = new(LastWriteTicks: fileInfo.LastWriteTime.Ticks, Length: fileInfo.Length, RelativePath: relativePath); results.Add(file); } return results.AsReadOnly(); @@ -81,61 +200,78 @@ internal static partial class Helper20241217 private static ReadOnlyCollection GetFiles(string searchPattern, string[] ignoreFileNames, Record record) => GetFiles(record.Directory, searchPattern, ignoreFileNames); - private static string? GetJsonIfNotEqual(string searchPattern, string[] ignoreFileNames, Record record, Job job, ReadOnlyCollection files) + private static Job GetJob(string searchPattern, string[] ignoreFileNames, Record record, ReadOnlyCollection files) { - string? result; - string? jsonNew; - string? jsonOld; - string fileName; - int ignoreCount = 0; - double filesTotalLengthNew = 0; - File[] filesArray = files.ToArray(); - double filesTotalLengthOld = job.FilesTotalLength; - foreach (File file in files) - filesTotalLengthNew += file.Length; - Job jobNew = new(job.AlternatePath, - record.Directory, - job.Extension, - filesArray, - files.Count, - filesTotalLengthNew, - job.Keep, - job.Targets); - result = JsonSerializer.Serialize(jobNew, JobSourceGenerationContext.Default.Job); - if (filesTotalLengthNew != filesTotalLengthOld) + Job result; + ReadOnlyCollection collection = GetFilteredFiles(searchPattern, ignoreFileNames, files); + double filesTotalLengthNew = collection.Select(l => l.Length).Sum(); + result = new(AlternatePath: record.Job.AlternatePath, + Directory: record.Directory, + Extension: record.Job.Extension, + Files: collection.ToArray(), + FilesCount: collection.Count, + FilesTotalLength: filesTotalLengthNew, + Keep: record.Job.Keep, + Targets: record.Job.Targets); + return result; + } + + private static bool GetAreTheyTheSame(ILogger logger, string searchPattern, string[] ignoreFileNames, Record record, Job jobNew) + { + bool result; + ReadOnlyCollection collection = GetFilteredFiles(searchPattern, ignoreFileNames, record.Job.Files.AsReadOnly()); + int filesCountOld = collection.Count; + int filesCountNew = jobNew.Files.Length; + if (filesCountNew != filesCountOld) { - filesTotalLengthOld = 0; - foreach (File file in job.Files) - { - fileName = Path.GetFileName(file.RelativePath); - if (fileName == searchPattern || ignoreFileNames.Any(l => l == fileName)) - { - ignoreCount += 1; - continue; - } - if (file.Length == 0) - { - ignoreCount += 1; - continue; - } - filesTotalLengthOld += file.Length; - } - } - if (filesTotalLengthNew != filesTotalLengthOld || files.Count != (job.Files.Length - ignoreCount)) - { - jsonNew = null; - jsonOld = null; + result = false; + logger.LogInformation("<{directory}> file count has changed {filesCountNew} != {filesCountOld}", record.Directory, filesCountNew, filesCountOld); } else { - jsonNew = JsonSerializer.Serialize((from l in filesArray orderby l.RelativePath.Length, l.RelativePath select l).ToArray(), FilesSourceGenerationContext.Default.FileArray); - jsonOld = JsonSerializer.Serialize((from l in job.Files orderby l.RelativePath.Length, l.RelativePath where l.RelativePath != searchPattern select l).ToArray(), FilesSourceGenerationContext.Default.FileArray); + double filesTotalLengthOld = collection.Select(l => l.Length).Sum(); + double filesTotalLengthNew = jobNew.Files.Select(l => l.Length).Sum(); + if (filesTotalLengthNew != filesTotalLengthOld) + { + result = false; + logger.LogInformation("<{directory}> file length has changed {filesTotalLengthNew} != {filesTotalLengthOld}", record.Directory, filesTotalLengthNew, filesTotalLengthOld); + } + else + { + string jsonNew = JsonSerializer.Serialize(jobNew.Files, FilesSourceGenerationContext.Default.FileArray); + string jsonOld = JsonSerializer.Serialize(collection.ToArray(), FilesSourceGenerationContext.Default.FileArray); + if (jsonNew == jsonOld) + result = true; + else + { + result = false; + WriteAllText(Path.Combine(Environment.CurrentDirectory, ".vscode", "helper", "old.json"), jsonOld); + WriteAllText(Path.Combine(Environment.CurrentDirectory, ".vscode", "helper", "new.json"), jsonNew); + logger.LogInformation("<{directory}> file serialized are different {filesTotalLengthNew} != {filesTotalLengthOld}", record.Directory, filesTotalLengthNew, filesTotalLengthOld); + } + } } - if (!string.IsNullOrEmpty(jsonNew) && !string.IsNullOrEmpty(jsonOld) && jsonNew == jsonOld) - result = null; return result; } + private static void WriteAllText(string path, string text) + { + string check = !System.IO.File.Exists(path) ? string.Empty : System.IO.File.ReadAllText(path); + if (check != text) + System.IO.File.WriteAllText(path, text); + } + + private static void WritePassedExtension(char destinationDriveLetter, Record record, ReadOnlyCollection files, string path) + { + string directoryName = Path.GetFileName(record.Directory); + if (record.Job.Extension.Equals(".iso", StringComparison.OrdinalIgnoreCase)) + WriteISO(destinationDriveLetter, record, files, path, directoryName); + else if (record.Job.Extension.Equals(".zip", StringComparison.OrdinalIgnoreCase)) + WriteZIP(destinationDriveLetter, record, files, path); + else + throw new NotImplementedException(); + } + private static void WriteISO(char destinationDriveLetter, Record record, ReadOnlyCollection files, string path, string directoryName) { string checkFile = $"{destinationDriveLetter}{path[1..]}"; @@ -169,167 +305,4 @@ internal static partial class Helper20241217 _ = zip.CreateEntryFromFile(Path.Combine(record.Directory, file.RelativePath), file.RelativePath); } - private static void WriteExtension(char destinationDriveLetter, Record record, Job job, ReadOnlyCollection files, string path) - { - string directoryName = Path.GetFileName(record.Directory); - if (job.Extension.Equals(".iso", StringComparison.OrdinalIgnoreCase)) - WriteISO(destinationDriveLetter, record, files, path, directoryName); - else if (job.Extension.Equals(".zip", StringComparison.OrdinalIgnoreCase)) - WriteZIP(destinationDriveLetter, record, files, path); - else - throw new NotImplementedException(); - } - - private static void PushTo(ServerMessageBlock serverMessageBlock, string path) - { - string remotePath = Path.Combine(serverMessageBlock.Path, Path.GetFileName(path)); - System.IO.File.Copy(path, remotePath); - } - - private static void PushTo(string directory, string path) - { - string remotePath = Path.Combine(directory, Path.GetFileName(path)); - System.IO.File.Copy(path, remotePath); - } - - private static ReadOnlyCollection PushTo(Job job, string path) - { - List results = []; - foreach (Target target in job.Targets) - { - if (target.SecureShell is not null) - continue; - else if (target.ServerMessageBlock is not null) - { - try - { PushTo(target.ServerMessageBlock, path); } - catch (Exception ex) - { - if (target.ServerMessageBlock.Required) - results.Add(ex); - } - } - else - throw new NotImplementedException(); - } - return results.AsReadOnly(); - } - - private static void DeleteOld(Job job, ServerMessageBlock serverMessageBlock, string path) - { - List results = []; - string[] files = Directory.GetFiles(serverMessageBlock.Path, $"*{job.Extension}", SearchOption.TopDirectoryOnly); - foreach (string file in files) - { - if (file == path) - continue; - results.Add(file); - } - for (int i = job.Keep - 1; i < results.Count; i++) - System.IO.File.Delete(results[i]); - } - - private static ReadOnlyCollection DeleteOld(Job job, string path) - { - List results = []; - foreach (Target target in job.Targets) - { - if (target.SecureShell is not null) - continue; - else if (target.ServerMessageBlock is not null) - { - try - { DeleteOld(job, target.ServerMessageBlock, path); } - catch (Exception ex) - { - if (target.ServerMessageBlock.Required) - results.Add(ex); - } - } - else - throw new NotImplementedException(); - } - return results.AsReadOnly(); - } - - private static void Verify(string searchPattern, string[] ignoreFileNames) - { - List targets = [ - new(new SecureShell(), null), - new(null, new ServerMessageBlock("\\\\mesfs.infineon.com\\EC_APC\\DEV", true)) - ]; - string directory = Path.Combine(Environment.CurrentDirectory, ".vscode", "helper"); - if (!Directory.Exists(directory)) - _ = Directory.CreateDirectory(directory); - ReadOnlyCollection files = GetFiles(directory, searchPattern, ignoreFileNames); - double filesTotalLength = 0; - foreach (File file in files) - filesTotalLength += file.Length; - Job job = new( - "C:/Users/phares", - directory, - "*.iso", - files.ToArray(), - files.Count, - filesTotalLength, - 3, - targets.ToArray()); - string json = JsonSerializer.Serialize(job, JobSourceGenerationContext.Default.Job); - System.IO.File.WriteAllText(Path.Combine(directory, "verify.json"), json); - } - - internal static void Backup(ILogger logger, List args) - { - string path; - string? json; - string directoryName; - ReadOnlyCollection files; - string searchPattern = args[2]; - ReadOnlyCollection exceptions; - string[] ignoreFileNames = args[3].Split('|'); - string sourceDirectory = Path.GetFullPath(args[0]); - char destinationDriveLetter = args[4].Split(':')[0][0]; - logger.LogInformation("Searching <{sourceDirectory}> with search pattern {searchPattern}", args[0], searchPattern); - if (Debugger.IsAttached) - Verify(searchPattern, ignoreFileNames); - IEnumerable records = GetRecords(sourceDirectory, searchPattern); - foreach (Record record in records) - { - if (record.Job is null || record.Job.Targets.Length == 0 || string.IsNullOrEmpty(record.Job.Extension)) - continue; - logger.LogInformation("Searching <{directory}>", record.Directory); - files = GetFiles(searchPattern, ignoreFileNames, record); - json = GetJsonIfNotEqual(searchPattern, ignoreFileNames, record, record.Job, files); - if (string.IsNullOrEmpty(json)) - continue; - directoryName = Path.GetFileName(record.Directory); - path = Path.Combine(record.Directory, $"{directoryName}-{DateTime.Now:yyyy-MM-dd-HH-mm-ss-fff}{record.Job.Extension}"); - logger.LogInformation("Writing <{directory}> extension", record.Directory); - WriteExtension(destinationDriveLetter, record, record.Job, files, path); - logger.LogInformation("Pushing <{directory}> extension", record.Directory); - exceptions = PushTo(record.Job, path); - if (exceptions.Count != 0) - { - foreach (Exception exception in exceptions) - logger.LogError(exception, exception.Message); - PushTo(record.Job.AlternatePath, path); - } - System.IO.File.WriteAllText(record.Path, json); - System.IO.File.Delete(path); - logger.LogInformation("Deleting old <{directory}> extension", record.Directory); - exceptions = DeleteOld(record.Job, path); - if (exceptions.Count != 0) - { - foreach (Exception exception in exceptions) - logger.LogError(exception, exception.Message); - } - } - if (Debugger.IsAttached && records.Count() == 0) - { - files = GetFiles(sourceDirectory, searchPattern, ignoreFileNames); - json = JsonSerializer.Serialize(files.ToArray(), FilesSourceGenerationContext.Default.FileArray); - System.IO.File.WriteAllText(Path.Combine(Environment.CurrentDirectory, ".vscode", "helper", ".json"), json); - } - } - } \ No newline at end of file