Mike Phares
fb9289a572
Better ISO support Only reviewing Files when comparing Extracted sections from UpdateSubTasksInMarkdownFiles
383 lines
16 KiB
C#
383 lines
16 KiB
C#
using DiscUtils.Iso9660;
|
|
using Microsoft.Extensions.Logging;
|
|
using Renci.SshNet;
|
|
using Renci.SshNet.Sftp;
|
|
using System.Collections.ObjectModel;
|
|
using System.Diagnostics;
|
|
using System.IO.Compression;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Serialization;
|
|
|
|
namespace File_Folder_Helper.ADO2024.PI4;
|
|
|
|
internal static partial class Helper20241217
|
|
{
|
|
|
|
private record Record(string Directory, Job? Job, string Path);
|
|
private record Job(string AlternatePath, string Directory, string Extension, File[] Files, int FilesCount, double FilesTotalLength, int Keep, Target[] Targets);
|
|
private record SecureShell(string Host, string Key, string Path, bool Required, string User);
|
|
private record ServerMessageBlock(string Path, bool Required);
|
|
private record Target(SecureShell? SecureShell, ServerMessageBlock? ServerMessageBlock);
|
|
private record File(long LastWriteTicks, long Length, string RelativePath);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(Job))]
|
|
private partial class JobSourceGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(File[]))]
|
|
private partial class FilesSourceGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
private static IEnumerable<Record> GetRecords(string directory, string searchPattern)
|
|
{
|
|
Job? job;
|
|
string json;
|
|
Record record;
|
|
string fileName;
|
|
string directoryName;
|
|
IEnumerable<string> files = Directory.EnumerateFiles(directory, searchPattern, new EnumerationOptions { IgnoreInaccessible = true, RecurseSubdirectories = true });
|
|
foreach (string file in files)
|
|
{
|
|
fileName = Path.GetFileName(file);
|
|
directoryName = Path.GetDirectoryName(file) ?? throw new Exception();
|
|
if (!fileName.StartsWith('.'))
|
|
{
|
|
System.IO.File.Move(file, Path.Combine(directoryName, $".{fileName}"));
|
|
continue;
|
|
}
|
|
json = System.IO.File.ReadAllText(file);
|
|
job = JsonSerializer.Deserialize(json, JobSourceGenerationContext.Default.Job);
|
|
record = new(directoryName, job, file);
|
|
yield return record;
|
|
}
|
|
}
|
|
|
|
private static ReadOnlyCollection<File> GetFiles(string directory, string searchPattern, string[] ignoreFileNames)
|
|
{
|
|
List<File> results = [];
|
|
File file;
|
|
string relativePath;
|
|
string[] files = Directory.GetFiles(directory, "*", SearchOption.AllDirectories);
|
|
FileInfo[] fileInfoCollection = files.Select(l => new FileInfo(l)).ToArray();
|
|
foreach (FileInfo fileInfo in fileInfoCollection)
|
|
{
|
|
if (fileInfo.Name == searchPattern)
|
|
continue;
|
|
if (ignoreFileNames.Any(l => l == fileInfo.Name))
|
|
continue;
|
|
if (!string.IsNullOrEmpty(fileInfo.LinkTarget))
|
|
continue;
|
|
relativePath = Path.GetRelativePath(directory, fileInfo.FullName).Replace(';', '_');
|
|
if (relativePath.StartsWith(".."))
|
|
relativePath = relativePath[3..];
|
|
file = new(fileInfo.LastWriteTime.Ticks, fileInfo.Length, relativePath);
|
|
results.Add(file);
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static ReadOnlyCollection<File> GetFiles(string searchPattern, string[] ignoreFileNames, Record record) =>
|
|
GetFiles(record.Directory, searchPattern, ignoreFileNames);
|
|
|
|
private static string? GetJsonIfNotEqual(string searchPattern, string[] ignoreFileNames, Record record, Job job, ReadOnlyCollection<File> files)
|
|
{
|
|
string? result;
|
|
string? jsonNew;
|
|
string? jsonOld;
|
|
string fileName;
|
|
int ignoreCount = 0;
|
|
double filesTotalLengthNew = 0;
|
|
File[] filesArray = files.ToArray();
|
|
double filesTotalLengthOld = job.FilesTotalLength;
|
|
foreach (File file in files)
|
|
filesTotalLengthNew += file.Length;
|
|
Job jobNew = new(job.AlternatePath,
|
|
record.Directory,
|
|
job.Extension,
|
|
filesArray,
|
|
files.Count,
|
|
filesTotalLengthNew,
|
|
job.Keep,
|
|
job.Targets);
|
|
result = JsonSerializer.Serialize(jobNew, JobSourceGenerationContext.Default.Job);
|
|
if (filesTotalLengthNew != filesTotalLengthOld)
|
|
{
|
|
filesTotalLengthOld = 0;
|
|
foreach (File file in job.Files)
|
|
{
|
|
fileName = Path.GetFileName(file.RelativePath);
|
|
if (fileName == searchPattern || ignoreFileNames.Any(l => l == fileName))
|
|
{
|
|
ignoreCount += 1;
|
|
continue;
|
|
}
|
|
if (file.Length == 0)
|
|
{
|
|
ignoreCount += 1;
|
|
continue;
|
|
}
|
|
filesTotalLengthOld += file.Length;
|
|
}
|
|
}
|
|
if (filesTotalLengthNew != filesTotalLengthOld || files.Count != (job.Files.Length - ignoreCount))
|
|
{
|
|
jsonNew = null;
|
|
jsonOld = null;
|
|
}
|
|
else
|
|
{
|
|
jsonNew = JsonSerializer.Serialize((from l in filesArray orderby l.RelativePath.Length, l.RelativePath select l).ToArray(), FilesSourceGenerationContext.Default.FileArray);
|
|
jsonOld = JsonSerializer.Serialize((from l in job.Files orderby l.RelativePath.Length, l.RelativePath where l.RelativePath != searchPattern select l).ToArray(), FilesSourceGenerationContext.Default.FileArray);
|
|
}
|
|
if (!string.IsNullOrEmpty(jsonNew) && !string.IsNullOrEmpty(jsonOld) && jsonNew == jsonOld)
|
|
result = null;
|
|
return result;
|
|
}
|
|
|
|
private static void WriteISO(Record record, ReadOnlyCollection<File> files, string path, string directoryName)
|
|
{
|
|
CDBuilder builder = new() { UseJoliet = true, VolumeIdentifier = directoryName.Length < 25 ? directoryName : directoryName[..25] };
|
|
foreach (File file in files)
|
|
_ = builder.AddFile(file.RelativePath, Path.Combine(record.Directory, file.RelativePath));
|
|
builder.Build(path);
|
|
}
|
|
|
|
private static void WriteZIP(Record record, ReadOnlyCollection<File> files, string path)
|
|
{
|
|
using ZipArchive zip = ZipFile.Open(path, ZipArchiveMode.Create);
|
|
string directoryEntry;
|
|
List<string> directoryEntries = [];
|
|
foreach (File file in files)
|
|
{
|
|
directoryEntry = Path.GetDirectoryName(file.RelativePath) ?? throw new Exception();
|
|
if (!directoryEntries.Contains(directoryEntry))
|
|
continue;
|
|
directoryEntries.Add(directoryEntry);
|
|
_ = zip.CreateEntry(file.RelativePath);
|
|
}
|
|
foreach (File file in files)
|
|
_ = zip.CreateEntryFromFile(Path.Combine(record.Directory, file.RelativePath), file.RelativePath);
|
|
}
|
|
|
|
private static void WriteExtension(Record record, Job job, ReadOnlyCollection<File> files, string path)
|
|
{
|
|
string directoryName = Path.GetFileName(record.Directory);
|
|
if (job.Extension.Equals(".iso", StringComparison.OrdinalIgnoreCase))
|
|
WriteISO(record, files, path, directoryName);
|
|
else if (job.Extension.Equals(".zip", StringComparison.OrdinalIgnoreCase))
|
|
WriteZIP(record, files, path);
|
|
else
|
|
throw new NotImplementedException();
|
|
}
|
|
|
|
private static void PushTo(Job job, SecureShell secureShell, string path)
|
|
{
|
|
string remotePath = string.Concat(secureShell.Path, '/', Path.GetFileName(path));
|
|
using SftpClient client = new(secureShell.Host, secureShell.User, new PrivateKeyFile(secureShell.Key));
|
|
client.Connect();
|
|
if (job.Files.Length == 0)
|
|
{
|
|
string directoryName = Path.GetDirectoryName(secureShell.Path) ?? throw new Exception();
|
|
try
|
|
{ client.CreateDirectory(Path.GetDirectoryName(directoryName) ?? throw new Exception()); }
|
|
catch (Exception) { }
|
|
try
|
|
{ client.CreateDirectory(directoryName); }
|
|
catch (Exception) { }
|
|
try
|
|
{ client.CreateDirectory(secureShell.Path); }
|
|
catch (Exception) { }
|
|
}
|
|
using FileStream fileStream = System.IO.File.OpenRead(path);
|
|
client.UploadFile(fileStream, remotePath);
|
|
}
|
|
|
|
private static void PushTo(ServerMessageBlock serverMessageBlock, string path)
|
|
{
|
|
string remotePath = Path.Combine(serverMessageBlock.Path, Path.GetFileName(path));
|
|
System.IO.File.Copy(path, remotePath);
|
|
}
|
|
|
|
private static void PushTo(string directory, string path)
|
|
{
|
|
string remotePath = Path.Combine(directory, Path.GetFileName(path));
|
|
System.IO.File.Copy(path, remotePath);
|
|
}
|
|
|
|
private static ReadOnlyCollection<Exception> PushTo(Job job, string path)
|
|
{
|
|
List<Exception> results = [];
|
|
foreach (Target target in job.Targets)
|
|
{
|
|
if (target.SecureShell is not null)
|
|
{
|
|
try
|
|
{ PushTo(job, target.SecureShell, path); }
|
|
catch (Exception ex)
|
|
{
|
|
if (target.SecureShell.Required)
|
|
results.Add(ex);
|
|
}
|
|
}
|
|
else if (target.ServerMessageBlock is not null)
|
|
{
|
|
try
|
|
{ PushTo(target.ServerMessageBlock, path); }
|
|
catch (Exception ex)
|
|
{
|
|
if (target.ServerMessageBlock.Required)
|
|
results.Add(ex);
|
|
}
|
|
}
|
|
else
|
|
throw new NotImplementedException();
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static void DeleteOld(Job job, SecureShell secureShell, string path)
|
|
{
|
|
List<string> results = [];
|
|
using SftpClient client = new(secureShell.Host, secureShell.User, new PrivateKeyFile(secureShell.Key));
|
|
client.Connect();
|
|
foreach (ISftpFile file in client.ListDirectory(secureShell.Path))
|
|
{
|
|
if (file.Name == path)
|
|
continue;
|
|
if (!file.Name.EndsWith(job.Extension, StringComparison.OrdinalIgnoreCase))
|
|
continue;
|
|
results.Add(file.FullName);
|
|
}
|
|
for (int i = job.Keep - 1; i < results.Count; i++)
|
|
client.DeleteFile(results[i]);
|
|
}
|
|
|
|
private static void DeleteOld(Job job, ServerMessageBlock serverMessageBlock, string path)
|
|
{
|
|
List<string> results = [];
|
|
string[] files = Directory.GetFiles(serverMessageBlock.Path, $"*{job.Extension}", SearchOption.TopDirectoryOnly);
|
|
foreach (string file in files)
|
|
{
|
|
if (file == path)
|
|
continue;
|
|
results.Add(file);
|
|
}
|
|
for (int i = job.Keep - 1; i < results.Count; i++)
|
|
System.IO.File.Delete(results[i]);
|
|
}
|
|
|
|
private static ReadOnlyCollection<Exception> DeleteOld(Job job, string path)
|
|
{
|
|
List<Exception> results = [];
|
|
foreach (Target target in job.Targets)
|
|
{
|
|
if (target.SecureShell is not null)
|
|
{
|
|
try
|
|
{ DeleteOld(job, target.SecureShell, path); }
|
|
catch (Exception ex)
|
|
{
|
|
if (target.SecureShell.Required)
|
|
results.Add(ex);
|
|
}
|
|
}
|
|
else if (target.ServerMessageBlock is not null)
|
|
{
|
|
try
|
|
{ DeleteOld(job, target.ServerMessageBlock, path); }
|
|
catch (Exception ex)
|
|
{
|
|
if (target.ServerMessageBlock.Required)
|
|
results.Add(ex);
|
|
}
|
|
}
|
|
else
|
|
throw new NotImplementedException();
|
|
}
|
|
return results.AsReadOnly();
|
|
}
|
|
|
|
private static void Verify(string searchPattern, string[] ignoreFileNames)
|
|
{
|
|
List<Target> targets = [
|
|
new(new SecureShell("free.file.sync.root", "C:/Users/phares/.ssh/id_ed25519", "\\home", true, "root"), null),
|
|
new(null, new ServerMessageBlock("\\\\mesfs.infineon.com\\EC_APC\\DEV", true))
|
|
];
|
|
string directory = Path.Combine(Environment.CurrentDirectory, ".vscode", "helper");
|
|
if (!Directory.Exists(directory))
|
|
_ = Directory.CreateDirectory(directory);
|
|
ReadOnlyCollection<File> files = GetFiles(directory, searchPattern, ignoreFileNames);
|
|
double filesTotalLength = 0;
|
|
foreach (File file in files)
|
|
filesTotalLength += file.Length;
|
|
Job job = new(
|
|
"C:/Users/phares",
|
|
directory,
|
|
"*.iso",
|
|
files.ToArray(),
|
|
files.Count,
|
|
filesTotalLength,
|
|
3,
|
|
targets.ToArray());
|
|
string json = JsonSerializer.Serialize(job, JobSourceGenerationContext.Default.Job);
|
|
System.IO.File.WriteAllText(Path.Combine(directory, "verify.json"), json);
|
|
}
|
|
|
|
internal static void Backup(ILogger<Worker> logger, List<string> args)
|
|
{
|
|
string path;
|
|
string? json;
|
|
string directoryName;
|
|
ReadOnlyCollection<File> files;
|
|
string searchPattern = args[2];
|
|
ReadOnlyCollection<Exception> exceptions;
|
|
string[] ignoreFileNames = args[3].Split('|');
|
|
string sourceDirectory = Path.GetFullPath(args[0]);
|
|
logger.LogInformation("Searching <{sourceDirectory}> with search pattern {searchPattern}", args[0], searchPattern);
|
|
if (Debugger.IsAttached)
|
|
Verify(searchPattern, ignoreFileNames);
|
|
IEnumerable<Record> records = GetRecords(sourceDirectory, searchPattern);
|
|
foreach (Record record in records)
|
|
{
|
|
if (record.Job is null || record.Job.Targets.Length == 0 || string.IsNullOrEmpty(record.Job.Extension))
|
|
continue;
|
|
logger.LogInformation("Searching <{directory}>", record.Directory);
|
|
files = GetFiles(searchPattern, ignoreFileNames, record);
|
|
json = GetJsonIfNotEqual(searchPattern, ignoreFileNames, record, record.Job, files);
|
|
if (string.IsNullOrEmpty(json))
|
|
continue;
|
|
directoryName = Path.GetFileName(record.Directory);
|
|
path = Path.Combine(record.Directory, $"{directoryName}-{DateTime.Now:yyyy-MM-dd-HH-mm-ss-fff}{record.Job.Extension}");
|
|
logger.LogInformation("Writing <{directory}> extension", record.Directory);
|
|
WriteExtension(record, record.Job, files, path);
|
|
logger.LogInformation("Pushing <{directory}> extension", record.Directory);
|
|
exceptions = PushTo(record.Job, path);
|
|
if (exceptions.Count != 0)
|
|
{
|
|
foreach (Exception exception in exceptions)
|
|
logger.LogError(exception, exception.Message);
|
|
PushTo(record.Job.AlternatePath, path);
|
|
}
|
|
System.IO.File.WriteAllText(record.Path, json);
|
|
System.IO.File.Delete(path);
|
|
logger.LogInformation("Deleting old <{directory}> extension", record.Directory);
|
|
exceptions = DeleteOld(record.Job, path);
|
|
if (exceptions.Count != 0)
|
|
{
|
|
foreach (Exception exception in exceptions)
|
|
logger.LogError(exception, exception.Message);
|
|
}
|
|
}
|
|
if (Debugger.IsAttached && records.Count() == 0)
|
|
{
|
|
files = GetFiles(sourceDirectory, searchPattern, ignoreFileNames);
|
|
json = JsonSerializer.Serialize(files.ToArray(), FilesSourceGenerationContext.Default.FileArray);
|
|
System.IO.File.WriteAllText(Path.Combine(Environment.CurrentDirectory, ".vscode", "helper", ".json"), json);
|
|
}
|
|
}
|
|
|
|
} |