phares3757/met08resihgcv
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Save check for file already present

Browse Source 
NoWaitDirectory
MoveArchive allow empty directory and continueOnError for clean files
CreatePointerFile and more on NoWaitDirectory
This commit is contained in:
Mike Phares 2023-03-16 10:51:51 -07:00
parent d5b2242f70
commit 5ec580e751
7 changed files with 100 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Adaptation/.editorconfig
View File

@ -76,6 +76,7 @@ csharp_using_directive_placement = outside_namespace
dotnet_code_quality_unused_parameters = all
dotnet_code_quality_unused_parameters = non_public # IDE0060: Remove unused parameter
dotnet_code_quality.CAXXXX.api_surface = private, internal
dotnet_diagnostic.CA1816.severity = none # CA1816: Call GC.SuppressFinalize correctly
dotnet_diagnostic.CA1825.severity = warning # CA1823: Avoid zero-length array allocations
dotnet_diagnostic.CA1829.severity = warning # CA1829: Use Length/Count property instead of Count() when available
dotnet_diagnostic.CA1834.severity = warning # CA1834: Consider using 'StringBuilder.Append(char)' when applicable

21
Adaptation/FileHandlers/Archive/FileRead.cs
View File

@ -115,7 +115,7 @@ public class FileRead : Shared.FileRead, IFileRead
}
}
private void MoveArchive(DateTime dateTime)
private void MoveArchive(string reportFullPath, DateTime dateTime)
{
if (dateTime == DateTime.MinValue)
{ }
@ -128,12 +128,17 @@ public class FileRead : Shared.FileRead, IFileRead
string jobIdDirectory = Path.Combine(_JobIdParentDirectory, _Logistics.JobID);
if (!Directory.Exists(jobIdDirectory))
_ = Directory.CreateDirectory(jobIdDirectory);
string[] matchDirectories = new string[] { GetDirectoriesRecursively(jobIdDirectory, logisticsSequence).FirstOrDefault() };
if ((matchDirectories is null) || matchDirectories.Length != 1)
throw new Exception("Didn't find directory by logistics sequence");
string sourceDirectory = Path.GetDirectoryName(matchDirectories[0]);
destinationArchiveDirectory = Path.Combine(destinationArchiveDirectory, Path.GetFileName(sourceDirectory));
Directory.Move(sourceDirectory, destinationArchiveDirectory);
if (!Directory.GetDirectories(jobIdDirectory).Any())
File.Copy(reportFullPath, Path.Combine(destinationArchiveDirectory, Path.GetFileName(reportFullPath)));
else
{
string[] matchDirectories = GetDirectoriesRecursively(jobIdDirectory, logisticsSequence).ToArray();
if (matchDirectories.Length != 1)
throw new Exception("Didn't find directory by logistics sequence");
string sourceDirectory = Path.GetDirectoryName(matchDirectories[0]);
destinationArchiveDirectory = Path.Combine(destinationArchiveDirectory, Path.GetFileName(sourceDirectory));
Directory.Move(sourceDirectory, destinationArchiveDirectory);
}
}
private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime dateTime)
@ -146,7 +151,7 @@ public class FileRead : Shared.FileRead, IFileRead
List<Shared.Properties.IDescription> descriptions = GetDuplicatorDescriptions(jsonElements);
Test[] tests = (from l in descriptions select (Test)l.Test).ToArray();
if (_IsEAFHosted && _FileConnectorConfiguration.FileScanningIntervalInSeconds > 0)
MoveArchive(dateTime);
MoveArchive(reportFullPath, dateTime);
results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(pdsf.Item1, tests, jsonElements, new List<FileInfo>());
return results;
}

63
Adaptation/FileHandlers/MoveMatchingFiles/FileRead.cs
View File

@ -7,6 +7,7 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading;
@ -153,18 +154,36 @@ public class FileRead : Shared.FileRead, IFileRead
return results;
}
private void MoveCollection(DateTime dateTime, List<(string matchingFile, string checkFile)> collection)
private static List<(string, string, string, string, string)> GetCollection(List<(string matchingFile, string checkFile)> collection)
{
long preWait;
List<(string, string, string, string, string)> results = new();
string errFile;
string checkDirectory;
string noWaitDirectory;
foreach ((string matchingFile, string checkFile) in collection)
{
errFile = string.Concat(checkFile, ".err");
checkDirectory = Path.GetDirectoryName(checkFile);
if (!Directory.Exists(checkDirectory))
_ = Directory.CreateDirectory(checkDirectory);
noWaitDirectory = Path.Combine(checkDirectory, "NoWaitDirectory");
results.Add(new(matchingFile, checkFile, errFile, checkDirectory, noWaitDirectory));
}
return results;
}
private void MoveCollection(DateTime dateTime, List<(string matchingFile, string checkFile)> collection)
{
long preWait;
List<(string checkFile, string errFile)> postCollection = new();
foreach ((string matchingFile, string checkFile, string errFile, string checkDirectory, string noWaitDirectory) in GetCollection(collection))
{
File.Move(matchingFile, checkFile);
if (Directory.Exists(noWaitDirectory))
{
postCollection.Add(new(checkFile, errFile));
continue;
}
if (_FileConnectorConfiguration?.FileHandleWaitTime is null)
preWait = DateTime.Now.AddMilliseconds(1234).Ticks;
else
@ -182,10 +201,45 @@ public class FileRead : Shared.FileRead, IFileRead
if (!File.Exists(checkFile))
break;
if (new TimeSpan(DateTime.Now.Ticks - dateTime.Ticks).TotalSeconds > _BreakAfterSeconds)
throw new Exception($"Not all files were consumned after {_BreakAfterSeconds} second(s)!");
throw new Exception($"Not all files were consumed after {_BreakAfterSeconds} second(s)!");
Thread.Sleep(500);
}
}
if (postCollection.Any())
{
Thread.Sleep(500);
StringBuilder stringBuilder = new();
foreach ((string checkFile, string errFile) in postCollection)
{
if (File.Exists(errFile))
_ = stringBuilder.AppendLine(File.ReadAllText(errFile));
if (File.Exists(checkFile))
_ = stringBuilder.AppendLine($"<{checkFile}> was not consumed by the end!");
}
if (stringBuilder.Length > 0)
throw new Exception(stringBuilder.ToString());
}
}
private static void CreatePointerFile(int numberLength, string parentDirectory, List<string> matchingFiles)
{
#nullable enable
string checkFile;
string writeFile;
string? directoryName;
int parentDirectoryLength = parentDirectory.Length;
foreach (string matchingFile in matchingFiles)
{
directoryName = Path.GetDirectoryName(matchingFile);
if (directoryName is null)
continue;
checkFile = $"{matchingFile[0]}{directoryName.Substring(parentDirectoryLength + numberLength + 1)}";
writeFile = Path.Combine(parentDirectory, $"{directoryName.Substring(parentDirectory.Length + 1, numberLength)}.txt");
if (File.Exists(writeFile))
continue;
File.AppendAllLines(writeFile, new string[] { parentDirectory, matchingFile, directoryName, checkFile });
}
#nullable disable
}
private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime dateTime)
@ -201,6 +255,9 @@ public class FileRead : Shared.FileRead, IFileRead
List<string> matchingFiles = GetMatchingFiles(ticks, reportFullPath, searchDirectories);
if (matchingFiles.Count != searchDirectories.Count)
throw new Exception($"Didn't find all files after {_BreakAfterSeconds} second(s)!");
try
{ CreatePointerFile(numberLength, parentParentDirectory, matchingFiles); }
catch (Exception) { }
List<(string matchingFile, string checkFile)> collection = GetCollection(numberLength, parentParentDirectory, matchingFiles);
MoveCollection(dateTime, collection);
return results;

31
Adaptation/FileHandlers/OpenInsight/FromIQS.cs
View File

@ -182,6 +182,7 @@ public class FromIQS
internal static void Save(string openInsightApiECDirectory, string openInsightApiIFXDirectory, Logistics logistics, string reportFullPath, string logisticLines, pcl.Description description, string lines, long? subGroupId, string weekOfYear)
{
string checkFile;
string fileName = Path.GetFileName(reportFullPath);
string json = GetJson(logistics, logisticLines, description);
string? ecPathRoot = Path.GetPathRoot(openInsightApiECDirectory);
@ -195,18 +196,24 @@ public class FromIQS
_ = Directory.CreateDirectory(ecDirectory);
if (ifxExists && !Directory.Exists(ifxDirectory))
_ = Directory.CreateDirectory(ifxDirectory);
if (ecExists)
File.Copy(reportFullPath, Path.Combine(ecDirectory, fileName));
if (ifxExists)
File.Copy(reportFullPath, Path.Combine(ifxDirectory, fileName));
if (ecExists)
File.WriteAllText(Path.Combine(ecDirectory, $"{logistics.DateTimeFromSequence.Ticks}.txt"), lines);
if (ifxExists)
File.WriteAllText(Path.Combine(ifxDirectory, $"{logistics.DateTimeFromSequence.Ticks}.txt"), lines);
if (ecExists)
File.WriteAllText(Path.Combine(ecDirectory, $"{logistics.DateTimeFromSequence.Ticks}.json"), json);
if (ifxExists)
File.WriteAllText(Path.Combine(ifxDirectory, $"{logistics.DateTimeFromSequence.Ticks}.json"), json);
checkFile = Path.Combine(ecDirectory, fileName);
if (ecExists && !File.Exists(checkFile))
File.Copy(reportFullPath, checkFile);
checkFile = Path.Combine(ifxDirectory, fileName);
if (ifxExists && !File.Exists(checkFile))
File.Copy(reportFullPath, checkFile);
checkFile = Path.Combine(ecDirectory, $"{logistics.DateTimeFromSequence.Ticks}.txt");
if (ecExists && !File.Exists(checkFile))
File.WriteAllText(checkFile, lines);
checkFile = Path.Combine(ifxDirectory, $"{logistics.DateTimeFromSequence.Ticks}.txt");
if (ifxExists && !File.Exists(checkFile))
File.WriteAllText(checkFile, lines);
checkFile = Path.Combine(ecDirectory, $"{logistics.DateTimeFromSequence.Ticks}.json");
if (ecExists && !File.Exists(checkFile))
File.WriteAllText(checkFile, json);
checkFile = Path.Combine(ifxDirectory, $"{logistics.DateTimeFromSequence.Ticks}.json");
if (ifxExists && !File.Exists(checkFile))
File.WriteAllText(checkFile, json);
}
#nullable disable

2
Adaptation/FileHandlers/Processed/FileRead.cs
View File

@ -118,7 +118,7 @@ public class FileRead : Shared.FileRead, IFileRead
if (!Directory.Exists(jobIdDirectory))
_ = Directory.CreateDirectory(jobIdDirectory);
string[] matchDirectories = GetInProcessDirectory(jobIdDirectory);
if ((matchDirectories is null) || matchDirectories.Length != 1)
if (matchDirectories.Length != 1)
throw new Exception("Didn't find directory by logistics sequence");
if (fileInfo.Exists && fileInfo.LastWriteTime < fileInfo.CreationTime)
File.SetLastWriteTime(reportFullPath, fileInfo.CreationTime);

5
Adaptation/MET08RESIHGCV-Development.yml
View File

@ -92,10 +92,11 @@ steps:
- script: '"C:\program files\dotnet\dotnet.exe" build --configuration $(Configuration) --source $(NugetSource)'
workingDirectory: Adaptation
displayName: "Core Build - Adaptation"
- powershell: Get-ChildItem .\ -include TestResults -Recurse | foreach ($_) { remove-item $_.fullname -Force -Recurse }
workingDirectory: "$(System.DefaultWorkingDirectory)/../../../$(Build.Repository.Name)"
displayName: 'PowerShell Script'
displayName: "PowerShell Script"
continueOnError: true
- script: "dotnet test --configuration $(Configuration)"
workingDirectory: Adaptation

5
Adaptation/MET08RESIHGCV.yml
View File

@ -89,10 +89,11 @@ steps:
- script: '"C:\program files\dotnet\dotnet.exe" build --configuration $(Configuration) --source $(NugetSource)'
workingDirectory: Adaptation
displayName: "Core Build - Adaptation"
- powershell: Get-ChildItem .\ -include TestResults -Recurse | foreach ($_) { remove-item $_.fullname -Force -Recurse }
workingDirectory: "$(System.DefaultWorkingDirectory)/../../../$(Build.Repository.Name)"
displayName: 'PowerShell Script'
displayName: "PowerShell Script"
continueOnError: true
- script: "dotnet test --configuration $(Configuration)"
workingDirectory: Adaptation