Save check for file already present

NoWaitDirectory
MoveArchive allow empty directory and continueOnError for clean files
CreatePointerFile and more on NoWaitDirectory

Adaptation/.editorconfig

@@ -76,6 +76,7 @@ csharp_using_directive_placement = outside_namespace
 dotnet_code_quality_unused_parameters = all
 dotnet_code_quality_unused_parameters = non_public # IDE0060: Remove unused parameter
 dotnet_code_quality.CAXXXX.api_surface = private, internal
+dotnet_diagnostic.CA1816.severity = none # CA1816: Call GC.SuppressFinalize correctly
 dotnet_diagnostic.CA1825.severity = warning # CA1823: Avoid zero-length array allocations
 dotnet_diagnostic.CA1829.severity = warning # CA1829: Use Length/Count property instead of Count() when available
 dotnet_diagnostic.CA1834.severity = warning # CA1834: Consider using 'StringBuilder.Append(char)' when applicable

Adaptation/FileHandlers/Archive/FileRead.cs

@@ -115,7 +115,7 @@ public class FileRead : Shared.FileRead, IFileRead
         }
     }
 
-    private void MoveArchive(DateTime dateTime)
+    private void MoveArchive(string reportFullPath, DateTime dateTime)
     {
         if (dateTime == DateTime.MinValue)
         { }
@@ -128,13 +128,18 @@ public class FileRead : Shared.FileRead, IFileRead
         string jobIdDirectory = Path.Combine(_JobIdParentDirectory, _Logistics.JobID);
         if (!Directory.Exists(jobIdDirectory))
             _ = Directory.CreateDirectory(jobIdDirectory);
-        string[] matchDirectories = new string[] { GetDirectoriesRecursively(jobIdDirectory, logisticsSequence).FirstOrDefault() };
+        if (!Directory.GetDirectories(jobIdDirectory).Any())
-        if ((matchDirectories is null) || matchDirectories.Length != 1)
+            File.Copy(reportFullPath, Path.Combine(destinationArchiveDirectory, Path.GetFileName(reportFullPath)));
+        else
+        {
+            string[] matchDirectories = GetDirectoriesRecursively(jobIdDirectory, logisticsSequence).ToArray();
+            if (matchDirectories.Length != 1)
                 throw new Exception("Didn't find directory by logistics sequence");
             string sourceDirectory = Path.GetDirectoryName(matchDirectories[0]);
             destinationArchiveDirectory = Path.Combine(destinationArchiveDirectory, Path.GetFileName(sourceDirectory));
             Directory.Move(sourceDirectory, destinationArchiveDirectory);
         }
+    }
 
     private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime dateTime)
     {
@@ -146,7 +151,7 @@ public class FileRead : Shared.FileRead, IFileRead
         List<Shared.Properties.IDescription> descriptions = GetDuplicatorDescriptions(jsonElements);
         Test[] tests = (from l in descriptions select (Test)l.Test).ToArray();
         if (_IsEAFHosted && _FileConnectorConfiguration.FileScanningIntervalInSeconds > 0)
-            MoveArchive(dateTime);
+            MoveArchive(reportFullPath, dateTime);
         results = new Tuple<string, Test[], JsonElement[], List<FileInfo>>(pdsf.Item1, tests, jsonElements, new List<FileInfo>());
         return results;
     }

Adaptation/FileHandlers/MoveMatchingFiles/FileRead.cs

@@ -7,6 +7,7 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Text.Json;
 using System.Threading;
 
@@ -153,18 +154,36 @@ public class FileRead : Shared.FileRead, IFileRead
         return results;
     }
 
-    private void MoveCollection(DateTime dateTime, List<(string matchingFile, string checkFile)> collection)
+    private static List<(string, string, string, string, string)> GetCollection(List<(string matchingFile, string checkFile)> collection)
     {
-        long preWait;
+        List<(string, string, string, string, string)> results = new();
         string errFile;
         string checkDirectory;
+        string noWaitDirectory;
         foreach ((string matchingFile, string checkFile) in collection)
         {
             errFile = string.Concat(checkFile, ".err");
             checkDirectory = Path.GetDirectoryName(checkFile);
             if (!Directory.Exists(checkDirectory))
                 _ = Directory.CreateDirectory(checkDirectory);
+            noWaitDirectory = Path.Combine(checkDirectory, "NoWaitDirectory");
+            results.Add(new(matchingFile, checkFile, errFile, checkDirectory, noWaitDirectory));
+        }
+        return results;
+    }
+
+    private void MoveCollection(DateTime dateTime, List<(string matchingFile, string checkFile)> collection)
+    {
+        long preWait;
+        List<(string checkFile, string errFile)> postCollection = new();
+        foreach ((string matchingFile, string checkFile, string errFile, string checkDirectory, string noWaitDirectory) in GetCollection(collection))
+        {
             File.Move(matchingFile, checkFile);
+            if (Directory.Exists(noWaitDirectory))
+            {
+                postCollection.Add(new(checkFile, errFile));
+                continue;
+            }
             if (_FileConnectorConfiguration?.FileHandleWaitTime is null)
                 preWait = DateTime.Now.AddMilliseconds(1234).Ticks;
             else
@@ -182,10 +201,45 @@ public class FileRead : Shared.FileRead, IFileRead
                 if (!File.Exists(checkFile))
                     break;
                 if (new TimeSpan(DateTime.Now.Ticks - dateTime.Ticks).TotalSeconds > _BreakAfterSeconds)
-                    throw new Exception($"Not all files were consumned after {_BreakAfterSeconds} second(s)!");
+                    throw new Exception($"Not all files were consumed after {_BreakAfterSeconds} second(s)!");
                 Thread.Sleep(500);
             }
         }
+        if (postCollection.Any())
+        {
+            Thread.Sleep(500);
+            StringBuilder stringBuilder = new();
+            foreach ((string checkFile, string errFile) in postCollection)
+            {
+                if (File.Exists(errFile))
+                    _ = stringBuilder.AppendLine(File.ReadAllText(errFile));
+                if (File.Exists(checkFile))
+                    _ = stringBuilder.AppendLine($"<{checkFile}> was not consumed by the end!");
+            }
+            if (stringBuilder.Length > 0)
+                throw new Exception(stringBuilder.ToString());
+        }
+    }
+
+    private static void CreatePointerFile(int numberLength, string parentDirectory, List<string> matchingFiles)
+    {
+#nullable enable
+        string checkFile;
+        string writeFile;
+        string? directoryName;
+        int parentDirectoryLength = parentDirectory.Length;
+        foreach (string matchingFile in matchingFiles)
+        {
+            directoryName = Path.GetDirectoryName(matchingFile);
+            if (directoryName is null)
+                continue;
+            checkFile = $"{matchingFile[0]}{directoryName.Substring(parentDirectoryLength + numberLength + 1)}";
+            writeFile = Path.Combine(parentDirectory, $"{directoryName.Substring(parentDirectory.Length + 1, numberLength)}.txt");
+            if (File.Exists(writeFile))
+                continue;
+            File.AppendAllLines(writeFile, new string[] { parentDirectory, matchingFile, directoryName, checkFile });
+        }
+#nullable disable
     }
 
     private Tuple<string, Test[], JsonElement[], List<FileInfo>> GetExtractResult(string reportFullPath, DateTime dateTime)
@@ -201,6 +255,9 @@ public class FileRead : Shared.FileRead, IFileRead
         List<string> matchingFiles = GetMatchingFiles(ticks, reportFullPath, searchDirectories);
         if (matchingFiles.Count != searchDirectories.Count)
             throw new Exception($"Didn't find all files after {_BreakAfterSeconds} second(s)!");
+        try
+        { CreatePointerFile(numberLength, parentParentDirectory, matchingFiles); }
+        catch (Exception) { }
         List<(string matchingFile, string checkFile)> collection = GetCollection(numberLength, parentParentDirectory, matchingFiles);
         MoveCollection(dateTime, collection);
         return results;

Adaptation/FileHandlers/OpenInsight/FromIQS.cs

@@ -182,6 +182,7 @@ public class FromIQS
 
     internal static void Save(string openInsightApiECDirectory, string openInsightApiIFXDirectory, Logistics logistics, string reportFullPath, string logisticLines, pcl.Description description, string lines, long? subGroupId, string weekOfYear)
     {
+        string checkFile;
         string fileName = Path.GetFileName(reportFullPath);
         string json = GetJson(logistics, logisticLines, description);
         string? ecPathRoot = Path.GetPathRoot(openInsightApiECDirectory);
@@ -195,18 +196,24 @@ public class FromIQS
             _ = Directory.CreateDirectory(ecDirectory);
         if (ifxExists && !Directory.Exists(ifxDirectory))
             _ = Directory.CreateDirectory(ifxDirectory);
-        if (ecExists)
+        checkFile = Path.Combine(ecDirectory, fileName);
-            File.Copy(reportFullPath, Path.Combine(ecDirectory, fileName));
+        if (ecExists && !File.Exists(checkFile))
-        if (ifxExists)
+            File.Copy(reportFullPath, checkFile);
-            File.Copy(reportFullPath, Path.Combine(ifxDirectory, fileName));
+        checkFile = Path.Combine(ifxDirectory, fileName);
-        if (ecExists)
+        if (ifxExists && !File.Exists(checkFile))
-            File.WriteAllText(Path.Combine(ecDirectory, $"{logistics.DateTimeFromSequence.Ticks}.txt"), lines);
+            File.Copy(reportFullPath, checkFile);
-        if (ifxExists)
+        checkFile = Path.Combine(ecDirectory, $"{logistics.DateTimeFromSequence.Ticks}.txt");
-            File.WriteAllText(Path.Combine(ifxDirectory, $"{logistics.DateTimeFromSequence.Ticks}.txt"), lines);
+        if (ecExists && !File.Exists(checkFile))
-        if (ecExists)
+            File.WriteAllText(checkFile, lines);
-            File.WriteAllText(Path.Combine(ecDirectory, $"{logistics.DateTimeFromSequence.Ticks}.json"), json);
+        checkFile = Path.Combine(ifxDirectory, $"{logistics.DateTimeFromSequence.Ticks}.txt");
-        if (ifxExists)
+        if (ifxExists && !File.Exists(checkFile))
-            File.WriteAllText(Path.Combine(ifxDirectory, $"{logistics.DateTimeFromSequence.Ticks}.json"), json);
+            File.WriteAllText(checkFile, lines);
+        checkFile = Path.Combine(ecDirectory, $"{logistics.DateTimeFromSequence.Ticks}.json");
+        if (ecExists && !File.Exists(checkFile))
+            File.WriteAllText(checkFile, json);
+        checkFile = Path.Combine(ifxDirectory, $"{logistics.DateTimeFromSequence.Ticks}.json");
+        if (ifxExists && !File.Exists(checkFile))
+            File.WriteAllText(checkFile, json);
     }
 
 #nullable disable

Adaptation/FileHandlers/Processed/FileRead.cs

@@ -118,7 +118,7 @@ public class FileRead : Shared.FileRead, IFileRead
         if (!Directory.Exists(jobIdDirectory))
             _ = Directory.CreateDirectory(jobIdDirectory);
         string[] matchDirectories = GetInProcessDirectory(jobIdDirectory);
-        if ((matchDirectories is null) || matchDirectories.Length != 1)
+        if (matchDirectories.Length != 1)
             throw new Exception("Didn't find directory by logistics sequence");
         if (fileInfo.Exists && fileInfo.LastWriteTime < fileInfo.CreationTime)
             File.SetLastWriteTime(reportFullPath, fileInfo.CreationTime);

Adaptation/MET08RESIHGCV-Development.yml

@@ -95,7 +95,8 @@ steps:
 
   - powershell: Get-ChildItem .\ -include TestResults -Recurse | foreach ($_) { remove-item $_.fullname -Force -Recurse }
     workingDirectory: "$(System.DefaultWorkingDirectory)/../../../$(Build.Repository.Name)"
-    displayName: 'PowerShell Script'
+    displayName: "PowerShell Script"
+    continueOnError: true
 
   - script: "dotnet test --configuration $(Configuration)"
     workingDirectory: Adaptation

Adaptation/MET08RESIHGCV.yml

@@ -92,7 +92,8 @@ steps:
 
   - powershell: Get-ChildItem .\ -include TestResults -Recurse | foreach ($_) { remove-item $_.fullname -Force -Recurse }
     workingDirectory: "$(System.DefaultWorkingDirectory)/../../../$(Build.Repository.Name)"
-    displayName: 'PowerShell Script'
+    displayName: "PowerShell Script"
+    continueOnError: true
 
   - script: "dotnet test --configuration $(Configuration)"
     workingDirectory: Adaptation