phares3757/file-folder-helper
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CSV Compare

Browse Source
This commit is contained in:
Mike Phares 2025-02-24 17:46:47 -07:00
parent dd4a16117c
commit d9e394f446
10 changed files with 6539 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.vscode/launch.json vendored
View File

@ -11,6 +11,18 @@
"preLaunchTask": "build",
"program": "${workspaceFolder}/bin/Debug/net8.0/win-x64/File-Folder-Helper.dll",
"args": [
"s",
"X",
"\\\\mesfs.infineon.com\\EC_Characterization_Si\\Archive\\HGCV1\\2025_Week_09\\2025-02-24",
"Day-Helper-2025-02-19",
"csv-*.pdsf",
"*.pdsf",
"Test,Count,MesEntity,HeaderUniqueId,UniqueId,Id,Recipe,Date,AreaDeltaFromLastRun,GLimit,HGCV1",
"Nine10mmEdgeMean,Nine4mmEdgeMean,NineCriticalPointsAverage,NineCriticalPointsPhaseAngleAverage,NineCriticalPointsStdDev,NineEdgeMeanDelta,NineMean,NineResRangePercent,AreaDeltaFromLastRun,Variation,Percentage HgCV 4PP Delta,HGCV1",
"RhoAvg01,RhoAvg02,RhoAvg03,RhoAvg04,RhoAvg05,RhoAvg06,RhoAvg07,RhoAvg08,RhoAvg09,HGCV1",
"FlatZMean|MeanFlatZ,GradeMean|MeanGrade,NAvgMean|MeanNAvg,NslMean|MeanNsl,PhaseMean|MeanPhase,RhoAvgMean|MeanRhoAvg,RhoslMean|MeanRhosl,RsMean|MeanRs,VdMean|MeanVd,FlatZRadialGradient|RadialGradientFlatZ,GradeRadialGradient|RadialGradientGrade,NAvgRadialGradient|RadialGradientNAvg,NslRadialGradient|RadialGradientNsl,PhaseRadialGradient|RadialGradientPhase,RhoAvgRadialGradient|RadialGradientRhoAvg,RhoslRadialGradient|RadialGradientRhosl,RsRadialGradient|RadialGradientRs,VdRadialGradient|RadialGradientVd,FlatZStdDev|StandardDeviationPercentageFlatZ,GradeStdDev|StandardDeviationPercentageGrade,NAvgStdDev|StandardDeviationPercentageNAvg,NslStdDev|StandardDeviationPercentageNsl,PhaseStdDev|StandardDeviationPercentagePhase,RhoAvgStdDev|StandardDeviationPercentageRhoAvg,RhoslStdDev|StandardDeviationPercentageRhosl,RsStdDev|StandardDeviationPercentageRs,VdStdDev|StandardDeviationPercentageVd,|HGCV1",
"888",
"999",
"s",
"X",
"D:/Tmp",

6
.vscode/tasks.json vendored
View File

@ -283,6 +283,12 @@
"type": "npm",
"script": "kanbn.board.json",
"problemMatcher": []
},
{
"label": "Jest",
"type": "shell",
"command": "npx jest",
"problemMatcher": []
}
]
}

2
ADO2024/PI2/Helper-2024-06-23.cs
View File

@ -1,9 +1,7 @@
using File_Folder_Helper.Helpers;
using File_Folder_Helper.Models;
using Microsoft.Extensions.Logging;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;

281
ADO2025/PI5/Helper-2025-02-19.cs Normal file
View File

@ -0,0 +1,281 @@
using Microsoft.Extensions.Logging;
using System.Collections.ObjectModel;
using System.Text;
using System.Text.Json;
namespace File_Folder_Helper.ADO2025.PI5;
internal static partial class Helper20250219
{
private record ProcessDataStandardFormat(ReadOnlyCollection<string> Body,
ReadOnlyCollection<string> Columns,
string Logistics);
private static ProcessDataStandardFormat GetLogisticsColumnsAndBody(string path, string[]? lines)
{
ProcessDataStandardFormat result;
string segment;
List<string> body = [];
List<string> columns = [];
StringBuilder logistics = new();
lines ??= File.ReadAllLines(path);
string[] segments;
if (lines.Length < 7)
segments = [];
else
segments = lines[6].Trim().Split('\t');
for (int c = 0; c < segments.Length; c++)
{
segment = segments[c][1..^1];
if (!columns.Contains(segment))
columns.Add(segment);
else
{
for (short i = 1; i < short.MaxValue; i++)
{
segment = string.Concat(segment, "_", i);
if (!columns.Contains(segment))
{
columns.Add(segment);
break;
}
}
}
}
bool lookForLogistics = false;
for (int r = 7; r < lines.Length; r++)
{
if (lines[r].StartsWith("NUM_DATA_ROWS"))
lookForLogistics = true;
if (!lookForLogistics)
{
body.Add(lines[r]);
continue;
}
if (lines[r].StartsWith("LOGISTICS_1"))
{
for (int i = r; i < lines.Length; i++)
{
if (lines[r].StartsWith("END_HEADER"))
break;
_ = logistics.AppendLine(lines[i]);
}
break;
}
}
result = new(Body: body.AsReadOnly(),
Columns: columns.AsReadOnly(),
logistics.ToString());
return result;
}
private static JsonElement[]? GetArray(ProcessDataStandardFormat processDataStandardFormat, bool lookForNumbers = false)
{
JsonElement[]? results;
if (processDataStandardFormat.Body.Count == 0 || !processDataStandardFormat.Body[0].Contains('\t'))
results = JsonSerializer.Deserialize<JsonElement[]>("[]") ?? throw new Exception();
else
{
string value;
string[] segments;
List<string> lines = [];
StringBuilder stringBuilder = new();
foreach (string bodyLine in processDataStandardFormat.Body)
{
_ = stringBuilder.Clear();
_ = stringBuilder.Append('{');
segments = bodyLine.Trim().Split('\t');
if (!lookForNumbers)
{
for (int c = 1; c < segments.Length; c++)
{
value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\");
_ = stringBuilder.Append('"').Append(processDataStandardFormat.Columns[c]).Append("\":\"").Append(value).Append("\",");
}
}
else
{
for (int c = 1; c < segments.Length; c++)
{
value = segments[c].Replace("\"", "\\\"").Replace("\\", "\\\\");
if (string.IsNullOrEmpty(value))
_ = stringBuilder.Append('"').Append(processDataStandardFormat.Columns[c]).Append("\":").Append(value).Append("null,");
else if (value.All(char.IsDigit))
_ = stringBuilder.Append('"').Append(processDataStandardFormat.Columns[c]).Append("\":").Append(value).Append(',');
else
_ = stringBuilder.Append('"').Append(processDataStandardFormat.Columns[c]).Append("\":\"").Append(value).Append("\",");
}
}
_ = stringBuilder.Remove(stringBuilder.Length - 1, 1);
_ = stringBuilder.AppendLine("}");
lines.Add(stringBuilder.ToString());
}
string json = $"[{string.Join(',', lines)}]";
results = JsonSerializer.Deserialize<JsonElement[]>(json);
}
return results;
}
private static int? TryGetPropertyIndex(JsonProperty[] jsonProperties, string propertyName)
{
int? result = null;
for (int i = 0; i < jsonProperties.Length; i++)
{
if (jsonProperties[i].Name != propertyName)
continue;
result = i;
break;
}
return result;
}
private static bool Compare(ILogger<Worker> logger, ReadOnlyCollection<string> ignore, ReadOnlyCollection<string> backfill, ReadOnlyCollection<string> indexOnly, ReadOnlyDictionary<string, string> keyValuePairs, string directory, JsonElement[] jsonElementsNew, JsonElement[] jsonElementsOld)
{
bool result;
int? q;
string valueNew;
string valueOld;
JsonProperty jsonPropertyOld;
JsonProperty jsonPropertyNew;
JsonProperty[] jsonPropertiesOld;
JsonProperty[] jsonPropertiesNew;
List<string> unknownColumns = [];
List<string> differentColumns = [];
int last = jsonElementsOld.Length - 1;
List<string> sameAfterSpaceSplitColumns = [];
for (int i = last; i > 0; i--)
{
if (jsonElementsOld[i].ValueKind != JsonValueKind.Object)
{
unknownColumns.Add(string.Empty);
break;
}
jsonPropertiesOld = jsonElementsOld[i].EnumerateObject().ToArray();
jsonPropertiesNew = jsonElementsNew[i].EnumerateObject().ToArray();
for (int p = 0; p < jsonPropertiesOld.Length; p++)
{
jsonPropertyOld = jsonPropertiesOld[p];
valueOld = jsonPropertyOld.Value.ToString();
if (ignore.Contains(jsonPropertyOld.Name))
{
if (i == last)
logger.LogDebug("{p} )) {jsonPropertyOld.Name} **", p, jsonPropertyOld.Name);
continue;
}
if (keyValuePairs.TryGetValue(jsonPropertyOld.Name, out string? name) && !string.IsNullOrEmpty(name))
{
q = TryGetPropertyIndex(jsonPropertiesNew, name);
if (q is null && i == 0)
unknownColumns.Add($"{jsonPropertyOld.Name}|{name}");
}
else
{
q = TryGetPropertyIndex(jsonPropertiesNew, jsonPropertyOld.Name);
if (q is null)
{
if (i == 0)
unknownColumns.Add(jsonPropertyOld.Name);
}
}
if (q is null)
{
if (i == last && !string.IsNullOrEmpty(valueOld))
logger.LogDebug("{p} )) {jsonPropertyOld.Name} ??", p, jsonPropertyOld.Name);
}
else
{
jsonPropertyNew = jsonPropertiesNew[q.Value];
valueNew = jsonPropertyNew.Value.ToString();
if (i == last)
logger.LogDebug("{p} )) {jsonPropertyOld.Name} ~~ {q.Value} => {jsonPropertyNew.Name}", p, jsonPropertyOld.Name, q.Value, jsonPropertyNew.Name);
if (valueNew != valueOld && !differentColumns.Contains(jsonPropertyOld.Name))
{
if (valueNew.Length >= 2 && valueNew.Split(' ')[0] == valueOld)
sameAfterSpaceSplitColumns.Add(jsonPropertyOld.Name);
else
{
if (backfill.Contains(jsonPropertyOld.Name) && i != last)
continue;
if (indexOnly.Contains(jsonPropertyOld.Name) && int.TryParse(jsonPropertyOld.Name[^2..], out int index) && i != index - 1)
continue;
logger.LogWarning("For [{jsonProperty.Name}] <{directory}> doesn't match ({valueNew} != {valueOld})!", jsonPropertyOld.Name, directory, valueNew, valueOld);
differentColumns.Add(jsonPropertyOld.Name);
}
}
}
}
}
result = unknownColumns.Count == 0 && differentColumns.Count == 0 && sameAfterSpaceSplitColumns.Count == 0;
return result;
}
private static void Compare(ILogger<Worker> logger, int sourceDirectoryLength, ReadOnlyCollection<string> ignore, ReadOnlyCollection<string> backfill, ReadOnlyCollection<string> indexOnly, ReadOnlyDictionary<string, string> keyValuePairs, string searchPattern, string[] files)
{
bool isMatch;
string directory;
string[] matches;
string directorySegment;
string[] directoryFiles;
JsonElement[]? jsonElementsNew;
JsonElement[]? jsonElementsOld;
ProcessDataStandardFormat processDataStandardFormat;
FileInfo[] collection = files.Select(l => new FileInfo(l)).ToArray();
string[] sorted = (from l in collection orderby l.CreationTime descending select l.FullName).ToArray();
foreach (string file in sorted)
{
directory = Path.GetDirectoryName(file) ?? throw new Exception();
directoryFiles = Directory.GetFiles(directory, searchPattern, SearchOption.TopDirectoryOnly);
matches = (from l in directoryFiles where l != file select l).ToArray();
if (matches.Length < 1)
continue;
directorySegment = directory[sourceDirectoryLength..];
processDataStandardFormat = GetLogisticsColumnsAndBody(file, lines: null);
jsonElementsNew = GetArray(processDataStandardFormat);
if (jsonElementsNew is null)
continue;
foreach (string match in matches)
{
processDataStandardFormat = GetLogisticsColumnsAndBody(match, lines: null);
jsonElementsOld = GetArray(processDataStandardFormat);
if (jsonElementsOld is null || jsonElementsOld.Length != jsonElementsNew.Length)
continue;
isMatch = Compare(logger, ignore, backfill, indexOnly, keyValuePairs, directorySegment, jsonElementsNew, jsonElementsOld);
if (!isMatch)
{
logger.LogWarning("! <{match}>", match);
continue;
}
logger.LogInformation("<{match}>", match);
}
}
}
internal static void Compare(ILogger<Worker> logger, List<string> args)
{
string[] segmentsB;
List<string> distinct = [];
string searchPattern = args[2];
string searchPatternB = args[3];
string[] segments = args[7].Split(',');
Dictionary<string, string> keyValuePairs = [];
ReadOnlyCollection<string> ignore = args[4].Split(',').AsReadOnly();
ReadOnlyCollection<string> backfill = args[5].Split(',').AsReadOnly();
ReadOnlyCollection<string> indexOnly = args[6].Split(',').AsReadOnly();
foreach (string segment in segments)
{
segmentsB = segment.Split('|');
if (segmentsB.Length != 2)
continue;
if (distinct.Contains(segmentsB[1]))
continue;
distinct.Add(segmentsB[1]);
keyValuePairs.Add(segmentsB[0], segmentsB[1]);
}
string sourceDirectory = Path.GetFullPath(args[0]);
string[] files = Directory.GetFiles(sourceDirectory, searchPattern, SearchOption.AllDirectories);
logger.LogInformation("<{files}>(s)", files.Length);
Compare(logger, sourceDirectory.Length, ignore, backfill, indexOnly, keyValuePairs.AsReadOnly(), searchPatternB, files);
}
}

2
Day/HelperDay.cs
View File

@ -135,6 +135,8 @@ internal static class HelperDay
ADO2025.PI4.Helper20250204.ExtractKanban(logger, args);
else if (args[1] == "Day-Helper-2025-02-18")
ADO2025.PI5.Helper20250218.MoveToArchive(logger, args);
else if (args[1] == "Day-Helper-2025-02-19")
ADO2025.PI5.Helper20250219.Compare(logger, args);
else
throw new Exception(appSettings.Company);
}

2
File-Folder-Helper.csproj
View File

@ -4,7 +4,7 @@
<OutputType>Exe</OutputType>
<ImplicitUsings>enable</ImplicitUsings>
<TargetFramework>net8.0</TargetFramework>
<RuntimeIdentifier>win-x64;linux-x64</RuntimeIdentifier>
<RuntimeIdentifier>win-x64</RuntimeIdentifier>
<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
<UserSecretsId>8da397d4-13ec-4576-9722-3c79cad25563</UserSecretsId>
<UserSecretsIdWindowsShortcut>eb9e8f58-fcb5-45bb-9d4d-54f064c485b1</UserSecretsIdWindowsShortcut>

88
Scripts/bio-rad.js Normal file
View File

@ -0,0 +1,88 @@
"use strict";
getValue($('gv.thicknessPoints', ''), $('dcp.BIORAD2/csv/Index', '0'));
function getCollectionParseFloat(collection) {
let result = [];
let value;
for (let i = 0; i < collection.length; i++) {
value = parseFloat(collection[i]);
result.push(value);
}
return result;
}
function getSum(collection) {
let result = 0;
if (!collection || collection.length === 0) {
result = 0;
}
else {
for (let i = 0; i < collection.length; i++) {
result += collection[i];
}
}
return result;
}
function getAverage(collection) {
let result = null;
if (collection == null || collection.length === 0)
result = 0;
else {
let sum = getSum(collection);
result = sum / collection.length;
}
return result;
}
function getValue(thicknessPoints, index) {
let result = null;
if (index === 13) {
if (thicknessPoints != undefined && thicknessPoints.length > 1) {
let collection = thicknessPoints[0] === '|' ? thicknessPoints.substring(1).split('|') : thicknessPoints.split('|');
let collectionParseFloat = getCollectionParseFloat(collection);
let thicknessFourteen3mmEdgeMean = getAverage([[collectionParseFloat[10], collectionParseFloat[11], collectionParseFloat[12], collectionParseFloat[13]]]);
let thicknessFourteenMeanFrom = getAverage([[collectionParseFloat[1], collectionParseFloat[2], collectionParseFloat[6], collectionParseFloat[7]]]);
result = (thicknessFourteen3mmEdgeMean - thicknessFourteenMeanFrom) / thicknessFourteenMeanFrom * 100;
}
}
return result;
}
function getVariance(collection) {
let result = null;
if (collection == null || collection.length === 0)
result = null;
else {
let variance = 0;
let t = collection[0];
for (let i = 1; i < collection.length; i++) {
t += collection[i];
const diff = ((i + 1) * collection[i]) - t;
variance += diff * diff / ((i + 1.0) * i);
}
result = variance / (collection.length - 1);
}
return result;
}
// $('gv.thicknessPoints', '') + '|' + $('dcp.BIORAD2/csv/Thickness', '')
// $('gv.thicknessPoints', '') + '|' + $('dcp.BIORAD3/csv/Thickness', '')
// $('gv.thicknessPoints', '') + '|' + $('dcp.BIORAD4/csv/Thickness', '')
// $('gv.thicknessPoints', '') + '|' + $('dcp.BIORAD5/b-csv/Thickness', '')
// \\mesfs.infineon.com\EC_Characterization_Si\Archive\BIORAD2\2025_Week_08\2025-02-20\64-659712-4626_2025-02-20_11;50_AM_5144331401\638756490128318288
// 0 1 2 3 4 5 6 7 8 9 10 11 12 13
// 1 2 3 4 5 6 7 8 9 10 11 12 13 14
const thicknessPoints = getCollectionParseFloat('|4.022|3.952|3.936|3.971|3.954|3.976|3.949|3.906|3.967|3.995|3.997|3.932|3.766|3.890'.substring(1).split('|'));
const thicknessTenPoints = thicknessPoints.slice(0, 10);
const thicknessFourteenCriticalPointsAverage = getAverage(thicknessTenPoints); // 15 // *3.962799999999999
const thicknessFourteenCriticalPointsStdDev = Math.sqrt(getVariance(thicknessTenPoints)); // 16 // *0.0318496467798311
const thicknessFourteenCenterMean = thicknessPoints[4]; // 17 // 3.954
const thicknessFourteenMeanFrom = getAverage([thicknessPoints[1], thicknessPoints[2], thicknessPoints[6], thicknessPoints[7]]); // 18 // *3.954
const thicknessFourteen5mmEdgeMean = getAverage([thicknessPoints[0], thicknessPoints[9]]); // 19 // *4.0085
const thicknessFourteen3mmEdgeMean = getAverage([thicknessPoints[10], thicknessPoints[11], thicknessPoints[12], thicknessPoints[13]]); // 20 // *3.89625
const thicknessFourteen5mmEdgePercent = (thicknessFourteen5mmEdgeMean - thicknessFourteenMeanFrom) / thicknessFourteenMeanFrom * 100; // 21 // *1.848440576764267
const thicknessFourteen3mmEdgePercent = (thicknessFourteen3mmEdgeMean - thicknessFourteenMeanFrom) / thicknessFourteenMeanFrom * 100; // 22 // *-1.0036206567998442
console.log(thicknessFourteenCriticalPointsAverage);

2
global.json
View File

@ -1,6 +1,6 @@
{
"sdk": {
"rollForward": "latestMinor",
"version": "8.0.112"
"version": "8.0.113"
}
}

6144
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

5
package.json
View File

@ -8,7 +8,8 @@
"prettier.write": "prettier . --write"
},
"devDependencies": {
"prettier": "3.0.0",
"axios": "^1.7.9"
"axios": "^1.7.9",
"jest": "^29.7.0",
"prettier": "3.0.0"
}
}