Mike Phares
2923e86a94
Download SSL Certificates Sort Subtasks of Markdown files Test BioRad EAF CopyDirectories json to Markdown Sort Day 2024 Q2 GitRemoteRemove Handle directoryInfo.LinkTarget better Remove StartAt Handle directoryInfo.LinkTarget
334 lines
17 KiB
C#
334 lines
17 KiB
C#
using Microsoft.Extensions.Logging;
|
|
using System.Collections.ObjectModel;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Serialization;
|
|
|
|
namespace File_Folder_Helper.Day;
|
|
|
|
internal static partial class Helper20240517
|
|
{
|
|
|
|
public record ContentSignature([property: JsonPropertyName("contentSignature")] string Value,
|
|
[property: JsonPropertyName("contentSignatureType")] string ContentSignatureType);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(ContentSignature))]
|
|
public partial class ContentSignatureGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record Type([property: JsonPropertyName("count")] int Count,
|
|
[property: JsonPropertyName("match")] string Match,
|
|
[property: JsonPropertyName("searchData")] SearchData SearchData);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(Type))]
|
|
public partial class TypeGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record ImageAmazon([property: JsonPropertyName("colorSpace")] string ColorSpace,
|
|
[property: JsonPropertyName("dateTime")] DateTime DateTime,
|
|
[property: JsonPropertyName("dateTimeDigitized")] DateTime DateTimeDigitized,
|
|
[property: JsonPropertyName("dateTimeOriginal")] DateTime DateTimeOriginal,
|
|
[property: JsonPropertyName("exposureMode")] string ExposureMode,
|
|
[property: JsonPropertyName("exposureProgram")] string ExposureProgram,
|
|
[property: JsonPropertyName("exposureTime")] string ExposureTime,
|
|
[property: JsonPropertyName("flash")] string Flash,
|
|
[property: JsonPropertyName("focalLength")] string FocalLength,
|
|
[property: JsonPropertyName("height")] int Height,
|
|
[property: JsonPropertyName("make")] string Make,
|
|
[property: JsonPropertyName("meteringMode")] string MeteringMode,
|
|
[property: JsonPropertyName("model")] string Model,
|
|
[property: JsonPropertyName("orientation")] string Orientation,
|
|
[property: JsonPropertyName("resolutionUnit")] string ResolutionUnit,
|
|
[property: JsonPropertyName("sensingMethod")] string SensingMethod,
|
|
[property: JsonPropertyName("sharpness")] string Sharpness,
|
|
[property: JsonPropertyName("software")] string Software,
|
|
[property: JsonPropertyName("subSecTime")] string SubSecTime,
|
|
[property: JsonPropertyName("subSecTimeDigitized")] string SubSecTimeDigitized,
|
|
[property: JsonPropertyName("subSecTimeOriginal")] string SubSecTimeOriginal,
|
|
[property: JsonPropertyName("whiteBalance")] string WhiteBalance,
|
|
[property: JsonPropertyName("width")] int Width,
|
|
[property: JsonPropertyName("apertureValue")] string ApertureValue);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(ImageAmazon))]
|
|
public partial class ImageAmazonGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record ContentProperties([property: JsonPropertyName("contentDate")] DateTime ContentDate,
|
|
[property: JsonPropertyName("contentSignatures")] IReadOnlyList<ContentSignature> ContentSignatures,
|
|
[property: JsonPropertyName("contentType")] string ContentType,
|
|
[property: JsonPropertyName("extension")] string Extension,
|
|
[property: JsonPropertyName("image")] ImageAmazon Image,
|
|
[property: JsonPropertyName("md5")] string Md5,
|
|
[property: JsonPropertyName("size")] int Size,
|
|
[property: JsonPropertyName("version")] int Version);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(ContentProperties))]
|
|
public partial class ContentPropertiesGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record XAccntParentMap();
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(XAccntParentMap))]
|
|
public partial class XAccntParentMapGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record Datum([property: JsonPropertyName("accessRuleIds")] IReadOnlyList<object> AccessRuleIds,
|
|
[property: JsonPropertyName("childAssetTypeInfo")] IReadOnlyList<object> ChildAssetTypeInfo,
|
|
[property: JsonPropertyName("contentProperties")] ContentProperties ContentProperties,
|
|
[property: JsonPropertyName("createdBy")] string CreatedBy,
|
|
[property: JsonPropertyName("createdDate")] DateTime CreatedDate,
|
|
[property: JsonPropertyName("eTagResponse")] string ETagResponse,
|
|
[property: JsonPropertyName("groupPermissions")] IReadOnlyList<object> GroupPermissions,
|
|
[property: JsonPropertyName("id")] string Id,
|
|
[property: JsonPropertyName("isRoot")] bool IsRoot,
|
|
[property: JsonPropertyName("isShared")] bool IsShared,
|
|
[property: JsonPropertyName("keywords")] IReadOnlyList<object> Keywords,
|
|
[property: JsonPropertyName("kind")] string Kind,
|
|
[property: JsonPropertyName("labels")] IReadOnlyList<object> Labels,
|
|
[property: JsonPropertyName("modifiedDate")] DateTime ModifiedDate,
|
|
[property: JsonPropertyName("name")] string Name,
|
|
[property: JsonPropertyName("ownerId")] string OwnerId,
|
|
[property: JsonPropertyName("parentMap")] ParentMap ParentMap,
|
|
[property: JsonPropertyName("parents")] IReadOnlyList<string> Parents,
|
|
[property: JsonPropertyName("protectedFolder")] bool ProtectedFolder,
|
|
[property: JsonPropertyName("restricted")] bool Restricted,
|
|
[property: JsonPropertyName("status")] string Status,
|
|
[property: JsonPropertyName("subKinds")] IReadOnlyList<object> SubKinds,
|
|
[property: JsonPropertyName("transforms")] IReadOnlyList<string> Transforms,
|
|
[property: JsonPropertyName("version")] int Version,
|
|
[property: JsonPropertyName("xAccntParentMap")] XAccntParentMap XAccntParentMap,
|
|
[property: JsonPropertyName("xAccntParents")] IReadOnlyList<object> XAccntParents,
|
|
[property: JsonPropertyName("match")] bool? Match);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(Datum))]
|
|
public partial class DatumGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(Dictionary<string, Datum>))]
|
|
public partial class DictionaryDatumGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record LocationAmazon([property: JsonPropertyName("count")] int Count,
|
|
[property: JsonPropertyName("match")] string Match,
|
|
[property: JsonPropertyName("searchData")] SearchData SearchData);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(LocationAmazon))]
|
|
public partial class LocationAmazonGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record LocationInfo([property: JsonPropertyName("city")] string City,
|
|
[property: JsonPropertyName("country")] string Country,
|
|
[property: JsonPropertyName("countryIso3Code")] string CountryIso3Code,
|
|
[property: JsonPropertyName("state")] string State,
|
|
[property: JsonPropertyName("stateCode")] string StateCode);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(LocationInfo))]
|
|
public partial class LocationInfoGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record SearchData([property: JsonPropertyName("clusterName")] string ClusterName,
|
|
[property: JsonPropertyName("locationId")] string LocationId,
|
|
[property: JsonPropertyName("locationInfo")] LocationInfo LocationInfo,
|
|
[property: JsonPropertyName("thingId")] string ThingId);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(SearchData))]
|
|
public partial class SearchDataGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record AllPerson([property: JsonPropertyName("count")] int Count,
|
|
[property: JsonPropertyName("match")] string Match,
|
|
[property: JsonPropertyName("searchData")] SearchData SearchData);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(AllPerson))]
|
|
public partial class AllPersonGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record PersonAmazon([property: JsonPropertyName("count")] int Count,
|
|
[property: JsonPropertyName("match")] string Match,
|
|
[property: JsonPropertyName("searchData")] SearchData SearchData);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(PersonAmazon))]
|
|
public partial class PersonAmazonGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record ClusterId([property: JsonPropertyName("count")] int Count,
|
|
[property: JsonPropertyName("match")] string Match,
|
|
[property: JsonPropertyName("searchData")] SearchData SearchData);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(ClusterId))]
|
|
public partial class ClusterIdGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record Thing([property: JsonPropertyName("count")] int Count,
|
|
[property: JsonPropertyName("match")] string Match,
|
|
[property: JsonPropertyName("searchData")] SearchData SearchData);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(Thing))]
|
|
public partial class ThingGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record Time([property: JsonPropertyName("count")] int Count,
|
|
[property: JsonPropertyName("match")] string Match,
|
|
[property: JsonPropertyName("searchData")] SearchData SearchData);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(Time))]
|
|
public partial class TimeGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record ParentMap([property: JsonPropertyName("FOLDER")] IReadOnlyList<string> FOLDER);
|
|
|
|
public record Aggregations([property: JsonPropertyName("allPeople")] IReadOnlyList<AllPerson> AllPeople,
|
|
[property: JsonPropertyName("clusterId")] IReadOnlyList<ClusterId> ClusterId,
|
|
[property: JsonPropertyName("location")] IReadOnlyList<LocationAmazon> Location,
|
|
[property: JsonPropertyName("people")] IReadOnlyList<PersonAmazon> People,
|
|
[property: JsonPropertyName("things")] IReadOnlyList<Thing> Things,
|
|
[property: JsonPropertyName("time")] IReadOnlyList<Time> Time,
|
|
[property: JsonPropertyName("type")] IReadOnlyList<Type> Type);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(Aggregations))]
|
|
public partial class AggregationsGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(ParentMap))]
|
|
public partial class ParentMapGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record RootAmazon([property: JsonPropertyName("aggregations")] Aggregations Aggregations,
|
|
[property: JsonPropertyName("count")] int Count,
|
|
[property: JsonPropertyName("data")] IReadOnlyList<Datum> Data);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(RootAmazon))]
|
|
public partial class RootAmazonGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
private static void SaveAmazon(string destination, string harFile)
|
|
{
|
|
string offset;
|
|
string personId;
|
|
RootAmazon amazon;
|
|
string? personName;
|
|
string personIdFile;
|
|
string personDirectory;
|
|
PersonAmazon personAmazon;
|
|
Dictionary<string, string> keyValuePairs = [];
|
|
ReadOnlyCollection<(string Url, string AggregationLine)> aggregationLines = GetAggregationLines(harFile);
|
|
foreach ((string url, string aggregationLine) in aggregationLines)
|
|
{
|
|
if (aggregationLine.Contains(",\"category\":\"allPeople\"}"))
|
|
continue;
|
|
amazon = JsonSerializer.Deserialize(aggregationLine, RootAmazonGenerationContext.Default.RootAmazon) ?? throw new Exception();
|
|
if (amazon.Aggregations?.People is null || amazon.Aggregations.People.Count < 1)
|
|
continue;
|
|
personAmazon = amazon.Aggregations.People[0];
|
|
if (!url.Contains(personAmazon.Match))
|
|
continue;
|
|
personDirectory = Path.Combine(destination, personAmazon.SearchData.ClusterName);
|
|
_ = Directory.CreateDirectory(personDirectory);
|
|
personIdFile = Path.Combine(personDirectory, $"000) {personAmazon.Match}.json");
|
|
_ = keyValuePairs.TryAdd(personAmazon.Match, personAmazon.SearchData.ClusterName);
|
|
SaveAmazon(amazon.Data, personIdFile);
|
|
}
|
|
foreach ((string url, string aggregationLine) in aggregationLines)
|
|
{
|
|
if (aggregationLine.Contains(",\"category\":\"allPeople\"}"))
|
|
continue;
|
|
amazon = JsonSerializer.Deserialize(aggregationLine, RootAmazonGenerationContext.Default.RootAmazon) ?? throw new Exception();
|
|
if (amazon.Aggregations?.People is not null && amazon.Aggregations.People.Count > 0)
|
|
continue;
|
|
if (!url.Contains("offset="))
|
|
continue;
|
|
offset = url.Split("offset=")[1];
|
|
if (!url.Contains("people%3A("))
|
|
continue;
|
|
personId = url.Split("people%3A(")[1].Split(')')[0];
|
|
if (!keyValuePairs.TryGetValue(personId, out personName))
|
|
continue;
|
|
personDirectory = Path.Combine(destination, personName);
|
|
_ = Directory.CreateDirectory(personDirectory);
|
|
personIdFile = Path.Combine(personDirectory, $"{offset.Split('&')[0]}) {personId}.json");
|
|
SaveAmazon(amazon.Data, personIdFile);
|
|
}
|
|
}
|
|
|
|
internal static void SaveAmazon(ILogger<Worker> logger, List<string> args)
|
|
{
|
|
string root = Path.GetFullPath(args[0]);
|
|
string destination = Path.GetFullPath(args[2]);
|
|
if (string.IsNullOrEmpty(root))
|
|
throw new NullReferenceException(nameof(root));
|
|
string[] harFiles = Directory.GetFiles(root, "*.har", SearchOption.TopDirectoryOnly);
|
|
foreach (string harFile in harFiles)
|
|
SaveAmazon(destination, harFile);
|
|
logger?.LogInformation("{harFiles} count", harFiles.Length);
|
|
}
|
|
|
|
private static void SaveAmazon(IReadOnlyList<Datum> data, string personIdFile)
|
|
{
|
|
string json;
|
|
Dictionary<string, Datum> keyValuePairs = [];
|
|
foreach (Datum datum in data)
|
|
_ = keyValuePairs.TryAdd(datum.Name.Split('.')[0], datum);
|
|
json = JsonSerializer.Serialize(keyValuePairs, DictionaryDatumGenerationContext.Default.DictionaryStringDatum);
|
|
File.WriteAllText(personIdFile, json);
|
|
}
|
|
|
|
private static ReadOnlyCollection<(string, string)> GetAggregationLines(string harFile)
|
|
{
|
|
List<(string, string)> results = [];
|
|
if (!File.Exists(harFile))
|
|
throw new Exception();
|
|
string lastUrl = string.Empty;
|
|
string text = "\"text\": \"{";
|
|
string[] lines = File.ReadAllLines(harFile);
|
|
foreach (string line in lines)
|
|
{
|
|
if (line.Contains("\"url\": \""))
|
|
lastUrl = line;
|
|
if (!line.Contains(text))
|
|
continue;
|
|
if (!line.Contains("aggregations"))
|
|
continue;
|
|
if (lastUrl.Contains("search?asset=NONE"))
|
|
continue;
|
|
results.Add(new(lastUrl, line.Trim()[(text.Length - 1)..^1].Replace("\\\"", "\"")));
|
|
lastUrl = string.Empty;
|
|
}
|
|
return new(results);
|
|
}
|
|
|
|
} |