Mike Phares
614588dcae
Some checks failed
Gitea Actions Demo / Explore-Gitea-Actions (push) Has been cancelled
Handle directoryInfo.LinkTarget GitRemoteRemove Handle directoryInfo.LinkTarget better Sort HelperMarkdown Sort Day 2024 Q2 u flag Better git remote management Gitea Echo
334 lines
17 KiB
C#
334 lines
17 KiB
C#
using Microsoft.Extensions.Logging;
|
|
using System.Collections.ObjectModel;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Serialization;
|
|
|
|
namespace File_Folder_Helper.Day;
|
|
|
|
internal static partial class Helper20240517
|
|
{
|
|
|
|
public record ContentSignature([property: JsonPropertyName("contentSignature")] string Value,
|
|
[property: JsonPropertyName("contentSignatureType")] string ContentSignatureType);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(ContentSignature))]
|
|
public partial class ContentSignatureGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record Type([property: JsonPropertyName("count")] int Count,
|
|
[property: JsonPropertyName("match")] string Match,
|
|
[property: JsonPropertyName("searchData")] SearchData SearchData);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(Type))]
|
|
public partial class TypeGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record ImageAmazon([property: JsonPropertyName("colorSpace")] string ColorSpace,
|
|
[property: JsonPropertyName("dateTime")] DateTime DateTime,
|
|
[property: JsonPropertyName("dateTimeDigitized")] DateTime DateTimeDigitized,
|
|
[property: JsonPropertyName("dateTimeOriginal")] DateTime DateTimeOriginal,
|
|
[property: JsonPropertyName("exposureMode")] string ExposureMode,
|
|
[property: JsonPropertyName("exposureProgram")] string ExposureProgram,
|
|
[property: JsonPropertyName("exposureTime")] string ExposureTime,
|
|
[property: JsonPropertyName("flash")] string Flash,
|
|
[property: JsonPropertyName("focalLength")] string FocalLength,
|
|
[property: JsonPropertyName("height")] int Height,
|
|
[property: JsonPropertyName("make")] string Make,
|
|
[property: JsonPropertyName("meteringMode")] string MeteringMode,
|
|
[property: JsonPropertyName("model")] string Model,
|
|
[property: JsonPropertyName("orientation")] string Orientation,
|
|
[property: JsonPropertyName("resolutionUnit")] string ResolutionUnit,
|
|
[property: JsonPropertyName("sensingMethod")] string SensingMethod,
|
|
[property: JsonPropertyName("sharpness")] string Sharpness,
|
|
[property: JsonPropertyName("software")] string Software,
|
|
[property: JsonPropertyName("subSecTime")] string SubSecTime,
|
|
[property: JsonPropertyName("subSecTimeDigitized")] string SubSecTimeDigitized,
|
|
[property: JsonPropertyName("subSecTimeOriginal")] string SubSecTimeOriginal,
|
|
[property: JsonPropertyName("whiteBalance")] string WhiteBalance,
|
|
[property: JsonPropertyName("width")] int Width,
|
|
[property: JsonPropertyName("apertureValue")] string ApertureValue);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(ImageAmazon))]
|
|
public partial class ImageAmazonGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record ContentProperties([property: JsonPropertyName("contentDate")] DateTime ContentDate,
|
|
[property: JsonPropertyName("contentSignatures")] IReadOnlyList<ContentSignature> ContentSignatures,
|
|
[property: JsonPropertyName("contentType")] string ContentType,
|
|
[property: JsonPropertyName("extension")] string Extension,
|
|
[property: JsonPropertyName("image")] ImageAmazon Image,
|
|
[property: JsonPropertyName("md5")] string Md5,
|
|
[property: JsonPropertyName("size")] int Size,
|
|
[property: JsonPropertyName("version")] int Version);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(ContentProperties))]
|
|
public partial class ContentPropertiesGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record XAccntParentMap();
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(XAccntParentMap))]
|
|
public partial class XAccntParentMapGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record Datum([property: JsonPropertyName("accessRuleIds")] IReadOnlyList<object> AccessRuleIds,
|
|
[property: JsonPropertyName("childAssetTypeInfo")] IReadOnlyList<object> ChildAssetTypeInfo,
|
|
[property: JsonPropertyName("contentProperties")] ContentProperties ContentProperties,
|
|
[property: JsonPropertyName("createdBy")] string CreatedBy,
|
|
[property: JsonPropertyName("createdDate")] DateTime CreatedDate,
|
|
[property: JsonPropertyName("eTagResponse")] string ETagResponse,
|
|
[property: JsonPropertyName("groupPermissions")] IReadOnlyList<object> GroupPermissions,
|
|
[property: JsonPropertyName("id")] string Id,
|
|
[property: JsonPropertyName("isRoot")] bool IsRoot,
|
|
[property: JsonPropertyName("isShared")] bool IsShared,
|
|
[property: JsonPropertyName("keywords")] IReadOnlyList<object> Keywords,
|
|
[property: JsonPropertyName("kind")] string Kind,
|
|
[property: JsonPropertyName("labels")] IReadOnlyList<object> Labels,
|
|
[property: JsonPropertyName("modifiedDate")] DateTime ModifiedDate,
|
|
[property: JsonPropertyName("name")] string Name,
|
|
[property: JsonPropertyName("ownerId")] string OwnerId,
|
|
[property: JsonPropertyName("parentMap")] ParentMap ParentMap,
|
|
[property: JsonPropertyName("parents")] IReadOnlyList<string> Parents,
|
|
[property: JsonPropertyName("protectedFolder")] bool ProtectedFolder,
|
|
[property: JsonPropertyName("restricted")] bool Restricted,
|
|
[property: JsonPropertyName("status")] string Status,
|
|
[property: JsonPropertyName("subKinds")] IReadOnlyList<object> SubKinds,
|
|
[property: JsonPropertyName("transforms")] IReadOnlyList<string> Transforms,
|
|
[property: JsonPropertyName("version")] int Version,
|
|
[property: JsonPropertyName("xAccntParentMap")] XAccntParentMap XAccntParentMap,
|
|
[property: JsonPropertyName("xAccntParents")] IReadOnlyList<object> XAccntParents,
|
|
[property: JsonPropertyName("match")] bool? Match);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(Datum))]
|
|
public partial class DatumGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(Dictionary<string, Datum>))]
|
|
public partial class DictionaryDatumGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record LocationAmazon([property: JsonPropertyName("count")] int Count,
|
|
[property: JsonPropertyName("match")] string Match,
|
|
[property: JsonPropertyName("searchData")] SearchData SearchData);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(LocationAmazon))]
|
|
public partial class LocationAmazonGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record LocationInfo([property: JsonPropertyName("city")] string City,
|
|
[property: JsonPropertyName("country")] string Country,
|
|
[property: JsonPropertyName("countryIso3Code")] string CountryIso3Code,
|
|
[property: JsonPropertyName("state")] string State,
|
|
[property: JsonPropertyName("stateCode")] string StateCode);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(LocationInfo))]
|
|
public partial class LocationInfoGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record SearchData([property: JsonPropertyName("clusterName")] string ClusterName,
|
|
[property: JsonPropertyName("locationId")] string LocationId,
|
|
[property: JsonPropertyName("locationInfo")] LocationInfo LocationInfo,
|
|
[property: JsonPropertyName("thingId")] string ThingId);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(SearchData))]
|
|
public partial class SearchDataGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record AllPerson([property: JsonPropertyName("count")] int Count,
|
|
[property: JsonPropertyName("match")] string Match,
|
|
[property: JsonPropertyName("searchData")] SearchData SearchData);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(AllPerson))]
|
|
public partial class AllPersonGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record PersonAmazon([property: JsonPropertyName("count")] int Count,
|
|
[property: JsonPropertyName("match")] string Match,
|
|
[property: JsonPropertyName("searchData")] SearchData SearchData);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(PersonAmazon))]
|
|
public partial class PersonAmazonGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record ClusterId([property: JsonPropertyName("count")] int Count,
|
|
[property: JsonPropertyName("match")] string Match,
|
|
[property: JsonPropertyName("searchData")] SearchData SearchData);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(ClusterId))]
|
|
public partial class ClusterIdGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record Thing([property: JsonPropertyName("count")] int Count,
|
|
[property: JsonPropertyName("match")] string Match,
|
|
[property: JsonPropertyName("searchData")] SearchData SearchData);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(Thing))]
|
|
public partial class ThingGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record Time([property: JsonPropertyName("count")] int Count,
|
|
[property: JsonPropertyName("match")] string Match,
|
|
[property: JsonPropertyName("searchData")] SearchData SearchData);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(Time))]
|
|
public partial class TimeGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record ParentMap([property: JsonPropertyName("FOLDER")] IReadOnlyList<string> FOLDER);
|
|
|
|
public record Aggregations([property: JsonPropertyName("allPeople")] IReadOnlyList<AllPerson> AllPeople,
|
|
[property: JsonPropertyName("clusterId")] IReadOnlyList<ClusterId> ClusterId,
|
|
[property: JsonPropertyName("location")] IReadOnlyList<LocationAmazon> Location,
|
|
[property: JsonPropertyName("people")] IReadOnlyList<PersonAmazon> People,
|
|
[property: JsonPropertyName("things")] IReadOnlyList<Thing> Things,
|
|
[property: JsonPropertyName("time")] IReadOnlyList<Time> Time,
|
|
[property: JsonPropertyName("type")] IReadOnlyList<Type> Type);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(Aggregations))]
|
|
public partial class AggregationsGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(ParentMap))]
|
|
public partial class ParentMapGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
public record RootAmazon([property: JsonPropertyName("aggregations")] Aggregations Aggregations,
|
|
[property: JsonPropertyName("count")] int Count,
|
|
[property: JsonPropertyName("data")] IReadOnlyList<Datum> Data);
|
|
|
|
[JsonSourceGenerationOptions(WriteIndented = true)]
|
|
[JsonSerializable(typeof(RootAmazon))]
|
|
public partial class RootAmazonGenerationContext : JsonSerializerContext
|
|
{
|
|
}
|
|
|
|
private static void SaveAmazon(string destination, string harFile)
|
|
{
|
|
string offset;
|
|
string personId;
|
|
RootAmazon amazon;
|
|
string? personName;
|
|
string personIdFile;
|
|
string personDirectory;
|
|
PersonAmazon personAmazon;
|
|
Dictionary<string, string> keyValuePairs = [];
|
|
ReadOnlyCollection<(string Url, string AggregationLine)> aggregationLines = GetAggregationLines(harFile);
|
|
foreach ((string url, string aggregationLine) in aggregationLines)
|
|
{
|
|
if (aggregationLine.Contains(",\"category\":\"allPeople\"}"))
|
|
continue;
|
|
amazon = JsonSerializer.Deserialize(aggregationLine, RootAmazonGenerationContext.Default.RootAmazon) ?? throw new Exception();
|
|
if (amazon.Aggregations?.People is null || amazon.Aggregations.People.Count < 1)
|
|
continue;
|
|
personAmazon = amazon.Aggregations.People[0];
|
|
if (!url.Contains(personAmazon.Match))
|
|
continue;
|
|
personDirectory = Path.Combine(destination, personAmazon.SearchData.ClusterName);
|
|
_ = Directory.CreateDirectory(personDirectory);
|
|
personIdFile = Path.Combine(personDirectory, $"000) {personAmazon.Match}.json");
|
|
_ = keyValuePairs.TryAdd(personAmazon.Match, personAmazon.SearchData.ClusterName);
|
|
SaveAmazon(amazon.Data, personIdFile);
|
|
}
|
|
foreach ((string url, string aggregationLine) in aggregationLines)
|
|
{
|
|
if (aggregationLine.Contains(",\"category\":\"allPeople\"}"))
|
|
continue;
|
|
amazon = JsonSerializer.Deserialize(aggregationLine, RootAmazonGenerationContext.Default.RootAmazon) ?? throw new Exception();
|
|
if (amazon.Aggregations?.People is not null && amazon.Aggregations.People.Count > 0)
|
|
continue;
|
|
if (!url.Contains("offset="))
|
|
continue;
|
|
offset = url.Split("offset=")[1];
|
|
if (!url.Contains("people%3A("))
|
|
continue;
|
|
personId = url.Split("people%3A(")[1].Split(')')[0];
|
|
if (!keyValuePairs.TryGetValue(personId, out personName))
|
|
continue;
|
|
personDirectory = Path.Combine(destination, personName);
|
|
_ = Directory.CreateDirectory(personDirectory);
|
|
personIdFile = Path.Combine(personDirectory, $"{offset.Split('&')[0]}) {personId}.json");
|
|
SaveAmazon(amazon.Data, personIdFile);
|
|
}
|
|
}
|
|
|
|
internal static void SaveAmazon(ILogger<Worker> logger, List<string> args)
|
|
{
|
|
string root = Path.GetFullPath(args[0]);
|
|
string destination = Path.GetFullPath(args[2]);
|
|
if (string.IsNullOrEmpty(root))
|
|
throw new NullReferenceException(nameof(root));
|
|
string[] harFiles = Directory.GetFiles(root, "*.har", SearchOption.TopDirectoryOnly);
|
|
foreach (string harFile in harFiles)
|
|
SaveAmazon(destination, harFile);
|
|
logger?.LogInformation("{harFiles} count", harFiles.Length);
|
|
}
|
|
|
|
private static void SaveAmazon(IReadOnlyList<Datum> data, string personIdFile)
|
|
{
|
|
string json;
|
|
Dictionary<string, Datum> keyValuePairs = [];
|
|
foreach (Datum datum in data)
|
|
_ = keyValuePairs.TryAdd(datum.Name.Split('.')[0], datum);
|
|
json = JsonSerializer.Serialize(keyValuePairs, DictionaryDatumGenerationContext.Default.DictionaryStringDatum);
|
|
File.WriteAllText(personIdFile, json);
|
|
}
|
|
|
|
private static ReadOnlyCollection<(string, string)> GetAggregationLines(string harFile)
|
|
{
|
|
List<(string, string)> results = [];
|
|
if (!File.Exists(harFile))
|
|
throw new Exception();
|
|
string lastUrl = string.Empty;
|
|
string text = "\"text\": \"{";
|
|
string[] lines = File.ReadAllLines(harFile);
|
|
foreach (string line in lines)
|
|
{
|
|
if (line.Contains("\"url\": \""))
|
|
lastUrl = line;
|
|
if (!line.Contains(text))
|
|
continue;
|
|
if (!line.Contains("aggregations"))
|
|
continue;
|
|
if (lastUrl.Contains("search?asset=NONE"))
|
|
continue;
|
|
results.Add(new(lastUrl, line.Trim()[(text.Length - 1)..^1].Replace("\\\"", "\"")));
|
|
lastUrl = string.Empty;
|
|
}
|
|
return new(results);
|
|
}
|
|
|
|
} |