phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Metadata-Query

Browse Source
This commit is contained in:
Mike Phares
2023-07-04 18:09:53 -07:00
parent 0bce2bf22b
commit fcd3f9030d
16 changed files with 709 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
Metadata-Query/Metadata-Query.csproj Normal file
View File

@ -0,0 +1,60 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<ImplicitUsings>enable</ImplicitUsings>
<LangVersion>10.0</LangVersion>
<Nullable>enable</Nullable>
<OutputType>Exe</OutputType>
<RuntimeIdentifier>win-x64</RuntimeIdentifier>
<TargetFramework>net7.0</TargetFramework>
<UserSecretsId>f89b7242-dbb0-4349-b950-657eb8cf87ef</UserSecretsId>
</PropertyGroup>
<PropertyGroup>
<PackageId>Phares.View.by.Distance.Metadata.Query</PackageId>
<GeneratePackageOnBuild>false</GeneratePackageOnBuild>
<Version>7.0.101.1</Version>
<Authors>Mike Phares</Authors>
<Company>Phares</Company>
<IncludeSymbols>true</IncludeSymbols>
<SymbolPackageFormat>snupkg</SymbolPackageFormat>
</PropertyGroup>
<PropertyGroup>
<IsWindows Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true'">true</IsWindows>
<IsOSX Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))' == 'true'">true</IsOSX>
<IsLinux Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true'">true</IsLinux>
</PropertyGroup>
<PropertyGroup Condition="'$(IsWindows)'=='true'">
<DefineConstants>Windows</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$(IsOSX)'=='true'">
<DefineConstants>OSX</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$(IsLinux)'=='true'">
<DefineConstants>Linux</DefineConstants>
</PropertyGroup>
<ItemGroup Condition="'$(RuntimeIdentifier)' == 'browser-wasm'">
<SupportedPlatform Include="browser" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Humanizer.Core" Version="2.14.1" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
<PackageReference Include="Serilog.Settings.Configuration" Version="7.0.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />
<PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />
<PackageReference Include="Serilog" Version="2.12.0" />
<PackageReference Include="System.Text.Json" Version="7.0.2" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Property\Property.csproj" />
<ProjectReference Include="..\Shared\View-by-Distance.Shared.csproj" />
</ItemGroup>
<ItemGroup>
<None Include="appsettings.Development.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<None Include="appsettings.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
</Project>

158
Metadata-Query/MetadataQuery.cs Normal file
View File

@ -0,0 +1,158 @@
using Microsoft.Extensions.Configuration;
using Phares.Shared;
using Serilog;
using ShellProgressBar;
using System.Text;
using System.Text.Json;
using View_by_Distance.Metadata.Query.Models;
using View_by_Distance.Shared.Models.Methods;
namespace View_by_Distance.Metadata.Query;
public class MetadataQuery
{
private readonly AppSettings _AppSettings;
private readonly string _WorkingDirectory;
private readonly Configuration _Configuration;
private readonly IsEnvironment _IsEnvironment;
private readonly IConfigurationRoot _ConfigurationRoot;
private readonly Property.Models.Configuration _PropertyConfiguration;
public MetadataQuery(List<string> args, IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, AppSettings appSettings, string workingDirectory, bool isSilent, IConsole console)
{
if (isSilent)
{ }
if (console is null)
{ }
_AppSettings = appSettings;
_IsEnvironment = isEnvironment;
long ticks = DateTime.Now.Ticks;
_WorkingDirectory = workingDirectory;
_ConfigurationRoot = configurationRoot;
ILogger? log = Log.ForContext<MetadataQuery>();
Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot);
Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration);
_PropertyConfiguration = propertyConfiguration;
_Configuration = configuration;
propertyConfiguration.Update();
log.Information(propertyConfiguration.RootDirectory);
Verify();
MetadataQueryFilesInDirectories(log, ticks);
}
private void Verify()
{
if (_AppSettings is null)
{ }
if (_IsEnvironment is null)
{ }
if (_Configuration is null)
{ }
if (_ConfigurationRoot is null)
{ }
if (_WorkingDirectory is null)
{ }
if (_PropertyConfiguration is null)
{ }
}
private List<(string FileName, string Count, string TagGroup, string TagIdName, string Value)> GetCollection(long ticks)
{
int count;
string json;
string message;
string fileName;
ProgressBar progressBar;
const string fileSearchFilter = "*";
const string directorySearchFilter = "*";
Dictionary<string, List<KeyValuePair<string, string>>>? dictionary;
List<(string FileName, string Count, string TagGroup, string TagIdName, string Value)> collection = new();
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
List<string[]> filesCollection = Shared.Models.Stateless.Methods.IDirectory.GetFilesCollection(_PropertyConfiguration.RootDirectory, directorySearchFilter, fileSearchFilter);
foreach (string[] files in filesCollection)
{
if (!files.Any())
continue;
message = $"{ticks}) Reading files for <{files.FirstOrDefault()}> - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
progressBar = new(files.Length, message, options);
foreach (string file in files)
{
count = 0;
progressBar.Tick();
json = File.ReadAllText(file);
fileName = Path.GetFileName(file);
dictionary = JsonSerializer.Deserialize<Dictionary<string, List<KeyValuePair<string, string>>>>(json);
if (dictionary is null)
continue;
foreach (KeyValuePair<string, List<KeyValuePair<string, string>>> keyValuePair in dictionary)
{
foreach (KeyValuePair<string, string> keyValue in keyValuePair.Value)
count++;
}
foreach (KeyValuePair<string, List<KeyValuePair<string, string>>> keyValuePair in dictionary)
{
foreach (KeyValuePair<string, string> keyValue in keyValuePair.Value)
collection.Add(new(fileName, count.ToString("000000"), keyValuePair.Key, keyValue.Key, keyValue.Value));
}
}
progressBar.Dispose();
}
return collection;
}
private static Dictionary<string, List<string>> GetKeyValuePairs(List<(string FileName, string Count, string TagGroup, string TagIdName, string Value)> matches)
{
string key;
string line;
List<string>? valuePairs;
Dictionary<string, List<string>> keyValuePairs = new();
foreach ((string fileName, string count, string tagGroup, string tagIdName, string value) in matches)
{
key = $"{tagGroup}\t{tagIdName}\t{value.Trim()}";
line = $"{tagGroup}\t{tagIdName}\t{value.Trim()}\t{count}\t{fileName}";
if (!keyValuePairs.TryGetValue(key, out valuePairs))
{
keyValuePairs.Add(key, new());
if (!keyValuePairs.TryGetValue(key, out valuePairs))
throw new Exception();
}
valuePairs.Add(line);
}
return keyValuePairs;
}
private void MetadataQueryFilesInDirectories(ILogger log, long ticks)
{
List<(string FileName, string Count, string TagGroup, string TagIdName, string Value)> collection = GetCollection(ticks);
log.Information($"Ready to query {collection.Count} entries?");
IEnumerable<(string FileName, string Count, string TagGroup, string TagIdName, string Value)> enumerable()
{
foreach ((string FileName, string Count, string TagGroup, string TagIdName, string Value) l in collection)
{
if (l.TagIdName.StartsWith("42016\t") && l.Value != "00000000000000000000000000000000")
{
yield return l;
}
}
}
List<(string FileName, string Count, string TagGroup, string TagIdName, string Value)> matches = enumerable().ToList();
if (matches.Any())
{
matches.Sort();
StringBuilder stringBuilder = new();
Dictionary<string, List<string>> keyValuePairs = GetKeyValuePairs(matches);
foreach (KeyValuePair<string, List<string>> keyValuePair in keyValuePairs)
{
if (keyValuePair.Value.Count != 2)
continue;
foreach (string line in keyValuePair.Value)
_ = stringBuilder.AppendLine(line);
}
string checkFile = $"D:/Tmp/Phares/{DateTime.Now.Ticks}.tsv";
string text = stringBuilder.ToString();
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(checkFile, text, updateToWhenMatches: null, compareBeforeWrite: true, updateDateWhenMatches: false);
}
}
}

16
Metadata-Query/Models/AppSettings.cs Normal file
View File

@ -0,0 +1,16 @@
using System.Text.Json;
namespace View_by_Distance.Metadata.Query.Models;
public record AppSettings(string Company,
int MaxDegreeOfParallelism,
string WorkingDirectoryName)
{
public override string ToString()
{
string result = JsonSerializer.Serialize(this, new JsonSerializerOptions() { WriteIndented = true });
return result;
}
}

44
Metadata-Query/Models/Binder/AppSettings.cs Normal file
View File

@ -0,0 +1,44 @@
using Microsoft.Extensions.Configuration;
using System.Text.Json;
namespace View_by_Distance.Metadata.Query.Models.Binder;
public class AppSettings
{
public string? Company { get; set; }
public int? MaxDegreeOfParallelism { get; set; }
public string? WorkingDirectoryName { get; set; }
public override string ToString()
{
string result = JsonSerializer.Serialize(this, new JsonSerializerOptions() { WriteIndented = true });
return result;
}
private static Models.AppSettings Get(AppSettings? appSettings)
{
Models.AppSettings result;
if (appSettings?.Company is null)
throw new NullReferenceException(nameof(appSettings.Company));
if (appSettings?.MaxDegreeOfParallelism is null)
throw new NullReferenceException(nameof(appSettings.MaxDegreeOfParallelism));
if (appSettings?.WorkingDirectoryName is null)
throw new NullReferenceException(nameof(appSettings.WorkingDirectoryName));
result = new(
appSettings.Company,
appSettings.MaxDegreeOfParallelism.Value,
appSettings.WorkingDirectoryName
);
return result;
}
public static Models.AppSettings Get(IConfigurationRoot configurationRoot)
{
Models.AppSettings result;
AppSettings? appSettings = configurationRoot.Get<AppSettings>();
result = Get(appSettings);
return result;
}
}

59
Metadata-Query/Models/Binder/Configuration.cs Normal file
View File

@ -0,0 +1,59 @@
using Microsoft.Extensions.Configuration;
using Phares.Shared;
using System.ComponentModel.DataAnnotations;
using System.Text.Json;
namespace View_by_Distance.Metadata.Query.Models.Binder;
public class Configuration
{
#nullable disable
[Display(Name = "Ignore Extensions"), Required] public string[] IgnoreExtensions { get; set; }
[Display(Name = "Property Configuration"), Required] public Property.Models.Configuration PropertyConfiguration { get; set; }
[Display(Name = "Person Birthday Format"), Required] public string PersonBirthdayFormat { get; set; }
#nullable restore
public override string ToString()
{
string result = JsonSerializer.Serialize(this, new JsonSerializerOptions() { WriteIndented = true });
return result;
}
private static Models.Configuration Get(Configuration? configuration)
{
Models.Configuration result;
if (configuration is null)
throw new NullReferenceException(nameof(configuration));
if (configuration.IgnoreExtensions is null)
throw new NullReferenceException(nameof(configuration.IgnoreExtensions));
if (configuration.PersonBirthdayFormat is null)
throw new NullReferenceException(nameof(configuration.PersonBirthdayFormat));
result = new(
configuration.IgnoreExtensions,
configuration.PersonBirthdayFormat,
configuration.PropertyConfiguration);
return result;
}
public static Models.Configuration Get(IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, Property.Models.Configuration propertyConfiguration)
{
Models.Configuration result;
Configuration? configuration;
if (isEnvironment is null)
configuration = configurationRoot.Get<Configuration>();
else
{
string environmentName = IsEnvironment.GetEnvironmentName(isEnvironment);
string section = string.Concat(environmentName, ":", nameof(Configuration));
IConfigurationSection configurationSection = configurationRoot.GetSection(section);
configuration = configurationSection.Get<Configuration>();
}
result = Get(configuration);
result.SetAndUpdate(propertyConfiguration);
return result;
}
}

38
Metadata-Query/Models/Configuration.cs Normal file
View File

@ -0,0 +1,38 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace View_by_Distance.Metadata.Query.Models;
public class Configuration
{
protected Property.Models.Configuration _PropertyConfiguration;
public string[] IgnoreExtensions { init; get; }
public string PersonBirthdayFormat { init; get; }
public Property.Models.Configuration PropertyConfiguration => _PropertyConfiguration;
[JsonConstructor]
public Configuration(
string[] ignoreExtensions,
string personBirthdayFormat,
Property.Models.Configuration propertyConfiguration)
{
IgnoreExtensions = ignoreExtensions;
PersonBirthdayFormat = personBirthdayFormat;
_PropertyConfiguration = propertyConfiguration;
}
public override string ToString()
{
string result = JsonSerializer.Serialize(this, new JsonSerializerOptions() { WriteIndented = true });
return result;
}
public void SetAndUpdate(Property.Models.Configuration configuration)
{
_PropertyConfiguration = configuration;
_PropertyConfiguration.Update();
}
}

10
Metadata-Query/Models/Stateless/SerilogExtensionMethods.cs Normal file
View File

@ -0,0 +1,10 @@
namespace View_by_Distance.Metadata.Query.Models.Stateless;
public static class SerilogExtensionMethods
{
internal static void Warn(this Serilog.ILogger log, string messageTemplate) => log.Warning(messageTemplate);
internal static void Info(this Serilog.ILogger log, string messageTemplate) => log.Information(messageTemplate);
}

71
Metadata-Query/Program.cs Normal file
View File

@ -0,0 +1,71 @@
using Microsoft.Extensions.Configuration;
using Phares.Shared;
using Serilog;
using System.Diagnostics;
using System.Reflection;
using View_by_Distance.Metadata.Query.Models;
using View_by_Distance.Shared.Models.Stateless.Methods;
namespace View_by_Distance.Metadata.Query;
public class Program
{
public static void Secondary(List<string> args)
{
LoggerConfiguration loggerConfiguration = new();
Assembly assembly = Assembly.GetExecutingAssembly();
bool debuggerWasAttachedAtLineZero = Debugger.IsAttached || assembly.Location.Contains(@"\bin\Debug");
IsEnvironment isEnvironment = new(processesCount: null, nullASPNetCoreEnvironmentIsDevelopment: debuggerWasAttachedAtLineZero, nullASPNetCoreEnvironmentIsProduction: !debuggerWasAttachedAtLineZero);
IConfigurationBuilder configurationBuilder = new ConfigurationBuilder()
.AddEnvironmentVariables()
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.AddJsonFile(isEnvironment.AppSettingsFileName, optional: false, reloadOnChange: true)
.AddUserSecrets<Program>();
IConfigurationRoot configurationRoot = configurationBuilder.Build();
AppSettings appSettings = Models.Binder.AppSettings.Get(configurationRoot);
if (appSettings.MaxDegreeOfParallelism > Environment.ProcessorCount)
throw new Exception("MaxDegreeOfParallelism must be =< Environment.ProcessorCount!");
if (string.IsNullOrEmpty(appSettings.WorkingDirectoryName))
throw new Exception("Working directory name must have a value!");
string workingDirectory = IWorkingDirectory.GetWorkingDirectory(assembly.GetName().Name, appSettings.WorkingDirectoryName);
Environment.SetEnvironmentVariable(nameof(workingDirectory), workingDirectory);
_ = ConfigurationLoggerConfigurationExtensions.Configuration(loggerConfiguration.ReadFrom, configurationRoot);
Log.Logger = loggerConfiguration.CreateLogger();
ILogger log = Log.ForContext<Program>();
int silentIndex = args.IndexOf("s");
if (silentIndex > -1)
args.RemoveAt(silentIndex);
try
{
if (args is null)
throw new Exception("args is null!");
Shared.Models.Console console = new();
_ = new MetadataQuery(args, isEnvironment, configurationRoot, appSettings, workingDirectory, silentIndex > -1, console);
}
catch (Exception ex)
{
log.Fatal(string.Concat(ex.Message, Environment.NewLine, ex.StackTrace));
}
finally
{
Log.CloseAndFlush();
}
if (silentIndex > -1)
log.Debug("Done. Bye");
else
{
log.Debug("Done. Press 'Enter' to end");
_ = Console.ReadLine();
}
}
public static void Main(string[] args)
{
if (args is not null)
Secondary(args.ToList());
else
Secondary(new List<string>());
}
}

10
Metadata-Query/appsettings.Development.json Normal file
View File

@ -0,0 +1,10 @@
{
"Logging": {
"LogLevel": {
"Log4netProvider": "Debug"
}
},
"Serilog": {
"MinimumLevel": "Debug"
}
}

127
Metadata-Query/appsettings.json Normal file
View File

@ -0,0 +1,127 @@
{
"Company": "Mike Phares",
"Destination": "L",
"Linux": {},
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft": "Warning",
"Log4netProvider": "Debug",
"Microsoft.Hosting.Lifetime": "Information"
}
},
"MaxDegreeOfParallelism": 6,
"Serilog": {
"Using": [
"Serilog.Sinks.Console",
"Serilog.Sinks.File"
],
"MinimumLevel": "Information",
"WriteTo": [
{
"Name": "Debug",
"Args": {
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level}] ({SourceContext}.{MethodName}) ({InstanceId}) ({RemoteIpAddress}) {Message}{NewLine}{Exception}"
}
},
{
"Name": "Console",
"Args": {
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level}] ({SourceContext}.{MethodName}) ({InstanceId}) ({RemoteIpAddress}) {Message}{NewLine}{Exception}"
}
},
{
"Name": "File",
"Args": {
"path": "%workingDirectory% - Log/log-.txt",
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} [{Level}] ({SourceContext}.{MethodName}) ({InstanceId}) ({RemoteIpAddress}) {Message}{NewLine}{Exception}",
"rollingInterval": "Hour"
}
}
],
"Enrich": [
"FromLogContext",
"WithMachineName",
"WithThreadId"
],
"Properties": {
"Application": "Sample"
}
},
"WorkingDirectoryName": "PharesApps",
"Windows": {
"Configuration": {
"DateGroup": "dd514b88",
"DiffPropertyDirectory": "",
"FileNameDirectorySeparator": ".Z.",
"ForcePropertyLastWriteTimeToCreationTime": false,
"MaxImagesInDirectoryForTopLevelFirstPass": 10,
"OutputExtension": ".jpg",
"Pattern": "[^ABCDEFGHIJKLMNOPQRSTUVWXYZbcdfghjklmnpqrstvwxyz0-9]",
"PersonBirthdayFormat": "yyyy-MM-dd_HH",
"PopulatePropertyId": true,
"PropertiesChangedForProperty": false,
"ResultAllInOne": "_ _ _",
"ResultAllInOneSubdirectoryLength": 2,
"ResultCollection": "[]",
"ResultContent": "()",
"ResultSingleton": "{}",
"RootDirectory": "D:/Images",
"WriteBitmapDataBytes": false,
"IgnoreExtensions": [
".gif",
".GIF",
".nef",
".NEF",
".pdf",
".PDF"
],
"ValidImageFormatExtensions": [
".bmp",
".BMP",
".gif",
".GIF",
".jpeg",
".JPEG",
".jpg",
".JPG",
".png",
".PNG",
".tiff",
".TIFF",
".tif",
".TIF"
],
"ValidMetadataExtensions": [
".3gp",
".3GP",
".avi",
".AVI",
".bmp",
".BMP",
".gif",
".GIF",
".ico",
".ICO",
".jpeg",
".JPEG",
".jpg",
".JPG",
".m4v",
".M4V",
".mov",
".MOV",
".mp4",
".MP4",
".mta",
".MTA",
".png",
".PNG",
".tiff",
".TIFF",
".tif",
".TIF"
]
}
}
}