phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Serilog

Browse Source
This commit is contained in:
Mike Phares
2023-10-15 12:13:48 -07:00
parent cd5ab223c9
commit 37b7ad2a1f
102 changed files with 631 additions and 975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
Instance/DlibDotNet.cs
View File

@ -1,4 +1,5 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Phares.Shared;
using ShellProgressBar;
using System.Collections.ObjectModel;
@ -29,11 +30,11 @@ public partial class DlibDotNet
private readonly F_Random _Random;
private readonly IConsole _Console;
private readonly E_Distance _Distance;
private readonly Serilog.ILogger? _Log;
private readonly IBlurHasher _BlurHasher;
private readonly D2_FaceParts _FaceParts;
private readonly AppSettings _AppSettings;
private readonly List<string> _Exceptions;
private readonly ILogger<Program>? _Logger;
private readonly IsEnvironment _IsEnvironment;
private readonly bool _PropertyRootExistedBefore;
private readonly Models.Configuration _Configuration;
@ -43,6 +44,7 @@ public partial class DlibDotNet
public DlibDotNet(
List<string> args,
ILogger<Program> logger,
IsEnvironment isEnvironment,
IConfigurationRoot configurationRoot,
AppSettings appSettings,
@ -51,6 +53,7 @@ public partial class DlibDotNet
IConsole console)
{
string message;
_Logger = logger;
_Console = console;
_AppSettings = appSettings;
_IsEnvironment = isEnvironment;
@ -59,11 +62,10 @@ public partial class DlibDotNet
_JLinkResolvedDirectories = new();
if (ticks.ToString().Last() == '0')
ticks += 1;
_Log = Serilog.Log.ForContext<DlibDotNet>();
ReadOnlyCollection<PersonContainer> personContainers;
Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot);
Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration);
_Log.Information(propertyConfiguration.RootDirectory);
_Logger?.LogInformation(propertyConfiguration.RootDirectory);
Property.Models.Configuration.Verify(propertyConfiguration, requireExist: false);
Verify(configuration);
VerifyExtra(args, propertyConfiguration, configuration);
@ -78,7 +80,7 @@ public partial class DlibDotNet
_ArgZeroIsConfigurationRootDirectory = propertyConfiguration.RootDirectory == argZero;
if (!Directory.Exists(argZero))
_ = Directory.CreateDirectory(argZero);
_Log.Information(configuration.ModelDirectory);
_Logger?.LogInformation(configuration.ModelDirectory);
{
(ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetPngLowQuality();
(ImageCodecInfo hiddenImageCodecInfo, EncoderParameters hiddenEncoderParameters, string hiddenFileNameExtension) = C_Resize.GetGifLowQuality();
@ -149,15 +151,15 @@ public partial class DlibDotNet
if (!_PropertyRootExistedBefore && !_IsEnvironment.Development && _Exceptions.Count == 0 && _ArgZeroIsConfigurationRootDirectory)
{
string d2FacePartsRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(D2_FaceParts));
_Log.Information(string.Concat("Cleaning <", d2FacePartsRootDirectory, ">"));
_Logger?.LogInformation(string.Concat("Cleaning <", d2FacePartsRootDirectory, ">"));
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(d2FacePartsRootDirectory, ticks);
}
message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}";
_Log.Information(message);
_Logger?.LogInformation(message);
if (_Exceptions.Count != 0)
throw new Exception(message);
if (_PropertyRootExistedBefore)
_Log.Information("First run completed. Run again if wanted");
_Logger?.LogInformation("First run completed. Run again if wanted");
}
private static void Verify(Models.Configuration configuration)
@ -427,8 +429,6 @@ public partial class DlibDotNet
private string SaveUrlAndGetNewRootDirectory(string[] files)
{
string result;
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
if (files.Length == 0)
throw new NotSupportedException();
string? sourceDirectory = Path.GetDirectoryName(files.First());
@ -448,7 +448,7 @@ public partial class DlibDotNet
int length = result.Length;
for (int y = 0; y < int.MaxValue; y++)
{
_Log.Information("Enter fileNameToCollection url for fileNameToCollection image");
_Logger?.LogInformation("Enter fileNameToCollection url for fileNameToCollection image");
line = _Console.ReadLine();
if (string.IsNullOrEmpty(line))
break;
@ -466,14 +466,12 @@ public partial class DlibDotNet
_ = new Item(fileHolder, relativePath, isValidImageFormatExtension);
// container.Items.Add(item);
}
_Log.Information(". . .");
_Logger?.LogInformation(". . .");
return result;
}
private void FullDoWork(string argZero, string propertyRoot, long ticks, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, int t, Container[] containers, A_Property propertyLogic, B_Metadata metadata, Dictionary<int, List<MappingFromPhotoPrism>> fileNameToCollection, MapLogic mapLogic)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
int total;
int notMapped;
string message;
@ -542,11 +540,11 @@ public partial class DlibDotNet
{
for (int y = 0; y < int.MaxValue; y++)
{
_Log.Information("Press \"Y\" key when ready to continue or close console");
_Logger?.LogInformation("Press \"Y\" key when ready to continue or close console");
if (_Console.ReadKey() == ConsoleKey.Y)
break;
}
_Log.Information(". . .");
_Logger?.LogInformation(". . .");
}
total += container.Items.Count;
}
@ -741,37 +739,22 @@ public partial class DlibDotNet
private void LogItemPropertyIsNull(Item item)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
if (!item.SourceDirectoryFileHolder.Exists)
_Log.Information(string.Concat("NoJson <", item.ImageFileHolder.FullName, '>'));
_Logger?.LogInformation(string.Concat("NoJson <", item.ImageFileHolder.FullName, '>'));
else if (item.FileSizeChanged.HasValue && item.FileSizeChanged.Value)
_Log.Information(string.Concat("FileSizeChanged <", item.ImageFileHolder.FullName, '>'));
_Logger?.LogInformation(string.Concat("FileSizeChanged <", item.ImageFileHolder.FullName, '>'));
else if (item.LastWriteTimeChanged.HasValue && item.LastWriteTimeChanged.Value)
_Log.Information(string.Concat("LastWriteTimeChanged <", item.ImageFileHolder.FullName, '>'));
_Logger?.LogInformation(string.Concat("LastWriteTimeChanged <", item.ImageFileHolder.FullName, '>'));
else if (item.Moved.HasValue && item.Moved.Value)
_Log.Information(string.Concat("Moved <", item.ImageFileHolder.FullName, '>'));
_Logger?.LogInformation(string.Concat("Moved <", item.ImageFileHolder.FullName, '>'));
}
private void LogNameWithoutExtensionIsIdFormatBut(Item item)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
_Log.Information($"Name without extension is Id format but doesn't match id <{item.ImageFileHolder.FullName}>");
_Logger?.LogInformation($"Name without extension is Id format but doesn't match id <{item.ImageFileHolder.FullName}>");
File.Move(item.ImageFileHolder.FullName, $"{item.ImageFileHolder.FullName}.rename");
}
private long LogDelta(long ticks, string? methodName)
{
long result;
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
double delta = new TimeSpan(DateTime.Now.Ticks - ticks).TotalMilliseconds;
_Log.Debug($"{methodName} took {Math.Floor(delta)} millisecond(s)");
result = DateTime.Now.Ticks;
return result;
}
private int GetNotMappedCountAndUpdateMappingFromPersonThenSetMapping(MapLogic mapLogic, Item item, bool? isFocusRelativePath, ReadOnlyCollection<LocationContainer<MetadataExtractor.Directory>> locationContainers, MappingFromItem mappingFromItem, List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection, List<Shared.Models.Face> faces)
{
int result;
@ -985,8 +968,6 @@ public partial class DlibDotNet
MappingFromItem mappingFromItem = IMappingFromItem.GetMappingFromItem(containerDateTimes, item, resizedFileHolder);
Map.Models.Stateless.Methods.IMapLogic.SetCreationTimeMaybeMoveToDecade(_Configuration.PropertyConfiguration, _Configuration.MoveToDecade && _Configuration.LocationContainerDistanceTolerance is null, mappingFromItem, locationContainers);
ReadOnlyDictionary<string, MetadataExtractorDirectory> metadataExtractorDirectories = metadata.GetMetadataCollection(subFileTuples, parseExceptions, changesFrom, mappingFromItem);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection));
if (_AppSettings.Places.Count > 0)
{
float latitude;
@ -1010,14 +991,8 @@ public partial class DlibDotNet
}
}
Dictionary<string, int[]> outputResolutionToResize = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, subFileTuples, parseExceptions, item.Property, mappingFromItem);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(C_Resize.GetResizeKeyValuePairs));
if (_Configuration.SaveResizedSubfiles)
{
_Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.Property, mappingFromItem, outputResolutionToResize);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(C_Resize.SaveResizedSubfile));
}
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
faces = new();
else if (!mappingFromItem.ResizedFileHolder.Exists && !File.Exists(mappingFromItem.ResizedFileHolder.FullName))
@ -1028,14 +1003,10 @@ public partial class DlibDotNet
if (!fileNameToCollection.TryGetValue(mappingFromItem.Id, out mappingFromPhotoPrismCollection))
mappingFromPhotoPrismCollection = null;
faces = _Faces.GetFaces(outputResolution, dResultsFullGroupDirectory, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionToResize, locationContainers, mappingFromPhotoPrismCollection);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D_Face.GetFaces));
result = GetNotMappedCountAndUpdateMappingFromPersonThenSetMapping(mapLogic, item, isFocusRelativePath, locationContainers, mappingFromItem, mappingFromPhotoPrismCollection, faces);
List<(Shared.Models.Face, FileInfo?, string, bool Saved)> faceCollection = _Faces.SaveFaces(_FaceParts.FileNameExtension, dResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem, faces);
if (_Configuration.CopyFacesAndSaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
_FaceParts.CopyFacesAndSaveFaceLandmarkImage(facePartsCollectionDirectory, mappingFromItem, faceCollection);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D_Face.SaveFaces));
if ((_Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch)
&& _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
&& locationContainers is not null && faceCollection.All(l => !l.Saved))
@ -1046,8 +1017,6 @@ public partial class DlibDotNet
bool saveRotated = false;
string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(_Configuration.PropertyConfiguration, container.SourceDirectory);
_FaceParts.SaveFaceLandmarkImages(_Configuration.PropertyConfiguration, subFileTuples, parseExceptions, mappingFromItem, faces, saveRotated);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D2_FaceParts.SaveFaceLandmarkImages));
}
}
lock (sourceDirectoryChanges)
@ -1073,8 +1042,6 @@ public partial class DlibDotNet
Item[] filteredItems,
string message)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
int result = 0;
int exceptionsCount = 0;
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
@ -1108,10 +1075,9 @@ public partial class DlibDotNet
if (!anyPropertiesChangedForX && (i == 0 || sourceDirectoryChanges.Count > 0))
progressBar.Tick();
}
catch (Exception ex)
catch (Exception)
{
exceptionsCount++;
_Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, ex.Message, Environment.NewLine, ex.StackTrace), ex);
if (exceptionsCount == filteredItems.Length)
throw new Exception(string.Concat("All in [", container.SourceDirectory, "] failed!"));
}
@ -1173,8 +1139,6 @@ public partial class DlibDotNet
private void SaveFaceDistances(long ticks, MapLogic mapLogic, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, ReadOnlyDictionary<int, ReadOnlyDictionary<int, Mapping>> idToWholePercentagesToMapping, ReadOnlyCollection<FaceDistance> faceDistanceEncodings, ReadOnlyCollection<FaceDistanceContainer> faceDistanceContainers)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
int? useFiltersCounter = null;
DistanceLimits distanceLimits;
ReadOnlyCollection<SortingContainer> sortingContainers;
@ -1183,7 +1147,7 @@ public partial class DlibDotNet
distanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaPermyriadTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh);
filteredFaceDistanceContainers = E_Distance.FilteredPostLoadFaceDistanceContainers(mapLogic, faceDistanceContainers, skipOlderThan, distanceLimits);
if (filteredFaceDistanceContainers.Length == 0)
_Log.Information("All images have been filtered!");
_Logger?.LogInformation("All images have been filtered!");
else
{
sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortedSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distanceLimits, faceDistanceEncodings, filteredFaceDistanceContainers);
@ -1194,7 +1158,7 @@ public partial class DlibDotNet
distanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaPermyriadTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh, useFiltersCounter);
filteredFaceDistanceContainers = E_Distance.FilteredPostLoadFaceDistanceContainers(mapLogic, faceDistanceContainers, skipOlderThan, distanceLimits);
if (filteredFaceDistanceContainers.Length == 0)
_Log.Information("All images have been filtered!");
_Logger?.LogInformation("All images have been filtered!");
else
{
sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortedSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distanceLimits, faceDistanceEncodings, filteredFaceDistanceContainers);