1137 lines
68 KiB
C#
1137 lines
68 KiB
C#
using Microsoft.Extensions.Configuration;
|
|
using Phares.Shared;
|
|
using ShellProgressBar;
|
|
using System.Diagnostics;
|
|
using System.Drawing;
|
|
using System.Drawing.Imaging;
|
|
using View_by_Distance.Distance.Models;
|
|
using View_by_Distance.Face.Models;
|
|
using View_by_Distance.FaceParts.Models;
|
|
using View_by_Distance.Instance.Models;
|
|
using View_by_Distance.Map.Models;
|
|
using View_by_Distance.Metadata.Models;
|
|
using View_by_Distance.PhotoPrism.Models;
|
|
using View_by_Distance.Property.Models;
|
|
using View_by_Distance.Resize.Models;
|
|
using View_by_Distance.Shared.Models;
|
|
using View_by_Distance.Shared.Models.Methods;
|
|
|
|
namespace View_by_Distance.Instance;
|
|
|
|
public partial class DlibDotNet
|
|
{
|
|
|
|
private readonly D_Face _Faces;
|
|
private readonly G_Index _Index;
|
|
private readonly C_Resize _Resize;
|
|
private readonly F_Random _Random;
|
|
private readonly E3_Rename _Rename;
|
|
private readonly IConsole _Console;
|
|
private readonly E_Distance _Distance;
|
|
private readonly Serilog.ILogger? _Log;
|
|
private readonly D2_FaceParts _FaceParts;
|
|
private readonly AppSettings _AppSettings;
|
|
private readonly List<string> _Exceptions;
|
|
private readonly IsEnvironment _IsEnvironment;
|
|
private readonly bool _PropertyRootExistedBefore;
|
|
private readonly PersonContainer[] _PersonContainers;
|
|
private readonly Models.Configuration _Configuration;
|
|
private readonly bool _ArgZeroIsConfigurationRootDirectory;
|
|
private readonly Map.Models.Configuration _MapConfiguration;
|
|
|
|
public DlibDotNet(
|
|
List<string> args,
|
|
IsEnvironment isEnvironment,
|
|
IConfigurationRoot configurationRoot,
|
|
AppSettings appSettings,
|
|
string workingDirectory,
|
|
bool isSilent,
|
|
IConsole console)
|
|
{
|
|
string message;
|
|
_Console = console;
|
|
_AppSettings = appSettings;
|
|
_IsEnvironment = isEnvironment;
|
|
long ticks = DateTime.Now.Ticks;
|
|
_Exceptions = new List<string>();
|
|
_Log = Serilog.Log.ForContext<DlibDotNet>();
|
|
Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot);
|
|
Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration);
|
|
_Log.Information(propertyConfiguration.RootDirectory);
|
|
Property.Models.Configuration.Verify(propertyConfiguration, requireExist: false);
|
|
Verify(configuration);
|
|
VerifyExtra(args, propertyConfiguration, configuration);
|
|
_Configuration = configuration;
|
|
_Index = new(configuration);
|
|
_Random = new(configuration);
|
|
_Rename = new(configuration);
|
|
if (configuration.IgnoreExtensions is null)
|
|
throw new NullReferenceException(nameof(configuration.IgnoreExtensions));
|
|
string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(propertyConfiguration, nameof(A_Property), create: false);
|
|
_PropertyRootExistedBefore = !Directory.Exists(propertyRoot);
|
|
string argZero = args.Count > 0 ? Path.GetFullPath(args[0]) : Path.GetFullPath(propertyConfiguration.RootDirectory);
|
|
_ArgZeroIsConfigurationRootDirectory = propertyConfiguration.RootDirectory == argZero;
|
|
if (!Directory.Exists(argZero))
|
|
_ = Directory.CreateDirectory(argZero);
|
|
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(argZero);
|
|
if (!Directory.Exists(argZero))
|
|
_ = Directory.CreateDirectory(argZero);
|
|
_Log.Information(configuration.ModelDirectory);
|
|
{
|
|
(ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetPngLowQuality();
|
|
(ImageCodecInfo hiddenImageCodecInfo, EncoderParameters hiddenEncoderParameters, string hiddenFileNameExtension) = C_Resize.GetGifLowQuality();
|
|
_Faces = new D_Face(
|
|
argZero,
|
|
configuration.CheckDFaceAndUpWriteDates,
|
|
configuration.PropertyConfiguration,
|
|
encoderParameters,
|
|
configuration.FaceDistanceHiddenImageFactor,
|
|
filenameExtension,
|
|
configuration.ForceFaceLastWriteTimeToCreationTime,
|
|
hiddenEncoderParameters,
|
|
hiddenFileNameExtension,
|
|
hiddenImageCodecInfo,
|
|
imageCodecInfo,
|
|
configuration.ModelDirectory,
|
|
configuration.ModelName,
|
|
configuration.OverrideForFaceImages,
|
|
configuration.PredictorModelName,
|
|
configuration.PropertiesChangedForFaces);
|
|
}
|
|
{
|
|
(ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetGifLowQuality();
|
|
_FaceParts = new D2_FaceParts(imageCodecInfo, encoderParameters, filenameExtension, configuration.CheckDFaceAndUpWriteDates, configuration.OverrideForFaceLandmarkImages);
|
|
}
|
|
_MapConfiguration = Get(configuration, _Faces.FileNameExtension, _Faces.HiddenFileNameExtension, _FaceParts.FileNameExtension);
|
|
_Distance = new(configuration.DistanceMoveUnableToMatch, configuration.DistanceRenameToMatch, _Configuration.FaceConfidencePercent, configuration.RangeDistanceTolerance, configuration.RangeFaceConfidence);
|
|
if (_PropertyRootExistedBefore || !_ArgZeroIsConfigurationRootDirectory)
|
|
_PersonContainers = Array.Empty<PersonContainer>();
|
|
else
|
|
{
|
|
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
|
|
message = $") Building People Collection - {totalSeconds} total second(s)";
|
|
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
|
|
using ProgressBar progressBar = new(1, message, options);
|
|
progressBar.Tick();
|
|
string rootDirectory = propertyConfiguration.RootDirectory;
|
|
string peopleRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(A2_People));
|
|
string? rootResultsDirectory = Path.GetDirectoryName(Path.GetDirectoryName(peopleRootDirectory));
|
|
if (rootResultsDirectory is null)
|
|
throw new Exception();
|
|
Storage storage = new(rootDirectory, rootResultsDirectory, peopleRootDirectory);
|
|
_PersonContainers = Shared.Models.Stateless.Methods.IPersonContainer.GetPersonContainers(storage, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), _Faces.FileNameExtension);
|
|
}
|
|
if (!isSilent && configuration.TestDistanceResults)
|
|
{
|
|
E2_Navigate e2Navigate = new(console, configuration, argZero);
|
|
e2Navigate.Navigate(propertyConfiguration, configuration.OutputResolutions[0]);
|
|
_Log.Information(propertyConfiguration.RootDirectory);
|
|
}
|
|
{
|
|
(ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetTuple(
|
|
configuration.OutputExtension,
|
|
configuration.OutputQuality);
|
|
_Resize = new C_Resize(
|
|
configuration.ForceResizeLastWriteTimeToCreationTime,
|
|
configuration.OverrideForResizeImages,
|
|
configuration.PropertiesChangedForResize,
|
|
configuration.ValidResolutions,
|
|
imageCodecInfo,
|
|
encoderParameters,
|
|
filenameExtension);
|
|
}
|
|
if (!configuration.SkipSearch)
|
|
Search(ticks, argZero, propertyRoot);
|
|
if (!_PropertyRootExistedBefore && !_IsEnvironment.Development && _Exceptions.Count == 0 && _ArgZeroIsConfigurationRootDirectory)
|
|
{
|
|
List<string[]> directoryCollections = _Rename.GetDirectoryRenameCollections(
|
|
propertyConfiguration,
|
|
relativePath: string.Empty,
|
|
newDirectoryName: string.Empty,
|
|
jsonFiles4InfoAny: false);
|
|
foreach (string[] directoryCollection in directoryCollections)
|
|
{
|
|
_Log.Information(string.Concat("Cleaning <", directoryCollection[0], ">"));
|
|
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(directoryCollection[0]);
|
|
}
|
|
string d2FacePartsRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(D2_FaceParts));
|
|
_Log.Information(string.Concat("Cleaning <", d2FacePartsRootDirectory, ">"));
|
|
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(d2FacePartsRootDirectory);
|
|
if (appSettings.MaxDegreeOfParallelism < 2)
|
|
ticks = LogDelta(ticks, nameof(Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories));
|
|
}
|
|
message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}";
|
|
_Log.Information(message);
|
|
if (_Exceptions.Count != 0)
|
|
throw new Exception(message);
|
|
if (_PropertyRootExistedBefore)
|
|
_Log.Information("First run completed. Run again if wanted");
|
|
}
|
|
|
|
private long LogDelta(long ticks, string? methodName)
|
|
{
|
|
long result;
|
|
if (_Log is null)
|
|
throw new NullReferenceException(nameof(_Log));
|
|
double delta = new TimeSpan(DateTime.Now.Ticks - ticks).TotalMilliseconds;
|
|
_Log.Debug($"{methodName} took {Math.Floor(delta)} millisecond(s)");
|
|
result = DateTime.Now.Ticks;
|
|
return result;
|
|
}
|
|
|
|
private long LogDeltaInSeconds(long ticks, string methodName)
|
|
{
|
|
long result;
|
|
if (_Log is null)
|
|
throw new NullReferenceException(nameof(_Log));
|
|
double delta = new TimeSpan(DateTime.Now.Ticks - ticks).Seconds;
|
|
_Log.Debug($"{methodName} took {Math.Floor(delta)} seconds(s)");
|
|
result = DateTime.Now.Ticks;
|
|
return result;
|
|
}
|
|
|
|
private long LogDeltaInMinutes(long ticks, string methodName)
|
|
{
|
|
long result;
|
|
if (_Log is null)
|
|
throw new NullReferenceException(nameof(_Log));
|
|
double delta = new TimeSpan(DateTime.Now.Ticks - ticks).Minutes;
|
|
_Log.Debug($"{methodName} took {Math.Floor(delta)} minutes(s)");
|
|
result = DateTime.Now.Ticks;
|
|
return result;
|
|
}
|
|
|
|
private void Verify(Models.Configuration configuration)
|
|
{
|
|
if (!configuration.OutputResolutions.Any() || string.IsNullOrEmpty(configuration.OutputResolutions[0]) || !configuration.ValidResolutions.Contains(configuration.OutputResolutions[0]))
|
|
throw new NullReferenceException($"{nameof(configuration.OutputResolutions)} must be fileNameToCollection valid outputResolution!");
|
|
if ((from l in configuration.OutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
|
throw new Exception($"One or more {nameof(configuration.OutputResolutions)} are not in the ValidResolutions list!");
|
|
if ((from l in configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
|
throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions)} are not in the ValidResolutions list!");
|
|
if ((from l in configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
|
throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions)} are not in the ValidResolutions list!");
|
|
if ((from l in configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
|
throw new Exception($"One or more {nameof(configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions)} are not in the ValidResolutions list!");
|
|
if ((from l in configuration.SaveShortcutsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
|
throw new Exception($"One or more {nameof(configuration.SaveShortcutsForOutputResolutions)} are not in the ValidResolutions list!");
|
|
if ((from l in configuration.SaveFaceLandmarkForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
|
throw new Exception($"One or more {nameof(configuration.SaveFaceLandmarkForOutputResolutions)} are not in the ValidResolutions list!");
|
|
if (string.IsNullOrEmpty(configuration.ModelName))
|
|
throw new NullReferenceException(nameof(configuration.ModelName));
|
|
if (string.IsNullOrEmpty(configuration.OutputExtension))
|
|
throw new NullReferenceException(nameof(configuration.OutputExtension));
|
|
if (string.IsNullOrEmpty(configuration.PredictorModelName))
|
|
throw new NullReferenceException(nameof(configuration.PredictorModelName));
|
|
if (string.IsNullOrEmpty(configuration.ModelDirectory) || !Directory.Exists(configuration.ModelDirectory))
|
|
throw new NullReferenceException(nameof(configuration.ModelDirectory));
|
|
if (configuration.MappedMaxIndex.HasValue && configuration.LoadOrCreateThenSaveIndex && !_IsEnvironment.DebuggerWasAttachedDuringConstructor)
|
|
throw new NullReferenceException(nameof(configuration.MappedMaxIndex));
|
|
}
|
|
|
|
private void VerifyExtra(List<string> args, Property.Models.Configuration propertyConfiguration, Models.Configuration configuration)
|
|
{
|
|
string[] sourceDirectoryNames;
|
|
if (!args.Any())
|
|
sourceDirectoryNames = Array.Empty<string>();
|
|
else
|
|
{
|
|
string? century;
|
|
string argZero = Path.GetFullPath(args[0]);
|
|
sourceDirectoryNames = argZero.Split(Path.DirectorySeparatorChar);
|
|
if (!argZero.StartsWith(propertyConfiguration.RootDirectory))
|
|
throw new Exception($"Source directory must be inside root directory! <{argZero}> <{propertyConfiguration.RootDirectory}>");
|
|
if (_ArgZeroIsConfigurationRootDirectory && propertyConfiguration.RootDirectory != argZero)
|
|
{
|
|
if (!configuration.MixedYearRelativePaths.Contains(sourceDirectoryNames[0]))
|
|
{
|
|
string[] segments = sourceDirectoryNames[0].Split(' ');
|
|
century = segments[^1].Length == 4 ? segments[^1][..2] : null;
|
|
if (segments.Length < 2 || century is null || (century != "18" && century != "19" && century != "20"))
|
|
throw new Exception("root subdirectory must have fileNameToCollection year at the end or directory name needs to be added to the exclude list!");
|
|
}
|
|
}
|
|
}
|
|
string[] resizeMatch = (from l in sourceDirectoryNames where configuration.ValidResolutions.Contains(l) select l).ToArray();
|
|
if (resizeMatch.Any())
|
|
throw new Exception("Input directory should be the source and not fileNameToCollection resized directory!");
|
|
if (configuration.LocationDigits != Shared.Models.Stateless.ILocation.Digits)
|
|
throw new Exception("Configuration has to match interface!");
|
|
if (configuration.LocationFactor != Shared.Models.Stateless.ILocation.Factor)
|
|
throw new Exception("Configuration has to match interface!");
|
|
}
|
|
|
|
private static Map.Models.Configuration Get(Models.Configuration configuration, string facesFileNameExtension, string facesHiddenFileNameExtension, string facePartsFileNameExtension)
|
|
{
|
|
Map.Models.Configuration result = new(
|
|
configuration.FaceConfidencePercent,
|
|
configuration.FaceDistancePermyriad,
|
|
configuration.MappingDefaultName,
|
|
configuration.PersonBirthdayFirstYear,
|
|
configuration.PersonBirthdayFormat,
|
|
configuration.PersonCharacters.ToArray(),
|
|
configuration.PersonCharactersCopyCount,
|
|
configuration.RangeDaysDeltaTolerance,
|
|
configuration.RangeDistanceTolerance,
|
|
configuration.SaveSortingWithoutPerson,
|
|
configuration.SortingMaximumPerKey,
|
|
configuration.SortingMinimumToUseSigma,
|
|
facesFileNameExtension,
|
|
facesHiddenFileNameExtension,
|
|
facePartsFileNameExtension);
|
|
return result;
|
|
}
|
|
|
|
private void FullParallelForWork(A_Property propertyLogic,
|
|
B_Metadata metadata,
|
|
Dictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers,
|
|
string outputResolution,
|
|
string cResultsFullGroupDirectory,
|
|
string dResultsDateGroupDirectory,
|
|
string dResultsFullGroupDirectory,
|
|
string eDistanceContentDirectory,
|
|
List<Tuple<string, DateTime>> sourceDirectoryChanges,
|
|
Dictionary<string, List<MappingFromPhotoPrism>> fileNameToCollection,
|
|
Container container,
|
|
int index,
|
|
Item item,
|
|
DateTime[] containerDateTimes)
|
|
{
|
|
if (_Log is null)
|
|
throw new NullReferenceException(nameof(_Log));
|
|
List<Shared.Models.Face> faces;
|
|
Shared.Models.Property property;
|
|
long ticks = DateTime.Now.Ticks;
|
|
DateTime dateTime = DateTime.Now;
|
|
List<string> parseExceptions = new();
|
|
List<Tuple<string, DateTime>> subFileTuples = new();
|
|
List<KeyValuePair<string, string>> metadataCollection;
|
|
if (item.Property is not null && item.Property.Id is not null && !item.Any())
|
|
{
|
|
property = item.Property;
|
|
if (item.SourceDirectoryFileHolder.LastWriteTime is not null)
|
|
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), item.SourceDirectoryFileHolder.LastWriteTime.Value));
|
|
else
|
|
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), new FileInfo(item.SourceDirectoryFileHolder.FullName).LastWriteTime));
|
|
bool nameWithoutExtensionIsIdFormat = Shared.Models.Stateless.Methods.IProperty.NameWithoutExtensionIsIdFormat(item.ImageFileHolder);
|
|
if (nameWithoutExtensionIsIdFormat && item.ImageFileHolder.NameWithoutExtension != item.Property.Id.ToString())
|
|
_Log.Information($"Name without extension is Id format but doesn't match id <{item.ImageFileHolder.FullName}>");
|
|
}
|
|
else
|
|
{
|
|
int? propertyHashCode = item.Property?.GetHashCode();
|
|
if (!item.SourceDirectoryFileHolder.Exists)
|
|
_Log.Information(string.Concat("NoJson <", item.ImageFileHolder.FullName, '>'));
|
|
else if (item.FileSizeChanged.HasValue && item.FileSizeChanged.Value)
|
|
_Log.Information(string.Concat("FileSizeChanged <", item.ImageFileHolder.FullName, '>'));
|
|
else if (item.LastWriteTimeChanged.HasValue && item.LastWriteTimeChanged.Value)
|
|
_Log.Information(string.Concat("LastWriteTimeChanged <", item.ImageFileHolder.FullName, '>'));
|
|
else if (item.Moved.HasValue && item.Moved.Value)
|
|
_Log.Information(string.Concat("Moved <", item.ImageFileHolder.FullName, '>'));
|
|
property = propertyLogic.GetProperty(item, subFileTuples, parseExceptions);
|
|
item.Update(property);
|
|
if (propertyHashCode is null)
|
|
{
|
|
lock (sourceDirectoryChanges)
|
|
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), DateTime.Now));
|
|
}
|
|
else if (propertyHashCode.Value != property.GetHashCode())
|
|
{
|
|
lock (sourceDirectoryChanges)
|
|
sourceDirectoryChanges.Add(new Tuple<string, DateTime>(nameof(A_Property), DateTime.Now));
|
|
}
|
|
}
|
|
if (property is null || item.Property is null)
|
|
throw new NullReferenceException(nameof(property));
|
|
FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(item);
|
|
item.SetResizedFileHolder(_Resize.FileNameExtension, resizedFileHolder);
|
|
string facesDirectory = _Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution) ? _Faces.GetFacesDirectory(dResultsFullGroupDirectory, item) : string.Empty;
|
|
string facePartsDirectory = _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution) ? _FaceParts.GetFacePartsDirectory(_Configuration.PropertyConfiguration, dResultsFullGroupDirectory, item) : string.Empty;
|
|
MappingFromItem mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item, resizedFileHolder);
|
|
(int metadataGroups, metadataCollection) = metadata.GetMetadataCollection(subFileTuples, parseExceptions, mappingFromItem);
|
|
if (_AppSettings.MaxDegreeOfParallelism < 2)
|
|
ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection));
|
|
Dictionary<string, int[]> outputResolutionToResize = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, subFileTuples, parseExceptions, metadataCollection, item.Property, mappingFromItem);
|
|
if (_AppSettings.MaxDegreeOfParallelism < 2)
|
|
ticks = LogDelta(ticks, nameof(C_Resize.GetResizeKeyValuePairs));
|
|
if (_Configuration.SaveResizedSubfiles)
|
|
{
|
|
_Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.Property, mappingFromItem, outputResolutionToResize);
|
|
if (_AppSettings.MaxDegreeOfParallelism < 2)
|
|
ticks = LogDelta(ticks, nameof(C_Resize.SaveResizedSubfile));
|
|
}
|
|
if (!mappingFromItem.ResizedFileHolder.Exists && !File.Exists(mappingFromItem.ResizedFileHolder.FullName))
|
|
faces = new();
|
|
else if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
|
|
faces = new();
|
|
else
|
|
{
|
|
List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection;
|
|
List<LocationContainer<MetadataExtractor.Directory>>? collection;
|
|
if (item.Property?.Id is null)
|
|
collection = null;
|
|
else
|
|
_ = idToLocationContainers.TryGetValue(item.Property.Id.Value, out collection);
|
|
if (!fileNameToCollection.TryGetValue(mappingFromItem.RelativePath[1..], out mappingFromPhotoPrismCollection))
|
|
mappingFromPhotoPrismCollection = null;
|
|
faces = _Faces.GetFaces(outputResolution, dResultsFullGroupDirectory, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionToResize, collection, mappingFromPhotoPrismCollection);
|
|
if (_AppSettings.MaxDegreeOfParallelism < 2)
|
|
ticks = LogDelta(ticks, nameof(D_Face.GetFaces));
|
|
bool anyFacesSaved = _Faces.SaveFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem, facesDirectory, faces);
|
|
if (_AppSettings.MaxDegreeOfParallelism < 2)
|
|
ticks = LogDelta(ticks, nameof(D_Face.SaveFaces));
|
|
if ((_Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch)
|
|
&& _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
|
|
&& !anyFacesSaved && collection is not null)
|
|
_Distance.LookForMatchFacesAndPossiblyRename(_Faces.FileNameExtension, eDistanceContentDirectory, mappingFromItem, faces, collection);
|
|
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
|
|
{
|
|
bool saveRotated = !_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution);
|
|
string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(_Configuration.PropertyConfiguration, container.SourceDirectory);
|
|
_FaceParts.SaveFaceLandmarkImages(_Configuration.PropertyConfiguration, facePartsDirectory, subFileTuples, parseExceptions, mappingFromItem, faces, saveRotated);
|
|
if (_AppSettings.MaxDegreeOfParallelism < 2)
|
|
ticks = LogDelta(ticks, nameof(D2_FaceParts.SaveFaceLandmarkImages));
|
|
}
|
|
}
|
|
lock (sourceDirectoryChanges)
|
|
{
|
|
item.Faces.AddRange(faces);
|
|
sourceDirectoryChanges.AddRange(from l in subFileTuples where l.Item2 > dateTime select l);
|
|
}
|
|
}
|
|
|
|
private int FullParallelWork(int maxDegreeOfParallelism,
|
|
A_Property propertyLogic,
|
|
B_Metadata metadata,
|
|
Dictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers,
|
|
string outputResolution,
|
|
string cResultsFullGroupDirectory,
|
|
string dResultsDateGroupDirectory,
|
|
string dResultsFullGroupDirectory,
|
|
string eDistanceContentDirectory,
|
|
List<Tuple<string, DateTime>> sourceDirectoryChanges,
|
|
Dictionary<string, List<MappingFromPhotoPrism>> fileNameToCollection,
|
|
Container container,
|
|
Item[] filteredItems,
|
|
string message)
|
|
{
|
|
if (_Log is null)
|
|
throw new NullReferenceException(nameof(_Log));
|
|
int result = 0;
|
|
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
|
|
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
|
|
DateTime[] containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems);
|
|
using ProgressBar progressBar = new(filteredItems.Length, message, options);
|
|
_ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) =>
|
|
{
|
|
try
|
|
{
|
|
FullParallelForWork(propertyLogic,
|
|
metadata,
|
|
idToLocationContainers,
|
|
outputResolution,
|
|
cResultsFullGroupDirectory,
|
|
dResultsDateGroupDirectory,
|
|
dResultsFullGroupDirectory,
|
|
eDistanceContentDirectory,
|
|
sourceDirectoryChanges,
|
|
fileNameToCollection,
|
|
container,
|
|
index: i,
|
|
filteredItems[i],
|
|
containerDateTimes);
|
|
if (i == 0 || sourceDirectoryChanges.Any())
|
|
progressBar.Tick();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
result += 1;
|
|
_Log.Error(string.Concat(container.SourceDirectory, Environment.NewLine, ex.Message, Environment.NewLine, ex.StackTrace), ex);
|
|
if (result == filteredItems.Length)
|
|
throw new Exception(string.Concat("All in [", container.SourceDirectory, "] failed!"));
|
|
}
|
|
});
|
|
return result;
|
|
}
|
|
|
|
private static void WriteTab(string checkDirectory, List<(string Id, string Line)> metadataIdLines, string fileName)
|
|
{
|
|
string text;
|
|
FileInfo fileInfo;
|
|
List<string> duplicates = new();
|
|
List<string> metadataIds = new();
|
|
fileInfo = new(Path.Combine(checkDirectory, "[()]", Path.ChangeExtension(fileName, "tsv")));
|
|
if (fileInfo?.Directory is null)
|
|
throw new Exception();
|
|
if (!fileInfo.Directory.Exists)
|
|
fileInfo.Directory.Create();
|
|
foreach ((string Id, string Line) metadataIdLine in metadataIdLines)
|
|
{
|
|
if (metadataIds.Contains(metadataIdLine.Id))
|
|
duplicates.Add(metadataIdLine.Id);
|
|
else
|
|
metadataIds.Add(metadataIdLine.Id);
|
|
}
|
|
for (int i = metadataIdLines.Count - 1; i > -1; i--)
|
|
{
|
|
if (duplicates.Contains(metadataIdLines[i].Id))
|
|
metadataIdLines.RemoveAt(i);
|
|
}
|
|
if (metadataIdLines.Any())
|
|
{
|
|
text = string.Join(Environment.NewLine, from l in metadataIdLines orderby l.Id select l.Line);
|
|
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, text, updateDateWhenMatches: true, compareBeforeWrite: true);
|
|
}
|
|
else
|
|
{
|
|
if (fileInfo.Exists)
|
|
File.Delete(fileInfo.FullName);
|
|
}
|
|
}
|
|
|
|
private (string, string) GetResultsFullGroupDirectories()
|
|
{
|
|
string aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
|
|
_Configuration.PropertyConfiguration,
|
|
nameof(A_Property),
|
|
string.Empty,
|
|
includeResizeGroup: false,
|
|
includeModel: false,
|
|
includePredictorModel: false);
|
|
string bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
|
|
_Configuration.PropertyConfiguration,
|
|
nameof(B_Metadata),
|
|
string.Empty,
|
|
includeResizeGroup: false,
|
|
includeModel: false,
|
|
includePredictorModel: false);
|
|
return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory);
|
|
}
|
|
|
|
private (string, string, string) GetResultsFullGroupDirectories(string outputResolution)
|
|
{
|
|
string cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
|
|
_Configuration.PropertyConfiguration,
|
|
nameof(C_Resize),
|
|
outputResolution,
|
|
includeResizeGroup: true,
|
|
includeModel: false,
|
|
includePredictorModel: false);
|
|
string dResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
|
|
_Configuration.PropertyConfiguration,
|
|
nameof(D_Face),
|
|
outputResolution,
|
|
includeResizeGroup: true,
|
|
includeModel: true,
|
|
includePredictorModel: true);
|
|
string d2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
|
|
_Configuration.PropertyConfiguration,
|
|
nameof(D2_FaceParts),
|
|
outputResolution,
|
|
includeResizeGroup: true,
|
|
includeModel: true,
|
|
includePredictorModel: true);
|
|
return new(cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory);
|
|
}
|
|
|
|
private void FullDoWork(string argZero, string propertyRoot, long ticks, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, int t, Container[] containers, A_Property propertyLogic, B_Metadata metadata, string eDistanceContentDirectory, Dictionary<string, List<MappingFromPhotoPrism>> fileNameToCollection, Dictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers)
|
|
{
|
|
if (_Log is null)
|
|
throw new NullReferenceException(nameof(_Log));
|
|
int total;
|
|
string message;
|
|
int totalSeconds;
|
|
int exceptionCount;
|
|
Container container;
|
|
Item[] filteredItems;
|
|
bool anyNullOrNoIsUniqueFileName;
|
|
string cResultsFullGroupDirectory;
|
|
string dResultsFullGroupDirectory;
|
|
string d2ResultsFullGroupDirectory;
|
|
int containersLength = containers.Length;
|
|
List<Tuple<string, DateTime>> sourceDirectoryChanges = new();
|
|
int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism;
|
|
string dResultsDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(D_Face));
|
|
foreach (string outputResolution in _Configuration.OutputResolutions)
|
|
{
|
|
total = 0;
|
|
(cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
|
|
// _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(dResultsFullGroupDirectory);
|
|
for (int i = 0; i < containers.Length; i++)
|
|
{
|
|
container = containers[i];
|
|
if (!container.Items.Any())
|
|
continue;
|
|
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
|
|
continue;
|
|
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container);
|
|
if (!filteredItems.Any())
|
|
continue;
|
|
sourceDirectoryChanges.Clear();
|
|
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
|
|
anyNullOrNoIsUniqueFileName = filteredItems.Any(l => l.IsUniqueFileName is null || !l.IsUniqueFileName.Value);
|
|
message = $"{i + 1:000} [{filteredItems.Length:000} collection] / {containersLength:000} - {total} / {t} total collection - {totalSeconds} total second(s) - {outputResolution} - {container.SourceDirectory}";
|
|
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
|
|
_Faces.SetAngleBracketCollection(dResultsFullGroupDirectory, container.SourceDirectory);
|
|
if (_Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
|
|
_FaceParts.SetAngleBracketCollection(_Configuration.PropertyConfiguration, d2ResultsFullGroupDirectory, container.SourceDirectory);
|
|
propertyLogic.SetAngleBracketCollection(aResultsFullGroupDirectory, container.SourceDirectory, anyNullOrNoIsUniqueFileName);
|
|
_Resize.SetAngleBracketCollection(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, container.SourceDirectory);
|
|
exceptionCount = FullParallelWork(maxDegreeOfParallelism,
|
|
propertyLogic,
|
|
metadata,
|
|
idToLocationContainers,
|
|
outputResolution,
|
|
cResultsFullGroupDirectory,
|
|
dResultsDateGroupDirectory,
|
|
dResultsFullGroupDirectory,
|
|
eDistanceContentDirectory,
|
|
sourceDirectoryChanges,
|
|
fileNameToCollection,
|
|
container,
|
|
filteredItems,
|
|
message);
|
|
if (exceptionCount != 0)
|
|
{
|
|
_Exceptions.Add(container.SourceDirectory);
|
|
continue;
|
|
}
|
|
if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any())
|
|
{
|
|
for (int y = 0; y < int.MaxValue; y++)
|
|
{
|
|
_Log.Information("Press \"Y\" key when ready to continue or close console");
|
|
if (_Console.ReadKey() == ConsoleKey.Y)
|
|
break;
|
|
}
|
|
_Log.Information(". . .");
|
|
}
|
|
// if (isSilent && container.SourceDirectory.EndsWith("Bohdi Ray 2016")) // 7#.1
|
|
// break;
|
|
// if (isSilent && container.SourceDirectory.EndsWith("Halloween 2013")) // 12#.1
|
|
// break;
|
|
// if (isSilent && container.SourceDirectory.EndsWith("zzz =2014.4 Winter Tracy Pictures")) // 30#.2
|
|
// break;
|
|
// if (isSilent && container.SourceDirectory.EndsWith("=2009.0 Winter Logan Michael")) //34#.2
|
|
// break;
|
|
// if (isSilent && container.SourceDirectory.EndsWith("Animal Kingdom")) //35#.3
|
|
// break;
|
|
// if (isSilent && container.SourceDirectory.EndsWith("Texas 2015")) //46#.4
|
|
// break;
|
|
total += container.Items.Count;
|
|
}
|
|
}
|
|
}
|
|
|
|
private void SetMapping(Dictionary<string, List<MappingFromPhotoPrism>> fileNameToCollection, string argZero, Container[] containers)
|
|
{
|
|
Mapping mapping;
|
|
int faceAreaPermille;
|
|
Item[] filteredItems;
|
|
int confidencePercent;
|
|
int normalizedRectangle;
|
|
DateTime[] containerDateTimes;
|
|
string deterministicHashCodeKey;
|
|
MappingFromItem mappingFromItem;
|
|
MappingFromLocation mappingFromLocation;
|
|
List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection;
|
|
foreach (Container container in containers)
|
|
{
|
|
if (!container.Items.Any())
|
|
continue;
|
|
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
|
|
continue;
|
|
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container);
|
|
if (!filteredItems.Any())
|
|
continue;
|
|
containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems);
|
|
foreach (Item item in filteredItems)
|
|
{
|
|
if (item.Property?.Id is null || item.ResizedFileHolder is null)
|
|
continue;
|
|
mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item);
|
|
foreach (Shared.Models.Face face in item.Faces)
|
|
{
|
|
if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
|
|
continue;
|
|
confidencePercent = Shared.Models.Stateless.Methods.ILocation.GetConfidencePercent(_Configuration.FaceConfidencePercent, _Configuration.RangeFaceConfidence, face.Location.Confidence);
|
|
faceAreaPermille = Shared.Models.Stateless.Methods.IMapping.GetAreaPermille(_Configuration.FaceAreaPermille, face.Location, face.OutputResolution);
|
|
normalizedRectangle = Shared.Models.Stateless.Methods.ILocation.GetNormalizedRectangle(face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution);
|
|
deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item.Property.Id.Value, face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution);
|
|
mappingFromLocation = new(faceAreaPermille, confidencePercent, deterministicHashCodeKey, normalizedRectangle);
|
|
if (!fileNameToCollection.TryGetValue(mappingFromItem.RelativePath[1..], out mappingFromPhotoPrismCollection))
|
|
mappingFromPhotoPrismCollection = null;
|
|
mapping = new(mappingFromItem, mappingFromLocation, mappingFromPhotoPrismCollection);
|
|
face.SetMapping(mapping);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private (List<int>, List<Shared.Models.Face>) GetFilteredDistinct(string argZero, Container[] containers)
|
|
{
|
|
List<int> resultIds = new();
|
|
List<Shared.Models.Face> resultFaces = new();
|
|
Item[] filteredItems;
|
|
bool isIgnoreRelativePath;
|
|
foreach (Container container in containers)
|
|
{
|
|
if (!container.Items.Any())
|
|
continue;
|
|
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
|
|
continue;
|
|
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container);
|
|
if (!filteredItems.Any())
|
|
continue;
|
|
isIgnoreRelativePath = _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory);
|
|
foreach (Item item in filteredItems)
|
|
{
|
|
if (item.Property?.Id is null || item.ResizedFileHolder is null)
|
|
continue;
|
|
if (resultIds.Contains(item.Property.Id.Value))
|
|
continue;
|
|
resultIds.Add(item.Property.Id.Value);
|
|
if (isIgnoreRelativePath)
|
|
continue;
|
|
foreach (Shared.Models.Face face in item.Faces)
|
|
{
|
|
if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
|
|
continue;
|
|
resultFaces.Add(face);
|
|
}
|
|
}
|
|
}
|
|
return new(resultIds, resultFaces);
|
|
}
|
|
|
|
private List<Item> GetItems(string argZero, Container[] containers)
|
|
{
|
|
List<Item> items = new();
|
|
Item[] filteredItems;
|
|
bool isIgnoreRelativePath;
|
|
List<Shared.Models.Face> collection = new();
|
|
foreach (Container container in containers)
|
|
{
|
|
if (!container.Items.Any())
|
|
continue;
|
|
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
|
|
continue;
|
|
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container);
|
|
if (!filteredItems.Any())
|
|
continue;
|
|
isIgnoreRelativePath = _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory);
|
|
foreach (Item item in filteredItems)
|
|
{
|
|
if (item.Property?.Id is null || item.ResizedFileHolder is null)
|
|
continue;
|
|
items.Add(item);
|
|
}
|
|
}
|
|
return items;
|
|
}
|
|
|
|
private void MapLogic(string argZero, long ticks, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string fPhotoPrismContentDirectory, MapLogicSupport mapLogicSupport, MapLogic mapLogic, string outputResolution, Dictionary<long, List<int>> personKeyToIds, List<Shared.Models.Face> distinctFilteredFaces, Mapping[] mappingCollection, int totalNotMapped)
|
|
{
|
|
int? useFiltersCounter = null;
|
|
SortingContainer[] sortingContainers;
|
|
string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()");
|
|
string d2FacePartsContentDirectory = Path.Combine(d2ResultsFullGroupDirectory, "()");
|
|
string dFacesCollectionDirectory = Path.Combine(dResultsFullGroupDirectory, "[]", _Configuration.PropertyConfiguration.ResultAllInOne);
|
|
Dictionary<int, Dictionary<int, Mapping>> idToNormalizedRectangleToMapping = MapLogicSupport.GetIdToNormalizedRectangleToFace(mappingCollection);
|
|
if (Directory.Exists(fPhotoPrismContentDirectory))
|
|
F_PhotoPrism.WriteMatches(fPhotoPrismContentDirectory, _Configuration.PersonBirthdayFormat, ticks, distinctFilteredFaces, mapLogic);
|
|
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
|
|
{
|
|
List<Item> filteredItems = GetItems(argZero, containers);
|
|
mapLogic.SaveShortcutsForOutputResolutionsDuringMapLogic(personKeyToIds, filteredItems, mappingCollection);
|
|
}
|
|
if (_Configuration.PersonCharactersCopyCount > 0 && !string.IsNullOrEmpty(_Configuration.PersonCharacters))
|
|
mapLogic.CopyAtLeastOneMappedFiles(dFacesContentDirectory, a2PeopleSingletonDirectory, mappingCollection);
|
|
mapLogic.CopyManualFiles(dFacesContentDirectory, idToNormalizedRectangleToMapping);
|
|
if (_Configuration.SaveMappedForOutputResolutions.Contains(outputResolution))
|
|
mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, personKeyToIds, mappingCollection, idToNormalizedRectangleToMapping, totalNotMapped);
|
|
if (_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution))
|
|
{
|
|
MapLogicSupport.SetFaceDistances(_AppSettings.MaxDegreeOfParallelism, ticks, distinctFilteredFaces);
|
|
List<Shared.Models.Face> distinctFurtherFilteredFaces = mapLogic.GetFurtherFilterBySkipCollection(distinctFilteredFaces);
|
|
Dictionary<int, Dictionary<int, PersonContainer[]>> missingIdThenNormalizedRectangleToPersonContainers = mapLogic.GetMissing(idToNormalizedRectangleToMapping);
|
|
List<FaceDistanceContainer> missingFaceDistanceContainers = _Distance.GetMissingFaceDistanceContainer(_AppSettings.MaxDegreeOfParallelism, ticks, dFacesCollectionDirectory, missingIdThenNormalizedRectangleToPersonContainers);
|
|
sortingContainers = mapLogicSupport.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distinctFurtherFilteredFaces, missingFaceDistanceContainers, useFiltersCounter);
|
|
if (!sortingContainers.Any())
|
|
{
|
|
for (useFiltersCounter = 1; useFiltersCounter < 17; useFiltersCounter++)
|
|
{
|
|
sortingContainers = mapLogicSupport.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distinctFurtherFilteredFaces, missingFaceDistanceContainers, useFiltersCounter);
|
|
if (sortingContainers.Any())
|
|
break;
|
|
}
|
|
}
|
|
MapLogicSupport.SaveFaceDistances(_Configuration.PropertyConfiguration, sortingContainers);
|
|
if (totalNotMapped > 0)
|
|
{
|
|
int updated = mapLogic.UpdateFromSortingContainers(sortingContainers);
|
|
List<SaveContainer> saveContainers = mapLogic.GetSaveContainers(dFacesContentDirectory, d2FacePartsContentDirectory, mappingCollection, idToNormalizedRectangleToMapping, useFiltersCounter, sortingContainers.Any());
|
|
mapLogic.SaveContainers(totalNotMapped, updated, saveContainers);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static Container? AreAllSameEndsWith(string argZero, Container[] containers)
|
|
{
|
|
Container? result = null;
|
|
string[] directoryNames = Shared.Models.Stateless.Methods.IPath.GetDirectoryNames(argZero);
|
|
if (directoryNames.Length > 2)
|
|
{
|
|
string directoryName;
|
|
string rootDirectoryName = directoryNames[^2];
|
|
if (!Directory.Exists(argZero))
|
|
throw new Exception();
|
|
foreach (Container container in containers)
|
|
{
|
|
if (container.SourceDirectory == argZero)
|
|
result = container;
|
|
directoryName = Path.GetFileName(container.SourceDirectory);
|
|
if (!directoryName.EndsWith(rootDirectoryName))
|
|
{
|
|
result = null;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
private string SaveUrlAndGetNewRootDirectory(Container container)
|
|
{
|
|
if (_Log is null)
|
|
throw new NullReferenceException(nameof(_Log));
|
|
string? result = Path.GetDirectoryName(Path.GetDirectoryName(container.SourceDirectory));
|
|
if (result is null)
|
|
throw new Exception();
|
|
Uri uri;
|
|
Item item;
|
|
string? line;
|
|
string fileName;
|
|
Task<byte[]> task;
|
|
string relativePath;
|
|
FileHolder fileHolder;
|
|
string extensionLowered;
|
|
string sourceDirectoryFile;
|
|
int length = result.Length;
|
|
HttpClient httpClient = new();
|
|
bool isValidImageFormatExtension;
|
|
for (int y = 0; y < int.MaxValue; y++)
|
|
{
|
|
_Log.Information("Enter fileNameToCollection url for fileNameToCollection image");
|
|
line = _Console.ReadLine();
|
|
if (string.IsNullOrEmpty(line))
|
|
break;
|
|
uri = new(line);
|
|
if (uri.HostNameType != UriHostNameType.Dns)
|
|
continue;
|
|
task = httpClient.GetByteArrayAsync(uri);
|
|
fileName = Path.GetFileName(uri.LocalPath);
|
|
sourceDirectoryFile = Path.Combine(container.SourceDirectory, fileName);
|
|
File.WriteAllBytes(sourceDirectoryFile, task.Result);
|
|
extensionLowered = Path.GetExtension(uri.LocalPath);
|
|
relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFile, length, forceExtensionToLower: true);
|
|
isValidImageFormatExtension = _Configuration.PropertyConfiguration.ValidImageFormatExtensions.Contains(extensionLowered);
|
|
fileHolder = new(sourceDirectoryFile);
|
|
item = new(fileHolder, relativePath, isValidImageFormatExtension);
|
|
container.Items.Add(item);
|
|
}
|
|
_Log.Information(". . .");
|
|
return result;
|
|
}
|
|
|
|
private void ParallelFor(string eDistanceContentDirectory, List<LocationContainer<MetadataExtractor.Directory>> collection, long personKey, string file)
|
|
{
|
|
const string lnk = ".lnk";
|
|
int? id, normalizedRectangle;
|
|
IReadOnlyList<MetadataExtractor.Directory> directories;
|
|
bool fromDistanceContent = !file.EndsWith(lnk) && file.Contains(eDistanceContentDirectory);
|
|
if (!file.EndsWith(lnk))
|
|
(id, normalizedRectangle) = Shared.Models.Stateless.Methods.IMapping.GetConverted(_MapConfiguration.FacesFileNameExtension, file);
|
|
else
|
|
(id, normalizedRectangle) = Shared.Models.Stateless.Methods.IMapping.GetConverted(_MapConfiguration.FacesFileNameExtension, file[..^4]);
|
|
if (id is null || normalizedRectangle is null)
|
|
return;
|
|
if (file.EndsWith(lnk) || (!_Configuration.DistanceMoveUnableToMatch && !_Configuration.DistanceRenameToMatch))
|
|
directories = new List<MetadataExtractor.Directory>();
|
|
else
|
|
directories = MetadataExtractor.ImageMetadataReader.ReadMetadata(file);
|
|
Rectangle? rectangle = Shared.Models.Stateless.Methods.ILocation.GetNormalizedRectangle(_Configuration.LocationDigits, normalizedRectangle.Value);
|
|
lock (collection)
|
|
collection.Add(new(fromDistanceContent, file, personKey, id.Value, normalizedRectangle.Value, directories, rectangle, null));
|
|
}
|
|
|
|
private void OpenPossibleDuplicates(List<(long, int, string, double?)> duplicates)
|
|
{
|
|
string personKeyFormatted;
|
|
foreach ((long personKey, int id, string file, double? percent) in duplicates)
|
|
{
|
|
if (percent is null)
|
|
continue;
|
|
_ = Process.Start("explorer.exe", string.Concat("\"", Path.GetDirectoryName(file), "\""));
|
|
personKeyFormatted = Shared.Models.Stateless.Methods.IPersonBirthday.GetFormatted(_Configuration.PersonBirthdayFormat, personKey);
|
|
}
|
|
foreach ((long personKey, int id, string file, double? percent) in duplicates)
|
|
{
|
|
if (percent is not null && percent.Value == 0)
|
|
continue;
|
|
_ = Process.Start("explorer.exe", string.Concat("\"", Path.GetDirectoryName(file), "\""));
|
|
personKeyFormatted = Shared.Models.Stateless.Methods.IPersonBirthday.GetFormatted(_Configuration.PersonBirthdayFormat, personKey);
|
|
}
|
|
foreach ((long personKey, int id, string file, double? percent) in duplicates)
|
|
{
|
|
if (percent is not null && percent.Value > 0)
|
|
continue;
|
|
_ = Process.Start("explorer.exe", string.Concat("\"", Path.GetDirectoryName(file), "\""));
|
|
personKeyFormatted = Shared.Models.Stateless.Methods.IPersonBirthday.GetFormatted(_Configuration.PersonBirthdayFormat, personKey);
|
|
}
|
|
}
|
|
|
|
private void LookForPossibleDuplicates(List<LocationContainer<MetadataExtractor.Directory>> collection)
|
|
{
|
|
string key;
|
|
double? percent;
|
|
Rectangle? rectangle;
|
|
List<string> delete = new();
|
|
Rectangle intersectRectangle;
|
|
(string File, int NormalizedRectangle) item;
|
|
Dictionary<string, (string, int)> distinct = new();
|
|
List<(long, int, string, double?)> duplicates = new();
|
|
foreach (LocationContainer<MetadataExtractor.Directory> locationContainer in collection)
|
|
{
|
|
key = string.Concat(locationContainer.PersonKey, locationContainer.Id);
|
|
if (distinct.TryGetValue(key, out item))
|
|
{
|
|
if (item.NormalizedRectangle == locationContainer.NormalizedRectangle)
|
|
continue;
|
|
if (locationContainer.Rectangle is null)
|
|
percent = null;
|
|
else
|
|
{
|
|
rectangle = Shared.Models.Stateless.Methods.ILocation.GetNormalizedRectangle(_Configuration.LocationDigits, item.NormalizedRectangle);
|
|
if (locationContainer.Rectangle is null || rectangle is null)
|
|
percent = null;
|
|
else
|
|
{
|
|
intersectRectangle = Rectangle.Intersect(locationContainer.Rectangle.Value, rectangle.Value);
|
|
percent = intersectRectangle.Width * intersectRectangle.Height;
|
|
}
|
|
}
|
|
delete.Add(item.File);
|
|
delete.Add(locationContainer.File);
|
|
duplicates.Add(new(locationContainer.PersonKey, locationContainer.Id, locationContainer.File, percent));
|
|
continue;
|
|
}
|
|
distinct.Add(key, new(locationContainer.File, locationContainer.NormalizedRectangle));
|
|
}
|
|
if (!_Configuration.DeletePossibleDuplicates)
|
|
{
|
|
if (duplicates.Any() && _IsEnvironment.Development)
|
|
OpenPossibleDuplicates(duplicates);
|
|
}
|
|
else
|
|
{
|
|
if (delete.Count > 5)
|
|
throw new Exception("Something maybe wrong!");
|
|
foreach (string file in delete)
|
|
{
|
|
if (File.Exists(file))
|
|
File.Delete(file);
|
|
}
|
|
}
|
|
}
|
|
|
|
private List<LocationContainer<MetadataExtractor.Directory>> GetCollection(long ticks, string? a2PeopleContentDirectory, string eDistanceContentDirectory)
|
|
{
|
|
string file;
|
|
List<(long PersonKey, string File)> collection = new();
|
|
List<LocationContainer<MetadataExtractor.Directory>> results = new();
|
|
collection.AddRange(Map.Models.Stateless.Methods.IMapLogic.GetDisplayDirectoryAllFiles(_PersonContainers));
|
|
collection.AddRange(Map.Models.Stateless.Methods.IMapLogic.DeleteEmptyDirectoriesAndGetMappedFaceFiles(_MapConfiguration, _PersonContainers, ticks, a2PeopleContentDirectory, eDistanceContentDirectory));
|
|
for (int i = 0; i < collection.Count; i++)
|
|
{
|
|
file = collection[i].File;
|
|
if (!file.EndsWith(".dup") && !file.EndsWith(".unk") && !file.EndsWith(".abd"))
|
|
continue;
|
|
if (!File.Exists(file))
|
|
continue;
|
|
File.Move(file, file[..^4]);
|
|
collection[i] = new(collection[i].PersonKey, file[..^4]);
|
|
}
|
|
if (collection.Any() && (_Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch))
|
|
{
|
|
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
|
|
string message = $") Building Mapped Face Files Collection - {totalSeconds} total second(s)";
|
|
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism };
|
|
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
|
|
using ProgressBar progressBar = new(collection.Count, message, options);
|
|
_ = Parallel.For(0, collection.Count, parallelOptions, (i, state) =>
|
|
{
|
|
progressBar.Tick();
|
|
ParallelFor(eDistanceContentDirectory, results, collection[i].PersonKey, collection[i].File);
|
|
});
|
|
}
|
|
LookForPossibleDuplicates(results);
|
|
return results;
|
|
}
|
|
|
|
private Dictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> GetDictionary(long ticks, string? a2PeopleContentDirectory, string eDistanceContentDirectory)
|
|
{
|
|
Dictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> results = new();
|
|
List<LocationContainer<MetadataExtractor.Directory>> collection = GetCollection(ticks, a2PeopleContentDirectory, eDistanceContentDirectory);
|
|
foreach (LocationContainer<MetadataExtractor.Directory> locationContainer in collection)
|
|
{
|
|
if (!results.ContainsKey(locationContainer.Id))
|
|
results.Add(locationContainer.Id, new());
|
|
results[locationContainer.Id].Add(locationContainer);
|
|
}
|
|
return results;
|
|
}
|
|
|
|
private static void LookForAbandoned(Dictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers, List<int> distinctFilteredIds)
|
|
{
|
|
List<string> renameCollection = new();
|
|
foreach (KeyValuePair<int, List<LocationContainer<MetadataExtractor.Directory>>> idToCollection in idToLocationContainers)
|
|
{
|
|
if (distinctFilteredIds.Contains(idToCollection.Key))
|
|
continue;
|
|
foreach (LocationContainer<MetadataExtractor.Directory> locationContainer in idToCollection.Value)
|
|
{
|
|
if (locationContainer.File.Contains('!'))
|
|
continue;
|
|
renameCollection.Add(locationContainer.File);
|
|
}
|
|
}
|
|
if (renameCollection.Any())
|
|
{
|
|
string checkFile;
|
|
foreach (string file in renameCollection)
|
|
{
|
|
if (!File.Exists(file))
|
|
continue;
|
|
checkFile = $"{file}.abd";
|
|
if (File.Exists(checkFile))
|
|
continue;
|
|
File.Move(file, checkFile);
|
|
}
|
|
throw new Exception($"Renamed {renameCollection.Count}(s) files!");
|
|
}
|
|
}
|
|
|
|
private void Search(long ticks, string argZero, string propertyRoot)
|
|
{
|
|
int t;
|
|
Container[] containers;
|
|
string eDistanceContentDirectory;
|
|
string? a2PeopleContentDirectory;
|
|
string cResultsFullGroupDirectory;
|
|
string dResultsFullGroupDirectory;
|
|
string d2ResultsFullGroupDirectory;
|
|
string fPhotoPrismContentDirectory;
|
|
string fPhotoPrismSingletonDirectory;
|
|
Dictionary<long, List<int>> personKeyToIds;
|
|
Dictionary<string, List<MappingFromPhotoPrism>> fileNameToCollection;
|
|
(string aResultsFullGroupDirectory, string bResultsFullGroupDirectory) = GetResultsFullGroupDirectories();
|
|
string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "{}");
|
|
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
|
|
string message = $") Building Container(s) - {totalSeconds} total second(s)";
|
|
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
|
|
using (ProgressBar progressBar = new(2, message, options))
|
|
{
|
|
progressBar.Tick();
|
|
string aPropertySingletonDirectory = Path.Combine(aResultsFullGroupDirectory, "{}");
|
|
if (!Directory.Exists(aPropertySingletonDirectory))
|
|
_ = Directory.CreateDirectory(aPropertySingletonDirectory);
|
|
(t, containers) = Shared.Models.Stateless.Methods.IContainer.GetContainers(_Configuration.PropertyConfiguration, aPropertySingletonDirectory);
|
|
progressBar.Tick();
|
|
}
|
|
Container? container = AreAllSameEndsWith(argZero, containers);
|
|
A_Property propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory);
|
|
B_Metadata metadata = new(_Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, bResultsFullGroupDirectory);
|
|
if (!_ArgZeroIsConfigurationRootDirectory || container is null)
|
|
{
|
|
a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "([])");
|
|
eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), "()");
|
|
fPhotoPrismContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), "()");
|
|
fPhotoPrismSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), "{}");
|
|
}
|
|
else
|
|
{
|
|
string resultsGroupDirectory;
|
|
a2PeopleContentDirectory = null;
|
|
eDistanceContentDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", "()");
|
|
fPhotoPrismContentDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", "()");
|
|
fPhotoPrismSingletonDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", "{}");
|
|
string? newRootDirectory = SaveUrlAndGetNewRootDirectory(container);
|
|
for (int i = 1; i < 10; i++)
|
|
{
|
|
resultsGroupDirectory = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, string.Empty, create: true);
|
|
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(resultsGroupDirectory);
|
|
}
|
|
argZero = newRootDirectory;
|
|
_Configuration.PropertyConfiguration.ChangeRootDirectory(newRootDirectory);
|
|
propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false);
|
|
propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory);
|
|
}
|
|
containers = Shared.Models.Stateless.Methods.IContainer.SortContainers(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, _ArgZeroIsConfigurationRootDirectory, argZero, containers);
|
|
MapLogicSupport mapLogicSupport = new(_Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeFaceAreaPermilleTolerance, _Configuration.SortingMaximumPerFaceShouldBeHigh);
|
|
MapLogic? mapLogic = _Configuration.DistanceMoveUnableToMatch ? null : new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, _PersonContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory, mapLogicSupport);
|
|
personKeyToIds = mapLogic is null ? new() : mapLogic.GetPersonKeyToIds();
|
|
fileNameToCollection = !Directory.Exists(fPhotoPrismSingletonDirectory) ? fileNameToCollection = new() : F_PhotoPrism.GetFileNameToCollection(fPhotoPrismSingletonDirectory);
|
|
Dictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers = GetDictionary(ticks, a2PeopleContentDirectory, eDistanceContentDirectory);
|
|
FullDoWork(argZero, propertyRoot, ticks, aResultsFullGroupDirectory, bResultsFullGroupDirectory, t, containers, propertyLogic, metadata, eDistanceContentDirectory, fileNameToCollection, idToLocationContainers);
|
|
_Distance.Clear();
|
|
mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, _PersonContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory, mapLogicSupport);
|
|
SetMapping(fileNameToCollection, argZero, containers);
|
|
if (!personKeyToIds.Any())
|
|
personKeyToIds = mapLogic.GetPersonKeyToIds();
|
|
(List<int> distinctFilteredIds, List<Shared.Models.Face> distinctFilteredFaces) = GetFilteredDistinct(argZero, containers);
|
|
LookForAbandoned(idToLocationContainers, distinctFilteredIds);
|
|
Mapping[] mappingCollection = MapLogicSupport.GetSelectedMappingCollection(distinctFilteredFaces);
|
|
int totalNotMapped = mapLogic.UpdateMappingFromPerson(mappingCollection);
|
|
string json = System.Text.Json.JsonSerializer.Serialize(mappingCollection);
|
|
File.WriteAllText(Path.Combine(eDistanceContentDirectory, $"{ticks}.json"), json);
|
|
if (a2PeopleContentDirectory is not null && false)
|
|
mapLogic.CreateTree(ticks, a2PeopleContentDirectory);
|
|
foreach (string outputResolution in _Configuration.OutputResolutions)
|
|
{
|
|
if (_PropertyRootExistedBefore || container is not null)
|
|
break;
|
|
if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
|
|
mapLogic.SaveShortcutsForOutputResolutionsPreMapLogic(eDistanceContentDirectory, personKeyToIds, idToLocationContainers, mappingCollection, totalNotMapped);
|
|
if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution))
|
|
mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, _PersonContainers, a2PeopleContentDirectory, personKeyToIds, mappingCollection, totalNotMapped);
|
|
(cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
|
|
if (_ArgZeroIsConfigurationRootDirectory
|
|
&& _Configuration.SaveResizedSubfiles
|
|
&& outputResolution == _Configuration.OutputResolutions[0]
|
|
&& _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
|
|
&& _Exceptions.Count == 0)
|
|
MapLogic(argZero, ticks, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, fPhotoPrismContentDirectory, mapLogicSupport, mapLogic, outputResolution, personKeyToIds, distinctFilteredFaces, mappingCollection, totalNotMapped);
|
|
if (_Configuration.SaveRandomForOutputResolutions.Contains(outputResolution))
|
|
_Random.Random(_Configuration.PropertyConfiguration, outputResolution, personKeyToIds, mappingCollection);
|
|
if (_IsEnvironment.Development)
|
|
continue;
|
|
G2_Identify identify = new(_Configuration);
|
|
List<G2_Identify> identifiedCollection = identify.GetIdentifiedCollection(_IsEnvironment, _Configuration.PropertyConfiguration, _Faces.FileNameExtension);
|
|
identify.WriteAllText(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0], identifiedCollection);
|
|
if (_Configuration.LoadOrCreateThenSaveIndex)
|
|
_Index.SetIndex(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0]);
|
|
if (!_IsEnvironment.Development)
|
|
{
|
|
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(aResultsFullGroupDirectory, "{}"));
|
|
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(bResultsFullGroupDirectory, "{}"));
|
|
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(cResultsFullGroupDirectory, "{}"));
|
|
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
|
|
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(dResultsFullGroupDirectory, "[]"));
|
|
}
|
|
}
|
|
}
|
|
|
|
internal void RenameQueue() => _Rename.RenameQueue(_Configuration.PropertyConfiguration);
|
|
|
|
} |