1643 lines
104 KiB
C#
1643 lines
104 KiB
C#
using Microsoft.Extensions.Configuration;
|
||
using Microsoft.Extensions.Logging;
|
||
using Phares.Shared;
|
||
using ShellProgressBar;
|
||
using System.Collections.ObjectModel;
|
||
using System.Drawing.Imaging;
|
||
using System.Text.Json;
|
||
using System.Text.RegularExpressions;
|
||
using View_by_Distance.Distance.Models;
|
||
using View_by_Distance.Face.Models;
|
||
using View_by_Distance.FaceParts.Models;
|
||
using View_by_Distance.Instance.Models;
|
||
using View_by_Distance.Map.Models;
|
||
using View_by_Distance.Metadata.Models;
|
||
using View_by_Distance.PhotoPrism.Models;
|
||
using View_by_Distance.Property.Models;
|
||
using View_by_Distance.Resize.Models;
|
||
using View_by_Distance.Shared.Models;
|
||
using View_by_Distance.Shared.Models.Methods;
|
||
using View_by_Distance.Shared.Models.Stateless.Methods;
|
||
using WindowsShortcutFactory;
|
||
|
||
namespace View_by_Distance.Instance;
|
||
|
||
public partial class DlibDotNet : IDlibDotNet, IDisposable
|
||
{
|
||
|
||
public record Record(ReadOnlyDictionary<int, ExifDirectory> ExifDirectoriesById,
|
||
string FilesCollectionRootDirectory,
|
||
bool FilesCollectionCountIsOne,
|
||
ReadOnlyCollection<ReadOnlyCollection<FilePath>> FilePathsCollection,
|
||
ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>>? IdToFilePaths,
|
||
ReadOnlyDictionary<int, Identifier>? SplatNineIdentifiers);
|
||
|
||
private readonly D_Face _Faces;
|
||
private ProgressBar? _ProgressBar;
|
||
private readonly C_Resize _Resize;
|
||
private readonly F_Random _Random;
|
||
private readonly IConsole _Console;
|
||
private readonly E_Distance _Distance;
|
||
private readonly D2_FaceParts _FaceParts;
|
||
private readonly IBlurHasher _BlurHasher;
|
||
private readonly AppSettings _AppSettings;
|
||
private readonly List<string> _Exceptions;
|
||
private readonly ILogger<Program>? _Logger;
|
||
private readonly IsEnvironment _IsEnvironment;
|
||
private readonly DistanceLimits _DistanceLimits;
|
||
private readonly bool _PropertyRootExistedBefore;
|
||
private readonly Models.Configuration _Configuration;
|
||
private readonly ProgressBarOptions _ProgressBarOptions;
|
||
private readonly bool _ArgZeroIsConfigurationRootDirectory;
|
||
private readonly Map.Models.Configuration _MapConfiguration;
|
||
private readonly List<(string Directory, long PersonKey)> _JLinkResolvedDirectories;
|
||
|
||
public DlibDotNet(
|
||
List<string> args,
|
||
ILogger<Program> logger,
|
||
IsEnvironment isEnvironment,
|
||
IConfigurationRoot configurationRoot,
|
||
AppSettings appSettings,
|
||
string workingDirectory,
|
||
bool isSilent,
|
||
IConsole console)
|
||
{
|
||
string message;
|
||
_Logger = logger;
|
||
_Exceptions = [];
|
||
_Console = console;
|
||
_AppSettings = appSettings;
|
||
_IsEnvironment = isEnvironment;
|
||
long ticks = DateTime.Now.Ticks;
|
||
_JLinkResolvedDirectories = [];
|
||
if (ticks.ToString().Last() == '0')
|
||
ticks += 1;
|
||
ReadOnlyCollection<PersonContainer> personContainers;
|
||
_ProgressBarOptions = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
|
||
Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot);
|
||
Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration);
|
||
_Logger?.LogInformation(propertyConfiguration.RootDirectory);
|
||
Property.Models.Configuration.Verify(propertyConfiguration, requireExist: false);
|
||
Verify(configuration);
|
||
VerifyExtra(args, propertyConfiguration, configuration);
|
||
_Configuration = configuration;
|
||
_Random = new(configuration);
|
||
if (configuration.IgnoreExtensions is null)
|
||
throw new NullReferenceException(nameof(configuration.IgnoreExtensions));
|
||
_BlurHasher = new BlurHash.Models.C2_BlurHasher(configuration.PropertyConfiguration);
|
||
string propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(propertyConfiguration, nameof(A_Property), create: false);
|
||
_PropertyRootExistedBefore = !Directory.Exists(propertyRoot);
|
||
string argZero = args.Count > 0 ? Path.GetFullPath(args[0]) : Path.GetFullPath(propertyConfiguration.RootDirectory);
|
||
_ArgZeroIsConfigurationRootDirectory = propertyConfiguration.RootDirectory == argZero;
|
||
if (!Directory.Exists(argZero))
|
||
_ = Directory.CreateDirectory(argZero);
|
||
_Logger?.LogInformation(configuration.ModelDirectory);
|
||
{
|
||
(ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetPngLowQuality();
|
||
(ImageCodecInfo hiddenImageCodecInfo, EncoderParameters hiddenEncoderParameters, string hiddenFileNameExtension) = C_Resize.GetJpegLowQuality();
|
||
_Faces = new D_Face(
|
||
argZero,
|
||
configuration.PropertyConfiguration,
|
||
configuration.CheckDFaceAndUpWriteDates,
|
||
encoderParameters,
|
||
configuration.FaceDistanceHiddenImageFactor,
|
||
filenameExtension,
|
||
configuration.ForceFaceLastWriteTimeToCreationTime,
|
||
hiddenEncoderParameters,
|
||
hiddenFileNameExtension,
|
||
hiddenImageCodecInfo,
|
||
imageCodecInfo,
|
||
configuration.LoadPhotoPrismLocations,
|
||
configuration.ModelDirectory,
|
||
configuration.ModelName,
|
||
configuration.OverrideForFaceImages,
|
||
configuration.PredictorModelName,
|
||
configuration.PropertiesChangedForFaces,
|
||
configuration.RectangleIntersectMinimums);
|
||
}
|
||
{
|
||
(ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetJpegLowQuality();
|
||
_FaceParts = new D2_FaceParts(_Configuration.PropertyConfiguration, imageCodecInfo, encoderParameters, filenameExtension, configuration.CheckDFaceAndUpWriteDates, configuration.OverrideForFaceLandmarkImages);
|
||
}
|
||
_DistanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh);
|
||
_MapConfiguration = Get(configuration, _DistanceLimits, _Faces.FileNameExtension, _Faces.HiddenFileNameExtension, _FaceParts.FileNameExtension);
|
||
_Distance = new(configuration.DistanceMoveUnableToMatch, configuration.DistanceRenameToMatch, configuration.FaceConfidencePercent, configuration.RangeDistanceTolerance, configuration.RectangleIntersectMinimums);
|
||
if (_PropertyRootExistedBefore || !_ArgZeroIsConfigurationRootDirectory)
|
||
personContainers = new([]);
|
||
else
|
||
{
|
||
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
|
||
message = $") Building People Collection - {totalSeconds} total second(s)";
|
||
using ProgressBar progressBar = new(1, message, _ProgressBarOptions);
|
||
progressBar.Tick();
|
||
string peopleRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(A2_People));
|
||
string? rootResultsDirectory = Path.GetDirectoryName(Path.GetDirectoryName(peopleRootDirectory)) ?? throw new Exception();
|
||
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(peopleRootDirectory, propertyConfiguration.ResultSingleton));
|
||
string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration.PropertyConfiguration, nameof(A2_People), "{}");
|
||
string a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration.PropertyConfiguration, nameof(A2_People), "([])");
|
||
personContainers = new(IPersonContainer.GetPersonContainers(a2PeopleSingletonDirectory, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), configuration.PropertyConfiguration, _Faces.FileNameExtension));
|
||
if (configuration.JLinks.Where(l => !string.IsNullOrEmpty(l)).Any())
|
||
{
|
||
_JLinkResolvedDirectories.AddRange(Map.Models.Stateless.Methods.IMapLogic.GetJLinkDirectories(configuration.GenealogicalDataCommunicationFile, configuration.JLinks, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), a2PeopleSingletonDirectory, a2PeopleContentDirectory));
|
||
if (_JLinkResolvedDirectories.Count == 0)
|
||
throw new Exception(nameof(Map.Models.Stateless.Methods.IMapLogic.GetJLinkDirectories));
|
||
}
|
||
}
|
||
{
|
||
(ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetTuple(
|
||
configuration.OutputExtension,
|
||
configuration.OutputQuality);
|
||
_Resize = new C_Resize(
|
||
configuration.PropertyConfiguration,
|
||
configuration.ForceResizeLastWriteTimeToCreationTime,
|
||
configuration.OverrideForResizeImages,
|
||
configuration.PropertiesChangedForResize,
|
||
configuration.ValidResolutions,
|
||
imageCodecInfo,
|
||
encoderParameters,
|
||
filenameExtension);
|
||
}
|
||
if (!configuration.SkipSearch)
|
||
Search(ticks, personContainers, argZero, propertyRoot);
|
||
if (!_PropertyRootExistedBefore && !_IsEnvironment.Development && _Exceptions.Count == 0 && _ArgZeroIsConfigurationRootDirectory)
|
||
{
|
||
string d2FacePartsRootDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(propertyConfiguration, nameof(D2_FaceParts));
|
||
_Logger?.LogInformation(string.Concat("Cleaning <", d2FacePartsRootDirectory, ">"));
|
||
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(d2FacePartsRootDirectory, ticks);
|
||
}
|
||
message = $"There were {_Exceptions.Count} exception(s) thrown! {Environment.NewLine}{string.Join(Environment.NewLine, _Exceptions)}";
|
||
_Logger?.LogInformation(message);
|
||
if (_Exceptions.Count != 0)
|
||
throw new Exception(message);
|
||
if (_PropertyRootExistedBefore)
|
||
_Logger?.LogInformation("First run completed. Run again if wanted");
|
||
}
|
||
|
||
private int FullParallelForWork(B_Metadata metadata,
|
||
MapLogic mapLogic,
|
||
string outputResolution,
|
||
bool outputResolutionHasNumber,
|
||
string cResultsFullGroupDirectory,
|
||
string d2ResultsFullGroupDirectory,
|
||
List<Tuple<string, DateTime>> sourceDirectoryChanges,
|
||
Dictionary<int, List<MappingFromPhotoPrism>> fileNameToCollection,
|
||
Record record,
|
||
Container.Models.Container container,
|
||
Item item,
|
||
DateTime[] containerDateTimes,
|
||
bool? isFocusRelativePath)
|
||
{
|
||
int result = 0;
|
||
bool? shouldIgnore;
|
||
List<Shared.Models.Face> faces;
|
||
long ticks = DateTime.Now.Ticks;
|
||
DateTime dateTime = DateTime.Now;
|
||
List<string> parseExceptions = [];
|
||
string[] changesFrom = [nameof(A_Property)];
|
||
List<Tuple<string, DateTime>> subFileTuples = [];
|
||
FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(cResultsFullGroupDirectory, item, outputResolutionHasNumber);
|
||
if (item.ExifDirectory is null || item.ExifDirectory.FilePath.Id is null || !item.SourceDirectoryFileHolder.Exists || item.SourceDirectoryFileHolder.CreationTime is null || item.SourceDirectoryFileHolder.LastWriteTime is null || item.Any())
|
||
throw new Exception();
|
||
if (_Configuration.PropertyConfiguration.ForcePropertyLastWriteTimeToCreationTime && item.SourceDirectoryFileHolder.LastWriteTime.Value != item.SourceDirectoryFileHolder.CreationTime.Value)
|
||
{
|
||
File.SetLastWriteTime(item.SourceDirectoryFileHolder.FullName, item.SourceDirectoryFileHolder.CreationTime.Value);
|
||
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), item.SourceDirectoryFileHolder.CreationTime.Value));
|
||
}
|
||
else if (item.SourceDirectoryFileHolder.LastWriteTime is not null)
|
||
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), item.SourceDirectoryFileHolder.LastWriteTime.Value));
|
||
else
|
||
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), new FileInfo(item.SourceDirectoryFileHolder.FullName).LastWriteTime));
|
||
if (resizedFileHolder.Exists && item.ExifDirectory.Width is not null && item.ExifDirectory.Width.Value > 4 && _Configuration.SaveBlurHashForOutputResolutions.Contains(outputResolution))
|
||
{
|
||
string? file = _BlurHasher.GetFile(item.FilePath);
|
||
if (file is not null && !File.Exists(file))
|
||
_ = _BlurHasher.EncodeAndSave(item.FilePath, resizedFileHolder);
|
||
}
|
||
if (item.FilePath.Id is null || !record.ExifDirectoriesById.TryGetValue(item.FilePath.Id.Value, out ExifDirectory? exifDirectory))
|
||
{
|
||
shouldIgnore = null;
|
||
exifDirectory = null;
|
||
}
|
||
else
|
||
{
|
||
ReadOnlyCollection<string> keywords = IMetaBase.GetKeywords(exifDirectory.ExifBaseDirectories);
|
||
shouldIgnore = _Configuration.PropertyConfiguration.IgnoreRulesKeyWords.Any(keywords.Contains);
|
||
if (shouldIgnore.Value)
|
||
{
|
||
FileInfo fileInfo = new(resizedFileHolder.FullName);
|
||
if (!fileInfo.Attributes.HasFlag(FileAttributes.Hidden))
|
||
File.SetAttributes(resizedFileHolder.FullName, FileAttributes.Hidden);
|
||
}
|
||
if (resizedFileHolder.Exists && item.FilePath.HasIgnoreKeyword is not null && item.FilePath.HasIgnoreKeyword.Value != shouldIgnore.Value)
|
||
{
|
||
if (item.FilePath.DirectoryFullPath.Contains("Results") && item.FilePath.DirectoryFullPath.Contains("Resize"))
|
||
File.Delete(resizedFileHolder.FullName);
|
||
else
|
||
throw new NotSupportedException($"Rename File! <{item.FilePath.FileNameFirstSegment}>");
|
||
}
|
||
}
|
||
item.SetResizedFileHolder(_Resize.FileNameExtension, resizedFileHolder);
|
||
MappingFromItem mappingFromItem = IMappingFromItem.GetMappingFromItem(containerDateTimes, item, resizedFileHolder);
|
||
exifDirectory ??= metadata.GetMetadataCollection(item.FilePath, subFileTuples, parseExceptions, changesFrom, mappingFromItem);
|
||
if (_AppSettings.Places.Count > 0)
|
||
{
|
||
float latitude;
|
||
float longitude;
|
||
double? distance;
|
||
MetadataExtractor.GeoLocation? geoLocation = Metadata.Models.Stateless.Methods.IMetadata.GeoLocation(exifDirectory);
|
||
foreach (Place place in _AppSettings.Places)
|
||
{
|
||
if (geoLocation is null)
|
||
continue;
|
||
latitude = Math.Abs(place.Latitude.Degrees) + (place.Latitude.Minutes / 60) + (place.Latitude.Seconds / 3600);
|
||
if (place.Latitude.Degrees < 0)
|
||
latitude *= -1;
|
||
longitude = Math.Abs(place.Longitude.Degrees) + (place.Longitude.Minutes / 60) + (place.Longitude.Seconds / 3600);
|
||
if (place.Longitude.Degrees < 0)
|
||
longitude *= -1;
|
||
distance = geoLocation is null ? null : Metadata.Models.Stateless.Methods.IMetadata.GetDistance(latitude, longitude, geoLocation.Latitude, geoLocation.Longitude);
|
||
if (distance is null or > 3)
|
||
continue;
|
||
distance += 1;
|
||
}
|
||
}
|
||
Dictionary<string, int[]> outputResolutionToResize = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, item.ExifDirectory, mappingFromItem);
|
||
if (_Configuration.SaveResizedSubfiles)
|
||
{
|
||
if (shouldIgnore is not null && item.FilePath.HasIgnoreKeyword is not null && item.FilePath.HasIgnoreKeyword.Value != shouldIgnore.Value)
|
||
faces = [];
|
||
else
|
||
_Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.ExifDirectory, mappingFromItem, outputResolutionToResize);
|
||
}
|
||
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
|
||
faces = [];
|
||
else if (!mappingFromItem.ResizedFileHolder.Exists && !File.Exists(mappingFromItem.ResizedFileHolder.FullName))
|
||
faces = [];
|
||
else
|
||
{
|
||
List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection;
|
||
if (!fileNameToCollection.TryGetValue(mappingFromItem.Id, out mappingFromPhotoPrismCollection))
|
||
mappingFromPhotoPrismCollection = null;
|
||
bool move = _Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch && _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution);
|
||
faces = _Faces.GetFaces(outputResolution, cResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, item.ExifDirectory, mappingFromItem, outputResolutionToResize, mappingFromPhotoPrismCollection);
|
||
result = GetNotMappedCountAndUpdateMappingFromPersonThenSetMapping(mapLogic, item, isFocusRelativePath, mappingFromItem, mappingFromPhotoPrismCollection, faces);
|
||
List<(Shared.Models.Face, FileHolder?, string, bool Saved)> faceCollection = _Faces.SaveFaces(item.FilePath, subFileTuples, parseExceptions, mappingFromItem, exifDirectory, faces);
|
||
if (move && faceCollection.All(l => !l.Saved))
|
||
{
|
||
ReadOnlyCollection<LocationContainer> locationContainers = mapLogic.GetLocationContainers(item);
|
||
if (locationContainers.Count > 0)
|
||
{
|
||
Map.Models.Stateless.Methods.IMapLogic.SetCreationTime(mappingFromItem, locationContainers);
|
||
if (_Configuration.LocationContainerDistanceTolerance is null && _Configuration.MoveToDecade)
|
||
Map.Models.Stateless.Methods.IMapLogic.MoveToDecade(_Configuration.PropertyConfiguration, mappingFromItem, locationContainers);
|
||
_Distance.LookForMatchFacesAndPossiblyRename(_Configuration.OverrideForFaceImages, _DistanceLimits, _Faces, item.FilePath, mappingFromItem, exifDirectory, faces, locationContainers);
|
||
}
|
||
}
|
||
(bool review, int[] eyesCollection) = Shared.Models.Stateless.Methods.IFace.GetEyeCollection(_Configuration.EyeThreshold, faces);
|
||
if (review || _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
|
||
{
|
||
bool saveRotated = false;
|
||
string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(_Configuration.PropertyConfiguration, container.SourceDirectory);
|
||
_FaceParts.SaveFaceLandmarkImages(_Configuration.PropertyConfiguration, d2ResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, mappingFromItem, exifDirectory, faces, saveRotated);
|
||
}
|
||
if (_Configuration.SaveFaceLandmarkForOutputResolutionsV2.Contains(outputResolution))
|
||
_FaceParts.SaveFaceLandmarkImages(d2ResultsFullGroupDirectory, mappingFromItem, exifDirectory, faces);
|
||
}
|
||
lock (sourceDirectoryChanges)
|
||
{
|
||
item.Faces.AddRange(faces);
|
||
sourceDirectoryChanges.AddRange(from l in subFileTuples where l.Item2 > dateTime select l);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
private (int, bool) FullParallelWork(int maxDegreeOfParallelism,
|
||
B_Metadata metadata,
|
||
MapLogic mapLogic,
|
||
string outputResolution,
|
||
bool outputResolutionHasNumber,
|
||
string cResultsFullGroupDirectory,
|
||
string d2ResultsFullGroupDirectory,
|
||
List<Tuple<string, DateTime>> sourceDirectoryChanges,
|
||
Dictionary<int, List<MappingFromPhotoPrism>> fileNameToCollection,
|
||
Record record,
|
||
Container.Models.Container container,
|
||
ReadOnlyCollection<Item> filteredItems,
|
||
string message)
|
||
{
|
||
int result = 0;
|
||
int exceptionsCount = 0;
|
||
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
|
||
DateTime[] containerDateTimes = Container.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems);
|
||
string focusRelativePath = Path.GetFullPath(string.Concat(_Configuration.PropertyConfiguration.RootDirectory, _Configuration.FocusDirectory));
|
||
bool? isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath);
|
||
bool anyPropertiesChangedForX = _Configuration.PropertyConfiguration.PropertiesChangedForProperty || _Configuration.PropertiesChangedForDistance || _Configuration.PropertiesChangedForFaces || _Configuration.PropertiesChangedForIndex || _Configuration.PropertiesChangedForMetadata || _Configuration.PropertiesChangedForResize;
|
||
using ProgressBar progressBar = new(filteredItems.Count, message, _ProgressBarOptions);
|
||
_ = Parallel.For(0, filteredItems.Count, parallelOptions, (i, state) =>
|
||
{
|
||
try
|
||
{
|
||
result += FullParallelForWork(metadata,
|
||
mapLogic,
|
||
outputResolution,
|
||
outputResolutionHasNumber,
|
||
cResultsFullGroupDirectory,
|
||
d2ResultsFullGroupDirectory,
|
||
sourceDirectoryChanges,
|
||
fileNameToCollection,
|
||
record,
|
||
container,
|
||
filteredItems[i],
|
||
containerDateTimes,
|
||
isFocusRelativePath);
|
||
if (!anyPropertiesChangedForX && (i == 0 || sourceDirectoryChanges.Count > 0))
|
||
progressBar.Tick();
|
||
}
|
||
catch (Exception)
|
||
{
|
||
exceptionsCount++;
|
||
if (exceptionsCount == filteredItems.Count)
|
||
throw new Exception(string.Concat("All in [", container.SourceDirectory, "] failed!"));
|
||
}
|
||
});
|
||
return (result, exceptionsCount > 0);
|
||
}
|
||
|
||
private void FullDoWork(string argZero,
|
||
string propertyRoot,
|
||
long ticks,
|
||
string fPhotoPrismSingletonDirectory,
|
||
int count,
|
||
B_Metadata metadata,
|
||
Record record,
|
||
ReadOnlyCollection<Container.Models.Container> readOnlyContainers,
|
||
MapLogic mapLogic)
|
||
{
|
||
int total;
|
||
int notMapped;
|
||
string message;
|
||
bool exceptions;
|
||
int totalSeconds;
|
||
int totalNotMapped = 0;
|
||
IDlibDotNet dlibDotNet = this;
|
||
bool outputResolutionHasNumber;
|
||
bool anyNullOrNoIsUniqueFileName;
|
||
string cResultsFullGroupDirectory;
|
||
string dResultsFullGroupDirectory;
|
||
string c2ResultsFullGroupDirectory;
|
||
string d2ResultsFullGroupDirectory;
|
||
Container.Models.Container container;
|
||
ReadOnlyCollection<Item> filteredItems;
|
||
List<Tuple<string, DateTime>> sourceDirectoryChanges = [];
|
||
int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism;
|
||
Dictionary<int, List<MappingFromPhotoPrism>> fileNameToCollection = !Directory.Exists(fPhotoPrismSingletonDirectory) ? fileNameToCollection = [] : F_PhotoPrism.GetFileNameToCollection(fPhotoPrismSingletonDirectory);
|
||
string dResultsDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(D_Face));
|
||
foreach (string outputResolution in _Configuration.OutputResolutions)
|
||
{
|
||
total = 0;
|
||
outputResolutionHasNumber = outputResolution.Any(char.IsNumber);
|
||
(cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
|
||
_Faces.Update(dResultsFullGroupDirectory);
|
||
_Resize.Update(cResultsFullGroupDirectory);
|
||
_FaceParts.Update(d2ResultsFullGroupDirectory);
|
||
_BlurHasher.Update(c2ResultsFullGroupDirectory);
|
||
for (int i = 0; i < readOnlyContainers.Count; i++)
|
||
{
|
||
container = readOnlyContainers[i];
|
||
if (container.Items.Count == 0)
|
||
continue;
|
||
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
|
||
continue;
|
||
filteredItems = Container.Models.Stateless.Methods.IContainer.GetValidImageItems(_Configuration.PropertyConfiguration, container);
|
||
if (filteredItems.Count == 0)
|
||
continue;
|
||
sourceDirectoryChanges.Clear();
|
||
anyNullOrNoIsUniqueFileName = filteredItems.Any(l => !l.IsUniqueFileName);
|
||
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
|
||
message = $"{totalSeconds} total second(s) - {outputResolution} - {i + 1:000} / {readOnlyContainers.Count:000} - {total} / {count} total - <{container.SourceDirectory}> [{filteredItems.Count:000}] - total not mapped {totalNotMapped:000000}";
|
||
if (outputResolutionHasNumber)
|
||
_Resize.SetAngleBracketCollection(cResultsFullGroupDirectory, container.SourceDirectory);
|
||
(notMapped, exceptions) = FullParallelWork(maxDegreeOfParallelism,
|
||
metadata,
|
||
mapLogic,
|
||
outputResolution,
|
||
outputResolutionHasNumber,
|
||
cResultsFullGroupDirectory,
|
||
d2ResultsFullGroupDirectory,
|
||
sourceDirectoryChanges,
|
||
fileNameToCollection,
|
||
record,
|
||
container,
|
||
filteredItems,
|
||
message);
|
||
totalNotMapped += notMapped;
|
||
if (exceptions)
|
||
{
|
||
_Exceptions.Add(container.SourceDirectory);
|
||
continue;
|
||
}
|
||
if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Length > 0)
|
||
{
|
||
for (int y = 0; y < int.MaxValue; y++)
|
||
{
|
||
_Logger?.LogInformation("Press \"Y\" key when ready to continue or close console");
|
||
if (_Console.ReadKey() == ConsoleKey.Y)
|
||
break;
|
||
}
|
||
_Logger?.LogInformation(". . .");
|
||
}
|
||
total += container.Items.Count;
|
||
}
|
||
totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
|
||
message = $"{totalSeconds} total second(s) - {outputResolution} - ### [###] / {readOnlyContainers.Count:000} - {total} / {count} total - <> - total not mapped {totalNotMapped:000000}";
|
||
using ProgressBar progressBar = new(1, message, _ProgressBarOptions);
|
||
progressBar.Tick();
|
||
}
|
||
}
|
||
|
||
void IDlibDotNet.Tick() =>
|
||
_ProgressBar?.Tick();
|
||
|
||
void IDisposable.Dispose()
|
||
{
|
||
_ProgressBar?.Dispose();
|
||
GC.SuppressFinalize(this);
|
||
}
|
||
|
||
(string, string) IDlibDotNet.GetResultsFullGroupDirectories()
|
||
{
|
||
string aResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
|
||
_Configuration.PropertyConfiguration,
|
||
nameof(A_Property),
|
||
string.Empty,
|
||
includeResizeGroup: false,
|
||
includeModel: false,
|
||
includePredictorModel: false);
|
||
string bResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
|
||
_Configuration.PropertyConfiguration,
|
||
nameof(B_Metadata),
|
||
string.Empty,
|
||
includeResizeGroup: false,
|
||
includeModel: false,
|
||
includePredictorModel: false);
|
||
return new(aResultsFullGroupDirectory, bResultsFullGroupDirectory);
|
||
}
|
||
|
||
void IDlibDotNet.ConstructProgressBar(int maxTicks, string message)
|
||
{
|
||
_ProgressBar?.Dispose();
|
||
_ProgressBar = new(maxTicks, message, _ProgressBarOptions);
|
||
}
|
||
|
||
(string, string, string, string) IDlibDotNet.GetResultsFullGroupDirectories(string outputResolution)
|
||
{
|
||
string cResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
|
||
_Configuration.PropertyConfiguration,
|
||
nameof(C_Resize),
|
||
outputResolution,
|
||
includeResizeGroup: true,
|
||
includeModel: false,
|
||
includePredictorModel: false);
|
||
string c2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
|
||
_Configuration.PropertyConfiguration,
|
||
nameof(BlurHash.Models.C2_BlurHasher),
|
||
outputResolution,
|
||
includeResizeGroup: true,
|
||
includeModel: false,
|
||
includePredictorModel: false);
|
||
string dResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
|
||
_Configuration.PropertyConfiguration,
|
||
nameof(D_Face),
|
||
outputResolution,
|
||
includeResizeGroup: true,
|
||
includeModel: true,
|
||
includePredictorModel: true);
|
||
string d2ResultsFullGroupDirectory = Property.Models.Stateless.IResult.GetResultsFullGroupDirectory(
|
||
_Configuration.PropertyConfiguration,
|
||
nameof(D2_FaceParts),
|
||
outputResolution,
|
||
includeResizeGroup: true,
|
||
includeModel: true,
|
||
includePredictorModel: true);
|
||
return new(cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory);
|
||
}
|
||
|
||
public ReadOnlyDictionary<string, LocationContainer> GetOnlyOne(IDistanceLimits distanceLimits, ReadOnlyCollection<LocationContainer> matrix)
|
||
{
|
||
Dictionary<string, LocationContainer> results = [];
|
||
List<string> added = [];
|
||
LocationContainer? tryGetValue;
|
||
foreach (LocationContainer locationContainer in matrix)
|
||
{
|
||
if (_Configuration.SaveIndividually)
|
||
break;
|
||
if (locationContainer.LengthSource is null)
|
||
continue;
|
||
if (_Configuration.UseExtraPersonKeyCheck)
|
||
{
|
||
if (results.TryGetValue(locationContainer.LengthSource.Name, out tryGetValue))
|
||
{
|
||
if (locationContainer.PersonKey is not null && tryGetValue.PersonKey is not null && locationContainer.PersonKey.Value != tryGetValue.PersonKey)
|
||
_ = results.Remove(locationContainer.LengthSource.Name);
|
||
continue;
|
||
}
|
||
}
|
||
if (added.Contains(locationContainer.LengthSource.Name))
|
||
continue;
|
||
added.Add(locationContainer.LengthSource.Name);
|
||
results.Add(locationContainer.LengthSource.Name, locationContainer);
|
||
}
|
||
return results.AsReadOnly();
|
||
}
|
||
|
||
private static Identifier GetIdentifier(Property.Models.Configuration propertyConfiguration, FilePath filePath, KeyValuePair<int, ReadOnlyCollection<FilePath>> keyValuePair)
|
||
{
|
||
Identifier result;
|
||
if (filePath.Id is null)
|
||
throw new Exception();
|
||
string[] directoryNames = keyValuePair.Value.Select(l => l.DirectoryFullPath.Replace('\\', '/')).ToArray();
|
||
string paddedId = IId.GetPaddedId(propertyConfiguration, filePath.Id.Value, filePath.ExtensionLowered, filePath.HasIgnoreKeyword, filePath.HasDateTimeOriginal, index: null);
|
||
result = new(directoryNames, filePath.ExtensionLowered, filePath.HasDateTimeOriginal, filePath.Id.Value, filePath.Length, paddedId, filePath.LastWriteTicks);
|
||
return result;
|
||
}
|
||
|
||
[GeneratedRegex(@"[\\,\/,\:,\*,\?,\"",\<,\>,\|]")]
|
||
private static partial Regex CameraRegex();
|
||
|
||
private static void Verify(Models.Configuration configuration)
|
||
{
|
||
if (configuration.RangeDaysDeltaTolerance.Length != 3)
|
||
throw new NullReferenceException(nameof(configuration.RangeDaysDeltaTolerance));
|
||
if (configuration.RangeDistanceTolerance.Length != 3)
|
||
throw new NullReferenceException(nameof(configuration.RangeDistanceTolerance));
|
||
if (configuration.RangeFaceAreaTolerance.Length != 3)
|
||
throw new NullReferenceException(nameof(configuration.RangeFaceAreaTolerance));
|
||
if (configuration.RangeFaceConfidence.Length != 3)
|
||
throw new NullReferenceException(nameof(configuration.RangeFaceConfidence));
|
||
if (configuration.LocationContainerDistanceTolerance is null && !string.IsNullOrEmpty(configuration.LocationContainerDebugDirectory))
|
||
throw new NullReferenceException($"{nameof(configuration.LocationContainerDistanceTolerance)} must have a value when {nameof(configuration.LocationContainerDebugDirectory)} is set!");
|
||
_ = DateTime.Now.AddDays(-configuration.RangeDaysDeltaTolerance[1]);
|
||
if (configuration.OutputResolutions.Length == 0 || string.IsNullOrEmpty(configuration.OutputResolutions[0]) || !configuration.ValidResolutions.Contains(configuration.OutputResolutions[0]))
|
||
throw new NullReferenceException($"{nameof(configuration.OutputResolutions)} must be fileNameToCollection valid outputResolution!");
|
||
if ((from l in configuration.OutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
||
throw new Exception($"One or more {nameof(configuration.OutputResolutions)} are not in the ValidResolutions list!");
|
||
if ((from l in configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
||
throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions)} are not in the ValidResolutions list!");
|
||
if ((from l in configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
||
throw new Exception($"One or more {nameof(configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions)} are not in the ValidResolutions list!");
|
||
if ((from l in configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
||
throw new Exception($"One or more {nameof(configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions)} are not in the ValidResolutions list!");
|
||
if ((from l in configuration.SaveShortcutsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
||
throw new Exception($"One or more {nameof(configuration.SaveShortcutsForOutputResolutions)} are not in the ValidResolutions list!");
|
||
if ((from l in configuration.SaveFaceLandmarkForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
|
||
throw new Exception($"One or more {nameof(configuration.SaveFaceLandmarkForOutputResolutions)} are not in the ValidResolutions list!");
|
||
if ((from l in configuration.SaveFaceLandmarkForOutputResolutionsV2 where !configuration.ValidResolutions.Contains(l) select false).Any())
|
||
throw new Exception($"One or more {nameof(configuration.SaveFaceLandmarkForOutputResolutionsV2)} are not in the ValidResolutions list!");
|
||
if (string.IsNullOrEmpty(configuration.ModelName))
|
||
throw new NullReferenceException(nameof(configuration.ModelName));
|
||
if (string.IsNullOrEmpty(configuration.OutputExtension))
|
||
throw new NullReferenceException(nameof(configuration.OutputExtension));
|
||
if (string.IsNullOrEmpty(configuration.PredictorModelName))
|
||
throw new NullReferenceException(nameof(configuration.PredictorModelName));
|
||
if (string.IsNullOrEmpty(configuration.ModelDirectory) || !Directory.Exists(configuration.ModelDirectory))
|
||
throw new NullReferenceException(nameof(configuration.ModelDirectory));
|
||
}
|
||
|
||
private static ExifDirectory? GetExifDirectory(FilePair filePair)
|
||
{
|
||
ExifDirectory? result;
|
||
if (filePair.Match is null)
|
||
result = null;
|
||
else
|
||
{
|
||
string json = File.ReadAllText(filePair.Match.FullName);
|
||
if (string.IsNullOrEmpty(json))
|
||
result = null;
|
||
else
|
||
result = JsonSerializer.Deserialize(json, ExifDirectorySourceGenerationContext.Default.ExifDirectory);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
private static void DeleteContinueFiles(ReadOnlyCollection<PersonContainer> personContainers)
|
||
{
|
||
foreach (PersonContainer personContainer in personContainers)
|
||
{
|
||
foreach (FilePath filePath in personContainer.DisplayDirectoryAllFilePaths)
|
||
{
|
||
if (filePath.ExtensionLowered != ".continue")
|
||
continue;
|
||
File.Delete(filePath.FullName);
|
||
}
|
||
}
|
||
}
|
||
|
||
private static void CheckForAllWindowsLinks(ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection)
|
||
{
|
||
string fileFullPath;
|
||
WindowsShortcut windowsShortcut;
|
||
foreach (ReadOnlyCollection<FilePath> filePaths in filePathsCollection)
|
||
{
|
||
if (filePaths.Count == 0)
|
||
continue;
|
||
if (filePaths.All(l => l.ExtensionLowered == ".lnk"))
|
||
{
|
||
foreach (FilePath filePath in filePaths)
|
||
{
|
||
windowsShortcut = WindowsShortcut.Load(filePath.FullName);
|
||
if (windowsShortcut.Path is null)
|
||
{
|
||
File.Delete(filePath.FullName);
|
||
continue;
|
||
}
|
||
fileFullPath = windowsShortcut.Path;
|
||
windowsShortcut.Dispose();
|
||
File.WriteAllText(Path.ChangeExtension(filePath.FullName, ".url"), fileFullPath);
|
||
File.Delete(filePath.FullName);
|
||
}
|
||
throw new NotSupportedException("All are Windows *.lnk files!");
|
||
}
|
||
}
|
||
}
|
||
|
||
private static bool GetFilesCollectionCountIsOne(ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection)
|
||
{
|
||
bool result = true;
|
||
int count = 0;
|
||
foreach (ReadOnlyCollection<FilePath> filePaths in filePathsCollection)
|
||
{
|
||
if (filePaths.Count == 0)
|
||
continue;
|
||
count += 1;
|
||
if (count > 1)
|
||
{
|
||
result = false;
|
||
break;
|
||
}
|
||
}
|
||
if (result)
|
||
CheckForAllWindowsLinks(filePathsCollection);
|
||
return result;
|
||
}
|
||
|
||
private static void SavePropertyShortcutsForOutputResolutions(string eDistanceContentDirectory, ReadOnlyCollection<Item> distinctValidImageItems)
|
||
{
|
||
#if VerifyItem
|
||
bool found;
|
||
List<Item> notFound = new();
|
||
foreach (Item item in distinctFilteredItems)
|
||
{
|
||
found = false;
|
||
if (item.ExifDirectory?.FilePath.Id is null)
|
||
continue;
|
||
foreach (Mapping mapping in distinctFilteredMappingCollection)
|
||
{
|
||
if (mapping.MappingFromItem.Id != item.ExifDirectory.FilePath.Id.Value)
|
||
continue;
|
||
found = true;
|
||
break;
|
||
}
|
||
if (!found)
|
||
notFound.Add(item);
|
||
}
|
||
if (notFound.Count > 0)
|
||
throw new NotSupportedException();
|
||
#endif
|
||
string model;
|
||
string fileName;
|
||
string directory;
|
||
bool? isWrongYear;
|
||
DateTime? dateTime;
|
||
string? exifDirectoryModel;
|
||
List<string> distinct = [];
|
||
WindowsShortcut windowsShortcut;
|
||
ReadOnlyCollection<DateTime> dateTimes;
|
||
List<(string, string, string)> collection = [];
|
||
foreach (Item item in distinctValidImageItems)
|
||
{
|
||
if (item.ExifDirectory?.FilePath.Id is null)
|
||
continue;
|
||
if (item.IsNotUniqueAndNeedsReview is null || !item.IsNotUniqueAndNeedsReview.Value)
|
||
continue;
|
||
directory = Path.Combine($"{eDistanceContentDirectory[..^1]}{nameof(item.IsNotUniqueAndNeedsReview)})", item.FilePath.NameWithoutExtension);
|
||
fileName = Path.Combine(directory, $"{item.FilePath.Length} {item.FilePath.LastWriteTicks}.lnk");
|
||
collection.Add((item.FilePath.FullName, directory, fileName));
|
||
if (distinct.Contains(directory))
|
||
continue;
|
||
distinct.Add(directory);
|
||
}
|
||
foreach (Item item in distinctValidImageItems)
|
||
{
|
||
if (item.ExifDirectory?.FilePath.Id is null)
|
||
continue;
|
||
dateTime = IDate.GetDateTimeOriginal(item.ExifDirectory);
|
||
if (dateTime is null)
|
||
continue;
|
||
dateTimes = IDate.GetDateTimes(item.ExifDirectory);
|
||
(isWrongYear, _) = Shared.Models.Stateless.Methods.IProperty.IsWrongYear(item.FilePath, dateTime, dateTimes.ToList());
|
||
if (isWrongYear is null || !isWrongYear.Value)
|
||
continue;
|
||
// Remove-Item -LiteralPath "\\?\D:\Tmp\a\EX-Z70 "
|
||
exifDirectoryModel = IMetaBase.GetModel(item.ExifDirectory?.ExifBaseDirectories);
|
||
model = string.IsNullOrEmpty(exifDirectoryModel) ? "Unknown" : CameraRegex().Replace(exifDirectoryModel.Trim(), "_");
|
||
directory = Path.Combine($"{eDistanceContentDirectory[..^1]}{nameof(Item)})", dateTime.Value.Year.ToString(), model);
|
||
fileName = item.IsNotUniqueAndNeedsReview is not null && item.IsNotUniqueAndNeedsReview.Value ? Path.Combine(directory, $"{item.FilePath.Name} {item.FilePath.Length}.lnk") : Path.Combine(directory, $"{item.FilePath.Name}.lnk");
|
||
collection.Add((item.FilePath.FullName, directory, fileName));
|
||
if (distinct.Contains(directory))
|
||
continue;
|
||
distinct.Add(directory);
|
||
}
|
||
#if Mapping
|
||
foreach (Mapping mapping in distinctFilteredMappingCollection)
|
||
{
|
||
if (mapping.MappingFromItem.IsWrongYear is null || !mapping.MappingFromItem.IsWrongYear.Value)
|
||
continue;
|
||
directory = Path.Combine($"{eDistanceContentDirectory[..^1]}{nameof(Mapping)})", mapping.MappingFromItem.MinimumDateTime.Year.ToString());
|
||
fileName = Path.Combine(directory, $"{mapping.MappingFromItem.ResizedFileHolder.Name}.lnk");
|
||
collection.Add((mapping.MappingFromItem.ResizedFileHolder.FullName, directory, fileName));
|
||
if (distinct.Contains(directory))
|
||
continue;
|
||
distinct.Add(directory);
|
||
}
|
||
#endif
|
||
foreach (string distinctDirectory in distinct)
|
||
{
|
||
if (!Directory.Exists(distinctDirectory))
|
||
_ = Directory.CreateDirectory(distinctDirectory);
|
||
}
|
||
foreach ((string path, string checkDirectory, string checkFile) in collection)
|
||
{
|
||
if (File.Exists(checkFile))
|
||
continue;
|
||
windowsShortcut = new() { Path = path };
|
||
windowsShortcut.Save(checkFile);
|
||
windowsShortcut.Dispose();
|
||
}
|
||
}
|
||
|
||
private static Map.Models.Configuration Get(Models.Configuration configuration, IDistanceLimits distanceLimits, string facesFileNameExtension, string facesHiddenFileNameExtension, string facePartsFileNameExtension)
|
||
{
|
||
Map.Models.Configuration result = new(distanceLimits,
|
||
configuration.PropertyConfiguration,
|
||
configuration.DeletePossibleDuplicates,
|
||
configuration.DistanceMoveUnableToMatch,
|
||
configuration.DistanceRenameToMatch,
|
||
configuration.FaceConfidencePercent,
|
||
configuration.FaceDistancePermyriad,
|
||
configuration.LinkedAlpha,
|
||
configuration.LocationContainerDebugDirectory,
|
||
configuration.LocationContainerDirectoryPattern,
|
||
configuration.LocationContainerDistanceGroupMinimum,
|
||
configuration.LocationContainerDistanceTake,
|
||
configuration.LocationContainerDistanceTolerance,
|
||
configuration.LocationDigits,
|
||
configuration.MappingDefaultName,
|
||
configuration.PersonBirthdayFirstYear,
|
||
configuration.PersonBirthdayFormat,
|
||
configuration.PersonCharacters.ToArray(),
|
||
configuration.RangeDaysDeltaTolerance,
|
||
configuration.RangeDistanceTolerance,
|
||
configuration.ReMap,
|
||
configuration.SaveIndividually,
|
||
configuration.SaveSortingWithoutPerson,
|
||
configuration.SkipNotSkipDirectories,
|
||
configuration.SortingMaximumPerKey,
|
||
configuration.SortingMinimumToUseSigma,
|
||
facesFileNameExtension,
|
||
facesHiddenFileNameExtension,
|
||
facePartsFileNameExtension);
|
||
return result;
|
||
}
|
||
|
||
private static ReadOnlyCollection<FilePath> HideSplatNineAndGetMovedDuplicatesWithSameSplatNine(Property.Models.Configuration propertyConfiguration, ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>> keyValuePairs)
|
||
{
|
||
List<FilePath> results = [];
|
||
string checkFile;
|
||
FileInfo fileInfo;
|
||
FilePath? filePath;
|
||
string checkDirectory;
|
||
List<FilePath> collection = [];
|
||
foreach (KeyValuePair<int, ReadOnlyCollection<FilePath>> keyValuePair in keyValuePairs)
|
||
{
|
||
collection.Clear();
|
||
for (int i = 0; i < keyValuePair.Value.Count; i++)
|
||
{
|
||
filePath = keyValuePair.Value[i];
|
||
if (!filePath.FullName.Contains(" !9"))
|
||
collection.Add(filePath);
|
||
else
|
||
{
|
||
fileInfo = new(filePath.FullName);
|
||
if (!fileInfo.Attributes.HasFlag(FileAttributes.Hidden))
|
||
File.SetAttributes(fileInfo.FullName, FileAttributes.Hidden);
|
||
}
|
||
}
|
||
if (keyValuePair.Value.Count == 1 || collection.Count == keyValuePair.Value.Count)
|
||
continue;
|
||
foreach (FilePath fp in collection)
|
||
{
|
||
if (fp.HasDateTimeOriginal is null || !fp.HasDateTimeOriginal.Value)
|
||
break;
|
||
checkDirectory = Path.Combine(propertyConfiguration.RootDirectory, $"{new DateTime(fp.LastWriteTicks):yyyy MM} Already in Splat Nine");
|
||
if (!Directory.Exists(checkDirectory))
|
||
_ = Directory.CreateDirectory(checkDirectory);
|
||
checkFile = Path.Combine(checkDirectory, fp.Name);
|
||
if (File.Exists(checkFile))
|
||
continue;
|
||
File.Move(fp.FullName, checkFile);
|
||
results.Add(fp);
|
||
}
|
||
}
|
||
return results.AsReadOnly();
|
||
}
|
||
|
||
private static ReadOnlyDictionary<int, Identifier> GetSplatNineIdentifiers(Property.Models.Configuration propertyConfiguration, string bResultsFullGroupDirectory, ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>> keyValuePairs)
|
||
{
|
||
Dictionary<int, Identifier> results = [];
|
||
string json;
|
||
FilePath? filePath;
|
||
Identifier identifier;
|
||
List<Identifier> identifiers = [];
|
||
string rootDirectory = propertyConfiguration.RootDirectory.Replace('\\', '/');
|
||
string bMetadataCollectionDirectory = Path.Combine(bResultsFullGroupDirectory, propertyConfiguration.ResultCollection);
|
||
if (!Directory.Exists(bMetadataCollectionDirectory))
|
||
_ = Directory.CreateDirectory(bMetadataCollectionDirectory);
|
||
foreach (KeyValuePair<int, ReadOnlyCollection<FilePath>> keyValuePair in keyValuePairs)
|
||
{
|
||
filePath = null;
|
||
for (int i = 0; i < keyValuePair.Value.Count; i++)
|
||
{
|
||
filePath = keyValuePair.Value[i];
|
||
if (filePath.FullName.Contains(" !9"))
|
||
break;
|
||
filePath = null;
|
||
}
|
||
if (filePath is null)
|
||
{
|
||
filePath = keyValuePair.Value[0];
|
||
if (filePath.Id is null)
|
||
continue;
|
||
identifier = GetIdentifier(propertyConfiguration, filePath, keyValuePair);
|
||
identifiers.Add(identifier);
|
||
continue;
|
||
}
|
||
else if (filePath.Id is null)
|
||
continue;
|
||
identifier = GetIdentifier(propertyConfiguration, filePath, keyValuePair);
|
||
identifiers.Add(identifier);
|
||
results.Add(keyValuePair.Key, identifier);
|
||
}
|
||
json = JsonSerializer.Serialize(results.Values.ToArray(), IdentifierCollectionSourceGenerationContext.Default.IdentifierArray);
|
||
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(Path.Combine(bMetadataCollectionDirectory, "!9.json"), json, updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null);
|
||
json = JsonSerializer.Serialize((from l in identifiers orderby l.PaddedId select l).ToArray(), IdentifierCollectionSourceGenerationContext.Default.IdentifierArray);
|
||
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(Path.Combine(bMetadataCollectionDirectory, ".json"), json.Replace(rootDirectory, string.Empty), updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null);
|
||
return results.AsReadOnly();
|
||
}
|
||
|
||
private static ReadOnlyDictionary<int, Identifier> GetSplatNineIdentifiersAndHideSplatNine(Property.Models.Configuration propertyConfiguration, string bResultsFullGroupDirectory, ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>> keyValuePairs)
|
||
{
|
||
ReadOnlyDictionary<int, Identifier> results;
|
||
if (keyValuePairs.Count == 0)
|
||
results = new(new Dictionary<int, Identifier>());
|
||
else
|
||
{
|
||
ReadOnlyCollection<FilePath> moved = HideSplatNineAndGetMovedDuplicatesWithSameSplatNine(propertyConfiguration, keyValuePairs);
|
||
if (moved.Count > 0)
|
||
throw new Exception($"House cleaning needed!{Environment.NewLine}{string.Join(Environment.NewLine, moved.Select(l => l.Id))}");
|
||
results = GetSplatNineIdentifiers(propertyConfiguration, bResultsFullGroupDirectory, keyValuePairs);
|
||
}
|
||
return results;
|
||
}
|
||
|
||
private bool? GetIsFocusModel(ExifDirectory? exifDirectory)
|
||
{
|
||
bool? result;
|
||
if (string.IsNullOrEmpty(_Configuration.FocusModel))
|
||
result = null;
|
||
else
|
||
{
|
||
string? model = IMetaBase.GetModel(exifDirectory?.ExifBaseDirectories);
|
||
if (exifDirectory is null || string.IsNullOrEmpty(model))
|
||
result = null;
|
||
else
|
||
result = model.Contains(_Configuration.FocusModel);
|
||
}
|
||
return result;
|
||
}
|
||
|
||
private void LogItemPropertyIsNull(Item item)
|
||
{
|
||
if (!item.SourceDirectoryFileHolder.Exists)
|
||
_Logger?.LogInformation(string.Concat("NoJson <", item.FilePath.FullName, '>'));
|
||
else if (item.FileSizeChanged.HasValue && item.FileSizeChanged.Value)
|
||
_Logger?.LogInformation(string.Concat("FileSizeChanged <", item.FilePath.FullName, '>'));
|
||
else if (item.LastWriteTimeChanged.HasValue && item.LastWriteTimeChanged.Value)
|
||
_Logger?.LogInformation(string.Concat("LastWriteTimeChanged <", item.FilePath.FullName, '>'));
|
||
else if (item.Moved.HasValue && item.Moved.Value)
|
||
_Logger?.LogInformation(string.Concat("Moved <", item.FilePath.FullName, '>'));
|
||
}
|
||
|
||
private string SaveUrlAndGetNewRootDirectory(ReadOnlyCollection<FilePath> filePaths)
|
||
{
|
||
string result;
|
||
if (filePaths.Count == 0)
|
||
throw new NotSupportedException();
|
||
string? sourceDirectory = filePaths[0].DirectoryFullPath;
|
||
if (string.IsNullOrEmpty(sourceDirectory))
|
||
throw new NotSupportedException();
|
||
Uri uri;
|
||
string? line;
|
||
string fileName;
|
||
FilePath filePath;
|
||
Task<byte[]> task;
|
||
string relativePath;
|
||
FileHolder fileHolder;
|
||
string extensionLowered;
|
||
string sourceDirectoryFile;
|
||
HttpClient httpClient = new();
|
||
bool isValidImageFormatExtension;
|
||
result = Path.GetDirectoryName(Path.GetDirectoryName(sourceDirectory)) ?? throw new NotSupportedException();
|
||
int length = result.Length;
|
||
for (int y = 0; y < int.MaxValue; y++)
|
||
{
|
||
_Logger?.LogInformation("Enter fileNameToCollection url for fileNameToCollection image");
|
||
line = _Console.ReadLine();
|
||
if (string.IsNullOrEmpty(line))
|
||
break;
|
||
uri = new(line);
|
||
if (uri.HostNameType != UriHostNameType.Dns)
|
||
continue;
|
||
task = httpClient.GetByteArrayAsync(uri);
|
||
fileName = Path.GetFileName(uri.LocalPath);
|
||
sourceDirectoryFile = Path.Combine(sourceDirectory, fileName);
|
||
File.WriteAllBytes(sourceDirectoryFile, task.Result);
|
||
extensionLowered = Path.GetExtension(uri.LocalPath);
|
||
relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFile, length, forceExtensionToLower: true);
|
||
isValidImageFormatExtension = _Configuration.PropertyConfiguration.ValidImageFormatExtensions.Contains(extensionLowered);
|
||
fileHolder = Shared.Models.Stateless.Methods.IFileHolder.Get(sourceDirectoryFile);
|
||
filePath = FilePath.Get(_Configuration.PropertyConfiguration, fileHolder, index: null);
|
||
_ = Item.Get(filePath, fileHolder, relativePath, isValidImageFormatExtension);
|
||
// container.Items.Add(item);
|
||
}
|
||
_Logger?.LogInformation(". . .");
|
||
return result;
|
||
}
|
||
|
||
private ReadOnlyCollection<FilePath> GetFilePath(long ticks, string dFacesContentDirectory)
|
||
{
|
||
List<FilePath> results = [];
|
||
FilePath filePath;
|
||
FileHolder fileHolder;
|
||
string[] files = Directory.GetFiles(dFacesContentDirectory, $"*{_Faces.FileNameExtension}", SearchOption.AllDirectories);
|
||
long? skipOlderThan = _Configuration.SkipOlderThanDays is null ? null : new DateTime(ticks).AddDays(-_Configuration.SkipOlderThanDays.Value).Ticks;
|
||
foreach (string file in files)
|
||
{
|
||
fileHolder = Shared.Models.Stateless.Methods.IFileHolder.Get(file);
|
||
filePath = FilePath.Get(_Configuration.PropertyConfiguration, fileHolder, index: null);
|
||
if (filePath.Id is null)
|
||
continue;
|
||
if (skipOlderThan is not null && (fileHolder.LastWriteTime is null || fileHolder.LastWriteTime.Value.Ticks < skipOlderThan.Value))
|
||
continue;
|
||
results.Add(filePath);
|
||
}
|
||
return results.AsReadOnly();
|
||
}
|
||
|
||
private void ParallelFor(IDlibDotNet dlibDotNet, FilePair filePair, Dictionary<int, ExifDirectory> results)
|
||
{
|
||
dlibDotNet?.Tick();
|
||
if (filePair.FilePath.Id is null || results.ContainsKey(filePair.FilePath.Id.Value))
|
||
return;
|
||
ExifDirectory? exifDirectory = GetExifDirectory(filePair);
|
||
if (exifDirectory is null)
|
||
return;
|
||
lock (results)
|
||
results.Add(filePair.FilePath.Id.Value, exifDirectory);
|
||
}
|
||
|
||
private bool GetRunToDoCollectionFirst(Models.Configuration configuration, long ticks, string[] checkDirectories)
|
||
{
|
||
bool result = configuration.SaveSortingWithoutPerson;
|
||
if (!result)
|
||
result = !IId.IsOffsetDeterministicHashCode(configuration.PropertyConfiguration);
|
||
if (!result)
|
||
{
|
||
string[] directories;
|
||
directories = Directory.GetDirectories(_Configuration.PropertyConfiguration.RootDirectory, "*", SearchOption.TopDirectoryOnly);
|
||
if (directories.Length == 0)
|
||
result = true;
|
||
else
|
||
{
|
||
string seasonDirectory;
|
||
DirectoryInfo directoryInfo;
|
||
DateTime dateTime = new(ticks);
|
||
string rootDirectory = _Configuration.PropertyConfiguration.RootDirectory;
|
||
string eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), _Configuration.PropertyConfiguration.ResultContent);
|
||
(int season, string seasonName) = Shared.Models.Stateless.Methods.IProperty.GetSeason(dateTime.DayOfYear);
|
||
FileSystemInfo fileSystemInfo = new DirectoryInfo(eDistanceContentDirectory);
|
||
foreach (string checkDirectory in checkDirectories)
|
||
{
|
||
if (!Directory.Exists(checkDirectory))
|
||
result = true;
|
||
if (checkDirectory == rootDirectory)
|
||
seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName}");
|
||
else
|
||
seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName} {Path.GetFileName(checkDirectory)}");
|
||
if (!Directory.Exists(seasonDirectory))
|
||
result = true;
|
||
if (result)
|
||
continue;
|
||
directories = Directory.GetDirectories(checkDirectory, "*", SearchOption.TopDirectoryOnly);
|
||
foreach (string directory in directories)
|
||
{
|
||
directoryInfo = new(directory);
|
||
if (directoryInfo.LastWriteTime > fileSystemInfo.LastWriteTime)
|
||
{
|
||
result = true;
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if (result)
|
||
result = true;
|
||
if (!result)
|
||
result = false;
|
||
return result;
|
||
}
|
||
|
||
private void Search(long ticks, ReadOnlyCollection<PersonContainer> personContainers, string argZero, string propertyRoot)
|
||
{
|
||
string message;
|
||
MapLogic? mapLogic;
|
||
Record? record = null;
|
||
string seasonDirectory;
|
||
A_Property propertyLogic;
|
||
IDlibDotNet dlibDotNet = this;
|
||
DateTime dateTime = new(ticks);
|
||
string eDistanceContentDirectory;
|
||
string? a2PeopleContentDirectory;
|
||
string aResultsFullGroupDirectory;
|
||
string bResultsFullGroupDirectory;
|
||
string cResultsFullGroupDirectory;
|
||
string fPhotoPrismContentDirectory;
|
||
const string fileSearchFilter = "*";
|
||
string fPhotoPrismSingletonDirectory;
|
||
const string directorySearchFilter = "*";
|
||
bool configurationOutputResolutionsHas = false;
|
||
ReadOnlyDictionary<long, List<int>> personKeyToIds;
|
||
(int season, string seasonName) = Shared.Models.Stateless.Methods.IProperty.GetSeason(dateTime.DayOfYear);
|
||
string[] checkDirectories =
|
||
[
|
||
Path.Combine(_Configuration.PropertyConfiguration.RootDirectory, "Ancestry"),
|
||
Path.Combine(_Configuration.PropertyConfiguration.RootDirectory, "Facebook"),
|
||
Path.Combine(_Configuration.PropertyConfiguration.RootDirectory, "LinkedIn")
|
||
];
|
||
bool runToDoCollectionFirst = GetRunToDoCollectionFirst(_Configuration, ticks, checkDirectories);
|
||
(aResultsFullGroupDirectory, bResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories();
|
||
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(_Configuration.PropertyConfiguration.RootDirectory, ticks);
|
||
a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "([])");
|
||
eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), _Configuration.PropertyConfiguration.ResultContent);
|
||
string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), _Configuration.PropertyConfiguration.ResultSingleton);
|
||
_ = Directory.CreateDirectory(Path.Combine(eDistanceContentDirectory, ticks.ToString()));
|
||
B_Metadata metadata = new(dlibDotNet, _Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, ticks, bResultsFullGroupDirectory);
|
||
if (runToDoCollectionFirst)
|
||
mapLogic = null;
|
||
else
|
||
mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _Distance, personContainers, ticks, a2PeopleContentDirectory, a2PeopleSingletonDirectory, eDistanceContentDirectory);
|
||
foreach (string outputResolution in _Configuration.OutputResolutions)
|
||
{
|
||
if (outputResolution.Any(char.IsNumber))
|
||
continue;
|
||
configurationOutputResolutionsHas = true;
|
||
if (!runToDoCollectionFirst)
|
||
break;
|
||
record = GetFilesCollectionThenCopyOrMove(dlibDotNet, ticks, fileSearchFilter, directorySearchFilter, bResultsFullGroupDirectory, outputResolution);
|
||
break;
|
||
}
|
||
fPhotoPrismContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), _Configuration.PropertyConfiguration.ResultContent);
|
||
fPhotoPrismSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(F_PhotoPrism), _Configuration.PropertyConfiguration.ResultSingleton);
|
||
propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory);
|
||
if (record is not null && record.FilesCollectionCountIsOne)
|
||
{
|
||
if (record.FilePathsCollection is null)
|
||
throw new NullReferenceException(nameof(record.FilePathsCollection));
|
||
string resultsGroupDirectory;
|
||
a2PeopleContentDirectory = null;
|
||
eDistanceContentDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", _Configuration.PropertyConfiguration.ResultContent);
|
||
fPhotoPrismContentDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", _Configuration.PropertyConfiguration.ResultContent);
|
||
fPhotoPrismSingletonDirectory = Path.Combine($"{Path.GetPathRoot(argZero)}", _Configuration.PropertyConfiguration.ResultSingleton);
|
||
for (int i = 1; i < 10; i++)
|
||
{
|
||
resultsGroupDirectory = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, string.Empty, create: true);
|
||
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(resultsGroupDirectory);
|
||
}
|
||
argZero = SaveUrlAndGetNewRootDirectory(record.FilePathsCollection.First());
|
||
_Configuration.PropertyConfiguration.ChangeRootDirectory(argZero);
|
||
(aResultsFullGroupDirectory, bResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories();
|
||
propertyRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(_Configuration.PropertyConfiguration, nameof(A_Property), create: false);
|
||
propertyLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _Resize.FileNameExtension, _Configuration.Reverse, aResultsFullGroupDirectory);
|
||
}
|
||
if (configurationOutputResolutionsHas)
|
||
{
|
||
foreach (string outputResolution in _Configuration.OutputResolutions)
|
||
{
|
||
if (outputResolution.Any(char.IsNumber))
|
||
continue;
|
||
Dictionary<int, ExifDirectory> exifDirectoriesById = [];
|
||
(cResultsFullGroupDirectory, _, _, _) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
|
||
string? filesCollectionRootDirectory = Path.Combine(cResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
|
||
ReadOnlyCollection<ReadOnlyCollection<FilePath>>? filePathsCollection = IDirectory.GetFilePathCollections(_Configuration.PropertyConfiguration, directorySearchFilter, fileSearchFilter, filesCollectionRootDirectory, useIgnoreExtensions: true, useCeilingAverage: true);
|
||
record = new(FilesCollectionRootDirectory: filesCollectionRootDirectory,
|
||
FilesCollectionCountIsOne: false,
|
||
FilePathsCollection: filePathsCollection,
|
||
ExifDirectoriesById: new(exifDirectoriesById),
|
||
IdToFilePaths: null,
|
||
SplatNineIdentifiers: null);
|
||
break;
|
||
}
|
||
}
|
||
if (string.IsNullOrEmpty(record?.FilesCollectionRootDirectory) || record.FilePathsCollection.Count == 0)
|
||
throw new NullReferenceException(nameof(record.FilePathsCollection));
|
||
foreach (string checkDirectory in checkDirectories)
|
||
{
|
||
seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName} {Path.GetFileName(checkDirectory)}");
|
||
if (!Directory.Exists(seasonDirectory))
|
||
_ = Directory.CreateDirectory(seasonDirectory);
|
||
}
|
||
int count = record.FilePathsCollection.Select(l => l.Count).Sum();
|
||
message = $") Building Container(s) - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
|
||
_ProgressBar = new(count, message, _ProgressBarOptions);
|
||
ReadOnlyCollection<Container.Models.Container> readOnlyContainers =
|
||
Container.Models.Stateless.Methods.IContainer.GetContainers(dlibDotNet,
|
||
_Configuration.PropertyConfiguration,
|
||
_Faces.FileNameExtension,
|
||
_Faces.HiddenFileNameExtension,
|
||
eDistanceContentDirectory,
|
||
record.FilesCollectionRootDirectory,
|
||
record.IdToFilePaths,
|
||
record.SplatNineIdentifiers,
|
||
record.FilePathsCollection,
|
||
record.ExifDirectoriesById);
|
||
_ProgressBar.Dispose();
|
||
mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, _Distance, personContainers, ticks, a2PeopleContentDirectory, a2PeopleSingletonDirectory, eDistanceContentDirectory);
|
||
DeleteContinueFiles(personContainers);
|
||
if (!runToDoCollectionFirst)
|
||
MapFaceFileLogic(ticks, personContainers, mapLogic, a2PeopleContentDirectory, eDistanceContentDirectory);
|
||
FullDoWork(argZero,
|
||
propertyRoot,
|
||
ticks,
|
||
fPhotoPrismSingletonDirectory,
|
||
count,
|
||
metadata,
|
||
record,
|
||
readOnlyContainers,
|
||
mapLogic);
|
||
ReadOnlyCollection<Item> distinctValidImageItems = Container.Models.Stateless.Methods.IContainer.GetValidImageItems(_Configuration.PropertyConfiguration, readOnlyContainers, distinctItems: true, filterItems: true);
|
||
if (_Configuration.LookForAbandoned)
|
||
{
|
||
string dResultsFullGroupDirectory;
|
||
string d2ResultsFullGroupDirectory;
|
||
foreach (string outputResolution in _Configuration.OutputResolutions)
|
||
{
|
||
_ProgressBar = new(5, nameof(mapLogic.LookForAbandoned), _ProgressBarOptions);
|
||
(cResultsFullGroupDirectory, _, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
|
||
mapLogic.LookForAbandoned(dlibDotNet, _Configuration.PropertyConfiguration, bResultsFullGroupDirectory, readOnlyContainers, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory);
|
||
_ProgressBar.Dispose();
|
||
}
|
||
}
|
||
_Distance.Clear();
|
||
ReadOnlyCollection<Shared.Models.Face> distinctValidImageFaces = Map.Models.Stateless.Methods.IMapLogic.GetFaces(distinctValidImageItems);
|
||
ReadOnlyCollection<Mapping> distinctValidImageMappingCollection = GetMappings(_Configuration.PropertyConfiguration, eDistanceContentDirectory, readOnlyContainers, mapLogic, distinctItems: true);
|
||
if (runToDoCollectionFirst)
|
||
{
|
||
if (!Directory.Exists(eDistanceContentDirectory))
|
||
_ = Directory.CreateDirectory(eDistanceContentDirectory);
|
||
string json = JsonSerializer.Serialize(distinctValidImageMappingCollection);
|
||
File.WriteAllText(Path.Combine(eDistanceContentDirectory, $"{ticks}.json"), json);
|
||
}
|
||
foreach (string outputResolution in _Configuration.OutputResolutions)
|
||
{
|
||
if (_PropertyRootExistedBefore)
|
||
break;
|
||
personKeyToIds = mapLogic.GetPersonKeyToIds();
|
||
if (_Configuration.SavePropertyShortcutsForOutputResolutions.Contains(outputResolution))
|
||
SavePropertyShortcutsForOutputResolutions(eDistanceContentDirectory, distinctValidImageItems);
|
||
if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
|
||
mapLogic.SaveShortcutsForOutputResolutionsPreMapLogic(eDistanceContentDirectory, personKeyToIds, distinctValidImageMappingCollection);
|
||
if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.JLinks.Where(l => !string.IsNullOrEmpty(l)).Any() && _Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution))
|
||
mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, personContainers, a2PeopleContentDirectory, personKeyToIds, distinctValidImageMappingCollection);
|
||
if (_ArgZeroIsConfigurationRootDirectory
|
||
&& _Configuration.SaveResizedSubfiles
|
||
&& outputResolution == _Configuration.OutputResolutions[0]
|
||
&& _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
|
||
&& _Exceptions.Count == 0)
|
||
MapLogic(ticks, readOnlyContainers, fPhotoPrismContentDirectory, mapLogic, outputResolution, new(personKeyToIds), distinctValidImageFaces, distinctValidImageMappingCollection);
|
||
if (runToDoCollectionFirst && _Configuration.SaveRandomForOutputResolutions.Contains(outputResolution) && personKeyToIds.Count > 0 && record?.SplatNineIdentifiers is not null && distinctValidImageMappingCollection.Count > 0)
|
||
_Random.Random(_Configuration.PropertyConfiguration, _Configuration.ImmichAssetsFile, _Configuration.ImmichOwnerId, _Configuration.ImmichRoot, _Configuration.RadomUseBirthdayMinimum, _Configuration.ValidKeyWordsToIgnoreInRandom, personKeyToIds, record.SplatNineIdentifiers, distinctValidImageMappingCollection);
|
||
if (_IsEnvironment.Development)
|
||
continue;
|
||
if (!_IsEnvironment.Development)
|
||
{
|
||
string dResultsFullGroupDirectory;
|
||
string c2ResultsFullGroupDirectory;
|
||
string d2ResultsFullGroupDirectory;
|
||
(cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
|
||
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(aResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton));
|
||
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(bResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton));
|
||
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(cResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton));
|
||
if (_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
|
||
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultCollection));
|
||
}
|
||
}
|
||
}
|
||
|
||
private Record GetFilesCollectionThenCopyOrMove(IDlibDotNet dlibDotNet, long ticks, string fileSearchFilter, string directorySearchFilter, string bResultsFullGroupDirectory, string outputResolution)
|
||
{
|
||
Record result;
|
||
int count;
|
||
string message;
|
||
ProgressBar progressBar;
|
||
const string extension = ".json";
|
||
ReadOnlyCollection<FilePair> filePairs;
|
||
int maxDegreeOfParallelism = Environment.ProcessorCount;
|
||
Dictionary<int, ExifDirectory> exifDirectoriesById = [];
|
||
string filesCollectionRootDirectory = _Configuration.PropertyConfiguration.RootDirectory;
|
||
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
|
||
(string cResultsFullGroupDirectory, _, _, _) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
|
||
string jsonGroupDirectory = Path.Combine(bResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultSingleton);
|
||
if (!Directory.Exists(jsonGroupDirectory))
|
||
_ = Directory.CreateDirectory(jsonGroupDirectory);
|
||
IReadOnlyDictionary<string, ReadOnlyDictionary<byte, ReadOnlyCollection<string>>> keyValuePairs =
|
||
Shared.Models.Stateless.Methods.IPath.GetKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, [_Configuration.PropertyConfiguration.ResultContent]);
|
||
ReadOnlyDictionary<byte, ReadOnlyCollection<string>> fileGroups = keyValuePairs[_Configuration.PropertyConfiguration.ResultContent];
|
||
ReadOnlyCollection<ReadOnlyCollection<FilePath>> filePathsCollection = IDirectory.GetFilePathCollections(_Configuration.PropertyConfiguration, directorySearchFilter, fileSearchFilter, filesCollectionRootDirectory, useIgnoreExtensions: true, useCeilingAverage: false);
|
||
count = filePathsCollection.Select(l => l.Count).Sum();
|
||
filePairs = IFilePair.GetFilePairs(_Configuration.PropertyConfiguration, directorySearchFilter, extension, jsonGroupDirectory, filePathsCollection);
|
||
message = $") Preloading ExifDirectory Dictionary - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
|
||
progressBar = new(count, message, _ProgressBarOptions);
|
||
_ = Parallel.For(0, filePairs.Count, parallelOptions, (i, state) => ParallelFor(dlibDotNet, filePairs[i], exifDirectoriesById));
|
||
progressBar.Dispose();
|
||
count = filePathsCollection.Select(l => l.Count).Sum();
|
||
bool filesCollectionCountIsOne = GetFilesCollectionCountIsOne(filePathsCollection);
|
||
message = $") Selecting for ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
|
||
progressBar = new(count, message, _ProgressBarOptions);
|
||
(string[] distinctDirectories, List<(FilePath, string)> toDoCollection) = IDirectory.GetToDoCollection(_Configuration.PropertyConfiguration, filePathsCollection, fileGroups, exifDirectoriesById, () => progressBar.Tick());
|
||
progressBar.Dispose();
|
||
foreach (string distinctDirectory in distinctDirectories)
|
||
{
|
||
if (!Directory.Exists(distinctDirectory))
|
||
_ = Directory.CreateDirectory(distinctDirectory);
|
||
}
|
||
message = $") Copying to ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
|
||
progressBar = new(count, message, _ProgressBarOptions);
|
||
_ = IDirectory.CopyOrMove(toDoCollection, move: false, moveBack: false, () => progressBar.Tick());
|
||
progressBar.Dispose();
|
||
ReadOnlyDictionary<int, ReadOnlyCollection<FilePath>> idToFilePaths = FilePath.GetKeyValuePairs(filePathsCollection);
|
||
ReadOnlyDictionary<int, Identifier> splatNineIdentifiers = GetSplatNineIdentifiersAndHideSplatNine(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, idToFilePaths);
|
||
result = new(ExifDirectoriesById: new(exifDirectoriesById),
|
||
FilesCollectionRootDirectory: filesCollectionRootDirectory,
|
||
FilesCollectionCountIsOne: filesCollectionCountIsOne,
|
||
FilePathsCollection: filePathsCollection,
|
||
IdToFilePaths: idToFilePaths,
|
||
SplatNineIdentifiers: splatNineIdentifiers);
|
||
return result;
|
||
}
|
||
|
||
private void MapFaceFileLogic(long ticks, ReadOnlyCollection<PersonContainer> personContainers, MapLogic mapLogic, string? a2PeopleContentDirectory, string eDistanceContentDirectory)
|
||
{
|
||
foreach (string outputResolution in _Configuration.OutputResolutions)
|
||
{
|
||
if (_Exceptions.Count != 0)
|
||
break;
|
||
if (!_Configuration.SaveResizedSubfiles)
|
||
continue;
|
||
if (!_ArgZeroIsConfigurationRootDirectory)
|
||
continue;
|
||
if (outputResolution != _Configuration.OutputResolutions[0])
|
||
continue;
|
||
if (_PropertyRootExistedBefore || a2PeopleContentDirectory is null)
|
||
break;
|
||
if (!_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution))
|
||
continue;
|
||
if (!_Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution))
|
||
continue;
|
||
List<SaveContainer> saveContainers = GetSaveContainers(ticks, personContainers, a2PeopleContentDirectory, eDistanceContentDirectory, mapLogic, outputResolution);
|
||
if (saveContainers.Count > 0)
|
||
{
|
||
int updated = 0;
|
||
mapLogic.SaveContainers(updated, saveContainers);
|
||
throw new NotSupportedException("Done Restart! :)");
|
||
}
|
||
}
|
||
}
|
||
|
||
private List<SaveContainer> GetSaveContainers(long ticks, ReadOnlyCollection<PersonContainer> personContainers, string a2PeopleSingletonDirectory, string eDistanceContentDirectory, MapLogic mapLogic, string outputResolution)
|
||
{
|
||
List<SaveContainer> results;
|
||
IDlibDotNet dlibDotNet = this;
|
||
long[] jLinkResolvedPersonKeys = _JLinkResolvedDirectories.Select(l => l.PersonKey).ToArray();
|
||
(string cResultsFullGroupDirectory, string _, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
|
||
string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
|
||
ReadOnlyDictionary<int, ReadOnlyDictionary<int, LocationContainer>> mapped = Map.Models.Stateless.Methods.IMapLogic.GetMapped(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, personContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory);
|
||
if (mapped.Count == 0 && !_Configuration.SaveSortingWithoutPerson)
|
||
throw new NotSupportedException($"Switch {nameof(_Configuration.SaveSortingWithoutPerson)}!");
|
||
ReadOnlyCollection<FilePath> filePaths = GetFilePath(ticks, dFacesContentDirectory);
|
||
List<LocationContainer> available = Map.Models.Stateless.Methods.IMapLogic.GetAvailable(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, _Faces, ticks, filePaths);
|
||
if (!string.IsNullOrEmpty(_Configuration.FocusDirectory) && _Configuration.FocusDirectory.Length != 2)
|
||
throw new NotSupportedException($"{nameof(_Configuration.FocusDirectory)} currently only works with output directory! Example 00.");
|
||
ReadOnlyDictionary<int, ReadOnlyDictionary<int, LocationContainer>> mappedWithEncoding = E_Distance.GetMappedWithEncoding(mapped);
|
||
if (mappedWithEncoding.Count == 0 && !_Configuration.SaveSortingWithoutPerson)
|
||
throw new NotSupportedException($"Switch {nameof(_Configuration.SaveSortingWithoutPerson)}!");
|
||
List<LocationContainer> preFiltered = E_Distance.GetPreFilterLocationContainer(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, _Configuration.FocusDirectory, _Configuration.FocusModel, _Configuration.SkipPersonWithMoreThen, ticks, mapLogic, jLinkResolvedPersonKeys, mapped, available);
|
||
if (preFiltered.Count == 0)
|
||
results = [];
|
||
else
|
||
{
|
||
DistanceLimits distanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh);
|
||
List<LocationContainer> postFiltered = E_Distance.GetPostFilterLocationContainer(mapLogic, preFiltered, distanceLimits);
|
||
if (postFiltered.Count == 0)
|
||
results = [];
|
||
else
|
||
{
|
||
string message = $") Building Matrix - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
|
||
_ProgressBar = new(postFiltered.Count, message, _ProgressBarOptions);
|
||
ReadOnlyCollection<LocationContainer> matrix = E_Distance.GetMatrixLocationContainers(dlibDotNet, _MapConfiguration, ticks, mapLogic, mappedWithEncoding, preFiltered, distanceLimits, postFiltered);
|
||
_ProgressBar.Dispose();
|
||
ReadOnlyDictionary<string, LocationContainer> onlyOne = GetOnlyOne(distanceLimits, matrix);
|
||
if (onlyOne.Count == 0)
|
||
results = [];
|
||
else
|
||
{
|
||
ReadOnlyDictionary<long, PersonContainer> personKeyToPersonContainer = PersonContainer.Extract(personContainers);
|
||
results = mapLogic.GetSaveContainers(cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, distanceLimits, onlyOne, personKeyToPersonContainer);
|
||
}
|
||
}
|
||
}
|
||
return results;
|
||
}
|
||
|
||
private ReadOnlyCollection<Mapping> GetMappings(Property.Models.Configuration propertyConfiguration, string eDistanceContentDirectory, ReadOnlyCollection<Container.Models.Container> readOnlyContainers, MapLogic mapLogic, bool distinctItems)
|
||
{
|
||
List<Mapping> results = [];
|
||
int count = 0;
|
||
int notMapped;
|
||
Mapping mapping;
|
||
bool anyValidFaces;
|
||
List<int> distinct = [];
|
||
string focusRelativePath;
|
||
bool? isFocusRelativePath;
|
||
DateTime[] containerDateTimes;
|
||
MappingFromItem mappingFromItem;
|
||
ReadOnlyCollection<Item> filteredItems;
|
||
foreach (Container.Models.Container container in readOnlyContainers)
|
||
{
|
||
if (container.Items.Count == 0)
|
||
continue;
|
||
filteredItems = Container.Models.Stateless.Methods.IContainer.GetValidImageItems(propertyConfiguration, container);
|
||
if (filteredItems.Count == 0)
|
||
continue;
|
||
containerDateTimes = Container.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems);
|
||
focusRelativePath = Path.GetFullPath(string.Concat(_Configuration.PropertyConfiguration.RootDirectory, _Configuration.FocusDirectory));
|
||
isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath);
|
||
foreach (Item item in filteredItems)
|
||
{
|
||
if (item.ExifDirectory?.FilePath.Id is null || item.ResizedFileHolder is null)
|
||
continue;
|
||
mappingFromItem = IMappingFromItem.GetMappingFromItem(containerDateTimes, item, item.ResizedFileHolder);
|
||
if (distinctItems)
|
||
{
|
||
if (distinct.Contains(item.ExifDirectory.FilePath.Id.Value))
|
||
continue;
|
||
distinct.Add(item.ExifDirectory.FilePath.Id.Value);
|
||
}
|
||
count++;
|
||
anyValidFaces = false;
|
||
foreach (Shared.Models.Face face in item.Faces)
|
||
{
|
||
if (face.Mapping is null)
|
||
throw new NotSupportedException();
|
||
if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
|
||
continue;
|
||
anyValidFaces = true;
|
||
results.Add(face.Mapping);
|
||
}
|
||
if (!anyValidFaces)
|
||
{
|
||
(mapping, notMapped) = GetMappingAndUpdateMappingFromPerson(mapLogic, item, isFocusRelativePath, mappingFromItem);
|
||
results.Add(mapping);
|
||
}
|
||
}
|
||
}
|
||
if (_Configuration.MoveToDecade && _Configuration.LocationContainerDistanceTolerance is null)
|
||
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(eDistanceContentDirectory);
|
||
return results.AsReadOnly();
|
||
}
|
||
|
||
private (Mapping, int) GetMappingAndUpdateMappingFromPerson(MapLogic mapLogic, Item item, bool? isFocusRelativePath, MappingFromItem mappingFromItem)
|
||
{
|
||
Mapping result;
|
||
int? eyeα = null;
|
||
bool? eyeReview = null;
|
||
int confidencePercent = 0;
|
||
int faceAreaPermyriad = 0;
|
||
bool? isFocusModel = GetIsFocusModel(item.ExifDirectory);
|
||
long[] jLinkResolvedPersonKeys = _JLinkResolvedDirectories.Select(l => l.PersonKey).ToArray();
|
||
int wholePercentRectangle = Shared.Models.Stateless.Methods.ILocation.GetWholePercentages(Shared.Models.Stateless.ILocation.Digits);
|
||
string deterministicHashCodeKey = IMapping.GetDeterministicHashCodeKey(item.FilePath, Shared.Models.Stateless.ILocation.Digits);
|
||
MappingFromLocation? mappingFromLocation = new(faceAreaPermyriad, confidencePercent, deterministicHashCodeKey, eyeα, eyeReview, wholePercentRectangle);
|
||
bool? inSkipCollection = mapLogic.InSkipCollection(mappingFromItem.Id, mappingFromLocation);
|
||
MappingFromFilterPre mappingFromFilterPre = new(inSkipCollection, isFocusModel, isFocusRelativePath);
|
||
ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>? wholePercentagesToPersonContainers = mapLogic.GetWholePercentagesToPersonContainers(mappingFromItem.Id);
|
||
bool? canReMap = Map.Models.Stateless.Methods.IMapLogic.CanReMap(jLinkResolvedPersonKeys, wholePercentagesToPersonContainers, mappingFromLocation);
|
||
bool? isFocusPerson = mapLogic.IsFocusPerson(_Configuration.SkipPersonWithMoreThen, jLinkResolvedPersonKeys, wholePercentagesToPersonContainers, mappingFromLocation);
|
||
MappingFromFilterPost mappingFromFilterPost = new(canReMap, inSkipCollection, isFocusPerson);
|
||
result = Mapping.Get(item.FilePath, mappingFromFilterPost, mappingFromFilterPre, mappingFromItem, mappingFromLocation, mappingFromPhotoPrismCollection: null);
|
||
int notMapped = mapLogic.UpdateMappingFromPerson(wholePercentagesToPersonContainers, result);
|
||
return (result, notMapped);
|
||
}
|
||
|
||
private void MapLogic(long ticks, ReadOnlyCollection<Container.Models.Container> containers, string fPhotoPrismContentDirectory, MapLogic mapLogic, string outputResolution, ReadOnlyDictionary<long, List<int>> personKeyToIds, ReadOnlyCollection<Shared.Models.Face> distinctValidImageFaces, ReadOnlyCollection<Mapping> distinctValidImageMappingCollection)
|
||
{
|
||
IDlibDotNet dlibDotNet = this;
|
||
(_, _, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory) = dlibDotNet.GetResultsFullGroupDirectories(outputResolution);
|
||
string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
|
||
string d2FacePartsContentDirectory = Path.Combine(d2ResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
|
||
string d2FacePartsContentCollectionDirectory = Path.Combine(d2ResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContentCollection);
|
||
if (distinctValidImageMappingCollection.Count > 0)
|
||
{
|
||
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(d2FacePartsContentDirectory, ticks);
|
||
if (Directory.Exists(d2FacePartsContentCollectionDirectory))
|
||
Shared.Models.Stateless.Methods.IPath.MakeHiddenIfAllItemsAreHidden(d2FacePartsContentCollectionDirectory);
|
||
}
|
||
if (Directory.Exists(fPhotoPrismContentDirectory))
|
||
F_PhotoPrism.WriteMatches(fPhotoPrismContentDirectory, _Configuration.PersonBirthdayFormat, _Configuration.RectangleIntersectMinimums, ticks, distinctValidImageFaces, mapLogic);
|
||
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
|
||
mapLogic.SaveShortcutsForOutputResolutionsDuringMapLogic(containers, personKeyToIds, dFacesContentDirectory, distinctValidImageMappingCollection);
|
||
ReadOnlyDictionary<int, ReadOnlyDictionary<int, Mapping>> idToWholePercentagesToMapping = Map.Models.Stateless.Methods.IMapLogic.GetIdToWholePercentagesToFace(distinctValidImageMappingCollection);
|
||
if (_Configuration.SaveMappedForOutputResolutions.Contains(outputResolution))
|
||
mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, personKeyToIds, distinctValidImageMappingCollection, idToWholePercentagesToMapping);
|
||
if (_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution))
|
||
SaveFaceDistances(ticks, mapLogic, distinctValidImageFaces, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping);
|
||
}
|
||
|
||
private void SaveFaceDistances(long ticks, MapLogic mapLogic, ReadOnlyCollection<Shared.Models.Face> distinctValidImageFaces, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, ReadOnlyDictionary<int, ReadOnlyDictionary<int, Mapping>> idToWholePercentagesToMapping)
|
||
{
|
||
E_Distance.PreFilterSetFaceDistances(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, distinctValidImageFaces);
|
||
ReadOnlyCollection<FaceDistanceContainer> faceDistanceContainers = E_Distance.GetFaceDistanceContainers(distinctValidImageFaces);
|
||
if (faceDistanceContainers.Count > 0)
|
||
{
|
||
List<FaceDistance> faceDistanceEncodings = [];
|
||
foreach (FaceDistanceContainer faceDistanceContainer in faceDistanceContainers)
|
||
{
|
||
if (faceDistanceContainer.FaceDistance.Encoding is null)
|
||
continue;
|
||
faceDistanceEncodings.Add(faceDistanceContainer.FaceDistance);
|
||
}
|
||
SaveFaceDistances(ticks, mapLogic, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, new(faceDistanceEncodings), faceDistanceContainers);
|
||
}
|
||
}
|
||
|
||
private void SaveFaceDistances(long ticks, MapLogic mapLogic, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, ReadOnlyDictionary<int, ReadOnlyDictionary<int, Mapping>> idToWholePercentagesToMapping, ReadOnlyCollection<FaceDistance> faceDistanceEncodings, ReadOnlyCollection<FaceDistanceContainer> faceDistanceContainers)
|
||
{
|
||
int? useFiltersCounter = null;
|
||
DistanceLimits distanceLimits;
|
||
ReadOnlyCollection<SortingContainer> sortingContainers;
|
||
FaceDistanceContainer[] filteredFaceDistanceContainers;
|
||
long? skipOlderThan = _Configuration.SkipOlderThanDays is null ? null : new DateTime(ticks).AddDays(-_Configuration.SkipOlderThanDays.Value).Ticks;
|
||
distanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh);
|
||
filteredFaceDistanceContainers = E_Distance.FilteredPostLoadFaceDistanceContainers(mapLogic, faceDistanceContainers, skipOlderThan, distanceLimits);
|
||
if (filteredFaceDistanceContainers.Length == 0)
|
||
_Logger?.LogInformation("All images have been filtered!");
|
||
else
|
||
{
|
||
sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortedSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distanceLimits, faceDistanceEncodings, filteredFaceDistanceContainers);
|
||
if (sortingContainers.Count == 0)
|
||
{
|
||
for (useFiltersCounter = 1; useFiltersCounter < _Configuration.UseFilterTries; useFiltersCounter++)
|
||
{
|
||
distanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh, useFiltersCounter);
|
||
filteredFaceDistanceContainers = E_Distance.FilteredPostLoadFaceDistanceContainers(mapLogic, faceDistanceContainers, skipOlderThan, distanceLimits);
|
||
if (filteredFaceDistanceContainers.Length == 0)
|
||
_Logger?.LogInformation("All images have been filtered!");
|
||
else
|
||
{
|
||
sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortedSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distanceLimits, faceDistanceEncodings, filteredFaceDistanceContainers);
|
||
if (sortingContainers.Count == 0)
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
sortingContainers = mapLogic.GetFilterSortingContainers(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, distanceLimits, sortingContainers);
|
||
if (sortingContainers.Count > 0)
|
||
E_Distance.SaveFaceDistances(_Configuration.PropertyConfiguration, sortingContainers);
|
||
if (filteredFaceDistanceContainers.Length > 0)
|
||
{
|
||
int updated = sortingContainers.Count == 0 ? 0 : mapLogic.UpdateFromSortingContainers(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, sortingContainers);
|
||
List<SaveContainer> saveContainers = mapLogic.GetSaveContainers(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, distanceLimits, useFiltersCounter, sortingContainers);
|
||
if (saveContainers.Count > 0)
|
||
mapLogic.SaveContainers(updated, saveContainers);
|
||
}
|
||
}
|
||
}
|
||
|
||
private void VerifyExtra(List<string> args, Property.Models.Configuration propertyConfiguration, Models.Configuration configuration)
|
||
{
|
||
string[] sourceDirectoryNames;
|
||
if (args.Count == 0)
|
||
sourceDirectoryNames = [];
|
||
else
|
||
{
|
||
string? century;
|
||
string argZero = Path.GetFullPath(args[0]);
|
||
sourceDirectoryNames = argZero.Split(Path.DirectorySeparatorChar);
|
||
if (!argZero.StartsWith(propertyConfiguration.RootDirectory))
|
||
throw new Exception($"Source directory must be inside root directory! <{argZero}> <{propertyConfiguration.RootDirectory}>");
|
||
if (_ArgZeroIsConfigurationRootDirectory && propertyConfiguration.RootDirectory != argZero)
|
||
{
|
||
if (!configuration.MixedYearRelativePaths.Contains(sourceDirectoryNames[0]))
|
||
{
|
||
string[] segments = sourceDirectoryNames[0].Split(' ');
|
||
century = segments[^1].Length == 4 ? segments[^1][..2] : null;
|
||
if (segments.Length < 2 || century is null || (century != "18" && century != "19" && century != "20"))
|
||
throw new Exception("root subdirectory must have fileNameToCollection year at the end or directory name needs to be added to the exclude list!");
|
||
}
|
||
}
|
||
}
|
||
string[] resizeMatch = (from l in sourceDirectoryNames where configuration.ValidResolutions.Contains(l) select l).ToArray();
|
||
if (resizeMatch.Length > 0)
|
||
throw new Exception("Input directory should be the source and not fileNameToCollection resized directory!");
|
||
if (configuration.LocationDigits != Shared.Models.Stateless.ILocation.Digits)
|
||
throw new Exception("Configuration has to match interface!");
|
||
if (configuration.LocationFactor != Shared.Models.Stateless.ILocation.Factor)
|
||
throw new Exception("Configuration has to match interface!");
|
||
if (configuration.SaveSortingWithoutPerson && configuration.JLinks.Length > 0)
|
||
throw new Exception("Configuration has SaveSortingWithoutPerson and JLinks!");
|
||
if (configuration.SaveSortingWithoutPerson && !string.IsNullOrEmpty(configuration.FocusModel))
|
||
throw new Exception("Configuration has SaveSortingWithoutPerson and FocusModel!");
|
||
if (configuration.SaveSortingWithoutPerson && !string.IsNullOrEmpty(configuration.FocusDirectory))
|
||
throw new Exception("Configuration has SaveSortingWithoutPerson and FocusDirectory!");
|
||
}
|
||
|
||
private int GetNotMappedCountAndUpdateMappingFromPersonThenSetMapping(MapLogic mapLogic, Item item, bool? isFocusRelativePath, MappingFromItem mappingFromItem, List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection, List<Shared.Models.Face> faces)
|
||
{
|
||
int result;
|
||
double? α;
|
||
int? eyeα;
|
||
bool? canReMap;
|
||
bool? eyeReview;
|
||
Mapping mapping;
|
||
int notMapped = 0;
|
||
bool? isFocusPerson;
|
||
int confidencePercent;
|
||
int faceAreaPermyriad;
|
||
bool? inSkipCollection;
|
||
int wholePercentRectangle;
|
||
string deterministicHashCodeKey;
|
||
MappingFromLocation? mappingFromLocation;
|
||
MappingFromFilterPre mappingFromFilterPre;
|
||
MappingFromFilterPost mappingFromFilterPost;
|
||
bool? isFocusModel = GetIsFocusModel(item.ExifDirectory);
|
||
ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>? wholePercentagesToPersonContainers = mapLogic.GetWholePercentagesToPersonContainers(item.ExifDirectory?.FilePath.Id);
|
||
long[] jLinkResolvedPersonKeys = _JLinkResolvedDirectories.Select(l => l.PersonKey).ToArray();
|
||
foreach (Shared.Models.Face face in faces)
|
||
{
|
||
if (item.ExifDirectory?.FilePath.Id is null || face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
|
||
{
|
||
canReMap = null;
|
||
isFocusPerson = null;
|
||
inSkipCollection = null;
|
||
mappingFromLocation = null;
|
||
mappingFromFilterPost = new(canReMap, inSkipCollection, isFocusPerson);
|
||
mappingFromFilterPre = new(inSkipCollection, isFocusModel, isFocusRelativePath);
|
||
}
|
||
else
|
||
{
|
||
if (face.FaceParts is null)
|
||
(eyeα, eyeReview) = (null, null);
|
||
else
|
||
{
|
||
(eyeReview, α) = Shared.Models.Stateless.Methods.IFace.GetEyeα(face.FaceParts);
|
||
eyeα = α is null ? null : (int)Math.Round(Math.Abs(α.Value));
|
||
}
|
||
confidencePercent = Shared.Models.Stateless.Methods.ILocation.GetConfidencePercent(_Configuration.FaceConfidencePercent, face.Location.Confidence);
|
||
faceAreaPermyriad = IMapping.GetAreaPermyriad(_Configuration.FaceAreaPermyriad, face.Location, face.OutputResolution);
|
||
wholePercentRectangle = Shared.Models.Stateless.Methods.ILocation.GetWholePercentages(face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution);
|
||
deterministicHashCodeKey = IMapping.GetDeterministicHashCodeKey(item.FilePath, face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution);
|
||
mappingFromLocation = new(faceAreaPermyriad, confidencePercent, deterministicHashCodeKey, eyeα, eyeReview, wholePercentRectangle);
|
||
inSkipCollection = mapLogic.InSkipCollection(item.ExifDirectory.FilePath.Id.Value, mappingFromLocation);
|
||
mappingFromFilterPre = new(inSkipCollection, isFocusModel, isFocusRelativePath);
|
||
canReMap = Map.Models.Stateless.Methods.IMapLogic.CanReMap(jLinkResolvedPersonKeys, wholePercentagesToPersonContainers, mappingFromLocation);
|
||
isFocusPerson = mapLogic.IsFocusPerson(_Configuration.SkipPersonWithMoreThen, jLinkResolvedPersonKeys, wholePercentagesToPersonContainers, mappingFromLocation);
|
||
mappingFromFilterPost = new(canReMap, inSkipCollection, isFocusPerson);
|
||
}
|
||
mapping = Mapping.Get(item.FilePath, mappingFromFilterPost, mappingFromFilterPre, mappingFromItem, mappingFromLocation, mappingFromPhotoPrismCollection);
|
||
notMapped += mapLogic.UpdateMappingFromPerson(wholePercentagesToPersonContainers, mapping);
|
||
face.SetMapping(mapping);
|
||
}
|
||
result = notMapped;
|
||
return result;
|
||
}
|
||
|
||
} |