phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PhotoPrism for more locations

Browse Source
This commit is contained in:
Mike Phares
2022-12-25 13:54:17 -07:00
parent a510019c1d
commit 37c7b6760d
31 changed files with 395 additions and 509 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
Instance/DlibDotNet.cs
View File

@ -33,9 +33,11 @@ public partial class DlibDotNet
private readonly List<string> _Exceptions;
private readonly IsEnvironment _IsEnvironment;
private readonly bool _PropertyRootExistedBefore;
private readonly PersonContainer[] _PersonContainers;
private readonly Models.Configuration _Configuration;
private readonly bool _ArgZeroIsConfigurationRootDirectory;
private readonly Map.Models.Configuration _MapConfiguration;
private readonly Dictionary<string, List<MappingFromPhotoPrism>> _FileNameToCollection;
public DlibDotNet(
List<string> args,
@ -46,13 +48,13 @@ public partial class DlibDotNet
bool isSilent,
IConsole console)
{
_Console = console;
string message;
_Console = console;
_AppSettings = appSettings;
_FileNameToCollection = new();
_IsEnvironment = isEnvironment;
long ticks = DateTime.Now.Ticks;
_Exceptions = new List<string>();
PersonContainer[] personContainers;
_Log = Serilog.Log.ForContext<DlibDotNet>();
Property.Models.Configuration propertyConfiguration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot);
Models.Configuration configuration = Models.Binder.Configuration.Get(isEnvironment, configurationRoot, propertyConfiguration);
@ -94,7 +96,6 @@ public partial class DlibDotNet
configuration.ModelDirectory,
configuration.ModelName,
configuration.OverrideForFaceImages,
configuration.RetryImagesWithoutAFaceLocation,
configuration.PredictorModelName,
configuration.PropertiesChangedForFaces);
}
@ -106,7 +107,7 @@ public partial class DlibDotNet
_MapConfiguration = Get(configuration, _Faces.FileNameExtension, _Faces.HiddenFileNameExtension, _FaceParts.FileNameExtension);
_Distance = new(configuration.DistanceMoveUnableToMatch, configuration.DistanceRenameToMatch, _Configuration.FaceConfidencePercent, configuration.RangeDistanceTolerance, configuration.RangeFaceConfidence);
if (_PropertyRootExistedBefore || !_ArgZeroIsConfigurationRootDirectory)
personContainers = Array.Empty<PersonContainer>();
_PersonContainers = Array.Empty<PersonContainer>();
else
{
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
@ -120,7 +121,7 @@ public partial class DlibDotNet
if (rootResultsDirectory is null)
throw new Exception();
Storage storage = new(rootDirectory, rootResultsDirectory, peopleRootDirectory);
personContainers = Shared.Models.Stateless.Methods.IPersonContainer.GetPersonContainers(storage, configuration.PersonBirthdayFormat, _Faces.FileNameExtension);
_PersonContainers = Shared.Models.Stateless.Methods.IPersonContainer.GetPersonContainers(storage, configuration.PersonBirthdayFormat, _Faces.FileNameExtension);
}
if (!isSilent && configuration.TestDistanceResults)
{
@ -142,7 +143,7 @@ public partial class DlibDotNet
filenameExtension);
}
if (!configuration.SkipSearch)
Search(ticks, argZero, propertyRoot, personContainers);
Search(ticks, argZero, propertyRoot);
if (!_PropertyRootExistedBefore && !_IsEnvironment.Development && _Exceptions.Count == 0 && _ArgZeroIsConfigurationRootDirectory)
{
List<string[]> directoryCollections = _Rename.GetDirectoryRenameCollections(
@ -205,7 +206,7 @@ public partial class DlibDotNet
private void Verify(Models.Configuration configuration)
{
if (!configuration.OutputResolutions.Any() || string.IsNullOrEmpty(configuration.OutputResolutions[0]) || !configuration.ValidResolutions.Contains(configuration.OutputResolutions[0]))
throw new NullReferenceException($"{nameof(configuration.OutputResolutions)} must be a valid outputResolution!");
throw new NullReferenceException($"{nameof(configuration.OutputResolutions)} must be _FileNameToCollection valid outputResolution!");
if ((from l in configuration.OutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
throw new Exception($"One or more {nameof(configuration.OutputResolutions)} are not in the ValidResolutions list!");
if ((from l in configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
@ -249,13 +250,13 @@ public partial class DlibDotNet
string[] segments = sourceDirectoryNames[0].Split(' ');
century = segments[^1].Length == 4 ? segments[^1][..2] : null;
if (segments.Length < 2 || century is null || (century != "18" && century != "19" && century != "20"))
throw new Exception("root subdirectory must have a year at the end or directory name needs to be added to the exclude list!");
throw new Exception("root subdirectory must have _FileNameToCollection year at the end or directory name needs to be added to the exclude list!");
}
}
}
string[] resizeMatch = (from l in sourceDirectoryNames where configuration.ValidResolutions.Contains(l) select l).ToArray();
if (resizeMatch.Any())
throw new Exception("Input directory should be the source and not a resized directory!");
throw new Exception("Input directory should be the source and not _FileNameToCollection resized directory!");
if (configuration.LocationDigits != Shared.Models.Stateless.ILocation.Digits)
throw new Exception("Configuration has to match interface!");
if (configuration.LocationFactor != Shared.Models.Stateless.ILocation.Factor)
@ -294,7 +295,6 @@ public partial class DlibDotNet
Dictionary<string, int[]> imageResizeKeyValuePairs;
List<Tuple<string, DateTime>> subFileTuples = new();
List<KeyValuePair<string, string>> metadataCollection;
// Dictionary<string, List<RelativeLocation>> relativeLocations;
if (item.Property is not null && item.Property.Id is not null && !item.Any())
{
property = item.Property;
@ -354,17 +354,21 @@ public partial class DlibDotNet
string path = Path.Combine(resizedFileHolder.DirectoryName, resizedFileHolder.NameWithoutExtension);
File.WriteAllBytes(path, bytes);
}
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
if (!mappingFromItem.ResizedFileHolder.Exists && !File.Exists(mappingFromItem.ResizedFileHolder.FullName))
faces = new();
else if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
faces = new();
else
{
List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>? collection;
int[] outputResolutionCollection = imageResizeKeyValuePairs[outputResolution];
int outputResolutionWidth = outputResolutionCollection[0];
int outputResolutionHeight = outputResolutionCollection[1];
int outputResolutionOrientation = outputResolutionCollection[2];
faces = _Faces.GetFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation);
// relativeLocations = _Faces.GetRelativeLocations(outputResolution, dResultsDateGroupDirectory, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation, original, imageResizeKeyValuePairs, faces);
List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection;
List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>? collection;
if (!_FileNameToCollection.TryGetValue(mappingFromItem.RelativePath[1..], out mappingFromPhotoPrismCollection))
mappingFromPhotoPrismCollection = null;
faces = _Faces.GetFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionWidth, outputResolutionHeight, outputResolutionOrientation, mappingFromPhotoPrismCollection);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D_Face.GetFaces));
bool anyFacesSaved = _Faces.SaveFaces(dResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem, facesDirectory, faces);
@ -439,25 +443,24 @@ public partial class DlibDotNet
{
try
{
FullParallelForWork(
propertyLogic,
idToMappedFaceFilesWithCollection,
outputResolution,
bResultsFullGroupDirectory,
cResultsFullGroupDirectory,
dResultsDateGroupDirectory,
dResultsFullGroupDirectory,
eDistanceContentDirectory,
sourceDirectoryChanges,
propertyFileHolderCollection,
propertyCollection,
metadataCollection,
resizeKeyValuePairs,
imageFaceCollections,
container,
index: i,
filteredItems[i],
containerDateTimes);
FullParallelForWork(propertyLogic,
idToMappedFaceFilesWithCollection,
outputResolution,
bResultsFullGroupDirectory,
cResultsFullGroupDirectory,
dResultsDateGroupDirectory,
dResultsFullGroupDirectory,
eDistanceContentDirectory,
sourceDirectoryChanges,
propertyFileHolderCollection,
propertyCollection,
metadataCollection,
resizeKeyValuePairs,
imageFaceCollections,
container,
index: i,
filteredItems[i],
containerDateTimes);
if (i == 0 || sourceDirectoryChanges.Any())
progressBar.Tick();
}
@ -627,7 +630,7 @@ public partial class DlibDotNet
_Metadata.SetAngleBracketCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, container.SourceDirectory);
}
private void FullDoWork(string argZero, string propertyRoot, long ticks, PersonContainer[] personContainers, A_Property propertyLogic, int t, Container[] containers, string? a2PeopleContentDirectory, string eDistanceContentDirectory)
private void FullDoWork(string argZero, string propertyRoot, long ticks, A_Property propertyLogic, int t, Container[] containers, string? a2PeopleContentDirectory, string eDistanceContentDirectory)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
@ -651,7 +654,7 @@ public partial class DlibDotNet
int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism;
List<Shared.Models.Property?> nullablePropertyCollection = new();
List<List<KeyValuePair<string, string>>> metadataCollection = new();
Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> idToMappedFaceFilesWithCollection = GetCollection(ticks, personContainers, a2PeopleContentDirectory, eDistanceContentDirectory);
Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> idToMappedFaceFilesWithCollection = GetCollection(ticks, a2PeopleContentDirectory, eDistanceContentDirectory);
string dResultsDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(D_Face));
foreach (string outputResolution in _Configuration.OutputResolutions)
{
@ -739,7 +742,7 @@ public partial class DlibDotNet
}
}
private void SetMapping(string argZero, Container[] containers)
private void SetMapping(Dictionary<string, List<MappingFromPhotoPrism>> fileNameToCollection, string argZero, Container[] containers)
{
Mapping mapping;
int faceAreaPermille;
@ -751,6 +754,7 @@ public partial class DlibDotNet
string deterministicHashCodeKey;
MappingFromItem mappingFromItem;
MappingFromLocation mappingFromLocation;
List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection;
foreach (Container container in containers)
{
if (!container.Items.Any())
@ -776,7 +780,9 @@ public partial class DlibDotNet
normalizedRectangle = Shared.Models.Stateless.Methods.ILocation.GetNormalizedRectangle(face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution);
deterministicHashCodeKey = Shared.Models.Stateless.Methods.IMapping.GetDeterministicHashCodeKey(item.Property.Id.Value, face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution);
mappingFromLocation = new(faceAreaPermille, confidencePercent, deterministicHashCodeKey, normalizedRectangle);
mapping = new(mappingFromItem, mappingFromLocation);
if (!fileNameToCollection.TryGetValue(mappingFromItem.RelativePath[1..], out mappingFromPhotoPrismCollection))
mappingFromPhotoPrismCollection = null;
mapping = new(mappingFromItem, mappingFromLocation, mappingFromPhotoPrismCollection);
face.SetMapping(mapping);
}
}
@ -847,7 +853,7 @@ public partial class DlibDotNet
return items;
}
private void MapLogic(string argZero, long ticks, PersonContainer[] personContainers, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, MapLogicSupport mapLogicSupport, MapLogic mapLogic, string outputResolution)
private void MapLogic(string argZero, long ticks, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, MapLogicSupport mapLogicSupport, MapLogic mapLogic, string outputResolution)
{
int? useFiltersCounter = null;
SortingContainer[] sortingContainers;
@ -857,17 +863,12 @@ public partial class DlibDotNet
Mapping[] mappingCollection = MapLogicSupport.GetSelectedMappingCollection(distinctFilteredFaces);
string dFacesCollectionDirectory = Path.Combine(dResultsFullGroupDirectory, "[]", _Configuration.PropertyConfiguration.ResultAllInOne);
(Dictionary<long, int> personKeyToCount, int totalNotMapped) = mapLogic.AddToMapping(mappingCollection);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution) || (!string.IsNullOrEmpty(_Configuration.PhotoPrismDirectory) && Directory.Exists(_Configuration.PhotoPrismDirectory)))
if (!string.IsNullOrEmpty(_Configuration.PhotoPrismDirectory) && Directory.Exists(_Configuration.PhotoPrismDirectory))
mapLogic.WriteMatches(ticks, distinctFilteredFaces);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
{
List<Item> filteredItems = GetItems(argZero, containers);
if (!string.IsNullOrEmpty(_Configuration.PhotoPrismDirectory))
{
mapLogic.SaveMarkers();
mapLogic.FindMatch(filteredItems);
mapLogic.LoadMatches();
}
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
mapLogic.SaveShortcutsForOutputResolutions(filteredItems, mappingCollection, personKeyToCount);
mapLogic.SaveShortcutsForOutputResolutions(filteredItems, mappingCollection, personKeyToCount);
}
Dictionary<int, Dictionary<int, Mapping>> idToNormalizedRectangleToMapping = MapLogicSupport.GetIdToNormalizedRectangleToFace(mappingCollection);
mapLogic.CopyManualFiles(dFacesContentDirectory, idToNormalizedRectangleToMapping);
@ -876,7 +877,7 @@ public partial class DlibDotNet
if (_Configuration.SaveMappedForOutputResolutions.Contains(outputResolution))
mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, mappingCollection, idToNormalizedRectangleToMapping, personKeyToCount, totalNotMapped);
if (_Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution))
mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, a2PeopleSingletonDirectory, personContainers, mappingCollection, personKeyToCount, totalNotMapped);
mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, _PersonContainers, a2PeopleSingletonDirectory, mappingCollection, personKeyToCount, totalNotMapped);
if (_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution))
{
mapLogic.UpdatedPersonKeyToRanges(_MapConfiguration, ticks, mappingCollection);
@ -952,7 +953,7 @@ public partial class DlibDotNet
bool isValidImageFormatExtension;
for (int y = 0; y < int.MaxValue; y++)
{
_Log.Information("Enter a url for a image");
_Log.Information("Enter _FileNameToCollection url for _FileNameToCollection image");
line = _Console.ReadLine();
if (string.IsNullOrEmpty(line))
break;
@ -1011,12 +1012,12 @@ public partial class DlibDotNet
collection.Add(new(value.MappedFaceFile, normalizedRectangle.Value, directories));
}
private Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> GetCollection(long ticks, PersonContainer[] personContainers, string? a2PeopleContentDirectory, string? eDistanceContentDirectory)
private Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> GetCollection(long ticks, string? a2PeopleContentDirectory, string? eDistanceContentDirectory)
{
Dictionary<int, List<(string, int, IReadOnlyList<MetadataExtractor.Directory>?)>> results = new();
int? id;
int? normalizedRectangle;
List<(string, string[], string)> collection = Map.Models.Stateless.Methods.IMapLogic.DeleteEmptyDirectoriesAndGetMappedFaceFiles(_MapConfiguration, ticks, a2PeopleContentDirectory, eDistanceContentDirectory, personContainers);
List<(string, string[], string)> collection = Map.Models.Stateless.Methods.IMapLogic.DeleteEmptyDirectoriesAndGetMappedFaceFiles(_MapConfiguration, _PersonContainers, ticks, a2PeopleContentDirectory, eDistanceContentDirectory);
foreach ((string personKeyFormatted, string[] personDisplayDirectoryNames, string mappedFaceFile) in collection)
{
(id, normalizedRectangle) = Shared.Models.Stateless.Methods.IMapping.GetConverted(_Faces.FileNameExtension, mappedFaceFile);
@ -1043,7 +1044,7 @@ public partial class DlibDotNet
return results;
}
private void Search(long ticks, string argZero, string propertyRoot, PersonContainer[] personContainers)
private void Search(long ticks, string argZero, string propertyRoot)
{
int j;
int f;
@ -1056,6 +1057,13 @@ public partial class DlibDotNet
string dResultsFullGroupDirectory;
string? eDistanceContentDirectory;
string d2ResultsFullGroupDirectory;
if (!string.IsNullOrEmpty(_Configuration.PhotoPrismDirectory) && Directory.Exists(_Configuration.PhotoPrismDirectory))
{
Map.Models.MapLogic.SaveMarkers(_Configuration.PhotoPrismDirectory);
Dictionary<string, List<MappingFromPhotoPrism>> fileNameToCollection = Map.Models.MapLogic.GetFileNameToCollection(_Configuration.PhotoPrismDirectory);
foreach (KeyValuePair<string, List<MappingFromPhotoPrism>> keyValuePair in fileNameToCollection)
_FileNameToCollection.Add(keyValuePair.Key, keyValuePair.Value);
}
string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "{}");
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string message = $") Building Container(s) - {totalSeconds} total second(s)";
@ -1092,10 +1100,10 @@ public partial class DlibDotNet
}
containers = Shared.Models.Stateless.Methods.IContainer.SortContainers(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, _ArgZeroIsConfigurationRootDirectory, argZero, containers);
MapLogicSupport mapLogicSupport = new(_Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeFaceAreaPermilleTolerance, _Configuration.SortingMaximumPerFaceShouldBeHigh);
MapLogic? mapLogic = _Configuration.DistanceMoveUnableToMatch ? null : new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, personContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory, mapLogicSupport);
FullDoWork(argZero, propertyRoot, ticks, personContainers, propertyLogic, t, containers, a2PeopleContentDirectory, eDistanceContentDirectory);
MapLogic? mapLogic = _Configuration.DistanceMoveUnableToMatch ? null : new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, _PersonContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory, mapLogicSupport);
FullDoWork(argZero, propertyRoot, ticks, propertyLogic, t, containers, a2PeopleContentDirectory, eDistanceContentDirectory);
_Distance.Clear();
mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, personContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory, mapLogicSupport);
mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, _PersonContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory, mapLogicSupport);
foreach (string outputResolution in _Configuration.OutputResolutions)
{
if (_PropertyRootExistedBefore || container is not null)
@ -1112,8 +1120,8 @@ public partial class DlibDotNet
{
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any())
break;
SetMapping(argZero, containers);
MapLogic(argZero, ticks, personContainers, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, mapLogicSupport, mapLogic, outputResolution);
SetMapping(_FileNameToCollection, argZero, containers);
MapLogic(argZero, ticks, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, mapLogicSupport, mapLogic, outputResolution);
if (_IsEnvironment.Development)
continue;
List<MappingFromItem> mappingFromItemCollection = GetMappingFromItemCollection(containers);