phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SaveShortcutsForOutputResolutions

Browse Source
This commit is contained in:
Mike Phares
2023-01-02 20:22:27 -07:00
parent d0cd52807d
commit 92288c9381
7 changed files with 252 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
Instance/DlibDotNet.cs
View File

@ -40,7 +40,6 @@ public partial class DlibDotNet
private readonly Models.Configuration _Configuration;
private readonly bool _ArgZeroIsConfigurationRootDirectory;
private readonly Map.Models.Configuration _MapConfiguration;
private readonly Dictionary<string, List<MappingFromPhotoPrism>> _FileNameToCollection;
public DlibDotNet(
List<string> args,
@ -54,7 +53,6 @@ public partial class DlibDotNet
string message;
_Console = console;
_AppSettings = appSettings;
_FileNameToCollection = new();
_IsEnvironment = isEnvironment;
long ticks = DateTime.Now.Ticks;
_Exceptions = new List<string>();
@ -209,7 +207,7 @@ public partial class DlibDotNet
private void Verify(Models.Configuration configuration)
{
if (!configuration.OutputResolutions.Any() || string.IsNullOrEmpty(configuration.OutputResolutions[0]) || !configuration.ValidResolutions.Contains(configuration.OutputResolutions[0]))
throw new NullReferenceException($"{nameof(configuration.OutputResolutions)} must be _FileNameToCollection valid outputResolution!");
throw new NullReferenceException($"{nameof(configuration.OutputResolutions)} must be fileNameToCollection valid outputResolution!");
if ((from l in configuration.OutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
throw new Exception($"One or more {nameof(configuration.OutputResolutions)} are not in the ValidResolutions list!");
if ((from l in configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
@ -253,13 +251,13 @@ public partial class DlibDotNet
string[] segments = sourceDirectoryNames[0].Split(' ');
century = segments[^1].Length == 4 ? segments[^1][..2] : null;
if (segments.Length < 2 || century is null || (century != "18" && century != "19" && century != "20"))
throw new Exception("root subdirectory must have _FileNameToCollection year at the end or directory name needs to be added to the exclude list!");
throw new Exception("root subdirectory must have fileNameToCollection year at the end or directory name needs to be added to the exclude list!");
}
}
}
string[] resizeMatch = (from l in sourceDirectoryNames where configuration.ValidResolutions.Contains(l) select l).ToArray();
if (resizeMatch.Any())
throw new Exception("Input directory should be the source and not _FileNameToCollection resized directory!");
throw new Exception("Input directory should be the source and not fileNameToCollection resized directory!");
if (configuration.LocationDigits != Shared.Models.Stateless.ILocation.Digits)
throw new Exception("Configuration has to match interface!");
if (configuration.LocationFactor != Shared.Models.Stateless.ILocation.Factor)
@ -287,7 +285,26 @@ public partial class DlibDotNet
return result;
}
private void FullParallelForWork(A_Property propertyLogic, Dictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToMappedFaceFilesWithCollection, string outputResolution, string bResultsFullGroupDirectory, string cResultsFullGroupDirectory, string dResultsDateGroupDirectory, string dResultsFullGroupDirectory, string eDistanceContentDirectory, List<Tuple<string, DateTime>> sourceDirectoryChanges, List<FileHolder?> propertyFileHolderCollection, List<Shared.Models.Property?> propertyCollection, List<List<KeyValuePair<string, string>>> metadataCollections, List<Dictionary<string, int[]>> resizeKeyValuePairs, List<List<Shared.Models.Face>> imageFaceCollections, Container container, int index, Item item, DateTime[] containerDateTimes)
private void FullParallelForWork(
A_Property propertyLogic,
Dictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToMappedFaceFilesWithCollection,
string outputResolution,
string bResultsFullGroupDirectory,
string cResultsFullGroupDirectory,
string dResultsDateGroupDirectory,
string dResultsFullGroupDirectory,
string eDistanceContentDirectory,
List<Tuple<string, DateTime>> sourceDirectoryChanges,
List<FileHolder?> propertyFileHolderCollection,
List<Shared.Models.Property?> propertyCollection,
List<List<KeyValuePair<string, string>>> metadataCollections,
List<Dictionary<string, int[]>> resizeKeyValuePairs,
Dictionary<string, List<MappingFromPhotoPrism>> fileNameToCollection,
List<List<Shared.Models.Face>> imageFaceCollections,
Container container,
int index,
Item item,
DateTime[] containerDateTimes)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
@ -364,7 +381,7 @@ public partial class DlibDotNet
collection = null;
else
_ = idToMappedFaceFilesWithCollection.TryGetValue(item.Property.Id.Value, out collection);
if (!_FileNameToCollection.TryGetValue(mappingFromItem.RelativePath[1..], out mappingFromPhotoPrismCollection))
if (!fileNameToCollection.TryGetValue(mappingFromItem.RelativePath[1..], out mappingFromPhotoPrismCollection))
mappingFromPhotoPrismCollection = null;
faces = _Faces.GetFaces(outputResolution, dResultsFullGroupDirectory, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionToResize, collection, mappingFromPhotoPrismCollection);
if (_AppSettings.MaxDegreeOfParallelism < 2)
@ -410,6 +427,7 @@ public partial class DlibDotNet
List<Shared.Models.Property?> propertyCollection,
List<List<KeyValuePair<string, string>>> metadataCollection,
List<Dictionary<string, int[]>> resizeKeyValuePairs,
Dictionary<string, List<MappingFromPhotoPrism>> fileNameToCollection,
List<List<Shared.Models.Face>> imageFaceCollections,
Container container,
Item[] filteredItems,
@ -453,6 +471,7 @@ public partial class DlibDotNet
propertyCollection,
metadataCollection,
resizeKeyValuePairs,
fileNameToCollection,
imageFaceCollections,
container,
index: i,
@ -627,7 +646,7 @@ public partial class DlibDotNet
_Metadata.SetAngleBracketCollection(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, container.SourceDirectory);
}
private void FullDoWork(string argZero, string propertyRoot, long ticks, A_Property propertyLogic, int t, Container[] containers, string? a2PeopleContentDirectory, string eDistanceContentDirectory)
private void FullDoWork(string argZero, string propertyRoot, long ticks, A_Property propertyLogic, int t, Container[] containers, string eDistanceContentDirectory, Dictionary<string, List<MappingFromPhotoPrism>> fileNameToCollection, Dictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToMappedFaceFilesWithCollection)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
@ -651,7 +670,6 @@ public partial class DlibDotNet
int maxDegreeOfParallelism = _AppSettings.MaxDegreeOfParallelism;
List<Shared.Models.Property?> nullablePropertyCollection = new();
List<List<KeyValuePair<string, string>>> metadataCollection = new();
Dictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToMappedFaceFilesWithCollection = GetDictionary(ticks, a2PeopleContentDirectory, eDistanceContentDirectory);
string dResultsDateGroupDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(D_Face));
foreach (string outputResolution in _Configuration.OutputResolutions)
{
@ -696,6 +714,7 @@ public partial class DlibDotNet
nullablePropertyCollection,
metadataCollection,
resizeKeyValuePairs,
fileNameToCollection,
imageFaceCollections,
container,
filteredItems,
@ -801,8 +820,16 @@ public partial class DlibDotNet
}
if (renameCollection.Any())
{
foreach (string rename in renameCollection)
File.Move(rename, $"{rename}.abd");
string checkFile;
foreach (string file in renameCollection)
{
if (!File.Exists(file))
continue;
checkFile = $"{file}.abd";
if (File.Exists(checkFile))
continue;
File.Move(file, checkFile);
}
throw new Exception($"Renamed {renameCollection.Count}(s) files!");
}
}
@ -824,18 +851,26 @@ public partial class DlibDotNet
}
if (renameCollection.Any())
{
foreach (string rename in renameCollection)
File.Move(rename, $"{rename}.mvd");
string checkFile;
foreach (string file in renameCollection)
{
if (!File.Exists(file))
continue;
checkFile = $"{file}.mvd";
if (File.Exists(checkFile))
continue;
File.Move(file, checkFile);
}
throw new Exception($"Renamed {renameCollection.Count}(s) files!");
}
}
private List<Shared.Models.Face> GetFilteredDistinctFaces(string argZero, Container[] containers)
private (List<int>, List<Shared.Models.Face>) GetFilteredDistinct(string argZero, Container[] containers)
{
List<Shared.Models.Face> results = new();
List<int> resultIds = new();
List<Shared.Models.Face> resultFaces = new();
Item[] filteredItems;
bool isIgnoreRelativePath;
List<int> distinct = new();
foreach (Container container in containers)
{
if (!container.Items.Any())
@ -850,20 +885,20 @@ public partial class DlibDotNet
{
if (item.Property?.Id is null || item.ResizedFileHolder is null)
continue;
if (distinct.Contains(item.Property.Id.Value))
if (resultIds.Contains(item.Property.Id.Value))
continue;
distinct.Add(item.Property.Id.Value);
resultIds.Add(item.Property.Id.Value);
if (isIgnoreRelativePath)
continue;
foreach (Shared.Models.Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
continue;
results.Add(face);
resultFaces.Add(face);
}
}
}
return results;
return new(resultIds, resultFaces);
}
private List<Item> GetItems(string argZero, Container[] containers)
@ -892,7 +927,7 @@ public partial class DlibDotNet
return items;
}
private void MapLogic(string argZero, long ticks, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string fPhotoPrismContentDirectory, MapLogicSupport mapLogicSupport, MapLogic mapLogic, string outputResolution, List<Shared.Models.Face> distinctFilteredFaces, Mapping[] mappingCollection, int totalNotMapped, Dictionary<long, int> personKeyToCount)
private void MapLogic(string argZero, long ticks, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string fPhotoPrismContentDirectory, MapLogicSupport mapLogicSupport, MapLogic mapLogic, string outputResolution, Dictionary<long, List<int>> personKeyToIds, List<Shared.Models.Face> distinctFilteredFaces, Mapping[] mappingCollection, int totalNotMapped)
{
int? useFiltersCounter = null;
SortingContainer[] sortingContainers;
@ -905,13 +940,13 @@ public partial class DlibDotNet
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
{
List<Item> filteredItems = GetItems(argZero, containers);
mapLogic.SaveShortcutsForOutputResolutions(filteredItems, mappingCollection, personKeyToCount);
mapLogic.SaveShortcutsForOutputResolutions(personKeyToIds, filteredItems, mappingCollection);
}
if (_Configuration.PersonCharactersCopyCount > 0 && !string.IsNullOrEmpty(_Configuration.PersonCharacters))
mapLogic.CopyAtLeastOneMappedFiles(dFacesContentDirectory, a2PeopleSingletonDirectory, mappingCollection);
mapLogic.CopyManualFiles(dFacesContentDirectory, idToNormalizedRectangleToMapping);
if (_Configuration.SaveMappedForOutputResolutions.Contains(outputResolution))
mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, mappingCollection, idToNormalizedRectangleToMapping, personKeyToCount, totalNotMapped);
mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, personKeyToIds, mappingCollection, idToNormalizedRectangleToMapping, totalNotMapped);
if (_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution))
{
MapLogicSupport.SetFaceDistances(_AppSettings.MaxDegreeOfParallelism, ticks, distinctFilteredFaces);
@ -984,7 +1019,7 @@ public partial class DlibDotNet
bool isValidImageFormatExtension;
for (int y = 0; y < int.MaxValue; y++)
{
_Log.Information("Enter _FileNameToCollection url for _FileNameToCollection image");
_Log.Information("Enter fileNameToCollection url for fileNameToCollection image");
line = _Console.ReadLine();
if (string.IsNullOrEmpty(line))
break;
@ -1154,6 +1189,36 @@ public partial class DlibDotNet
return results;
}
private static void LookForAbandoned(Dictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToMappedFaceFilesWithCollection, List<int> distinctFilteredIds)
{
List<string> renameCollection = new();
foreach (KeyValuePair<int, List<LocationContainer<MetadataExtractor.Directory>>> idToCollection in idToMappedFaceFilesWithCollection)
{
if (distinctFilteredIds.Contains(idToCollection.Key))
continue;
foreach (LocationContainer<MetadataExtractor.Directory> locationContainer in idToCollection.Value)
{
if (locationContainer.File.Contains('!'))
continue;
renameCollection.Add(locationContainer.File);
}
}
if (renameCollection.Any())
{
string checkFile;
foreach (string file in renameCollection)
{
if (!File.Exists(file))
continue;
checkFile = $"{file}.abd";
if (File.Exists(checkFile))
continue;
File.Move(file, checkFile);
}
throw new Exception($"Renamed {renameCollection.Count}(s) files!");
}
}
private void Search(long ticks, string argZero, string propertyRoot)
{
int f;
@ -1169,6 +1234,8 @@ public partial class DlibDotNet
string d2ResultsFullGroupDirectory;
string fPhotoPrismContentDirectory;
string fPhotoPrismSingletonDirectory;
Dictionary<long, List<int>> personKeyToIds;
Dictionary<string, List<MappingFromPhotoPrism>> fileNameToCollection;
string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "{}");
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string message = $") Building Container(s) - {totalSeconds} total second(s)";
@ -1210,19 +1277,18 @@ public partial class DlibDotNet
containers = Shared.Models.Stateless.Methods.IContainer.SortContainers(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, _ArgZeroIsConfigurationRootDirectory, argZero, containers);
MapLogicSupport mapLogicSupport = new(_Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeFaceAreaPermilleTolerance, _Configuration.SortingMaximumPerFaceShouldBeHigh);
MapLogic? mapLogic = _Configuration.DistanceMoveUnableToMatch ? null : new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, _PersonContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory, mapLogicSupport);
if (Directory.Exists(fPhotoPrismSingletonDirectory))
{
Dictionary<string, List<MappingFromPhotoPrism>> fileNameToCollection = F_PhotoPrism.GetFileNameToCollection(fPhotoPrismSingletonDirectory);
foreach (KeyValuePair<string, List<MappingFromPhotoPrism>> keyValuePair in fileNameToCollection)
_FileNameToCollection.Add(keyValuePair.Key, keyValuePair.Value);
}
FullDoWork(argZero, propertyRoot, ticks, propertyLogic, t, containers, a2PeopleContentDirectory, eDistanceContentDirectory);
personKeyToIds = mapLogic is null ? new() : mapLogic.GetPersonKeyToIds();
fileNameToCollection = !Directory.Exists(fPhotoPrismSingletonDirectory) ? fileNameToCollection = new() : F_PhotoPrism.GetFileNameToCollection(fPhotoPrismSingletonDirectory);
Dictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToMappedFaceFilesWithCollection = GetDictionary(ticks, a2PeopleContentDirectory, eDistanceContentDirectory);
FullDoWork(argZero, propertyRoot, ticks, propertyLogic, t, containers, eDistanceContentDirectory, fileNameToCollection, idToMappedFaceFilesWithCollection);
_Distance.Clear();
mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, ticks, _PersonContainers, a2PeopleSingletonDirectory, eDistanceContentDirectory, mapLogicSupport);
SetMapping(_FileNameToCollection, argZero, containers);
List<Shared.Models.Face> distinctFilteredFaces = GetFilteredDistinctFaces(argZero, containers);
SetMapping(fileNameToCollection, argZero, containers);
if (!personKeyToIds.Any())
personKeyToIds = mapLogic.GetPersonKeyToIds();
(List<int> distinctFilteredIds, List<Shared.Models.Face> distinctFilteredFaces) = GetFilteredDistinct(argZero, containers);
LookForAbandoned(idToMappedFaceFilesWithCollection, distinctFilteredIds);
Mapping[] mappingCollection = MapLogicSupport.GetSelectedMappingCollection(distinctFilteredFaces);
Dictionary<long, int> personKeyToCount = mapLogic.GetPersonKeyToCount(mappingCollection);
int totalNotMapped = mapLogic.UpdateMappingFromPerson(mappingCollection);
if (a2PeopleContentDirectory is not null && false)
mapLogic.CreateTree(ticks, a2PeopleContentDirectory);
@ -1230,17 +1296,19 @@ public partial class DlibDotNet
{
if (_PropertyRootExistedBefore || container is not null)
break;
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
mapLogic.SaveShortcutsForOutputResolutions(a2PeopleContentDirectory, personKeyToIds, mappingCollection, totalNotMapped);
if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution))
mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, _PersonContainers, a2PeopleContentDirectory, mappingCollection, personKeyToCount, totalNotMapped);
mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, _PersonContainers, a2PeopleContentDirectory, personKeyToIds, mappingCollection, totalNotMapped);
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
if (_ArgZeroIsConfigurationRootDirectory
&& _Configuration.SaveResizedSubfiles
&& outputResolution == _Configuration.OutputResolutions[0]
&& _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
&& _Exceptions.Count == 0)
MapLogic(argZero, ticks, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, fPhotoPrismContentDirectory, mapLogicSupport, mapLogic, outputResolution, distinctFilteredFaces, mappingCollection, totalNotMapped, personKeyToCount);
MapLogic(argZero, ticks, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, fPhotoPrismContentDirectory, mapLogicSupport, mapLogic, outputResolution, personKeyToIds, distinctFilteredFaces, mappingCollection, totalNotMapped);
if (_Configuration.SaveRandomForOutputResolutions.Contains(outputResolution))
_Random.Random(_Configuration.PropertyConfiguration, mapLogic, outputResolution, mappingCollection);
_Random.Random(_Configuration.PropertyConfiguration, outputResolution, personKeyToIds, mappingCollection);
if (_IsEnvironment.Development)
continue;
G2_Identify identify = new(_Configuration);