phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

DistanceLimits

Browse Source
This commit is contained in:
Mike Phares
2023-02-25 23:01:08 -07:00
parent db43eb5459
commit 0f025b5e2e
26 changed files with 426 additions and 392 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
Instance/DlibDotNet.cs
View File

@ -252,6 +252,7 @@ public partial class DlibDotNet
configuration.RangeDaysDeltaTolerance,
configuration.RangeDistanceTolerance,
configuration.SaveSortingWithoutPerson,
configuration.SkipNotSkipDirectories,
configuration.SortingMaximumPerKey,
configuration.SortingMinimumToUseSigma,
facesFileNameExtension,
@ -325,7 +326,8 @@ public partial class DlibDotNet
item.SetResizedFileHolder(_Resize.FileNameExtension, resizedFileHolder);
string facesDirectory = _Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution) ? _Faces.GetFacesDirectory(dResultsFullGroupDirectory, item) : string.Empty;
string facePartsDirectory = _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution) ? _FaceParts.GetFacePartsDirectory(_Configuration.PropertyConfiguration, dResultsFullGroupDirectory, item) : string.Empty;
MappingFromItem mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item, resizedFileHolder);
bool isIgnoreRelativePath = _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory);
MappingFromItem mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item, resizedFileHolder, isIgnoreRelativePath);
(int metadataGroups, metadataCollection) = metadata.GetMetadataCollection(subFileTuples, parseExceptions, mappingFromItem);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(B_Metadata.GetMetadataCollection));
@ -595,6 +597,7 @@ public partial class DlibDotNet
Item[] filteredItems;
int confidencePercent;
int normalizedRectangle;
bool isIgnoreRelativePath;
DateTime[] containerDateTimes;
string deterministicHashCodeKey;
MappingFromItem mappingFromItem;
@ -610,11 +613,12 @@ public partial class DlibDotNet
if (!filteredItems.Any())
continue;
containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems);
isIgnoreRelativePath = _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory);
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null || item.ResizedFileHolder is null)
continue;
mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item);
mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item, isIgnoreRelativePath);
foreach (Shared.Models.Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
@ -641,7 +645,6 @@ public partial class DlibDotNet
List<int> resultIds = new();
List<Shared.Models.Face> resultFaces = new();
Item[] filteredItems;
bool isIgnoreRelativePath;
foreach (Container container in containers)
{
if (!container.Items.Any())
@ -651,7 +654,6 @@ public partial class DlibDotNet
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container);
if (!filteredItems.Any())
continue;
isIgnoreRelativePath = _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory);
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null || item.ResizedFileHolder is null)
@ -659,8 +661,6 @@ public partial class DlibDotNet
if (resultIds.Contains(item.Property.Id.Value))
continue;
resultIds.Add(item.Property.Id.Value);
if (isIgnoreRelativePath)
continue;
foreach (Shared.Models.Face face in item.Faces)
{
if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
@ -676,8 +676,6 @@ public partial class DlibDotNet
{
List<Item> items = new();
Item[] filteredItems;
bool isIgnoreRelativePath;
List<Shared.Models.Face> collection = new();
foreach (Container container in containers)
{
if (!container.Items.Any())
@ -687,7 +685,6 @@ public partial class DlibDotNet
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container);
if (!filteredItems.Any())
continue;
isIgnoreRelativePath = _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory);
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null || item.ResizedFileHolder is null)
@ -705,7 +702,7 @@ public partial class DlibDotNet
string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()");
string d2FacePartsContentDirectory = Path.Combine(d2ResultsFullGroupDirectory, "()");
string dFacesCollectionDirectory = Path.Combine(dResultsFullGroupDirectory, "[]", _Configuration.PropertyConfiguration.ResultAllInOne);
Dictionary<int, Dictionary<int, Mapping>> idToNormalizedRectangleToMapping = MapLogicSupport.GetIdToNormalizedRectangleToFace(mappingCollection);
Dictionary<int, Dictionary<int, Mapping>> idToNormalizedRectangleToMapping = Map.Models.Stateless.Methods.IMapLogic.GetIdToNormalizedRectangleToFace(mappingCollection);
if (Directory.Exists(fPhotoPrismContentDirectory))
F_PhotoPrism.WriteMatches(fPhotoPrismContentDirectory, _Configuration.PersonBirthdayFormat, ticks, distinctFilteredFaces, mapLogic);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
@ -720,27 +717,26 @@ public partial class DlibDotNet
mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, personKeyToIds, mappingCollection, idToNormalizedRectangleToMapping, totalNotMapped);
if (_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution))
{
MapLogicSupport mapLogicSupport;
MapLogicSupport.SetFaceDistances(_AppSettings.MaxDegreeOfParallelism, ticks, distinctFilteredFaces);
List<Shared.Models.Face> distinctFurtherFilteredFaces = mapLogic.GetFurtherFilterBySkipCollection(distinctFilteredFaces);
DistanceLimits distanceLimits;
E_Distance.SetFaceDistances(_AppSettings.MaxDegreeOfParallelism, ticks, distinctFilteredFaces);
Dictionary<int, Dictionary<int, PersonContainer[]>> missingIdThenNormalizedRectangleToPersonContainers = mapLogic.GetMissing(idToNormalizedRectangleToMapping);
List<FaceDistanceContainer> missingFaceDistanceContainers = _Distance.GetMissingFaceDistanceContainer(_AppSettings.MaxDegreeOfParallelism, ticks, dFacesCollectionDirectory, missingIdThenNormalizedRectangleToPersonContainers);
mapLogicSupport = new(_Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaPermilleTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh);
sortingContainers = mapLogicSupport.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distinctFurtherFilteredFaces, missingFaceDistanceContainers);
distanceLimits = new(_Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaPermilleTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh);
sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distanceLimits, distinctFilteredFaces, missingFaceDistanceContainers);
if (!sortingContainers.Any())
{
for (useFiltersCounter = 1; useFiltersCounter < 17; useFiltersCounter++)
for (useFiltersCounter = 1; useFiltersCounter < _Configuration.UseFilterTries; useFiltersCounter++)
{
mapLogicSupport = new(_Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaPermilleTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh, useFiltersCounter);
sortingContainers = mapLogicSupport.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distinctFurtherFilteredFaces, missingFaceDistanceContainers);
distanceLimits = new(_Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaPermilleTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh, useFiltersCounter);
sortingContainers = E_Distance.SetFaceMappingSortingCollectionThenGetSortingContainers(_AppSettings.MaxDegreeOfParallelism, _MapConfiguration, ticks, mapLogic, distanceLimits, distinctFilteredFaces, missingFaceDistanceContainers);
if (sortingContainers.Any())
break;
}
}
MapLogicSupport.SaveFaceDistances(_Configuration.PropertyConfiguration, sortingContainers);
E_Distance.SaveFaceDistances(_Configuration.PropertyConfiguration, sortingContainers);
if (totalNotMapped > 0)
{
int updated = mapLogic.UpdateFromSortingContainers(mapLogicSupport, sortingContainers);
int updated = mapLogic.UpdateFromSortingContainers(distanceLimits, sortingContainers);
List<SaveContainer> saveContainers = mapLogic.GetSaveContainers(dFacesContentDirectory, d2FacePartsContentDirectory, mappingCollection, idToNormalizedRectangleToMapping, useFiltersCounter, sortingContainers.Any());
mapLogic.SaveContainers(totalNotMapped, updated, saveContainers);
}
@ -1061,7 +1057,7 @@ public partial class DlibDotNet
personKeyToIds = mapLogic.GetPersonKeyToIds();
(List<int> distinctFilteredIds, List<Shared.Models.Face> distinctFilteredFaces) = GetFilteredDistinct(argZero, containers);
LookForAbandoned(idToLocationContainers, distinctFilteredIds);
Mapping[] mappingCollection = MapLogicSupport.GetSelectedMappingCollection(distinctFilteredFaces);
Mapping[] mappingCollection = Map.Models.Stateless.Methods.IMapLogic.GetSelectedMappingCollection(distinctFilteredFaces);
int totalNotMapped = mapLogic.UpdateMappingFromPerson(mappingCollection);
string json = System.Text.Json.JsonSerializer.Serialize(mappingCollection);
File.WriteAllText(Path.Combine(eDistanceContentDirectory, $"{ticks}.json"), json);

7
Instance/Models/Binder/Configuration.cs
View File

@ -72,11 +72,13 @@ public class Configuration
[Display(Name = "Save Resized Subfiles"), Required] public bool? SaveResizedSubfiles { get; set; }
[Display(Name = "Save Shortcuts"), Required] public string[] SaveShortcutsForOutputResolutions { get; set; }
[Display(Name = "Save Sorting Without Person"), Required] public bool? SaveSortingWithoutPerson { get; set; }
[Display(Name = "Skip Not Skip Directories"), Required] public string[] SkipNotSkipDirectories { get; set; }
[Display(Name = "Skip Search"), Required] public bool? SkipSearch { get; set; }
[Display(Name = "SortingMaximumPerFaceShould be High"), Required] public int? SortingMaximumPerFaceShouldBeHigh { get; set; }
[Display(Name = "Sorting Maximum Per Key"), Required] public int? SortingMaximumPerKey { get; set; }
[Display(Name = "Sorting Minimum to use Sigma"), Required] public int? SortingMinimumToUseSigma { get; set; }
[Display(Name = "Test Distance Results"), Required] public bool? TestDistanceResults { get; set; }
[Display(Name = "Use Filter Tries"), Required] public int? UseFilterTries { get; set; }
[Display(Name = "Valid Resolutions"), Required] public string[] ValidResolutions { get; set; }
#nullable restore
@ -195,6 +197,7 @@ public class Configuration
throw new NullReferenceException(nameof(configuration.SaveResizedSubfiles));
configuration.SaveShortcutsForOutputResolutions ??= Array.Empty<string>();
configuration.SaveShortcutsForOutputResolutions ??= Array.Empty<string>();
configuration.SkipNotSkipDirectories ??= Array.Empty<string>();
if (configuration.SaveSortingWithoutPerson is null)
throw new NullReferenceException(nameof(configuration.SaveSortingWithoutPerson));
if (configuration.SkipSearch is null)
@ -207,6 +210,8 @@ public class Configuration
throw new NullReferenceException(nameof(configuration.SortingMinimumToUseSigma));
if (configuration.TestDistanceResults is null)
throw new NullReferenceException(nameof(configuration.TestDistanceResults));
if (configuration.UseFilterTries is null)
throw new NullReferenceException(nameof(configuration.UseFilterTries));
if (configuration.ValidResolutions is null)
throw new NullReferenceException(nameof(configuration.ValidResolutions));
_ = DateTime.Now.AddDays(-configuration.RangeDaysDeltaTolerance[1]);
@ -270,11 +275,13 @@ public class Configuration
configuration.SaveResizedSubfiles.Value,
configuration.SaveShortcutsForOutputResolutions,
configuration.SaveSortingWithoutPerson.Value,
configuration.SkipNotSkipDirectories,
configuration.SkipSearch.Value,
configuration.SortingMaximumPerFaceShouldBeHigh.Value,
configuration.SortingMaximumPerKey.Value,
configuration.SortingMinimumToUseSigma.Value,
configuration.TestDistanceResults.Value,
configuration.UseFilterTries.Value,
configuration.ValidResolutions);
return result;
}

6
Instance/Models/Configuration.cs
View File

@ -68,11 +68,13 @@ public class Configuration
public bool SaveResizedSubfiles { init; get; }
public string[] SaveShortcutsForOutputResolutions { init; get; }
public bool SaveSortingWithoutPerson { init; get; }
public string[] SkipNotSkipDirectories { init; get; }
public bool SkipSearch { init; get; }
public int SortingMaximumPerFaceShouldBeHigh { init; get; }
public int SortingMaximumPerKey { init; get; }
public int SortingMinimumToUseSigma { init; get; }
public bool TestDistanceResults { init; get; }
public int UseFilterTries { init; get; }
public string[] ValidResolutions { init; get; }
[JsonConstructor]
@ -136,11 +138,13 @@ public class Configuration
bool saveResizedSubfiles,
string[] saveShortcutsForOutputResolutions,
bool saveSortingWithoutPerson,
string[] skipNotSkipDirectories,
bool skipSearch,
int sortingMaximumPerFaceShouldBeHigh,
int sortingMaximumPerKey,
int sortingSigma,
bool testDistanceResults,
int useFilterTries,
string[] validResolutions)
{
_PropertyConfiguration = propertyConfiguration;
@ -203,11 +207,13 @@ public class Configuration
SaveRandomForOutputResolutions = saveRandomForOutputResolutions;
SaveShortcutsForOutputResolutions = saveShortcutsForOutputResolutions;
SaveSortingWithoutPerson = saveSortingWithoutPerson;
SkipNotSkipDirectories = skipNotSkipDirectories;
SkipSearch = skipSearch;
SortingMaximumPerFaceShouldBeHigh = sortingMaximumPerFaceShouldBeHigh;
SortingMaximumPerKey = sortingMaximumPerKey;
SortingMinimumToUseSigma = sortingSigma;
TestDistanceResults = testDistanceResults;
UseFilterTries = useFilterTries;
ValidResolutions = validResolutions;
}

20
Instance/Models/_F_Random.cs
View File

@ -25,24 +25,6 @@ internal class F_Random
return result;
}
private bool IsIgnoreRelativePath(string directory)
{
bool result = false;
string? checkDirectory = Path.GetFullPath(directory);
for (int i = 0; i < int.MaxValue; i++)
{
if (_Configuration.IgnoreRelativePaths.Contains(Path.GetFileName(checkDirectory)))
{
result = true;
break;
}
checkDirectory = Path.GetDirectoryName(checkDirectory);
if (string.IsNullOrEmpty(checkDirectory) || checkDirectory == _Configuration.PropertyConfiguration.RootDirectory)
break;
}
return result;
}
private static Dictionary<string, List<string>> Get(Dictionary<long, List<int>> personKeyToIds, Shared.Models.Mapping[] mappingCollection, string dateFormat)
{
Dictionary<string, List<string>> results = new();
@ -95,7 +77,7 @@ internal class F_Random
{
if (mapping.MappingFromItem.ImageFileHolder.DirectoryName is null)
continue;
if (_Configuration.IgnoreRelativePaths.Any(l => mapping.MappingFromItem.ImageFileHolder.DirectoryName.Contains(l)) && IsIgnoreRelativePath(mapping.MappingFromItem.ImageFileHolder.DirectoryName))
if (mapping.MappingFromItem.IsIgnoreRelativePath)
continue;
relativePaths.Add(mapping.MappingFromItem.RelativePath);
}

19
Instance/appsettings.Development.json
View File

@ -65,6 +65,25 @@
"SaveMappedForOutputResolutions": [],
"SaveRandomForOutputResolutions": [],
"SaveShortcutsForOutputResolutions": [],
"SkipNotSkipDirectories": [],
"xSkipNotSkipDirectories": [
"/!/_ Images-To-Do",
"/!/_ Test",
"/!/Disney",
"/!/Chelsea's Friends",
"/!/Jason's Friends",
"/!/Johnny's Friends",
"/!/Julie's Friends",
"/!/Kristy's Family",
"/!/Kristy's Friends",
"/!/Logan Friends",
"/!/Mackenzie Friends",
"/!/Mike's Family",
"/!/Mike's Friends",
"/!/Statue",
"/!/Tracy's Friend",
"/!/Wax"
],
"IgnoreRelativePaths": [
"3757 W Whitman 2017",
"501 Playful Meadows 2006",

2
Instance/appsettings.json
View File

@ -111,6 +111,7 @@
"SortingMaximumPerKey": 7,
"SortingMinimumToUseSigma": 10,
"TestDistanceResults": true,
"UseFilterTries": 0,
"WriteBitmapDataBytes": false,
"IgnoreExtensions": [
".gif",
@ -155,6 +156,7 @@
"SaveMappedForOutputResolutions": [],
"SaveRandomForOutputResolutions": [],
"SaveShortcutsForOutputResolutions": [],
"SkipNotSkipDirectories": [],
"VerifyToSeason": [],
"ValidImageFormatExtensions": [
".bmp",