phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Removed GetMissingFaceDistanceContainer

Browse Source
This commit is contained in:
Mike Phares 2023-08-07 16:30:05 -07:00
parent 2eb5cfd42f
commit 9991d2bfac
3 changed files with 28 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
Distance/Models/_E_Distance.cs
View File

@ -315,59 +315,6 @@ public partial class E_Distance : IDistance<MetadataExtractor.Directory>
_DuplicateMappedFaceFiles.Clear();
}
public ReadOnlyCollection<FaceDistanceContainer> GetMissingFaceDistanceContainer(int maxDegreeOfParallelism, long ticks, string dFacesCollectionDirectory, ReadOnlyDictionary<int, ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>> missingIdThenWholePercentagesToPersonContainers)
{
List<FaceDistanceContainer> results = new();
string[] files;
List<Face>? faces;
int wholePercentages;
int confidencePercent;
bool? isWrongYear = null;
FaceDistance faceDistance;
List<(int id, string json)> collection = new();
FaceDistanceContainer faceDistanceContainer;
foreach (KeyValuePair<int, ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>> keyValuePair in missingIdThenWholePercentagesToPersonContainers)
{
files = Directory.GetFiles(dFacesCollectionDirectory, $"{keyValuePair.Key}*.json", SearchOption.TopDirectoryOnly);
if (files.Length != 1)
continue;
collection.Add(new(keyValuePair.Key, Shared.Models.Stateless.Methods.IFace.GetJson(files[0])));
}
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxDegreeOfParallelism };
string message = $") {collection.Count:000} Setting missing distance containers - {totalSeconds} total second(s)";
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
using ProgressBar progressBar = new(collection.Count, message, options);
_ = Parallel.For(0, collection.Count, parallelOptions, (i, state) =>
{
progressBar.Tick();
int id = collection[i].id;
string json = collection[i].json;
faces = JsonSerializer.Deserialize<List<Face>>(json);
if (faces is null)
throw new NullReferenceException(nameof(faces));
foreach (Face face in faces)
{
if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
continue;
confidencePercent = Shared.Models.Stateless.Methods.ILocation.GetConfidencePercent(_FaceConfidencePercent, _RangeFaceConfidence, face.Location.Confidence);
wholePercentages = Shared.Models.Stateless.Methods.ILocation.GetWholePercentages(face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution);
if (face.FaceDistance?.Encoding is not null && face.FaceDistance.Encoding is FaceRecognitionDotNet.FaceEncoding faceEncoding)
faceDistance = new(confidencePercent, face.DateTime, faceEncoding, id, isWrongYear, wholePercentages);
else
{
faceEncoding = FaceRecognition.LoadFaceEncoding(face.FaceEncoding.RawEncoding);
faceDistance = new(confidencePercent, face.DateTime, faceEncoding, id, isWrongYear, wholePercentages);
face.SetFaceDistance(faceDistance);
}
faceDistanceContainer = new(face, faceDistance);
lock (results)
results.Add(faceDistanceContainer);
}
});
return new(results);
}
public static void SaveFaceDistances(Property.Models.Configuration configuration, SortingContainer[] sortingContainers)
{
string eDistanceContentCollectionDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(E_Distance), "([])");
@ -387,7 +334,7 @@ public partial class E_Distance : IDistance<MetadataExtractor.Directory>
{
if (face.Mapping?.MappingFromFilter is null)
throw new NotSupportedException();
if (face.Mapping.MappingFromFilter.IsFocusPerson is not null && !face.Mapping.MappingFromFilter.IsFocusPerson.Value)
if (face.Mapping.MappingFromFilter.IsUsed is not null && face.Mapping.MappingFromFilter.IsUsed.Value && face.Mapping.MappingFromFilter.IsFocusPerson is not null && !face.Mapping.MappingFromFilter.IsFocusPerson.Value)
continue;
if (face.FaceEncoding is not null && face.FaceDistance?.Encoding is not null && face.FaceDistance.Encoding is FaceRecognitionDotNet.FaceEncoding _)
continue;
@ -443,11 +390,11 @@ public partial class E_Distance : IDistance<MetadataExtractor.Directory>
return results;
}
private static List<Sorting> GetSortingCollection(Map.Models.MapLogic mapLogic, ReadOnlyCollection<FaceDistance> faceDistanceEncodings, int i, FaceDistance faceDistanceEncoding)
private static List<Sorting> GetSortingCollection(Map.Models.MapLogic mapLogic, ReadOnlyCollection<FaceDistance> faceDistanceEncodings, int i, FaceDistance faceDistanceEncoding, Face face)
{
List<Sorting> results;
List<FaceDistance> faceDistanceLengths = FaceRecognition.FaceDistances(faceDistanceEncodings, faceDistanceEncoding);
results = mapLogic.GetSortingCollection(i, faceDistanceEncoding, faceDistanceLengths);
results = mapLogic.GetSortingCollection(i, faceDistanceEncoding, face, faceDistanceLengths);
return results;
}
@ -471,24 +418,6 @@ public partial class E_Distance : IDistance<MetadataExtractor.Directory>
return results;
}
public static ReadOnlyCollection<FaceDistance> GetFaceDistanceEncodings(ReadOnlyCollection<FaceDistanceContainer> faceDistanceContainers, ReadOnlyCollection<FaceDistanceContainer> missingFaceDistanceContainers)
{
List<FaceDistance> faceDistanceEncodings = new();
foreach (FaceDistanceContainer faceDistanceContainer in faceDistanceContainers)
{
if (faceDistanceContainer.FaceDistance.Encoding is null)
continue;
faceDistanceEncodings.Add(faceDistanceContainer.FaceDistance);
}
foreach (FaceDistanceContainer faceDistanceContainer in missingFaceDistanceContainers)
{
if (faceDistanceContainer.FaceDistance.Encoding is null)
continue;
faceDistanceEncodings.Add(faceDistanceContainer.FaceDistance);
}
return new(faceDistanceEncodings);
}
public static FaceDistanceContainer[] FilteredFaceDistanceContainers(Map.Models.MapLogic mapLogic, ReadOnlyCollection<FaceDistanceContainer> faceDistanceContainers, long? skipOlderThan, DistanceLimits distanceLimits)
{
List<FaceDistanceContainer> results = new();
@ -531,7 +460,7 @@ public partial class E_Distance : IDistance<MetadataExtractor.Directory>
progressBar.Tick();
FaceDistance faceDistanceEncoding = filteredFaceDistanceContainers[i].FaceDistance;
Face face = filteredFaceDistanceContainers[i].Face;
List<Sorting> sortingCollection = GetSortingCollection(mapLogic, faceDistanceEncodings, i, faceDistanceEncoding);
List<Sorting> sortingCollection = GetSortingCollection(mapLogic, faceDistanceEncodings, i, faceDistanceEncoding, face);
if (sortingCollection.Count == 0)
return;
List<SortingContainer> sortingContainers = GetSortingContainers(mapConfiguration, distanceLimits, face, faceDistanceEncoding, sortingCollection);

19
Instance/DlibDotNet.cs
View File

@ -816,15 +816,21 @@ public partial class DlibDotNet
}
}
private void SaveFaceDistances(long ticks, MapLogic mapLogic, ReadOnlyCollection<Shared.Models.Face> distinctFilteredFaces, ReadOnlyCollection<Mapping> mappingCollection, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, string dFacesCollectionDirectory, ReadOnlyDictionary<int, ReadOnlyDictionary<int, Mapping>> idToWholePercentagesToMapping)
private void SaveFaceDistances(long ticks, MapLogic mapLogic, ReadOnlyCollection<Shared.Models.Face> distinctFilteredFaces, ReadOnlyCollection<Mapping> mappingCollection, string dFacesContentDirectory, string d2FacePartsContentDirectory, string d2FacePartsContentCollectionDirectory, ReadOnlyDictionary<int, ReadOnlyDictionary<int, Mapping>> idToWholePercentagesToMapping)
{
E_Distance.SetFaceDistances(_AppSettings.MaxDegreeOfParallelism, ticks, distinctFilteredFaces);
ReadOnlyCollection<FaceDistanceContainer> faceDistanceContainers = E_Distance.GetFaceDistanceContainers(distinctFilteredFaces);
ReadOnlyDictionary<int, ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>> missingIdThenWholePercentagesToPersonContainers = mapLogic.GetMissing(idToWholePercentagesToMapping);
ReadOnlyCollection<FaceDistanceContainer> missingFaceDistanceContainers = _Distance.GetMissingFaceDistanceContainer(_AppSettings.MaxDegreeOfParallelism, ticks, dFacesCollectionDirectory, missingIdThenWholePercentagesToPersonContainers);
ReadOnlyCollection<FaceDistance> faceDistanceEncodings = E_Distance.GetFaceDistanceEncodings(faceDistanceContainers, missingFaceDistanceContainers);
if (faceDistanceContainers.Count > 0)
SaveFaceDistances(ticks, mapLogic, mappingCollection, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, faceDistanceEncodings, faceDistanceContainers);
{
List<FaceDistance> faceDistanceEncodings = new();
foreach (FaceDistanceContainer faceDistanceContainer in faceDistanceContainers)
{
if (faceDistanceContainer.FaceDistance.Encoding is null)
continue;
faceDistanceEncodings.Add(faceDistanceContainer.FaceDistance);
}
SaveFaceDistances(ticks, mapLogic, mappingCollection, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, new(faceDistanceEncodings), faceDistanceContainers);
}
}
private void MapLogic(long ticks, ReadOnlyCollection<Container> containers, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string fPhotoPrismContentDirectory, MapLogic mapLogic, string outputResolution, ReadOnlyDictionary<long, List<int>> personKeyToIds, ReadOnlyCollection<Shared.Models.Face> distinctFilteredFaces, ReadOnlyCollection<Mapping> distinctFilteredMappingCollection)
@ -832,7 +838,6 @@ public partial class DlibDotNet
string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
string d2FacePartsContentDirectory = Path.Combine(d2ResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
string d2FacePartsContentCollectionDirectory = Path.Combine(d2ResultsFullGroupDirectory, "[()]");
string dFacesCollectionDirectory = Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultCollection, _Configuration.PropertyConfiguration.ResultAllInOne);
if (distinctFilteredMappingCollection.Count > 0)
{
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(d2FacePartsContentDirectory, ticks);
@ -848,7 +853,7 @@ public partial class DlibDotNet
if (_Configuration.SaveMappedForOutputResolutions.Contains(outputResolution))
mapLogic.SaveMapped(dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, personKeyToIds, distinctFilteredMappingCollection, idToWholePercentagesToMapping);
if (_Configuration.SaveFaceDistancesForOutputResolutions.Contains(outputResolution))
SaveFaceDistances(ticks, mapLogic, distinctFilteredFaces, distinctFilteredMappingCollection, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, dFacesCollectionDirectory, idToWholePercentagesToMapping);
SaveFaceDistances(ticks, mapLogic, distinctFilteredFaces, distinctFilteredMappingCollection, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping);
}
private string SaveUrlAndGetNewRootDirectory(string[] files)

35
Map/Models/MapLogic.cs
View File

@ -582,7 +582,7 @@ public class MapLogic : Shared.Models.Methods.IMapLogic
Stateless.MapLogic.SaveMappingShortcuts(mappingDirectory);
}
public List<Sorting> GetSortingCollection(int i, FaceDistance faceDistanceEncoding, List<FaceDistance> faceDistanceLengths)
public List<Sorting> GetSortingCollection(int i, FaceDistance faceDistanceEncoding, Face face, List<FaceDistance> faceDistanceLengths)
{
if (_Configuration is null)
throw new NullReferenceException(nameof(_Configuration));
@ -591,22 +591,20 @@ public class MapLogic : Shared.Models.Methods.IMapLogic
FaceDistance faceDistanceLength;
List<int>? wholePercentagesCollection;
bool skipNotSkipCollectionAny = _SkipNotSkipCollection.Count > 0;
ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>? wholePercentagesToPersonContainers;
for (int j = 0; j < faceDistanceLengths.Count; j++)
{
if (faceDistanceEncoding.WholePercentages is null)
throw new NotSupportedException();
if (face.Mapping?.MappingFromFilter is null)
throw new NotSupportedException();
if (j == i)
continue;
if (_SkipCollection.TryGetValue(faceDistanceEncoding.Id, out wholePercentagesCollection) && wholePercentagesCollection.Contains(faceDistanceEncoding.WholePercentages.Value))
continue;
if (skipNotSkipCollectionAny && (!_SkipNotSkipCollection.TryGetValue(faceDistanceEncoding.Id, out wholePercentagesCollection) || !wholePercentagesCollection.Contains(faceDistanceEncoding.WholePercentages.Value)))
continue;
if (_IdThenWholePercentagesToPersonContainers.TryGetValue(faceDistanceEncoding.Id, out wholePercentagesToPersonContainers))
{
if (wholePercentagesToPersonContainers.ContainsKey(faceDistanceEncoding.WholePercentages.Value))
continue;
}
if (face.Mapping.MappingFromFilter.IsUsed is not null && face.Mapping.MappingFromFilter.IsUsed.Value)
continue;
faceDistanceLength = faceDistanceLengths[j];
if (faceDistanceLength.WholePercentages is null || faceDistanceLength.Length is null)
throw new NotSupportedException();
@ -1223,18 +1221,6 @@ public class MapLogic : Shared.Models.Methods.IMapLogic
}
}
public ReadOnlyDictionary<int, ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>> GetMissing(ReadOnlyDictionary<int, ReadOnlyDictionary<int, Mapping>> idToWholePercentagesToMapping)
{
Dictionary<int, ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>> results = new();
foreach (KeyValuePair<int, ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>> idToCollection in _IdThenWholePercentagesToPersonContainers)
{
if (idToWholePercentagesToMapping.ContainsKey(idToCollection.Key))
continue;
results.Add(idToCollection.Key, idToCollection.Value);
}
return new(results);
}
public ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>? GetWholePercentagesToPersonContainers(int? id)
{
ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>? result;
@ -1245,23 +1231,26 @@ public class MapLogic : Shared.Models.Methods.IMapLogic
return result;
}
public bool IsUsed(bool ignoreXMatches, int id, ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>? wholePercentagesToPersonContainers, MappingFromLocation mappingFromLocation)
private bool IsUsed(bool ignoreXMatches, int id, ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>? wholePercentagesToPersonContainers, int wholePercentages)
{
bool result;
List<int>? wholePercentagesCollection;
ReadOnlyCollection<PersonContainer>? personContainers;
result = _SkipCollection.TryGetValue(id, out wholePercentagesCollection) && wholePercentagesCollection.Contains(mappingFromLocation.WholePercentages);
result = _SkipCollection.TryGetValue(id, out wholePercentagesCollection) && wholePercentagesCollection.Contains(wholePercentages);
if (!result && wholePercentagesToPersonContainers is not null)
{
if (wholePercentagesToPersonContainers.TryGetValue(mappingFromLocation.WholePercentages, out personContainers))
if (wholePercentagesToPersonContainers.TryGetValue(wholePercentages, out personContainers))
{
if (!ignoreXMatches || personContainers.All(l => IPerson.IsDefaultName(l)))
if (!ignoreXMatches || !personContainers.Any(l => IPerson.IsDefaultName(l)))
result = true;
}
}
return result;
}
public bool IsUsed(bool ignoreXMatches, int id, ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>? wholePercentagesToPersonContainers, MappingFromLocation mappingFromLocation) =>
IsUsed(ignoreXMatches, id, wholePercentagesToPersonContainers, mappingFromLocation.WholePercentages);
public bool InSkipCollection(int id, MappingFromLocation mappingFromLocation) =>
_SkipCollection.TryGetValue(id, out List<int>? wholePercentagesCollection) && wholePercentagesCollection.Contains(mappingFromLocation.WholePercentages);