@ -338,7 +338,7 @@ public partial class DlibDotNet
}
_Distance . Clear ( ) ;
ReadOnlyCollection < Shared . Models . Face > distinctFilteredFaces = Map . Models . Stateless . Methods . IMapLogic . GetFaces ( distinctFilteredItems ) ;
ReadOnlyCollection < Mapping > distinctFilteredMappingCollection = SetCreationTimeAnd GetMappings( _Configuration . PropertyConfiguration , eDistanceContentDirectory , containers , mapLogic , distinctItems : true ) ;
ReadOnlyCollection < Mapping > distinctFilteredMappingCollection = GetMappings ( _Configuration . PropertyConfiguration , eDistanceContentDirectory , containers , mapLogic , distinctItems : true ) ;
if ( runToDoCollectionFirst )
{
string json = JsonSerializer . Serialize ( distinctFilteredMappingCollection ) ;
@ -556,7 +556,7 @@ public partial class DlibDotNet
}
}
private ReadOnlyCollection < Mapping > SetCreationTimeAnd GetMappings( Property . Models . Configuration propertyConfiguration , string eDistanceContentDirectory , Container [ ] containers , MapLogic mapLogic , bool distinctItems )
private ReadOnlyCollection < Mapping > GetMappings ( Property . Models . Configuration propertyConfiguration , string eDistanceContentDirectory , Container [ ] containers , MapLogic mapLogic , bool distinctItems )
{
ReadOnlyCollection < Mapping > results ;
int count = 0 ;
@ -570,7 +570,6 @@ public partial class DlibDotNet
IEnumerable < Item > filteredItems ;
MappingFromItem mappingFromItem ;
List < Mapping > mappingCollection = new ( ) ;
ReadOnlyCollection < string > ? locationContainersFiles = null ;
foreach ( Container container in containers )
{
if ( container . Items . Count = = 0 )
@ -600,15 +599,10 @@ public partial class DlibDotNet
continue ;
anyValidFaces = true ;
mappingCollection . Add ( face . Mapping ) ;
if ( face . Mapping . MappingFromPerson is null | | face . Mapping . MappingFromPerson . LocationContainersFiles . Count = = 0 )
continue ;
Map . Models . Stateless . Methods . IMapLogic . SetCreationTime ( face . Mapping . MappingFromItem , face . Mapping . MappingFromPerson ) ;
if ( _Configuration . MoveToDecade & & _Configuration . LocationContainerDistanceTolerance is null )
Map . Models . Stateless . Methods . IMapLogic . MoveToDecade ( propertyConfiguration , face . Mapping . MappingFromItem , face . Mapping . MappingFromPerson ) ;
}
if ( ! anyValidFaces )
{
( mapping , notMapped ) = GetMapping ( mapLogic, locationContainersFiles , item , isFocusRelativePath , mappingFromItem ) ;
( mapping , notMapped ) = GetMappingAndUpdateMappingFromPerson ( mapLogic , item , isFocusRelativePath , mappingFromItem ) ;
mappingCollection . Add ( mapping ) ;
}
}
@ -728,7 +722,7 @@ public partial class DlibDotNet
if ( _Configuration . SaveMappedForOutputResolutions . Contains ( outputResolution ) )
mapLogic . SaveMapped ( dFacesContentDirectory , d2FacePartsContentDirectory , d2FacePartsContentCollectionDirectory , personKeyToIds , distinctFilteredMappingCollection , idToWholePercentagesToMapping ) ;
if ( _Configuration . SaveFaceDistancesForOutputResolutions . Contains ( outputResolution ) )
SaveFaceDistances ( ticks , mapLogic , distinctFilteredFaces , distinctFilteredMappingCollection , dFacesContentDirectory , d2FacePartsContentDirectory , d2FacePartsContentCollectionDirectory , idToWholePercentagesToMapping ) ;
SaveFaceDistances ( ticks , mapLogic , distinctFilteredFaces , dFacesContentDirectory , d2FacePartsContentDirectory , d2FacePartsContentCollectionDirectory , idToWholePercentagesToMapping ) ;
}
private bool? GetIsFocusModel ( Shared . Models . Property ? property )
@ -776,12 +770,11 @@ public partial class DlibDotNet
return result ;
}
private int GetNotMappedCountAndSetMapping ( MapLogic mapLogic , Item item , bool? isFocusRelativePath , ReadOnlyCollection < LocationContainer < MetadataExtractor . Directory > > locationContainers , MappingFromItem mappingFromItem , List < MappingFromPhotoPrism > ? mappingFromPhotoPrismCollection , List < Shared . Models . Face > faces )
private int GetNotMappedCountAndUpdateMappingFromPersonThen SetMapping ( MapLogic mapLogic , Item item , bool? isFocusRelativePath , ReadOnlyCollection < LocationContainer < MetadataExtractor . Directory > > locationContainers , MappingFromItem mappingFromItem , List < MappingFromPhotoPrism > ? mappingFromPhotoPrismCollection , List < Shared . Models . Face > faces )
{
int result ;
double? α ;
int? eyeα ;
bool? isUsed ;
bool? canReMap ;
bool? eyeReview ;
Mapping mapping ;
@ -792,12 +785,10 @@ public partial class DlibDotNet
bool? inSkipCollection ;
int wholePercentRectangle ;
string deterministicHashCodeKey ;
MappingFromFilter mappingFromFilter ;
MappingFromLocation ? mappingFromLocation ;
MappingFromFilterPre mappingFromFilterPre ;
MappingFromFilterPost mappingFromFilterPost ;
bool? isFocusModel = GetIsFocusModel ( item . Property ) ;
bool ignoreXMatches = _JLinkResolvedDirectories . Count > 0 ;
long [ ] jLinkResolvedPersonKeys = _JLinkResolvedDirectories . Select ( l = > l . PersonKey ) . ToArray ( ) ;
ReadOnlyDictionary < int , ReadOnlyCollection < PersonContainer > > ? wholePercentagesToPersonContainers ;
ReadOnlyCollection < string > locationContainersFiles = new ( ( from l in locationContainers select l . File ) . ToArray ( ) ) ;
@ -807,13 +798,11 @@ public partial class DlibDotNet
if ( item . Property ? . Id is null | | face . FaceEncoding is null | | face . Location is null | | face . OutputResolution is null )
{
canReMap = null ;
isUsed = null ;
isFocusPerson = null ;
inSkipCollection = null ;
mappingFromLocation = null ;
mappingFromFilterPost = new ( canReMap , inSkipCollection , isFocusPerson ) ;
mappingFromFilterPre = new ( inSkipCollection , isFocusModel , isFocusRelativePath ) ;
mappingFromFilter = new ( isFocusModel , isFocusPerson , isFocusRelativePath , inSkipCollection , isUsed ) ;
}
else
{
@ -832,14 +821,11 @@ public partial class DlibDotNet
inSkipCollection = mapLogic . InSkipCollection ( item . Property . Id . Value , mappingFromLocation ) ;
mappingFromFilterPre = new ( inSkipCollection , isFocusModel , isFocusRelativePath ) ;
canReMap = Map . Models . Stateless . Methods . IMapLogic . CanReMap ( jLinkResolvedPersonKeys , wholePercentagesToPersonContainers , mappingFromLocation ) ;
isUsed = mapLogic . IsUsed ( ignoreXMatches , item . Property . Id . Value , wholePercentagesToPersonContainers , mappingFromLocation ) ;
isFocusPerson = mapLogic . IsFocusPersonOld ( _Configuration . SkipPersonWithMoreThen , _JLinkResolvedDirectories , wholePercentagesToPersonContainers , mappingFromLocation ) ;
mappingFromFilter = new ( isFocusModel , isFocusPerson , isFocusRelativePath , inSkipCollection , isUsed ) ;
isFocusPerson = mapLogic . IsFocusPerson ( _Configuration . SkipPersonWithMoreThen , jLinkResolvedPersonKeys , wholePercentagesToPersonContainers , mappingFromLocation ) ;
mappingFromFilterPost = new ( canReMap , inSkipCollection , isFocusPerson ) ;
}
mapping = new ( mappingFromFilter , mappingFromFilterPost, mappingFromFilterPre , mappingFromItem , mappingFromLocation , mappingFromPhotoPrismCollection ) ;
notMapped + = mapLogic . UpdateMappingFromPerson ( locationContainersFiles , wholePercentagesToPersonContainers, mapping ) ;
mapping = new ( mappingFromFilterPost , mappingFromFilterPre , mappingFromItem , mappingFromLocation , mappingFromPhotoPrismCollection ) ;
notMapped + = mapLogic . UpdateMappingFromPerson ( wholePercentagesToPersonContainers , mapping ) ;
face . SetMapping ( mapping ) ;
}
result = notMapped ;
@ -862,6 +848,8 @@ public partial class DlibDotNet
configuration . PersonCharacters . ToArray ( ) ,
configuration . RangeDaysDeltaTolerance ,
configuration . RangeDistanceTolerance ,
configuration . ReMap ,
configuration . SaveIndividually ,
configuration . SaveSortingWithoutPerson ,
configuration . SkipNotSkipDirectories ,
configuration . SortingMaximumPerKey ,
@ -872,10 +860,9 @@ public partial class DlibDotNet
return result ;
}
private ( Mapping , int ) GetMapping ( MapLogic mapLogic, ReadOnlyCollection < string > ? locationContainersFiles , Item item , bool? isFocusRelativePath , MappingFromItem mappingFromItem )
private ( Mapping , int ) GetMappingAndUpdateMappingFromPerson ( MapLogic mapLogic , Item item , bool? isFocusRelativePath , MappingFromItem mappingFromItem )
{
Mapping result ;
bool? isUsed ;
bool? canReMap ;
int? eyeα = null ;
bool? isFocusPerson ;
@ -885,24 +872,20 @@ public partial class DlibDotNet
int faceAreaPermyriad = 0 ;
int wholePercentRectangle ;
string deterministicHashCodeKey ;
MappingFromFilter mappingFromFilter ;
MappingFromLocation ? mappingFromLocation ;
MappingFromFilterPre mappingFromFilterPre ;
MappingFromFilterPost mappingFromFilterPost ;
bool? isFocusModel = GetIsFocusModel ( item . Property ) ;
bool ignoreXMatches = _JLinkResolvedDirectories . Count > 0 ;
long [ ] jLinkResolvedPersonKeys = _JLinkResolvedDirectories . Select ( l = > l . PersonKey ) . ToArray ( ) ;
ReadOnlyDictionary < int , ReadOnlyCollection < PersonContainer > > ? wholePercentagesToPersonContainers = mapLogic . GetWholePercentagesToPersonContainers ( item . Property ? . Id ) ;
if ( item . Property ? . Id is null )
{
isUsed = null ;
canReMap = null ;
isFocusPerson = null ;
inSkipCollection = null ;
mappingFromLocation = null ;
mappingFromFilterPost = new ( canReMap , inSkipCollection , isFocusPerson ) ;
mappingFromFilterPre = new ( inSkipCollection , isFocusModel , isFocusRelativePath ) ;
mappingFromFilter = new ( isFocusModel , isFocusPerson , isFocusRelativePath , inSkipCollection , isUsed ) ;
}
else
{
@ -912,14 +895,11 @@ public partial class DlibDotNet
inSkipCollection = mapLogic . InSkipCollection ( item . Property . Id . Value , mappingFromLocation ) ;
mappingFromFilterPre = new ( inSkipCollection , isFocusModel , isFocusRelativePath ) ;
canReMap = Map . Models . Stateless . Methods . IMapLogic . CanReMap ( jLinkResolvedPersonKeys , wholePercentagesToPersonContainers , mappingFromLocation ) ;
isUsed = mapLogic . IsUsed ( ignoreXMatches , item . Property . Id . Value , wholePercentagesToPersonContainers , mappingFromLocation ) ;
isFocusPerson = mapLogic . IsFocusPersonOld ( _Configuration . SkipPersonWithMoreThen , _JLinkResolvedDirectories , wholePercentagesToPersonContainers , mappingFromLocation ) ;
mappingFromFilter = new ( isFocusModel , isFocusPerson , isFocusRelativePath , inSkipCollection , isUsed ) ;
isFocusPerson = mapLogic . IsFocusPerson ( _Configuration . SkipPersonWithMoreThen , jLinkResolvedPersonKeys , wholePercentagesToPersonContainers , mappingFromLocation ) ;
mappingFromFilterPost = new ( canReMap , inSkipCollection , isFocusPerson ) ;
}
result = new ( mappingFromFilter , mappingFromFilterPost, mappingFromFilterPre , mappingFromItem , mappingFromLocation , mappingFromPhotoPrismCollection : null ) ;
int notMapped = mapLogic . UpdateMappingFromPerson ( locationContainersFiles , wholePercentagesToPersonContainers, result ) ;
result = new ( mappingFromFilterPost , mappingFromFilterPre , mappingFromItem , mappingFromLocation , mappingFromPhotoPrismCollection : null ) ;
int notMapped = mapLogic . UpdateMappingFromPerson ( wholePercentagesToPersonContainers , result ) ;
return ( result , notMapped ) ;
}
@ -998,6 +978,7 @@ public partial class DlibDotNet
throw new NullReferenceException ( nameof ( property ) ) ;
item . SetResizedFileHolder ( _Resize . FileNameExtension , resizedFileHolder ) ;
MappingFromItem mappingFromItem = IMappingFromItem . GetMappingFromItem ( containerDateTimes , item , resizedFileHolder ) ;
Map . Models . Stateless . Methods . IMapLogic . SetCreationTimeMaybeMoveToDecade ( _Configuration . PropertyConfiguration , _Configuration . MoveToDecade & & _Configuration . LocationContainerDistanceTolerance is null , mappingFromItem , locationContainers ) ;
( int metadataGroups , metadataCollection ) = metadata . GetMetadataCollection ( subFileTuples , parseExceptions , changesFrom , mappingFromItem ) ;
if ( _AppSettings . MaxDegreeOfParallelism < 2 )
ticks = LogDelta ( ticks , nameof ( B_Metadata . GetMetadataCollection ) ) ;
@ -1022,7 +1003,7 @@ public partial class DlibDotNet
faces = _Faces . GetFaces ( outputResolution , dResultsFullGroupDirectory , subFileTuples , parseExceptions , property , mappingFromItem , outputResolutionToResize , locationContainers , mappingFromPhotoPrismCollection ) ;
if ( _AppSettings . MaxDegreeOfParallelism < 2 )
ticks = LogDelta ( ticks , nameof ( D_Face . GetFaces ) ) ;
result = GetNotMappedCountAndSetMapping ( mapLogic , item , isFocusRelativePath , locationContainers , mappingFromItem , mappingFromPhotoPrismCollection , faces ) ;
result = GetNotMappedCountAndUpdateMappingFromPersonThen SetMapping ( mapLogic , item , isFocusRelativePath , locationContainers , mappingFromItem , mappingFromPhotoPrismCollection , faces ) ;
List < ( Shared . Models . Face , FileInfo ? , string , bool Saved ) > faceCollection = _Faces . SaveFaces ( _FaceParts . FileNameExtension , dResultsFullGroupDirectory , subFileTuples , parseExceptions , mappingFromItem , faces ) ;
if ( _Configuration . CopyFacesAndSaveFaceLandmarkForOutputResolutions . Contains ( outputResolution ) )
_FaceParts . CopyFacesAndSaveFaceLandmarkImage ( facePartsCollectionDirectory , mappingFromItem , faceCollection ) ;
@ -1069,7 +1050,6 @@ public partial class DlibDotNet
throw new NullReferenceException ( nameof ( _Log ) ) ;
int result = 0 ;
int exceptionsCount = 0 ;
bool ignoreXMatches = _JLinkResolvedDirectories . Count > 0 ;
ParallelOptions parallelOptions = new ( ) { MaxDegreeOfParallelism = maxDegreeOfParallelism } ;
DateTime [ ] containerDateTimes = Shared . Models . Stateless . Methods . IContainer . GetContainerDateTimes ( filteredItems ) ;
ProgressBarOptions options = new ( ) { ProgressCharacter = '─' , ProgressBarOnBottom = true , DisableBottomPercentage = true } ;
@ -1164,49 +1144,53 @@ public partial class DlibDotNet
return new ( cResultsFullGroupDirectory , c2ResultsFullGroupDirectory , dResultsFullGroupDirectory , d2ResultsFullGroupDirectory ) ;
}
private void SaveFaceDistances ( long ticks , MapLogic mapLogic , ReadOnlyCollection < Mapping > mappingCollection , string dFacesContentDirectory , string d2FacePartsContentDirectory , string d2FacePartsContentCollectionDirectory , ReadOnlyDictionary < int , ReadOnlyDictionary < int , Mapping > > idToWholePercentagesToMapping , ReadOnlyCollection < FaceDistance > faceDistanceEncodings , ReadOnlyCollection < FaceDistanceContainer > faceDistanceContainers )
private void SaveFaceDistances ( long ticks , MapLogic mapLogic , string dFacesContentDirectory , string d2FacePartsContentDirectory , string d2FacePartsContentCollectionDirectory , ReadOnlyDictionary < int , ReadOnlyDictionary < int , Mapping > > idToWholePercentagesToMapping , ReadOnlyCollection < FaceDistance > faceDistanceEncodings , ReadOnlyCollection < FaceDistanceContainer > faceDistanceContainers )
{
if ( _Log is null )
throw new NullReferenceException ( nameof ( _Log ) ) ;
int? useFiltersCounter = null ;
DistanceLimits distanceLimits ;
SortingContainer [ ] sortingContainers ;
ReadOnlyCollection < SortingContainer> sortingContainers ;
FaceDistanceContainer [ ] filteredFaceDistanceContainers ;
long? skipOlderThan = _Configuration . SkipOlderThanDays is null ? null : new DateTime ( ticks ) . AddDays ( - _Configuration . SkipOlderThanDays . Value ) . Ticks ;
distanceLimits = new ( _Configuration . FaceAreaPermyriad , _Configuration . FaceConfidencePercent , _Configuration . FaceDistancePermyriad , _Configuration . RangeDaysDeltaTolerance , _Configuration . RangeDistanceTolerance , _Configuration . RangeFaceAreaPermyriadTolerance , _Configuration . RangeFaceConfidence , _Configuration . SortingMaximumPerFaceShouldBeHigh ) ;
filteredFaceDistanceContainers = E_Distance . FilteredFaceDistanceContainers ( mapLogic , faceDistanceContainers , skipOlderThan , distanceLimits ) ;
filteredFaceDistanceContainers = E_Distance . FilteredPostLoad FaceDistanceContainers ( mapLogic , faceDistanceContainers , skipOlderThan , distanceLimits ) ;
if ( filteredFaceDistanceContainers . Length = = 0 )
_Log . Information ( "All images have been filtered!" ) ;
else
{
sortingContainers = E_Distance . SetFaceMappingSortingCollectionThenGetSortingContainers ( _AppSettings . MaxDegreeOfParallelism , _MapConfiguration , ticks , mapLogic , distanceLimits , faceDistanceEncodings , filteredFaceDistanceContainers ) ;
if ( sortingContainers . Length = = 0 )
sortingContainers = E_Distance . SetFaceMappingSortingCollectionThenGetSortedSort ingContainers ( _AppSettings . MaxDegreeOfParallelism , _MapConfiguration , ticks , mapLogic , distanceLimits , faceDistanceEncodings , filteredFaceDistanceContainers ) ;
if ( sortingContainers . Count = = 0 )
{
for ( useFiltersCounter = 1 ; useFiltersCounter < _Configuration . UseFilterTries ; useFiltersCounter + + )
{
distanceLimits = new ( _Configuration . FaceAreaPermyriad , _Configuration . FaceConfidencePercent , _Configuration . FaceDistancePermyriad , _Configuration . RangeDaysDeltaTolerance , _Configuration . RangeDistanceTolerance , _Configuration . RangeFaceAreaPermyriadTolerance , _Configuration . RangeFaceConfidence , _Configuration . SortingMaximumPerFaceShouldBeHigh , useFiltersCounter ) ;
filteredFaceDistanceContainers = E_Distance . FilteredFaceDistanceContainers ( mapLogic , faceDistanceContainers , skipOlderThan , distanceLimits ) ;
filteredFaceDistanceContainers = E_Distance . FilteredPostLoad FaceDistanceContainers ( mapLogic , faceDistanceContainers , skipOlderThan , distanceLimits ) ;
if ( filteredFaceDistanceContainers . Length = = 0 )
_Log . Information ( "All images have been filtered!" ) ;
else
{
sortingContainers = E_Distance . SetFaceMappingSortingCollectionThenGetSortingContainers ( _AppSettings . MaxDegreeOfParallelism , _MapConfiguration , ticks , mapLogic , distanceLimits , faceDistanceEncodings , filteredFaceDistanceContainers ) ;
if ( sortingContainers . Length = = 0 )
sortingContainers = E_Distance . SetFaceMappingSortingCollectionThenGetSortedSort ingContainers ( _AppSettings . MaxDegreeOfParallelism , _MapConfiguration , ticks , mapLogic , distanceLimits , faceDistanceEncodings , filteredFaceDistanceContainers ) ;
if ( sortingContainers . Count = = 0 )
break ;
}
}
}
E_Distance . SaveFaceDistances ( _Configuration . PropertyConfiguration , sortingContainers ) ;
sortingContainers = mapLogic . GetFilterSortingContainers ( dFacesContentDirectory , d2FacePartsContentDirectory , d2FacePartsContentCollectionDirectory , idToWholePercentagesToMapping , distanceLimits , sortingContainers ) ;
if ( sortingContainers . Count > 0 )
E_Distance . SaveFaceDistances ( _Configuration . PropertyConfiguration , sortingContainers ) ;
if ( filteredFaceDistanceContainers . Length > 0 )
{
int updated = sortingContainers . Length = = 0 ? 0 : mapLogic . UpdateFromSortingContainers ( _Configuration . SaveIndividuall y, idToWholePercentagesToMapping , distanceLimits , sortingContainers ) ;
List < SaveContainer > saveContainers = mapLogic . GetSaveContainers ( _Configuration . SaveIndividually , dFacesContentDirectory , d2FacePartsContentDirectory , d2FacePartsContentCollectionDirectory , mappingCollection , idToWholePercentagesToMapping , useFiltersCounter , sortingContainers . Length > 0 ) ;
mapLogic . SaveContainers ( _Configuration . SaveIndividually , updated , save Containers) ;
int updated = sortingContainers . Count = = 0 ? 0 : mapLogic . UpdateFromSortingContainers ( dFacesContentDirectory , d2FacePartsContentDirectory , d2FacePartsContentCollectionDirector y, idToWholePercentagesToMapping , sortingContainers ) ;
List < SaveContainer > saveContainers ;
saveContainers = mapLogic. Get SaveContainers( dFacesContentDirectory , d2FacePartsContentDirectory , d2FacePartsContentCollectionDirectory , idToWholePercentagesToMapping , distanceLimits , useFiltersCounter , sorting Containers) ;
if ( saveContainers . Count > 0 )
mapLogic . SaveContainers ( updated , saveContainers ) ;
}
}
}
private void SaveFaceDistances ( long ticks , MapLogic mapLogic , ReadOnlyCollection < Shared . Models . Face > distinctFilteredFaces , ReadOnlyCollection < Mapping > mappingCollection , string dFacesContentDirectory , string d2FacePartsContentDirectory , string d2FacePartsContentCollectionDirectory , ReadOnlyDictionary < int , ReadOnlyDictionary < int , Mapping > > idToWholePercentagesToMapping )
private void SaveFaceDistances ( long ticks , MapLogic mapLogic , ReadOnlyCollection < Shared . Models . Face > distinctFilteredFaces , string dFacesContentDirectory , string d2FacePartsContentDirectory , string d2FacePartsContentCollectionDirectory , ReadOnlyDictionary < int , ReadOnlyDictionary < int , Mapping > > idToWholePercentagesToMapping )
{
E_Distance . PreFilterSetFaceDistances ( _AppSettings . MaxDegreeOfParallelism , ticks , distinctFilteredFaces ) ;
ReadOnlyCollection < FaceDistanceContainer > faceDistanceContainers = E_Distance . GetFaceDistanceContainers ( distinctFilteredFaces ) ;
@ -1219,7 +1203,7 @@ public partial class DlibDotNet
continue ;
faceDistanceEncodings . Add ( faceDistanceContainer . FaceDistance ) ;
}
SaveFaceDistances ( ticks , mapLogic , mappingCollection , dFacesContentDirectory , d2FacePartsContentDirectory , d2FacePartsContentCollectionDirectory , idToWholePercentagesToMapping , new ( faceDistanceEncodings ) , faceDistanceContainers ) ;
SaveFaceDistances ( ticks , mapLogic , dFacesContentDirectory , d2FacePartsContentDirectory , d2FacePartsContentCollectionDirectory , idToWholePercentagesToMapping , new ( faceDistanceEncodings ) , faceDistanceContainers ) ;
}
}