phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Any

Browse Source 
MoveToDecade
This commit is contained in:
Mike Phares
2023-08-06 17:19:06 -07:00
parent 6d17380430
commit d4cbea3835
41 changed files with 974 additions and 637 deletions
Download Patch File Download Diff File Expand all files Collapse all files

234
Instance/DlibDotNet.cs
View File

@ -128,7 +128,7 @@ public partial class DlibDotNet
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(peopleRootDirectory, propertyConfiguration.ResultSingleton));
string a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration.PropertyConfiguration, nameof(A2_People), "([])");
personContainers = new(IPersonContainer.GetPersonContainers(storage, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), _Faces.FileNameExtension));
if (configuration.JLinks.Length > 0)
if (configuration.JLinks.Where(l => !string.IsNullOrEmpty(l)).Any())
{
string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration.PropertyConfiguration, nameof(A2_People), "{}");
_JLinkResolvedDirectories.AddRange(Map.Models.Stateless.Methods.IMapLogic.GetJLinkDirectories(configuration.GenealogicalDataCommunicationFile, configuration.JLinks, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), a2PeopleSingletonDirectory, a2PeopleContentDirectory));
@ -201,7 +201,7 @@ public partial class DlibDotNet
private static void Verify(Models.Configuration configuration)
{
if (!configuration.OutputResolutions.Any() || string.IsNullOrEmpty(configuration.OutputResolutions[0]) || !configuration.ValidResolutions.Contains(configuration.OutputResolutions[0]))
if (configuration.OutputResolutions.Length == 0 || string.IsNullOrEmpty(configuration.OutputResolutions[0]) || !configuration.ValidResolutions.Contains(configuration.OutputResolutions[0]))
throw new NullReferenceException($"{nameof(configuration.OutputResolutions)} must be fileNameToCollection valid outputResolution!");
if ((from l in configuration.OutputResolutions where !configuration.ValidResolutions.Contains(l) select false).Any())
throw new Exception($"One or more {nameof(configuration.OutputResolutions)} are not in the ValidResolutions list!");
@ -228,7 +228,7 @@ public partial class DlibDotNet
private void VerifyExtra(List<string> args, Property.Models.Configuration propertyConfiguration, Models.Configuration configuration)
{
string[] sourceDirectoryNames;
if (!args.Any())
if (args.Count == 0)
sourceDirectoryNames = Array.Empty<string>();
else
{
@ -249,7 +249,7 @@ public partial class DlibDotNet
}
}
string[] resizeMatch = (from l in sourceDirectoryNames where configuration.ValidResolutions.Contains(l) select l).ToArray();
if (resizeMatch.Any())
if (resizeMatch.Length > 0)
throw new Exception("Input directory should be the source and not fileNameToCollection resized directory!");
if (configuration.LocationDigits != Shared.Models.Stateless.ILocation.Digits)
throw new Exception("Configuration has to match interface!");
@ -294,7 +294,7 @@ public partial class DlibDotNet
return result;
}
private void SetMapping(MapLogic mapLogic, Item item, bool? isFocusRelativePath, bool? isIgnoreRelativePath, MappingFromItem mappingFromItem, List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection, List<Shared.Models.Face> faces)
private void SetMapping(MapLogic mapLogic, Item item, bool? isFocusRelativePath, bool? isIgnoreRelativePath, ReadOnlyCollection<LocationContainer<MetadataExtractor.Directory>> locationContainers, MappingFromItem mappingFromItem, List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection, List<Shared.Models.Face> faces)
{
double? α;
int? eyeα;
@ -311,6 +311,7 @@ public partial class DlibDotNet
MappingFromLocation? mappingFromLocation;
bool? isFocusModel = GetIsFocusModel(item.Property);
bool ignoreXMatches = _JLinkResolvedDirectories.Count > 0;
ReadOnlyCollection<string> locationContainersFiles = new((from l in locationContainers select l.File).ToArray());
foreach (Shared.Models.Face face in faces)
{
if (item.Property?.Id is null || face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
@ -337,16 +338,16 @@ public partial class DlibDotNet
mappingFromLocation = new(faceAreaPermyriad, confidencePercent, deterministicHashCodeKey, eyeα, eyeReview, wholePercentRectangle);
isUsed = mapLogic.IsUsed(ignoreXMatches, item.Property.Id.Value, mappingFromLocation);
inSkipCollection = mapLogic.InSkipCollection(item.Property.Id.Value, mappingFromLocation);
isFocusPerson = mapLogic.IsFocusPerson(_JLinkResolvedDirectories, item.Property.Id.Value, mappingFromLocation);
isFocusPerson = mapLogic.IsFocusPerson(_Configuration.SkipPersonWithMoreThen, _JLinkResolvedDirectories, item.Property.Id.Value, mappingFromLocation);
mappingFromFilter = new(isFocusModel, isFocusPerson, isFocusRelativePath, isIgnoreRelativePath, inSkipCollection, isUsed);
}
mapping = new(mappingFromItem, mappingFromFilter, mappingFromLocation, mappingFromPhotoPrismCollection);
_ = mapLogic.UpdateMappingFromPerson(mapping);
_ = mapLogic.UpdateMappingFromPerson(locationContainersFiles, mapping);
face.SetMapping(mapping);
}
}
private Mapping GetMapping(MapLogic mapLogic, Item item, bool? isFocusRelativePath, bool? isIgnoreRelativePath, MappingFromItem mappingFromItem, List<(string, long)> jLinkResolvedDirectories)
private Mapping GetMapping(MapLogic mapLogic, Item item, bool? isFocusRelativePath, bool? isIgnoreRelativePath, MappingFromItem mappingFromItem)
{
Mapping result;
bool? isUsed;
@ -361,7 +362,7 @@ public partial class DlibDotNet
MappingFromFilter mappingFromFilter;
MappingFromLocation? mappingFromLocation;
bool? isFocusModel = GetIsFocusModel(item.Property);
bool ignoreXMatches = jLinkResolvedDirectories.Count > 0;
bool ignoreXMatches = _JLinkResolvedDirectories.Count > 0;
if (item.Property?.Id is null)
{
isUsed = null;
@ -377,7 +378,7 @@ public partial class DlibDotNet
mappingFromLocation = new(faceAreaPermyriad, confidencePercent, deterministicHashCodeKey, eyeα, eyeReview, wholePercentRectangle);
isUsed = mapLogic.IsUsed(ignoreXMatches, item.Property.Id.Value, mappingFromLocation);
inSkipCollection = mapLogic.InSkipCollection(item.Property.Id.Value, mappingFromLocation);
isFocusPerson = mapLogic.IsFocusPerson(jLinkResolvedDirectories, item.Property.Id.Value, mappingFromLocation);
isFocusPerson = mapLogic.IsFocusPerson(_Configuration.SkipPersonWithMoreThen, _JLinkResolvedDirectories, item.Property.Id.Value, mappingFromLocation);
mappingFromFilter = new(isFocusModel, isFocusPerson, isFocusRelativePath, isIgnoreRelativePath, inSkipCollection, isUsed);
}
result = new(mappingFromItem, mappingFromFilter, mappingFromLocation, mappingFromPhotoPrismCollection: null);
@ -408,7 +409,6 @@ public partial class DlibDotNet
private void FullParallelForWork(A_Property propertyLogic,
B_Metadata metadata,
ReadOnlyDictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers,
MapLogic mapLogic,
string outputResolution,
bool outputResolutionHasNumber,
@ -434,6 +434,7 @@ public partial class DlibDotNet
List<KeyValuePair<string, string>> metadataCollection;
string[] changesFrom = new string[] { nameof(A_Property) };
FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(cResultsFullGroupDirectory, item, outputResolutionHasNumber);
ReadOnlyCollection<LocationContainer<MetadataExtractor.Directory>> locationContainers = mapLogic.GetLocationContainers(item);
if (item.Property is null || item.Property.Id is null || item.Any())
{
LogItemPropertyIsNull(item);
@ -495,17 +496,12 @@ public partial class DlibDotNet
else
{
List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection;
List<LocationContainer<MetadataExtractor.Directory>>? locationContainers;
if (item.Property?.Id is null)
locationContainers = null;
else
_ = idToLocationContainers.TryGetValue(item.Property.Id.Value, out locationContainers);
if (!fileNameToCollection.TryGetValue(mappingFromItem.Id, out mappingFromPhotoPrismCollection))
mappingFromPhotoPrismCollection = null;
faces = _Faces.GetFaces(outputResolution, dResultsFullGroupDirectory, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionToResize, locationContainers, mappingFromPhotoPrismCollection);
if (_AppSettings.MaxDegreeOfParallelism < 2)
ticks = LogDelta(ticks, nameof(D_Face.GetFaces));
SetMapping(mapLogic, item, isFocusRelativePath, isIgnoreRelativePath, mappingFromItem, mappingFromPhotoPrismCollection, faces);
SetMapping(mapLogic, item, isFocusRelativePath, isIgnoreRelativePath, locationContainers, mappingFromItem, mappingFromPhotoPrismCollection, faces);
List<(Shared.Models.Face, FileInfo?, string, bool Saved)> faceCollection = _Faces.SaveFaces(_FaceParts.FileNameExtension, dResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem, faces);
if (_Configuration.CopyFacesAndSaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
_FaceParts.CopyFacesAndSaveFaceLandmarkImage(facePartsCollectionDirectory, mappingFromItem, faceCollection);
@ -535,7 +531,6 @@ public partial class DlibDotNet
private int FullParallelWork(int maxDegreeOfParallelism,
A_Property propertyLogic,
B_Metadata metadata,
ReadOnlyDictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers,
MapLogic mapLogic,
string outputResolution,
bool outputResolutionHasNumber,
@ -558,7 +553,7 @@ public partial class DlibDotNet
string focusRelativePath = Path.GetFullPath(string.Concat(_Configuration.PropertyConfiguration.RootDirectory, _Configuration.FocusDirectory));
bool? isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath);
string facePartsCollectionDirectory = _Configuration.CopyFacesAndSaveFaceLandmarkForOutputResolutions.Contains(outputResolution) ? _FaceParts.GetFacePartsDirectory(_Configuration.PropertyConfiguration, d2ResultsFullGroupDirectory, item: filteredItems.First(), includeNameWithoutExtension: false) : string.Empty;
bool? isIgnoreRelativePath = !_Configuration.IgnoreRelativePaths.Any() ? null : _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory);
bool? isIgnoreRelativePath = _Configuration.IgnoreRelativePaths.Length == 0 ? null : _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory);
using ProgressBar progressBar = new(filteredItems.Length, message, options);
_ = Parallel.For(0, filteredItems.Length, parallelOptions, (i, state) =>
{
@ -566,7 +561,6 @@ public partial class DlibDotNet
{
FullParallelForWork(propertyLogic,
metadata,
idToLocationContainers,
mapLogic,
outputResolution,
outputResolutionHasNumber,
@ -582,7 +576,7 @@ public partial class DlibDotNet
isFocusRelativePath,
isIgnoreRelativePath,
facePartsCollectionDirectory);
if (i == 0 || sourceDirectoryChanges.Any())
if (i == 0 || sourceDirectoryChanges.Count > 0)
progressBar.Tick();
}
catch (Exception ex)
@ -619,7 +613,7 @@ public partial class DlibDotNet
if (duplicates.Contains(metadataIdLines[i].Id))
metadataIdLines.RemoveAt(i);
}
if (metadataIdLines.Any())
if (metadataIdLines.Count > 0)
{
text = string.Join(Environment.NewLine, from l in metadataIdLines orderby l.Id select l.Line);
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(fileInfo.FullName, text, updateDateWhenMatches: true, compareBeforeWrite: true);
@ -683,7 +677,7 @@ public partial class DlibDotNet
return new(cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory);
}
private void FullDoWork(string argZero, string propertyRoot, long ticks, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, int t, Container[] containers, A_Property propertyLogic, B_Metadata metadata, Dictionary<int, List<MappingFromPhotoPrism>> fileNameToCollection, ReadOnlyDictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers, MapLogic mapLogic)
private void FullDoWork(string argZero, string propertyRoot, long ticks, string aResultsFullGroupDirectory, string bResultsFullGroupDirectory, int t, Container[] containers, A_Property propertyLogic, B_Metadata metadata, Dictionary<int, List<MappingFromPhotoPrism>> fileNameToCollection, MapLogic mapLogic)
{
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
@ -715,12 +709,12 @@ public partial class DlibDotNet
for (int i = 0; i < containers.Length; i++)
{
container = containers[i];
if (!container.Items.Any())
if (container.Items.Count == 0)
continue;
if (!_ArgZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
continue;
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container);
if (!filteredItems.Any())
if (filteredItems.Length == 0)
continue;
sourceDirectoryChanges.Clear();
anyNullOrNoIsUniqueFileName = filteredItems.Any(l => !l.IsUniqueFileName);
@ -732,7 +726,6 @@ public partial class DlibDotNet
exceptionCount = FullParallelWork(maxDegreeOfParallelism,
propertyLogic,
metadata,
idToLocationContainers,
mapLogic,
outputResolution,
outputResolutionHasNumber,
@ -749,7 +742,7 @@ public partial class DlibDotNet
_Exceptions.Add(container.SourceDirectory);
continue;
}
if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Any())
if (Directory.GetFiles(propertyRoot, "*.txt", SearchOption.TopDirectoryOnly).Length > 0)
{
for (int y = 0; y < int.MaxValue; y++)
{
@ -786,7 +779,7 @@ public partial class DlibDotNet
{
distanceLimits = new(_Configuration.FaceAreaPermyriad, _Configuration.FaceConfidencePercent, _Configuration.FaceDistancePermyriad, _Configuration.RangeDaysDeltaTolerance, _Configuration.RangeDistanceTolerance, _Configuration.RangeFaceAreaTolerance, _Configuration.RangeFaceConfidence, _Configuration.SortingMaximumPerFaceShouldBeHigh, useFiltersCounter);
filteredFaceDistanceContainers = E_Distance.FilteredFaceDistanceContainers(mapLogic, faceDistanceContainers, skipOlderThan, distanceLimits);
if (filteredFaceDistanceContainers.Any())
if (filteredFaceDistanceContainers.Length == 0)
_Log.Information("All images have been filtered!");
else
{
@ -800,7 +793,7 @@ public partial class DlibDotNet
if (filteredFaceDistanceContainers.Length > 0)
{
int updated = mapLogic.UpdateFromSortingContainers(_Configuration.SaveIndividually, distanceLimits, sortingContainers);
List<SaveContainer> saveContainers = mapLogic.GetSaveContainers(_Configuration.SaveIndividually, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, mappingCollection, idToWholePercentagesToMapping, useFiltersCounter, sortingContainers.Any());
List<SaveContainer> saveContainers = mapLogic.GetSaveContainers(_Configuration.SaveIndividually, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, mappingCollection, idToWholePercentagesToMapping, useFiltersCounter, sortingContainers.Length > 0);
mapLogic.SaveContainers(_Configuration.SaveIndividually, filteredFaceDistanceContainers.Length, updated, saveContainers);
}
}
@ -810,10 +803,10 @@ public partial class DlibDotNet
{
E_Distance.SetFaceDistances(_AppSettings.MaxDegreeOfParallelism, ticks, distinctFilteredFaces);
FaceDistanceContainer[] faceDistanceContainers = E_Distance.GetFaceDistanceContainers(distinctFilteredFaces);
Dictionary<int, Dictionary<int, PersonContainer[]>> missingIdThenWholePercentagesToPersonContainers = mapLogic.GetMissing(idToWholePercentagesToMapping);
Dictionary<int, ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>> missingIdThenWholePercentagesToPersonContainers = mapLogic.GetMissing(idToWholePercentagesToMapping);
List<FaceDistanceContainer> missingFaceDistanceContainers = _Distance.GetMissingFaceDistanceContainer(_AppSettings.MaxDegreeOfParallelism, ticks, dFacesCollectionDirectory, missingIdThenWholePercentagesToPersonContainers);
List<FaceDistance> faceDistanceEncodings = E_Distance.GetFaceDistanceEncodings(faceDistanceContainers, missingFaceDistanceContainers);
if (faceDistanceContainers.Any())
if (faceDistanceContainers.Length > 0)
SaveFaceDistances(ticks, mapLogic, mappingCollection, dFacesContentDirectory, d2FacePartsContentDirectory, d2FacePartsContentCollectionDirectory, idToWholePercentagesToMapping, faceDistanceEncodings, faceDistanceContainers);
}
@ -823,7 +816,7 @@ public partial class DlibDotNet
string d2FacePartsContentDirectory = Path.Combine(d2ResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
string d2FacePartsContentCollectionDirectory = Path.Combine(d2ResultsFullGroupDirectory, "[()]");
string dFacesCollectionDirectory = Path.Combine(dResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultCollection, _Configuration.PropertyConfiguration.ResultAllInOne);
if (distinctFilteredMappingCollection.Any())
if (distinctFilteredMappingCollection.Length > 0)
{
Shared.Models.Stateless.Methods.IPath.ChangeDateForEmptyDirectories(d2FacePartsContentDirectory, ticks);
if (Directory.Exists(d2FacePartsContentCollectionDirectory))
@ -852,7 +845,7 @@ public partial class DlibDotNet
string result;
if (_Log is null)
throw new NullReferenceException(nameof(_Log));
if (!files.Any())
if (files.Length == 0)
throw new NotSupportedException();
string? sourceDirectory = Path.GetDirectoryName(files.First());
if (string.IsNullOrEmpty(sourceDirectory))
@ -893,108 +886,6 @@ public partial class DlibDotNet
return result;
}
private static void LookForAbandoned(List<int> distinctFilteredIds, string directory, string directoryName)
{
string fileNameWithoutExtension;
bool nameWithoutExtensionIsIdFormat;
List<string> renameCollection = new();
bool nameWithoutExtensionIsPaddedIdFormat;
int sortOrderOnlyLengthIndex = IDirectory.GetSortOrderOnlyLengthIndex();
string[] distinctFilteredIdsValues = distinctFilteredIds.Select(l => l.ToString()).ToArray();
string[] files = Directory.GetFiles(directory, "*", SearchOption.AllDirectories);
foreach (string file in files)
{
fileNameWithoutExtension = Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(file)));
nameWithoutExtensionIsIdFormat = Shared.Models.Stateless.Methods.IProperty.NameWithoutExtensionIsIdFormat(fileNameWithoutExtension);
nameWithoutExtensionIsPaddedIdFormat = IDirectory.NameWithoutExtensionIsPaddedIdFormat(fileNameWithoutExtension, sortOrderOnlyLengthIndex);
if (!nameWithoutExtensionIsIdFormat && !nameWithoutExtensionIsPaddedIdFormat)
continue;
if (distinctFilteredIdsValues.Contains(fileNameWithoutExtension))
continue;
renameCollection.Add(file);
}
if (renameCollection.Any())
{
if (directoryName.Length == 2)
IDirectory.MoveFiles(renameCollection, directoryName, $"{directoryName[0]}abd{directoryName[^1]}");
else if (directoryName.Length == 4)
IDirectory.MoveFiles(renameCollection, directoryName, $"{directoryName[..2]}abd{directoryName[^2..]}");
else
throw new NotSupportedException();
}
}
private static void LookForAbandoned(string bResultsFullGroupDirectory, List<int> distinctFilteredIds)
{
string[] directories = Directory.GetDirectories(bResultsFullGroupDirectory, "*", SearchOption.TopDirectoryOnly);
foreach (string directory in directories)
{
string? directoryName = Path.GetFileName(directory);
if (string.IsNullOrEmpty(directoryName) || (directoryName.Length != 2 && directoryName.Length != 4))
continue;
LookForAbandoned(distinctFilteredIds, directory, directoryName);
}
}
private void LookForAbandoned(ReadOnlyDictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers, List<int> distinctFilteredIds)
{
List<string> renameCollection = new();
foreach (KeyValuePair<int, List<LocationContainer<MetadataExtractor.Directory>>> keyValuePair in idToLocationContainers)
{
if (distinctFilteredIds.Contains(keyValuePair.Key))
continue;
foreach (LocationContainer<MetadataExtractor.Directory> locationContainer in keyValuePair.Value)
{
if (locationContainer.File.Contains('!'))
continue;
renameCollection.Add(locationContainer.File);
}
}
if (renameCollection.Any())
IDirectory.MoveFiles(renameCollection, _Configuration.PropertyConfiguration.ResultContent, "(abd)");
}
private void LookForAbandoned(string bResultsFullGroupDirectory, Container[] containers, ReadOnlyDictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers)
{
string[] directories;
string? directoryName;
string cResultsFullGroupDirectory;
string dResultsFullGroupDirectory;
string c2ResultsFullGroupDirectory;
string d2ResultsFullGroupDirectory;
List<int> distinctFilteredIds = Shared.Models.Stateless.Methods.IContainer.GetFilteredDistinctIds(_Configuration.PropertyConfiguration, containers);
LookForAbandoned(idToLocationContainers, distinctFilteredIds);
LookForAbandoned(bResultsFullGroupDirectory, distinctFilteredIds);
foreach (string outputResolution in _Configuration.OutputResolutions)
{
(cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
directories = Directory.GetDirectories(cResultsFullGroupDirectory, "*", SearchOption.TopDirectoryOnly);
foreach (string directory in directories)
{
directoryName = Path.GetFileName(directory);
if (string.IsNullOrEmpty(directoryName) || (directoryName.Length != 2 && directoryName.Length != 4))
continue;
LookForAbandoned(distinctFilteredIds, directory, directoryName);
}
directories = Directory.GetDirectories(dResultsFullGroupDirectory, "*", SearchOption.TopDirectoryOnly);
foreach (string directory in directories)
{
directoryName = Path.GetFileName(directory);
if (string.IsNullOrEmpty(directoryName) || (directoryName.Length != 2 && directoryName.Length != 4))
continue;
LookForAbandoned(distinctFilteredIds, directory, directoryName);
}
directories = Directory.GetDirectories(d2ResultsFullGroupDirectory, "*", SearchOption.TopDirectoryOnly);
foreach (string directory in directories)
{
directoryName = Path.GetFileName(directory);
if (string.IsNullOrEmpty(directoryName) || (directoryName.Length != 2 && directoryName.Length != 4))
continue;
LookForAbandoned(distinctFilteredIds, directory, directoryName);
}
}
}
private Mapping[] GetMappings(Property.Models.Configuration propertyConfiguration, Container[] containers, MapLogic mapLogic, bool distinctItems)
{
Mapping[] results;
@ -1011,7 +902,7 @@ public partial class DlibDotNet
List<Mapping> mappingCollection = new();
foreach (Container container in containers)
{
if (!container.Items.Any())
if (container.Items.Count == 0)
continue;
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(propertyConfiguration, container);
if (!filteredItems.Any())
@ -1019,7 +910,7 @@ public partial class DlibDotNet
containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems);
focusRelativePath = Path.GetFullPath(string.Concat(_Configuration.PropertyConfiguration.RootDirectory, _Configuration.FocusDirectory));
isFocusRelativePath = string.IsNullOrEmpty(_Configuration.FocusDirectory) ? null : container.SourceDirectory.StartsWith(focusRelativePath);
isIgnoreRelativePath = !_Configuration.IgnoreRelativePaths.Any() ? null : _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory);
isIgnoreRelativePath = _Configuration.IgnoreRelativePaths.Length == 0 ? null : _Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory);
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null || item.ResizedFileHolder is null)
@ -1042,7 +933,7 @@ public partial class DlibDotNet
}
if (!anyValidFaces)
{
mapping = GetMapping(mapLogic, item, isFocusRelativePath, isIgnoreRelativePath, mappingFromItem, _JLinkResolvedDirectories);
mapping = GetMapping(mapLogic, item, isFocusRelativePath, isIgnoreRelativePath, mappingFromItem);
mappingCollection.Add(mapping);
}
}
@ -1071,7 +962,7 @@ public partial class DlibDotNet
if (!found)
notFound.Add(item);
}
if (notFound.Any())
if (notFound.Count > 0)
throw new NotSupportedException();
#endif
string model;
@ -1140,6 +1031,44 @@ public partial class DlibDotNet
}
}
private static bool GetRunToDoCollectionFirst(long ticks, string rootDirectory, FileSystemInfo fileSystemInfo)
{
bool result = false;
string[] directories;
string seasonDirectory;
DirectoryInfo directoryInfo;
DateTime dateTime = new(ticks);
(int season, string seasonName) = Shared.Models.Stateless.Methods.IProperty.GetSeason(dateTime.DayOfYear);
string[] checkDirectories = new string[]
{
Path.Combine(rootDirectory, "Ancestry"),
Path.Combine(rootDirectory, "Facebook"),
Path.Combine(rootDirectory, "LinkedIn"),
rootDirectory,
};
foreach (string checkDirectory in checkDirectories)
{
if (!Directory.Exists(checkDirectory))
_ = Directory.CreateDirectory(checkDirectory);
seasonDirectory = Path.Combine(checkDirectory, $"{dateTime.Year}.{season} {seasonName} {Path.GetFileName(checkDirectory)}");
if (!Directory.Exists(seasonDirectory))
_ = Directory.CreateDirectory(seasonDirectory);
if (result)
continue;
directories = Directory.GetDirectories(checkDirectory, "*", SearchOption.TopDirectoryOnly);
foreach (string directory in directories)
{
directoryInfo = new(directory);
if (directoryInfo.LastWriteTime > fileSystemInfo.LastWriteTime)
{
result = true;
break;
}
}
}
return result;
}
private void Search(long ticks, ReadOnlyCollection<PersonContainer> personContainers, string argZero, string propertyRoot)
{
int t;
@ -1170,8 +1099,8 @@ public partial class DlibDotNet
a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "([])");
eDistanceContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(E_Distance), _Configuration.PropertyConfiguration.ResultContent);
string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), _Configuration.PropertyConfiguration.ResultSingleton);
TimeSpan eLastWriteTimeTimeSpan = new(ticks - new DirectoryInfo(eDistanceContentDirectory).LastWriteTime.Ticks);
if (eLastWriteTimeTimeSpan.TotalDays > 1)
bool runToDoCollectionFirst = GetRunToDoCollectionFirst(ticks, _Configuration.PropertyConfiguration.RootDirectory, new DirectoryInfo(eDistanceContentDirectory));
if (runToDoCollectionFirst)
mapLogic = null;
else
mapLogic = new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, personContainers, ticks, a2PeopleSingletonDirectory, eDistanceContentDirectory);
@ -1180,7 +1109,7 @@ public partial class DlibDotNet
if (outputResolution.Any(l => char.IsNumber(l)))
continue;
configurationOutputResolutionsHas = true;
if (eLastWriteTimeTimeSpan.TotalDays < 1)
if (!runToDoCollectionFirst)
break;
ProgressBar progressBar;
(cResultsFullGroupDirectory, _, _, _) = GetResultsFullGroupDirectories(outputResolution);
@ -1249,16 +1178,21 @@ public partial class DlibDotNet
B_Metadata metadata = new(_Configuration.PropertyConfiguration, _Configuration.ForceMetadataLastWriteTimeToCreationTime, _Configuration.PropertiesChangedForMetadata, bResultsFullGroupDirectory);
mapLogic ??= new(_AppSettings.MaxDegreeOfParallelism, _Configuration.PropertyConfiguration, _MapConfiguration, personContainers, ticks, a2PeopleSingletonDirectory, eDistanceContentDirectory);
containers = Shared.Models.Stateless.Methods.IContainer.SortContainers(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, _ArgZeroIsConfigurationRootDirectory, argZero, containers);
ReadOnlyDictionary<int, List<LocationContainer<MetadataExtractor.Directory>>> idToLocationContainers = mapLogic.GetIdToLocationContainers();
FullDoWork(argZero, propertyRoot, ticks, aResultsFullGroupDirectory, bResultsFullGroupDirectory, t, containers, propertyLogic, metadata, fileNameToCollection, idToLocationContainers, mapLogic);
if (_Configuration.LookForAbandoned)
LookForAbandoned(bResultsFullGroupDirectory, containers, idToLocationContainers);
_Distance.Clear();
FullDoWork(argZero, propertyRoot, ticks, aResultsFullGroupDirectory, bResultsFullGroupDirectory, t, containers, propertyLogic, metadata, fileNameToCollection, mapLogic);
List<Item> distinctFilteredItems = Shared.Models.Stateless.Methods.IContainer.GetItems(_Configuration.PropertyConfiguration, containers, distinctItems: true, filterItems: true);
if (_Configuration.LookForAbandoned)
{
foreach (string outputResolution in _Configuration.OutputResolutions)
{
(cResultsFullGroupDirectory, _, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
mapLogic.LookForAbandoned(_Configuration.PropertyConfiguration, bResultsFullGroupDirectory, containers, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory);
}
}
_Distance.Clear();
Verify(eDistanceContentDirectory, distinctFilteredItems);
List<Shared.Models.Face> distinctFilteredFaces = Map.Models.Stateless.Methods.IMapLogic.GetFaces(distinctFilteredItems);
Mapping[] distinctFilteredMappingCollection = GetMappings(_Configuration.PropertyConfiguration, containers, mapLogic, distinctItems: true);
int totalNotMapped = mapLogic.UpdateMappingFromPerson(distinctFilteredMappingCollection);
int totalNotMapped = mapLogic.UpdateMappingFromPerson(_Configuration.PropertyConfiguration, distinctFilteredMappingCollection);
string json = JsonSerializer.Serialize(distinctFilteredMappingCollection);
File.WriteAllText(Path.Combine(eDistanceContentDirectory, $"{ticks}.json"), json);
foreach (string outputResolution in _Configuration.OutputResolutions)
@ -1268,7 +1202,7 @@ public partial class DlibDotNet
personKeyToIds = mapLogic.GetPersonKeyToIds();
if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
mapLogic.SaveShortcutsForOutputResolutionsPreMapLogic(eDistanceContentDirectory, personKeyToIds, distinctFilteredMappingCollection);
if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.JLinks.Length > 0 && _Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution))
if (!string.IsNullOrEmpty(a2PeopleContentDirectory) && _Configuration.JLinks.Where(l => !string.IsNullOrEmpty(l)).Any() && _Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution))
mapLogic.SaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, personContainers, a2PeopleContentDirectory, personKeyToIds, distinctFilteredMappingCollection, totalNotMapped);
(cResultsFullGroupDirectory, c2ResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
if (_ArgZeroIsConfigurationRootDirectory
@ -1277,7 +1211,7 @@ public partial class DlibDotNet
&& _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
&& _Exceptions.Count == 0)
MapLogic(ticks, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, fPhotoPrismContentDirectory, mapLogic, outputResolution, personKeyToIds, distinctFilteredFaces, distinctFilteredMappingCollection, totalNotMapped);
if (_Configuration.SaveRandomForOutputResolutions.Contains(outputResolution) && personKeyToIds.Any() && distinctFilteredMappingCollection.Any())
if (_Configuration.SaveRandomForOutputResolutions.Contains(outputResolution) && personKeyToIds.Count > 0 && distinctFilteredMappingCollection.Length > 0)
_Random.Random(_Configuration.PropertyConfiguration, outputResolution, personKeyToIds, distinctFilteredMappingCollection);
if (_IsEnvironment.Development)
continue;

2
Instance/Models/Binder/Configuration.cs
View File

@ -84,6 +84,7 @@ public class Configuration
[Display(Name = "Save Sorting Without Person"), Required] public bool? SaveSortingWithoutPerson { get; set; }
[Display(Name = "Skip Not Skip Directories"), Required] public string[] SkipNotSkipDirectories { get; set; }
[Display(Name = "Skip Older Than Days"), Required] public int? SkipOlderThanDays { get; set; }
[Display(Name = "Skip Person With More Then"), Required] public int? SkipPersonWithMoreThen { get; set; }
[Display(Name = "Skip Search"), Required] public bool? SkipSearch { get; set; }
[Display(Name = "SortingMaximumPerFaceShould be High"), Required] public int? SortingMaximumPerFaceShouldBeHigh { get; set; }
[Display(Name = "Sorting Maximum Per Key"), Required] public int? SortingMaximumPerKey { get; set; }
@ -314,6 +315,7 @@ public class Configuration
configuration.SaveSortingWithoutPerson.Value,
configuration.SkipNotSkipDirectories,
configuration.SkipOlderThanDays,
configuration.SkipPersonWithMoreThen,
configuration.SkipSearch.Value,
configuration.SortingMaximumPerFaceShouldBeHigh.Value,
configuration.SortingMaximumPerKey.Value,

15
Instance/Models/Configuration.cs
View File

@ -14,15 +14,14 @@ public class Configuration
public bool CheckJsonForDistanceResults { init; get; }
public string[] CopyFacesAndSaveFaceLandmarkForOutputResolutions { init; get; }
public int CrossDirectoryMaxItemsInDistanceCollection { init; get; }
public bool DeletePossibleDuplicates { get; internal set; }
public int DistanceFactor { init; get; }
public bool DistanceMoveUnableToMatch { init; get; }
public int DistancePixelDistanceTolerance { init; get; }
public bool DistanceRenameToMatch { init; get; }
public int EyeThreshold { init; get; }
public int FaceAreaPermyriad { init; get; }
public int FaceDistanceHiddenImageFactor { init; get; }
public int FaceConfidencePercent { init; get; }
public int FaceDistanceHiddenImageFactor { init; get; }
public int FaceDistancePermyriad { init; get; }
public string FocusDirectory { init; get; }
public string FocusModel { init; get; }
@ -72,22 +71,24 @@ public class Configuration
public string[] SaveFaceDistancesForOutputResolutions { init; get; }
public string[] SaveFaceLandmarkForOutputResolutions { init; get; }
public string[] SaveFilteredOriginalImagesFromJLinksForOutputResolutions { init; get; }
public bool SaveIndividually { init; get; }
public bool SaveFullYearOfRandomFiles { init; get; }
public bool SaveIndividually { init; get; }
public string[] SaveMappedForOutputResolutions { init; get; }
public string[] SaveRandomForOutputResolutions { init; get; }
public bool SaveResizedSubfiles { init; get; }
public string[] SaveShortcutsForOutputResolutions { init; get; }
public bool SaveSortingWithoutPerson { init; get; }
public string[] SkipNotSkipDirectories { init; get; }
public bool SkipSearch { init; get; }
public int? SkipOlderThanDays { init; get; }
public int? SkipPersonWithMoreThen { init; get; }
public bool SkipSearch { init; get; }
public int SortingMaximumPerFaceShouldBeHigh { init; get; }
public int SortingMaximumPerKey { init; get; }
public int SortingMinimumToUseSigma { init; get; }
public bool TestDistanceResults { init; get; }
public int UseFilterTries { init; get; }
public string[] ValidResolutions { init; get; }
public bool DeletePossibleDuplicates { get; internal set; }
[JsonConstructor]
public Configuration(Property.Models.Configuration propertyConfiguration,
@ -162,6 +163,7 @@ public class Configuration
bool saveSortingWithoutPerson,
string[] skipNotSkipDirectories,
int? skipOlderThanDays,
int? skipPersonWithMoreThen,
bool skipSearch,
int sortingMaximumPerFaceShouldBeHigh,
int sortingMaximumPerKey,
@ -182,8 +184,8 @@ public class Configuration
DistanceRenameToMatch = distanceRenameToMatch;
EyeThreshold = eyeThreshold;
FaceAreaPermyriad = faceAreaPermyriad;
FaceDistanceHiddenImageFactor = faceDistanceHiddenImageFactor;
FaceConfidencePercent = faceConfidencePercent;
FaceDistanceHiddenImageFactor = faceDistanceHiddenImageFactor;
FaceDistancePermyriad = faceDistancePermyriad;
FocusDirectory = focusDirectory;
FocusModel = focusModel;
@ -233,8 +235,8 @@ public class Configuration
SaveFaceDistancesForOutputResolutions = saveFaceDistancesForOutputResolutions;
SaveFaceLandmarkForOutputResolutions = saveFaceLandmarkForOutputResolutions;
SaveFilteredOriginalImagesFromJLinksForOutputResolutions = saveFilteredOriginalImagesFromJLinksForOutputResolutions;
SaveIndividually = saveIndividually;
SaveFullYearOfRandomFiles = saveFullYearOfRandomFiles;
SaveIndividually = saveIndividually;
SaveMappedForOutputResolutions = saveMappedForOutputResolutions;
SaveRandomForOutputResolutions = saveRandomForOutputResolutions;
SaveResizedSubfiles = saveResizedSubfiles;
@ -242,6 +244,7 @@ public class Configuration
SaveSortingWithoutPerson = saveSortingWithoutPerson;
SkipNotSkipDirectories = skipNotSkipDirectories;
SkipOlderThanDays = skipOlderThanDays;
SkipPersonWithMoreThen = skipPersonWithMoreThen;
SkipSearch = skipSearch;
SortingMaximumPerFaceShouldBeHigh = sortingMaximumPerFaceShouldBeHigh;
SortingMaximumPerKey = sortingMaximumPerKey;

2
Instance/Models/_F_Random.cs
View File

@ -85,7 +85,7 @@ internal class F_Random
relativePaths.Add(mapping.MappingFromItem.RelativePath);
distinctCollection.Add(mapping.MappingFromItem.Id);
}
if (relativePaths.Any())
if (relativePaths.Count > 0)
{
for (int i = 0; i < 366; i++)
{