phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

FilePath ready to test

Browse Source
This commit is contained in:
Mike Phares
2023-12-24 11:29:36 -07:00
parent af491371a3
commit 7007a9df2e
60 changed files with 960 additions and 857 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
Instance/DlibDotNet.cs
View File

@ -127,7 +127,7 @@ public partial class DlibDotNet
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(peopleRootDirectory, propertyConfiguration.ResultSingleton));
string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration.PropertyConfiguration, nameof(A2_People), "{}");
string a2PeopleContentDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration.PropertyConfiguration, nameof(A2_People), "([])");
personContainers = new(IPersonContainer.GetPersonContainers(a2PeopleSingletonDirectory, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), _Faces.FileNameExtension));
personContainers = new(IPersonContainer.GetPersonContainers(a2PeopleSingletonDirectory, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), configuration.PropertyConfiguration, _Faces.FileNameExtension));
if (configuration.JLinks.Where(l => !string.IsNullOrEmpty(l)).Any())
{
_JLinkResolvedDirectories.AddRange(Map.Models.Stateless.Methods.IMapLogic.GetJLinkDirectories(configuration.GenealogicalDataCommunicationFile, configuration.JLinks, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), a2PeopleSingletonDirectory, a2PeopleContentDirectory));
@ -237,6 +237,7 @@ public partial class DlibDotNet
private ReadOnlyCollection<int> GetNotNineCollection(ReadOnlyCollection<string[]> filesCollection)
{
List<int> results = [];
FilePath filePath;
FileHolder fileHolder;
foreach (string[] files in filesCollection)
{
@ -244,11 +245,11 @@ public partial class DlibDotNet
{
if (!file.Contains(" !9"))
continue;
fileHolder = new(file);
(_, int? id) = IDirectory.GetId(_Configuration.PropertyConfiguration.Offset, fileHolder);
if (id is null)
fileHolder = Shared.Models.Stateless.Methods.IFileHolder.Get(file);
filePath = FilePath.Get(_Configuration.PropertyConfiguration, fileHolder, index: null);
if (filePath.Id is null)
continue;
results.Add(id.Value);
results.Add(filePath.Id.Value);
}
}
return new(results);
@ -332,7 +333,7 @@ public partial class DlibDotNet
if (outputResolution.Any(char.IsNumber))
continue;
(cResultsFullGroupDirectory, _, _, _) = GetResultsFullGroupDirectories(outputResolution);
filesCollectionRootDirectory = Path.Combine(cResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent, _Configuration.PropertyConfiguration.ResultAllInOne);
filesCollectionRootDirectory = Path.Combine(cResultsFullGroupDirectory, _Configuration.PropertyConfiguration.ResultContent);
filesCollection = IDirectory.GetFilesCollection(filesCollectionRootDirectory, directorySearchFilter, fileSearchFilter, useCeilingAverage: true);
count = filesCollection.Select(l => l.Length).Sum();
break;
@ -370,6 +371,8 @@ public partial class DlibDotNet
ReadOnlyCollection<Mapping> distinctFilteredMappingCollection = GetMappings(_Configuration.PropertyConfiguration, eDistanceContentDirectory, containers, mapLogic, distinctItems: true);
if (runToDoCollectionFirst)
{
if (!Directory.Exists(eDistanceContentDirectory))
_ = Directory.CreateDirectory(eDistanceContentDirectory);
string json = JsonSerializer.Serialize(distinctFilteredMappingCollection);
File.WriteAllText(Path.Combine(eDistanceContentDirectory, $"{ticks}.json"), json);
}
@ -462,6 +465,7 @@ public partial class DlibDotNet
Uri uri;
string? line;
string fileName;
FilePath filePath;
Task<byte[]> task;
string relativePath;
FileHolder fileHolder;
@ -487,8 +491,9 @@ public partial class DlibDotNet
extensionLowered = Path.GetExtension(uri.LocalPath);
relativePath = Shared.Models.Stateless.Methods.IPath.GetRelativePath(sourceDirectoryFile, length, forceExtensionToLower: true);
isValidImageFormatExtension = _Configuration.PropertyConfiguration.ValidImageFormatExtensions.Contains(extensionLowered);
fileHolder = new(sourceDirectoryFile);
_ = new Item(fileHolder, relativePath, isValidImageFormatExtension);
fileHolder = Shared.Models.Stateless.Methods.IFileHolder.Get(sourceDirectoryFile);
filePath = FilePath.Get(_Configuration.PropertyConfiguration, fileHolder, index: null);
_ = new Item(filePath, fileHolder, relativePath, isValidImageFormatExtension);
// container.Items.Add(item);
}
_Logger?.LogInformation(". . .");
@ -774,12 +779,6 @@ public partial class DlibDotNet
_Logger?.LogInformation(string.Concat("Moved <", item.ImageFileHolder.FullName, '>'));
}
private void LogNameWithoutExtensionIsIdFormatBut(Item item)
{
_Logger?.LogInformation($"Name without extension is Id format but doesn't match id <{item.ImageFileHolder.FullName}>");
File.Move(item.ImageFileHolder.FullName, $"{item.ImageFileHolder.FullName}.rename");
}
private int GetNotMappedCountAndUpdateMappingFromPersonThenSetMapping(MapLogic mapLogic, Item item, bool? isFocusRelativePath, ReadOnlyCollection<LocationContainer<MetadataExtractor.Directory>> locationContainers, MappingFromItem mappingFromItem, List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection, List<Shared.Models.Face> faces)
{
int result;
@ -801,7 +800,7 @@ public partial class DlibDotNet
bool? isFocusModel = GetIsFocusModel(item.Property);
long[] jLinkResolvedPersonKeys = _JLinkResolvedDirectories.Select(l => l.PersonKey).ToArray();
ReadOnlyDictionary<int, ReadOnlyCollection<PersonContainer>>? wholePercentagesToPersonContainers;
ReadOnlyCollection<string> locationContainersFiles = new((from l in locationContainers select l.File).ToArray());
ReadOnlyCollection<string> locationContainersFiles = new((from l in locationContainers select l.FilePath.FullName).ToArray());
foreach (Shared.Models.Face face in faces)
{
wholePercentagesToPersonContainers = mapLogic.GetWholePercentagesToPersonContainers(item.Property?.Id);
@ -826,7 +825,7 @@ public partial class DlibDotNet
confidencePercent = Shared.Models.Stateless.Methods.ILocation.GetConfidencePercent(_Configuration.FaceConfidencePercent, face.Location.Confidence);
faceAreaPermyriad = IMapping.GetAreaPermyriad(_Configuration.FaceAreaPermyriad, face.Location, face.OutputResolution);
wholePercentRectangle = Shared.Models.Stateless.Methods.ILocation.GetWholePercentages(face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution);
deterministicHashCodeKey = IMapping.GetDeterministicHashCodeKey(item.Property.Id.Value, face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution);
deterministicHashCodeKey = IMapping.GetDeterministicHashCodeKey(item.FilePath, face.Location, Shared.Models.Stateless.ILocation.Digits, face.OutputResolution);
mappingFromLocation = new(faceAreaPermyriad, confidencePercent, deterministicHashCodeKey, eyeα, eyeReview, wholePercentRectangle);
inSkipCollection = mapLogic.InSkipCollection(item.Property.Id.Value, mappingFromLocation);
mappingFromFilterPre = new(inSkipCollection, isFocusModel, isFocusRelativePath);
@ -834,7 +833,7 @@ public partial class DlibDotNet
isFocusPerson = mapLogic.IsFocusPerson(_Configuration.SkipPersonWithMoreThen, jLinkResolvedPersonKeys, wholePercentagesToPersonContainers, mappingFromLocation);
mappingFromFilterPost = new(canReMap, inSkipCollection, isFocusPerson);
}
mapping = new(mappingFromFilterPost, mappingFromFilterPre, mappingFromItem, mappingFromLocation, mappingFromPhotoPrismCollection);
mapping = new(item.FilePath, mappingFromFilterPost, mappingFromFilterPre, mappingFromItem, mappingFromLocation, mappingFromPhotoPrismCollection);
notMapped += mapLogic.UpdateMappingFromPerson(wholePercentagesToPersonContainers, mapping);
face.SetMapping(mapping);
}
@ -904,7 +903,7 @@ public partial class DlibDotNet
else
{
wholePercentRectangle = Shared.Models.Stateless.Methods.ILocation.GetWholePercentages(Shared.Models.Stateless.ILocation.Digits);
deterministicHashCodeKey = IMapping.GetDeterministicHashCodeKey(item.Property.Id.Value, Shared.Models.Stateless.ILocation.Digits);
deterministicHashCodeKey = IMapping.GetDeterministicHashCodeKey(item.FilePath, Shared.Models.Stateless.ILocation.Digits);
mappingFromLocation = new(faceAreaPermyriad, confidencePercent, deterministicHashCodeKey, eyeα, eyeReview, wholePercentRectangle);
inSkipCollection = mapLogic.InSkipCollection(item.Property.Id.Value, mappingFromLocation);
mappingFromFilterPre = new(inSkipCollection, isFocusModel, isFocusRelativePath);
@ -912,7 +911,7 @@ public partial class DlibDotNet
isFocusPerson = mapLogic.IsFocusPerson(_Configuration.SkipPersonWithMoreThen, jLinkResolvedPersonKeys, wholePercentagesToPersonContainers, mappingFromLocation);
mappingFromFilterPost = new(canReMap, inSkipCollection, isFocusPerson);
}
result = new(mappingFromFilterPost, mappingFromFilterPre, mappingFromItem, mappingFromLocation, mappingFromPhotoPrismCollection: null);
result = new(item.FilePath, mappingFromFilterPost, mappingFromFilterPre, mappingFromItem, mappingFromLocation, mappingFromPhotoPrismCollection: null);
int notMapped = mapLogic.UpdateMappingFromPerson(wholePercentagesToPersonContainers, result);
return (result, notMapped);
}
@ -940,8 +939,8 @@ public partial class DlibDotNet
DateTime dateTime = DateTime.Now;
Shared.Models.Property? property;
List<string> parseExceptions = [];
List<Tuple<string, DateTime>> subFileTuples = [];
string[] changesFrom = [nameof(A_Property)];
List<Tuple<string, DateTime>> subFileTuples = [];
FileHolder resizedFileHolder = _Resize.GetResizedFileHolder(cResultsFullGroupDirectory, item, outputResolutionHasNumber);
ReadOnlyCollection<LocationContainer<MetadataExtractor.Directory>> locationContainers = mapLogic.GetLocationContainers(item);
if (item.Property is null || item.Property.Id is null || !item.SourceDirectoryFileHolder.Exists || item.SourceDirectoryFileHolder.CreationTime is null || item.SourceDirectoryFileHolder.LastWriteTime is null || item.Any())
@ -973,18 +972,12 @@ public partial class DlibDotNet
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), item.SourceDirectoryFileHolder.LastWriteTime.Value));
else
subFileTuples.Add(new Tuple<string, DateTime>(nameof(A_Property), new FileInfo(item.SourceDirectoryFileHolder.FullName).LastWriteTime));
short sortOrderOnlyLengthIndex = IDirectory.GetSortOrderOnlyLengthIndex(_Configuration.PropertyConfiguration.Offset);
bool nameWithoutExtensionIsIdFormat = Shared.Models.Stateless.Methods.IProperty.NameWithoutExtensionIsIdFormat(item.ImageFileHolder);
bool nameWithoutExtensionIsPaddedIdFormat = IDirectory.NameWithoutExtensionIsPaddedIdFormat(item.ImageFileHolder, sortOrderOnlyLengthIndex);
if (nameWithoutExtensionIsIdFormat && item.ImageFileHolder.NameWithoutExtension != item.Property.Id.ToString())
LogNameWithoutExtensionIsIdFormatBut(item);
if (nameWithoutExtensionIsPaddedIdFormat && item.ImageFileHolder.NameWithoutExtension.EndsWith(item.Property.Id.Value.ToString()[1..]))
LogNameWithoutExtensionIsIdFormatBut(item);
FilePath filePath = FilePath.Get(_Configuration.PropertyConfiguration, item.ImageFileHolder, index: null);
if (resizedFileHolder.Exists && item.Property.Width is not null && item.Property.Width.Value > 4 && _Configuration.SaveBlurHashForOutputResolutions.Contains(outputResolution))
{
string? file = _BlurHasher.GetFile(resizedFileHolder);
string? file = _BlurHasher.GetFile(item.FilePath);
if (file is not null && !File.Exists(file))
_ = _BlurHasher.EncodeAndSave(resizedFileHolder);
_ = _BlurHasher.EncodeAndSave(item.FilePath, resizedFileHolder);
}
}
if (property is null || item.Property is null)
@ -992,7 +985,7 @@ public partial class DlibDotNet
item.SetResizedFileHolder(_Resize.FileNameExtension, resizedFileHolder);
MappingFromItem mappingFromItem = IMappingFromItem.GetMappingFromItem(containerDateTimes, item, resizedFileHolder);
Map.Models.Stateless.Methods.IMapLogic.SetCreationTimeMaybeMoveToDecade(_Configuration.PropertyConfiguration, _Configuration.MoveToDecade && _Configuration.LocationContainerDistanceTolerance is null, mappingFromItem, locationContainers);
ReadOnlyDictionary<string, MetadataExtractorDirectory> metadataExtractorDirectories = metadata.GetMetadataCollection(subFileTuples, parseExceptions, changesFrom, mappingFromItem);
ReadOnlyDictionary<string, MetadataExtractorDirectory> metadataExtractorDirectories = metadata.GetMetadataCollection(item.FilePath, subFileTuples, parseExceptions, changesFrom, mappingFromItem);
if (_AppSettings.Places.Count > 0)
{
float latitude;
@ -1015,7 +1008,7 @@ public partial class DlibDotNet
distance += 1;
}
}
Dictionary<string, int[]> outputResolutionToResize = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, subFileTuples, parseExceptions, item.Property, mappingFromItem);
Dictionary<string, int[]> outputResolutionToResize = _Resize.GetResizeKeyValuePairs(_Configuration.PropertyConfiguration, cResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, item.Property, mappingFromItem);
if (_Configuration.SaveResizedSubfiles)
_Resize.SaveResizedSubfile(_Configuration.PropertyConfiguration, outputResolution, cResultsFullGroupDirectory, subFileTuples, item, item.Property, mappingFromItem, outputResolutionToResize);
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Contains(outputResolution))
@ -1027,21 +1020,21 @@ public partial class DlibDotNet
List<MappingFromPhotoPrism>? mappingFromPhotoPrismCollection;
if (!fileNameToCollection.TryGetValue(mappingFromItem.Id, out mappingFromPhotoPrismCollection))
mappingFromPhotoPrismCollection = null;
faces = _Faces.GetFaces(outputResolution, dResultsFullGroupDirectory, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionToResize, locationContainers, mappingFromPhotoPrismCollection);
faces = _Faces.GetFaces(outputResolution, dResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, property, mappingFromItem, outputResolutionToResize, locationContainers, mappingFromPhotoPrismCollection);
result = GetNotMappedCountAndUpdateMappingFromPersonThenSetMapping(mapLogic, item, isFocusRelativePath, locationContainers, mappingFromItem, mappingFromPhotoPrismCollection, faces);
List<(Shared.Models.Face, FileInfo?, string, bool Saved)> faceCollection = _Faces.SaveFaces(_FaceParts.FileNameExtension, dResultsFullGroupDirectory, subFileTuples, parseExceptions, mappingFromItem, faces);
List<(Shared.Models.Face, FileInfo?, string, bool Saved)> faceCollection = _Faces.SaveFaces(_FaceParts.FileNameExtension, dResultsFullGroupDirectory, item.FilePath, subFileTuples, parseExceptions, mappingFromItem, faces);
if (_Configuration.CopyFacesAndSaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
_FaceParts.CopyFacesAndSaveFaceLandmarkImage(facePartsCollectionDirectory, mappingFromItem, faceCollection);
if ((_Configuration.DistanceMoveUnableToMatch || _Configuration.DistanceRenameToMatch)
&& _Configuration.LoadOrCreateThenSaveDistanceResultsForOutputResolutions.Contains(outputResolution)
&& locationContainers is not null && faceCollection.All(l => !l.Saved))
_Distance.LookForMatchFacesAndPossiblyRename(_Faces.FileNameExtension, mappingFromItem, faces, locationContainers);
_Distance.LookForMatchFacesAndPossiblyRename(_Faces.FileNameExtension, item.FilePath, mappingFromItem, faces, locationContainers);
(bool review, int[] eyesCollection) = Shared.Models.Stateless.Methods.IFace.GetEyeCollection(_Configuration.EyeThreshold, faces);
if (review || _Configuration.SaveFaceLandmarkForOutputResolutions.Contains(outputResolution))
{
bool saveRotated = false;
string sourceDirectorySegment = Property.Models.Stateless.IResult.GetRelativePath(_Configuration.PropertyConfiguration, container.SourceDirectory);
_FaceParts.SaveFaceLandmarkImages(_Configuration.PropertyConfiguration, subFileTuples, parseExceptions, mappingFromItem, faces, saveRotated);
_FaceParts.SaveFaceLandmarkImages(_Configuration.PropertyConfiguration, item.FilePath, subFileTuples, parseExceptions, mappingFromItem, faces, saveRotated);
}
}
lock (sourceDirectoryChanges)
@ -1233,7 +1226,7 @@ public partial class DlibDotNet
int count = filesCollection.Select(l => l.Length).Sum();
string message = $") Selecting for ## pattern directory - {(int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds)} total second(s)";
progressBar = new(count, message, options);
(string[] distinctDirectories, List<(FileHolder, string?, string)> toDoCollection) = IDirectory.GetToDoCollection(_Configuration.PropertyConfiguration, filesCollection, fileGroups[_Configuration.PropertyConfiguration.ResultContent], () => progressBar.Tick());
(string[] distinctDirectories, List<(FilePath, string)> toDoCollection) = IDirectory.GetToDoCollection(_Configuration.PropertyConfiguration, filesCollection, fileGroups[_Configuration.PropertyConfiguration.ResultContent], () => progressBar.Tick());
progressBar.Dispose();
foreach (string distinctDirectory in distinctDirectories)
{