From ea25d268ffdf1adab918e47f44c3fa21a73090e6 Mon Sep 17 00:00:00 2001 From: Mike Phares Date: Sun, 18 Jun 2023 18:51:30 -0700 Subject: [PATCH] _RangeDistanceToleranceAverage VerifyPersonContainersDisplayDirectoryAllFiles usePersonCharactersCopyCount --- .kanbn/index.md | 5 +-- .kanbn/tasks/find-incorrectly-mapped-faces.md | 16 ++++++++ .kanbn/tasks/import-know-faces-into-db.md | 20 ++++++++- Distance/Models/_E_Distance.cs | 6 ++- Instance/DlibDotNet.cs | 31 ++++++++++++-- Map/Models/MapLogic.cs | 41 +++++++++++++------ Tests/UnitTestCalculations.cs | 4 +- Tests/UnitTestHardCoded.cs | 9 ++++ 8 files changed, 109 insertions(+), 23 deletions(-) create mode 100644 .kanbn/tasks/find-incorrectly-mapped-faces.md diff --git a/.kanbn/index.md b/.kanbn/index.md index 297ad83..32a82e1 100644 --- a/.kanbn/index.md +++ b/.kanbn/index.md @@ -13,13 +13,12 @@ completedColumns: ## Todo - -- [import-know-faces-into-db](tasks/import-know-faces-into-db.md) - ## In Progress +- [import-know-faces-into-db](tasks/import-know-faces-into-db.md) - [setup-photo-prism-again-in-wsl-docker](tasks/setup-photo-prism-again-in-wsl-docker.md) - [run-scan-originals](tasks/run-scan-originals.md) +- [find-incorrectly-mapped-faces](tasks/find-incorrectly-mapped-faces.md) ## Done diff --git a/.kanbn/tasks/find-incorrectly-mapped-faces.md b/.kanbn/tasks/find-incorrectly-mapped-faces.md new file mode 100644 index 0000000..50e9089 --- /dev/null +++ b/.kanbn/tasks/find-incorrectly-mapped-faces.md @@ -0,0 +1,16 @@ +--- +created: 2023-06-12T15:44:11.932Z +updated: 2023-06-12T15:45:59.891Z +assigned: "" +progress: 0 +tags: [] +--- + +# find-incorrectly-mapped-faces + +- Added _RangeDistanceToleranceAverage to rename ~200 files to *.ukn + +## Sub-tasks + +- [ ] Kristy has some Mike Phares Jr +- [ ] Ethan B has some Phares slides diff --git a/.kanbn/tasks/import-know-faces-into-db.md b/.kanbn/tasks/import-know-faces-into-db.md index e73b69a..42a5e69 100644 --- a/.kanbn/tasks/import-know-faces-into-db.md +++ b/.kanbn/tasks/import-know-faces-into-db.md @@ -8,6 +8,24 @@ tags: [] # import-know-faces-into-db +```bash +# locating data volumes in docker desktop windows +# https://stackoverflow.com/questions/43181654/locating-data-volumes-in-docker-desktop-windows +# phpmyadmin mariadb docker compose +# https://david.dev/how-to-install-mariadb-phpmyadmin-with-docker-compose +# https://docs.photoprism.app/user-guide/backups/ +docker compose exec photoprism photoprism backup -i -f +# https://docs.photoprism.app/user-guide/backups/restore/ +docker compose exec photoprism photoprism restore -i -f +``` + ## Sub-tasks -- [ ] import-know-faces-into-db +- [x] Backup DB +- [x] Add PhpMyAdmin to docker +- [x] Restore DB +- [x] Update code 5ec29e81a095cca0bbc76161bc8ef2a60b911274 and 8282821c82025c662c1d3f45ce68fc183f231789 +- [x] Run code then execute 638226889564490047-marker_name_update.sql +- [x] Re-index (NOT complete) +- [x] Execute 638226889564490047-subject_alias_update.sql +- [x] Backup DB diff --git a/Distance/Models/_E_Distance.cs b/Distance/Models/_E_Distance.cs index 905387f..acf0562 100644 --- a/Distance/Models/_E_Distance.cs +++ b/Distance/Models/_E_Distance.cs @@ -19,9 +19,10 @@ public partial class E_Distance private readonly bool _DistanceMoveUnableToMatch; private readonly List _AllMappedFaceFiles; private readonly List _AllMappedFaceFileNames; + private readonly double _RangeDistanceToleranceAverage; private readonly List _DuplicateMappedFaceFiles; - public E_Distance(bool distanceMoveUnableToMatch, bool distanceRenameToMatch, int faceConfidencePercent, float[] rangeFaceConfidence) + public E_Distance(bool distanceMoveUnableToMatch, bool distanceRenameToMatch, int faceConfidencePercent, float[] rangeDistanceTolerance, float[] rangeFaceConfidence) { _Debug = new(); _Moved = new(); @@ -34,6 +35,7 @@ public partial class E_Distance _DistanceRenameToMatch = distanceRenameToMatch; _FaceConfidencePercent = faceConfidencePercent; _DistanceMoveUnableToMatch = distanceMoveUnableToMatch; + _RangeDistanceToleranceAverage = rangeDistanceTolerance.Average(); } private static void MoveUnableToMatch(string file) @@ -117,7 +119,7 @@ public partial class E_Distance { (Face, double?)[] results; List<(Face Face, double? Length)> collection = GetValues(mappingFromItem, intersectFaces, modelsFaceEncoding); - results = (from l in collection orderby l.Length select l).Take(1).ToArray(); + results = (from l in collection where l.Length < _RangeDistanceToleranceAverage orderby l.Length select l).Take(1).ToArray(); if (results.Any()) { (Face _, double? length) = results.First(); diff --git a/Instance/DlibDotNet.cs b/Instance/DlibDotNet.cs index d784604..3dae3c5 100644 --- a/Instance/DlibDotNet.cs +++ b/Instance/DlibDotNet.cs @@ -106,7 +106,7 @@ public partial class DlibDotNet _FaceParts = new D2_FaceParts(imageCodecInfo, encoderParameters, filenameExtension, configuration.CheckDFaceAndUpWriteDates, configuration.OverrideForFaceLandmarkImages); } _MapConfiguration = Get(configuration, _Faces.FileNameExtension, _Faces.HiddenFileNameExtension, _FaceParts.FileNameExtension); - _Distance = new(configuration.DistanceMoveUnableToMatch, configuration.DistanceRenameToMatch, _Configuration.FaceConfidencePercent, configuration.RangeFaceConfidence); + _Distance = new(configuration.DistanceMoveUnableToMatch, configuration.DistanceRenameToMatch, _Configuration.FaceConfidencePercent, configuration.RangeDistanceTolerance, configuration.RangeFaceConfidence); if (_PropertyRootExistedBefore || !_ArgZeroIsConfigurationRootDirectory) { _GenealogicalDataCommunicationFooterLines = null; @@ -127,6 +127,7 @@ public partial class DlibDotNet _ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(peopleRootDirectory, "{}")); (_GenealogicalDataCommunicationHeaderLines, Dictionary> individuals, _GenealogicalDataCommunicationFooterLines) = Shared.Models.Stateless.Methods.IGenealogicalDataCommunication.GetIndividuals(configuration.GenealogicalDataCommunicationFile, requireNickName: true); _PersonContainers = Shared.Models.Stateless.Methods.IPersonContainer.GetPersonContainers(storage, configuration.MappingDefaultName, configuration.PersonBirthdayFormat, configuration.PersonCharacters.ToArray(), _Faces.FileNameExtension, individuals); + VerifyPersonContainersDisplayDirectoryAllFiles(); } { (ImageCodecInfo imageCodecInfo, EncoderParameters encoderParameters, string filenameExtension) = C_Resize.GetTuple( @@ -802,9 +803,7 @@ public partial class DlibDotNet { if (_Log is null) throw new NullReferenceException(nameof(_Log)); - string? result = Path.GetDirectoryName(Path.GetDirectoryName(container.SourceDirectory)); - if (result is null) - throw new Exception(); + string? result = Path.GetDirectoryName(Path.GetDirectoryName(container.SourceDirectory)) ?? throw new Exception(); Uri uri; Item item; string? line; @@ -994,6 +993,30 @@ public partial class DlibDotNet return results; } + private void VerifyPersonContainersDisplayDirectoryAllFiles() + { + WindowsShortcut windowsShortcut; + foreach (PersonContainer personContainer in _PersonContainers) + { + foreach (string file in personContainer.DisplayDirectoryAllFiles) + { + if (!file.EndsWith(".lnk")) + continue; + try + { + windowsShortcut = WindowsShortcut.Load(file); + if (windowsShortcut.Path is null) + continue; + if (!File.Exists(windowsShortcut.Path)) + File.Delete(file); + windowsShortcut.Dispose(); + } + catch (Exception) + { } + } + } + } + private static void Verify(string eDistanceContentDirectory, List distinctFilteredItems) { #if VerifyItem diff --git a/Map/Models/MapLogic.cs b/Map/Models/MapLogic.cs index 75fe8d2..d77f5bb 100644 --- a/Map/Models/MapLogic.cs +++ b/Map/Models/MapLogic.cs @@ -20,9 +20,6 @@ public class MapLogic : Shared.Models.Methods.IMapLogic protected readonly List> _LocationContainers; protected readonly ReadOnlyDictionary> _IdThenNormalizedRectangleToPersonContainers; - public Dictionary KeyValuePairs => throw new NotImplementedException(); - public Dictionary IndicesFromNew => throw new NotImplementedException(); - private readonly long _Ticks; private readonly Serilog.ILogger? _Log; private readonly Configuration? _Configuration; @@ -746,7 +743,7 @@ public class MapLogic : Shared.Models.Methods.IMapLogic string successful = $"_ {nameof(Shared.Models.Stateless.IMapLogic.ManualCopy).Humanize(LetterCasing.Title)} Successful"; foreach (KeyValuePair personKeyToPersonContainer in _PersonKeyToPersonContainer) { - if (personKeyToPersonContainer.Value.Key is null || personKeyToPersonContainer.Value.Birthdays is null || !personKeyToPersonContainer.Value.Birthdays.Any()) + if (personKeyToPersonContainer.Value.Key is null || personKeyToPersonContainer.Value.Birthdays is null || !personKeyToPersonContainer.Value.Birthdays.Any() || personKeyToPersonContainer.Value.PersonDirectory is null) continue; personBirthday = personKeyToPersonContainer.Value.Birthdays[zero]; foreach (string personDisplayDirectoryAllFile in personKeyToPersonContainer.Value.DisplayDirectoryAllFiles) @@ -805,7 +802,10 @@ public class MapLogic : Shared.Models.Methods.IMapLogic continue; if (File.Exists(shortcutFile)) continue; - windowsShortcut = new() { Path = faceFile }; + if (personKeyToPersonContainer.Value.PersonDirectory.Char == ']') + windowsShortcut = new() { Path = faceFile }; + else + windowsShortcut = new() { Path = checkFile }; windowsShortcut.Save(shortcutFile); windowsShortcut.Dispose(); if (!File.Exists(shortcutFile)) @@ -892,6 +892,7 @@ public class MapLogic : Shared.Models.Methods.IMapLogic { if (_Configuration is null) throw new NullReferenceException(nameof(_Configuration)); + string year; long personKey; string fileName; string faceFile; @@ -901,6 +902,8 @@ public class MapLogic : Shared.Models.Methods.IMapLogic string? facesDirectory; PersonContainer? personContainer; Dictionary personKeyToCount = new(); + bool usePersonCharactersCopyCount = personCharactersCopyCount != int.MaxValue; + string delete = $"_ {nameof(Shared.Models.Stateless.IMapLogic.ManualCopy).Humanize(LetterCasing.Title)} Delete"; foreach (Mapping mapping in mappingCollection) { directoryName = Path.GetDirectoryName(mapping.MappingFromItem.RelativePath); @@ -922,10 +925,13 @@ public class MapLogic : Shared.Models.Methods.IMapLogic faceFileName = $"{mapping.MappingFromLocation.DeterministicHashCodeKey}{mapping.MappingFromItem.ImageFileHolder.ExtensionLowered}{_Configuration.FacesFileNameExtension}"; faceFile = Path.Combine(facesDirectory, faceFileName); personKey = mapping.MappingFromPerson.PersonBirthday.Value.Ticks; - if (!personKeyToCount.ContainsKey(personKey)) - personKeyToCount.Add(personKey, 0); - if (personKeyToCount[personKey] > personCharactersCopyCount) - continue; + if (usePersonCharactersCopyCount) + { + if (!personKeyToCount.ContainsKey(personKey)) + personKeyToCount.Add(personKey, 0); + if (personKeyToCount[personKey] > personCharactersCopyCount) + continue; + } if (!_PersonKeyToPersonContainer.TryGetValue(personKey, out personContainer)) continue; if (personContainer.PersonDirectory is null || !_Configuration.PersonCharacters.Contains(personContainer.PersonDirectory.Char)) @@ -935,12 +941,23 @@ public class MapLogic : Shared.Models.Methods.IMapLogic if (!File.Exists(faceFile)) continue; directory = Path.Combine(a2PeopleSingletonDirectory, personContainer.PersonDirectory.Char.ToString(), personContainer.PersonDirectory.Group, personContainer.DisplayDirectoryName); - if (!Directory.Exists(directory)) - continue; + if (usePersonCharactersCopyCount) + { + if (!Directory.Exists(directory)) + continue; + } + else + { + year = mapping.MappingFromItem.IsWrongYear is null || mapping.MappingFromItem.IsWrongYear.Value ? "0000" : mapping.MappingFromItem.MinimumDateTime.ToString("yyyy"); + directory = Path.Combine(directory, delete, year); + if (!Directory.Exists(directory)) + _ = Directory.CreateDirectory(directory); + } fileName = Path.Combine(directory, faceFileName); if (File.Exists(fileName)) continue; - personKeyToCount[personKey] += 1; + if (usePersonCharactersCopyCount) + personKeyToCount[personKey] += 1; File.Copy(faceFile, fileName); } } diff --git a/Tests/UnitTestCalculations.cs b/Tests/UnitTestCalculations.cs index 301ded0..eed7993 100644 --- a/Tests/UnitTestCalculations.cs +++ b/Tests/UnitTestCalculations.cs @@ -119,8 +119,10 @@ public partial class UnitTestCalculations [TestMethod] public void TestMethodHumanize() { - string successful = $"_ {"ManualCopy".Humanize(LetterCasing.Title)} Successful"; + string successful = $"_ {nameof(Shared.Models.Stateless.IMapLogic.ManualCopy).Humanize(LetterCasing.Title)} Successful"; Assert.IsTrue(successful == "_ Manual Copy Successful"); + string verify = $"_ {nameof(Shared.Models.Stateless.IMapLogic.ManualCopy).Humanize(LetterCasing.Title)} Verify"; + Assert.IsTrue(verify == "_ Manual Copy Verify"); } [TestMethod] diff --git a/Tests/UnitTestHardCoded.cs b/Tests/UnitTestHardCoded.cs index 153378a..a64fe89 100644 --- a/Tests/UnitTestHardCoded.cs +++ b/Tests/UnitTestHardCoded.cs @@ -773,4 +773,13 @@ public partial class UnitTestHardCoded NonThrowTryCatch(); } + [TestMethod] + public void TestMethodRenameForUnkown() + { + string[] files = Directory.GetFiles(@"D:\1) Images A\Images-1e85c0ba-Results\E) Distance\1e85c0ba\(Bad-2023-06-18-less-0.4)", "*.unk", SearchOption.AllDirectories); + foreach (string file in files) + File.Move(file, file[..^4]); + NonThrowTryCatch(); + } + } \ No newline at end of file