phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SaveRandomForOutputResolutions

Browse Source
This commit is contained in:
Mike Phares
2022-12-29 15:24:23 -07:00
parent 26edd826d5
commit 06a1207285
9 changed files with 209 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
Instance/DlibDotNet.cs
View File

@ -796,13 +796,12 @@ public partial class DlibDotNet
}
}
private Shared.Models.Face[] GetFilteredDistinctFaces(string argZero, Container[] containers)
private List<Shared.Models.Face> GetFilteredDistinctFaces(string argZero, Container[] containers)
{
Shared.Models.Face[] results;
List<Shared.Models.Face> results = new();
Item[] filteredItems;
bool isIgnoreRelativePath;
List<int> distinct = new();
List<Shared.Models.Face> collection = new();
foreach (Container container in containers)
{
if (!container.Items.Any())
@ -826,11 +825,10 @@ public partial class DlibDotNet
{
if (face.FaceEncoding is null || face.Location is null || face.OutputResolution is null)
continue;
collection.Add(face);
results.Add(face);
}
}
}
results = (from l in collection orderby l.Mapping?.MappingFromItem.Id select l).ToArray();
return results;
}
@ -860,16 +858,15 @@ public partial class DlibDotNet
return items;
}
private void MapLogic(string argZero, long ticks, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string fPhotoPrismContentDirectory, MapLogicSupport mapLogicSupport, MapLogic mapLogic, string outputResolution)
private void MapLogic(string argZero, long ticks, Container[] containers, string a2PeopleSingletonDirectory, string dResultsFullGroupDirectory, string d2ResultsFullGroupDirectory, string fPhotoPrismContentDirectory, MapLogicSupport mapLogicSupport, MapLogic mapLogic, string outputResolution, List<Shared.Models.Face> distinctFilteredFaces, Mapping[] mappingCollection, int totalNotMapped)
{
int? useFiltersCounter = null;
SortingContainer[] sortingContainers;
string dFacesContentDirectory = Path.Combine(dResultsFullGroupDirectory, "()");
string d2FacePartsContentDirectory = Path.Combine(d2ResultsFullGroupDirectory, "()");
Shared.Models.Face[] distinctFilteredFaces = GetFilteredDistinctFaces(argZero, containers);
Mapping[] mappingCollection = MapLogicSupport.GetSelectedMappingCollection(distinctFilteredFaces);
Dictionary<long, int> personKeyToCount = mapLogic.GetPersonKeyToCount(mappingCollection);
string dFacesCollectionDirectory = Path.Combine(dResultsFullGroupDirectory, "[]", _Configuration.PropertyConfiguration.ResultAllInOne);
(Dictionary<long, int> personKeyToCount, int totalNotMapped) = mapLogic.AddToMapping(mappingCollection);
Dictionary<int, Dictionary<int, Mapping>> idToNormalizedRectangleToMapping = MapLogicSupport.GetIdToNormalizedRectangleToFace(mappingCollection);
if (Directory.Exists(fPhotoPrismContentDirectory))
F_PhotoPrism.WriteMatches(fPhotoPrismContentDirectory, _Configuration.PersonBirthdayFormat, ticks, distinctFilteredFaces, mapLogic);
if (_Configuration.SaveShortcutsForOutputResolutions.Contains(outputResolution))
@ -879,7 +876,6 @@ public partial class DlibDotNet
}
if (_Configuration.PersonCharactersToCopyTo.Length == 1 && _Configuration.PersonCharacters.ToArray().Contains(_Configuration.PersonCharactersToCopyTo[0]))
mapLogic.CopyAtLeastOneMappedFiles(_Configuration.PersonCharactersToCopyTo[0], dFacesContentDirectory, a2PeopleSingletonDirectory, mappingCollection);
Dictionary<int, Dictionary<int, Mapping>> idToNormalizedRectangleToMapping = MapLogicSupport.GetIdToNormalizedRectangleToFace(mappingCollection);
mapLogic.CopyManualFiles(dFacesContentDirectory, idToNormalizedRectangleToMapping);
if (_Configuration.SaveNotMappedForOutputResolutions.Contains(outputResolution))
mapLogic.CopyNotMappedFaces(_Configuration.RangeFaceAreaPermilleTolerance, dFacesContentDirectory, idToNormalizedRectangleToMapping);
@ -906,7 +902,7 @@ public partial class DlibDotNet
MapLogicSupport.SaveFaceDistances(_Configuration.PropertyConfiguration, sortingContainers);
if (totalNotMapped > 0)
{
bool saveNullPerson = !personKeyToCount.Any() || _Configuration.RangeDistanceTolerance[1] < (_Configuration.RangeDistanceTolerance[2] * .5);
bool saveNullPerson = !personKeyToCount.Any() || _Configuration.RangeDistanceTolerance[1] < (_Configuration.RangeDistanceTolerance[2] * .66);
int updated = mapLogic.UpdateFromSortingContainers(sortingContainers, saveNullPerson);
List<SaveContainer> saveContainers = mapLogic.GetSaveContainers(dFacesContentDirectory, d2FacePartsContentDirectory, mappingCollection, idToNormalizedRectangleToMapping, useFiltersCounter, saveNullPerson);
mapLogic.SaveContainers(totalNotMapped, updated, saveContainers);
@ -984,33 +980,6 @@ public partial class DlibDotNet
return result;
}
private List<MappingFromItem> GetMappingFromItemCollection(Container[] containers)
{
List<MappingFromItem> results = new();
Item[] filteredItems;
DateTime[] containerDateTimes;
MappingFromItem mappingFromItem;
foreach (Container container in containers)
{
if (!container.Items.Any())
continue;
if (_Configuration.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(_Configuration.PropertyConfiguration, _Configuration.IgnoreRelativePaths, container.SourceDirectory))
continue;
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(_Configuration.PropertyConfiguration, container);
if (!filteredItems.Any())
continue;
containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems);
foreach (Item item in filteredItems)
{
if (item.Property?.Id is null)
continue;
mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item);
results.Add(mappingFromItem);
}
}
return results;
}
private void ParallelFor(string eDistanceContentDirectory, List<(bool, string, int, int, IReadOnlyList<MetadataExtractor.Directory>)> collection, string file)
{
const string lnk = ".lnk";
@ -1082,12 +1051,14 @@ public partial class DlibDotNet
private void Search(long ticks, string argZero, string propertyRoot)
{
int j;
int f;
int j;
int t;
int totalNotMapped;
Container[] containers;
string? a2PeopleContentDirectory;
Mapping[] mappingCollection;
string eDistanceContentDirectory;
string? a2PeopleContentDirectory;
string aResultsFullGroupDirectory;
string bResultsFullGroupDirectory;
string cResultsFullGroupDirectory;
@ -1095,6 +1066,7 @@ public partial class DlibDotNet
string d2ResultsFullGroupDirectory;
string fPhotoPrismContentDirectory;
string fPhotoPrismSingletonDirectory;
List<Shared.Models.Face> distinctFilteredFaces;
string a2PeopleSingletonDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(_Configuration.PropertyConfiguration, nameof(A2_People), "{}");
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
string message = $") Building Container(s) - {totalSeconds} total second(s)";
@ -1150,6 +1122,10 @@ public partial class DlibDotNet
(aResultsFullGroupDirectory, bResultsFullGroupDirectory, cResultsFullGroupDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory) = GetResultsFullGroupDirectories(outputResolution);
if (_Configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions.Contains(outputResolution))
MapLogicSupport.BeforeSaveFilteredOriginalImagesFromJLinks(_Configuration.JLinks, a2PeopleSingletonDirectory);
SetMapping(_FileNameToCollection, argZero, containers);
distinctFilteredFaces = GetFilteredDistinctFaces(argZero, containers);
mappingCollection = MapLogicSupport.GetSelectedMappingCollection(distinctFilteredFaces);
totalNotMapped = mapLogic.UpdateMappingFromPerson(mappingCollection);
if (_ArgZeroIsConfigurationRootDirectory
&& _Configuration.SaveResizedSubfiles
&& outputResolution == _Configuration.OutputResolutions[0]
@ -1158,18 +1134,17 @@ public partial class DlibDotNet
{
if (!_Configuration.LoadOrCreateThenSaveImageFacesResultsForOutputResolutions.Any())
break;
SetMapping(_FileNameToCollection, argZero, containers);
MapLogic(argZero, ticks, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, fPhotoPrismContentDirectory, mapLogicSupport, mapLogic, outputResolution);
MapLogic(argZero, ticks, containers, a2PeopleSingletonDirectory, dResultsFullGroupDirectory, d2ResultsFullGroupDirectory, fPhotoPrismContentDirectory, mapLogicSupport, mapLogic, outputResolution, distinctFilteredFaces, mappingCollection, totalNotMapped);
if (_IsEnvironment.Development)
continue;
List<MappingFromItem> mappingFromItemCollection = GetMappingFromItemCollection(containers);
_Random.Random(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0], mappingFromItemCollection);
G2_Identify identify = new(_Configuration);
List<G2_Identify> identifiedCollection = identify.GetIdentifiedCollection(_IsEnvironment, _Configuration.PropertyConfiguration, _Faces.FileNameExtension);
identify.WriteAllText(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0], identifiedCollection);
if (_Configuration.LoadOrCreateThenSaveIndex)
_Index.SetIndex(_Configuration.PropertyConfiguration, _Configuration.OutputResolutions[0]);
}
if (_Configuration.SaveRandomForOutputResolutions.Contains(outputResolution))
_Random.Random(_Configuration.PropertyConfiguration, mapLogic, outputResolution, mappingCollection);
if (!_IsEnvironment.Development)
{
_ = Shared.Models.Stateless.Methods.IPath.DeleteEmptyDirectories(Path.Combine(aResultsFullGroupDirectory, "{}"));

9
Instance/Models/Binder/Configuration.cs
View File

@ -70,6 +70,7 @@ public class Configuration
[Display(Name = "Save Mapped"), Required] public string[] SaveMappedForOutputResolutions { get; set; }
[Display(Name = "Save Not Mapped"), Required] public string[] SaveNotMappedForOutputResolutions { get; set; }
[Display(Name = "Save Resized Images by Person Key Formatted"), Required] public string[] SaveFilteredOriginalImagesFromJLinksForOutputResolutions { get; set; }
[Display(Name = "Save Random For Output Resolutions"), Required] public string[] SaveRandomForOutputResolutions { get; set; }
[Display(Name = "Save Resized Subfiles"), Required] public bool? SaveResizedSubfiles { get; set; }
[Display(Name = "Save Shortcuts"), Required] public string[] SaveShortcutsForOutputResolutions { get; set; }
[Display(Name = "Skip Search"), Required] public bool? SkipSearch { get; set; }
@ -191,6 +192,7 @@ public class Configuration
configuration.SaveMappedForOutputResolutions ??= Array.Empty<string>();
configuration.SaveNotMappedForOutputResolutions ??= Array.Empty<string>();
configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions ??= Array.Empty<string>();
configuration.SaveRandomForOutputResolutions ??= Array.Empty<string>();
if (configuration.SaveResizedSubfiles is null)
throw new NullReferenceException(nameof(configuration.SaveResizedSubfiles));
configuration.SaveShortcutsForOutputResolutions ??= Array.Empty<string>();
@ -217,8 +219,8 @@ public class Configuration
configuration.DistancePixelDistanceTolerance.Value,
configuration.DistanceRenameToMatch.Value,
configuration.FaceAreaPermille.Value,
configuration.FaceDistanceHiddenImageFactor.Value,
configuration.FaceConfidencePercent.Value,
configuration.FaceDistanceHiddenImageFactor.Value,
configuration.FaceDistancePermyriad.Value,
configuration.ForceFaceLastWriteTimeToCreationTime.Value,
configuration.ForceMetadataLastWriteTimeToCreationTime.Value,
@ -256,16 +258,17 @@ public class Configuration
configuration.PropertiesChangedForMetadata.Value,
configuration.PropertiesChangedForResize.Value,
configuration.RangeDaysDeltaTolerance,
configuration.RangeDistanceTolerance,
configuration.RangeFaceAreaPermilleTolerance,
configuration.RangeFaceConfidence,
configuration.RangeDistanceTolerance,
configuration.Reverse.Value,
configuration.SaveFaceDistancesForOutputResolutions,
configuration.SaveFaceLandmarkForOutputResolutions,
configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions,
configuration.SaveFullYearOfRandomFiles.Value,
configuration.SaveMappedForOutputResolutions,
configuration.SaveNotMappedForOutputResolutions,
configuration.SaveFilteredOriginalImagesFromJLinksForOutputResolutions,
configuration.SaveRandomForOutputResolutions,
configuration.SaveResizedSubfiles.Value,
configuration.SaveShortcutsForOutputResolutions,
configuration.SkipSearch.Value,

9
Instance/Models/Configuration.cs
View File

@ -66,6 +66,7 @@ public class Configuration
public string[] SaveMappedForOutputResolutions { init; get; }
public string[] SaveNotMappedForOutputResolutions { init; get; }
public string[] SaveFilteredOriginalImagesFromJLinksForOutputResolutions { init; get; }
public string[] SaveRandomForOutputResolutions { init; get; }
public bool SaveResizedSubfiles { init; get; }
public string[] SaveShortcutsForOutputResolutions { init; get; }
public bool SkipSearch { init; get; }
@ -85,8 +86,8 @@ public class Configuration
int distancePixelDistanceTolerance,
bool distanceRenameToMatch,
int faceAreaPermille,
int faceDistanceHiddenImageFactor,
int faceConfidencePercent,
int faceDistanceHiddenImageFactor,
int faceDistancePermyriad,
bool forceFaceLastWriteTimeToCreationTime,
bool forceMetadataLastWriteTimeToCreationTime,
@ -124,16 +125,17 @@ public class Configuration
bool propertiesChangedForMetadata,
bool propertiesChangedForResize,
int[] rangeDaysDeltaTolerance,
double[] rangeDistanceTolerance,
int[] rangeFaceAreaPermilleTolerance,
double[] rangeFaceConfidence,
double[] rangeDistanceTolerance,
bool reverse,
string[] saveFaceDistancesForOutputResolutions,
string[] saveFaceLandmarkForOutputResolutions,
string[] saveFilteredOriginalImagesFromJLinksForOutputResolutions,
bool saveFullYearOfRandomFiles,
string[] saveMappedForOutputResolutions,
string[] saveNotMappedForOutputResolutions,
string[] saveFilteredOriginalImagesFromJLinksForOutputResolutions,
string[] saveRandomForOutputResolutions,
bool saveResizedSubfiles,
string[] saveShortcutsForOutputResolutions,
bool skipSearch,
@ -202,6 +204,7 @@ public class Configuration
SaveNotMappedForOutputResolutions = saveNotMappedForOutputResolutions;
SaveFilteredOriginalImagesFromJLinksForOutputResolutions = saveFilteredOriginalImagesFromJLinksForOutputResolutions;
SaveResizedSubfiles = saveResizedSubfiles;
SaveRandomForOutputResolutions = saveRandomForOutputResolutions;
SaveShortcutsForOutputResolutions = saveShortcutsForOutputResolutions;
SkipSearch = skipSearch;
SortingMaximumPerFaceShouldBeHigh = sortingMaximumPerFaceShouldBeHigh;

66
Instance/Models/_F_Random.cs
View File

@ -43,46 +43,74 @@ internal class F_Random
return result;
}
internal void Random(Property.Models.Configuration configuration, string outputResolution, List<Shared.Models.MappingFromItem> mappingFromItemCollection)
private static Dictionary<string, List<string>> Get(Map.Models.MapLogic mapLogic, Shared.Models.Mapping[] mappingCollection, string dateFormat)
{
Dictionary<string, List<string>> results = new();
string key;
DateTime dateTime;
List<long>? personKeys;
List<string>? relativePaths;
Dictionary<int, List<long>> idToPersonKeys = mapLogic.GetIdToPeronKeys();
foreach (Shared.Models.Mapping mapping in mappingCollection)
{
if (mapping.MappingFromItem.ImageFileHolder.DirectoryName is null || mapping.MappingFromPerson is null)
continue;
if (!idToPersonKeys.TryGetValue(mapping.MappingFromItem.Id, out personKeys))
continue;
if (!personKeys.Contains(mapping.MappingFromPerson.PersonBirthday.Value.Ticks))
continue;
dateTime = new(mapping.MappingFromPerson.PersonBirthday.Value.Ticks);
key = dateTime.ToString(dateFormat);
if (!results.TryGetValue(key, out relativePaths))
{
results.Add(key, new());
if (!results.TryGetValue(key, out relativePaths))
throw new Exception();
}
relativePaths.Add(mapping.MappingFromItem.RelativePath);
}
return results;
}
internal void Random(Property.Models.Configuration configuration, Map.Models.MapLogic mapLogic, string outputResolution, Shared.Models.Mapping[] mappingCollection)
{
string key;
string json;
string jsonFile;
Random random = new();
List<string>? collection;
string dateFormat = "MM-dd";
List<string> relativePaths = new();
List<string> ignoreRelativePaths = new();
DateTime dateTime = new(2024, 1, 1); //Leap year
DateTime dateTime = new(2024, 1, 1); //Leap year
Dictionary<string, List<string>> dayToRelativePaths = Get(mapLogic, mappingCollection, dateFormat);
string fRandomCollectionDirectory = Property.Models.Stateless.IResult.GetResultsDateGroupDirectory(configuration, nameof(F_Random), "[]");
string[] files = Directory.GetFiles(fRandomCollectionDirectory, "*", SearchOption.TopDirectoryOnly);
foreach (string file in files)
File.Delete(file);
foreach (Shared.Models.MappingFromItem mappingFromItem in mappingFromItemCollection)
foreach (Shared.Models.Mapping mapping in mappingCollection)
{
if (mappingFromItem.ImageFileHolder.DirectoryName is null)
if (mapping.MappingFromItem.ImageFileHolder.DirectoryName is null)
continue;
if (!_Configuration.IgnoreRelativePaths.Any(l => mappingFromItem.ImageFileHolder.DirectoryName.Contains(l)) || !IsIgnoreRelativePath(mappingFromItem.ImageFileHolder.DirectoryName))
relativePaths.Add(mappingFromItem.RelativePath);
else
ignoreRelativePaths.Add(mappingFromItem.RelativePath);
if (_Configuration.IgnoreRelativePaths.Any(l => mapping.MappingFromItem.ImageFileHolder.DirectoryName.Contains(l)) && IsIgnoreRelativePath(mapping.MappingFromItem.ImageFileHolder.DirectoryName))
continue;
relativePaths.Add(mapping.MappingFromItem.RelativePath);
}
if (relativePaths.Any())
{
for (int i = 0; i < 366; i++)
{
relativePaths = (from l in relativePaths orderby random.NextDouble() select l).ToList();
jsonFile = Path.Combine(fRandomCollectionDirectory, $"{dateTime.AddDays(i):MM-dd}.json");
json = JsonSerializer.Serialize(relativePaths, _WriteIndentedJsonSerializerOptions);
key = dateTime.AddDays(i).ToString(dateFormat);
if (dayToRelativePaths.TryGetValue(key, out collection) && collection.Count > 10)
collection = (from l in collection orderby random.NextDouble() select l).ToList();
else
collection = (from l in relativePaths orderby random.NextDouble() select l).ToList();
jsonFile = Path.Combine(fRandomCollectionDirectory, $"{key}.json");
json = JsonSerializer.Serialize(collection, _WriteIndentedJsonSerializerOptions);
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(jsonFile, json, updateDateWhenMatches: false, compareBeforeWrite: false);
if (!_Configuration.SaveFullYearOfRandomFiles)
break;
}
}
if (ignoreRelativePaths.Any())
{
ignoreRelativePaths = (from l in ignoreRelativePaths orderby random.NextDouble() select l).ToList();
jsonFile = Path.Combine(fRandomCollectionDirectory, "01-01.txt");
json = JsonSerializer.Serialize(ignoreRelativePaths, _WriteIndentedJsonSerializerOptions);
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(jsonFile, json, updateDateWhenMatches: false, compareBeforeWrite: false);
}
}
}

12
Instance/appsettings.Development.json
View File

@ -13,6 +13,7 @@
"DistanceRenameToMatch": true,
"DistanceMoveUnableToMatch": true,
"PersonCharacters": "!#]^_`~+",
"PersonCharactersToCopyTo": "x",
"xPersonCharactersToCopyTo": "!",
"xxPersonCharactersToCopyTo": "#",
"xxxPersonCharactersToCopyTo": "]",
@ -22,9 +23,9 @@
"xxxxxxxPersonCharactersToCopyTo": "~",
"xxxxxxxxPersonCharactersToCopyTo": "+",
"xRootDirectory": "D:/Tmp/phares/Pictures",
"xxRootDirectory": "D:/Tmp/Phares/Compare/Not-Copy-Copy-37c7b67",
"xxxRootDirectory": "D:/Tmp/Phares/Compare/Corrupt",
"RootDirectory": "D:/1) Images A/Images-37c7b67",
"xxRootDirectory": "D:/Tmp/Phares/Compare/Corrupt",
"RootDirectory": "D:/Tmp/Phares/Compare/Not-Copy-Copy-37c7b67",
"xxxxRootDirectory": "D:/1) Images A/Images-37c7b67",
"xxxxxRootDirectory": "D:/1) Images A/Images-37c7b67/Facebook/=2022.3 Facebook",
"JLinks": [
"Julie"
@ -63,9 +64,12 @@
"7680 x 4320"
],
"SaveFaceLandmarkForOutputResolutions": [],
"SaveFilteredOriginalImagesFromJLinksForOutputResolutions": [],
"SaveMappedForOutputResolutions": [],
"SaveNotMappedForOutputResolutions": [],
"SaveFilteredOriginalImagesFromJLinksForOutputResolutions": [],
"SaveRandomForOutputResolutions": [
"7680 x 4320"
],
"SaveShortcutsForOutputResolutions": [],
"IgnoreRelativePaths": [
"3757 W Whitman 2017",

13
Instance/appsettings.json
View File

@ -147,14 +147,15 @@
0.2,
100
],
"SaveFaceDistancesForOutputResolutions": [],
"SaveFaceLandmarkForOutputResolutions": [],
"SaveMappedForOutputResolutions": [],
"SaveNotMappedForOutputResolutions": [],
"SaveFilteredOriginalImagesFromJLinksForOutputResolutions": [],
"SaveShortcutsForOutputResolutions": [],
"IgnoreRelativePaths": [],
"MixedYearRelativePaths": [],
"SaveFaceDistancesForOutputResolutions": [],
"SaveFaceLandmarkForOutputResolutions": [],
"SaveFilteredOriginalImagesFromJLinksForOutputResolutions": [],
"SaveMappedForOutputResolutions": [],
"SaveNotMappedForOutputResolutions": [],
"SaveRandomForOutputResolutions": [],
"SaveShortcutsForOutputResolutions": [],
"VerifyToSeason": [],
"ValidImageFormatExtensions": [
".bmp",