Mass push
This commit is contained in:
.vscode
Compare
Date-Group
Delete-By-Relative
.vscode
Delete-By-Relative.csprojDeleteByRelative.csModels
Program.csappsettings.Development.jsonappsettings.jsonDistance
Drag-Drop
Duplicate-Search
.vscode
Duplicate-Search.csprojDuplicateSearch.csModels
Program.csappsettings.Development.jsonappsettings.jsonFace
FaceParts
FaceRecognitionDotNet
Instance
DlibDotNet.csInstance.csproj
Models
Program.csappsettings.Development.jsonappsettings.Staging.jsonappsettings.jsonMap
Metadata
Not-Copy-Copy
PrepareForOld
Property-Compare
Property
Models
Property.csprojResize
Shared
Models
FaceDistance.csFileHolder.csItem.csLocation.csMapping.csMarker.csMarkerWith.cs
View-by-Distance.Shared.csprojProperties
IFaceDistance.csIItem.csIMapping.csIProperty.csIPropertyConfiguration.csIRelativeLocation.csISorting.cs
Property.csRelativeLocation.csSorting.csSortingContainer.csStateless
Tests
Models
Tests.csprojUnitTestCalculations.csUnitTestExample.csUnitTestIsEnvironment.csUnitTestResize.csTestsWithFaceRecognitionDotNet
View-by-Distance-MKLink-Console.sln
255
Duplicate-Search/DuplicateSearch.cs
Normal file
255
Duplicate-Search/DuplicateSearch.cs
Normal file
@ -0,0 +1,255 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Phares.Shared;
|
||||
using Serilog;
|
||||
using ShellProgressBar;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using View_by_Distance.Duplicate.Search.Models;
|
||||
using View_by_Distance.Property.Models;
|
||||
using View_by_Distance.Shared.Models;
|
||||
using View_by_Distance.Shared.Models.Methods;
|
||||
|
||||
namespace View_by_Distance.Duplicate.Search;
|
||||
|
||||
public class DuplicateSearch
|
||||
{
|
||||
|
||||
public DuplicateSearch(List<string> args, IsEnvironment isEnvironment, IConfigurationRoot configurationRoot, AppSettings appSettings, string workingDirectory, bool isSilent, IConsole console)
|
||||
{
|
||||
if (isSilent)
|
||||
{ }
|
||||
if (console is null)
|
||||
{ }
|
||||
string searchPattern = "*";
|
||||
long ticks = DateTime.Now.Ticks;
|
||||
ILogger? log = Log.ForContext<DuplicateSearch>();
|
||||
Configuration configuration = Property.Models.Binder.Configuration.Get(isEnvironment, configurationRoot);
|
||||
configuration.Update();
|
||||
log.Information(configuration.RootDirectory);
|
||||
if (appSettings.IndexOnly)
|
||||
WriteIndexData(ticks, configuration, searchPattern);
|
||||
else
|
||||
{
|
||||
Configuration.Verify(configuration, requireExist: false);
|
||||
string argZero = args.Count > 0 ? Path.GetFullPath(args[0]) : Path.GetFullPath(configuration.RootDirectory);
|
||||
bool argZeroIsConfigurationRootDirectory = configuration.RootDirectory == argZero;
|
||||
Container[] containers = GetContainers(appSettings, ticks, argZero, configuration, argZeroIsConfigurationRootDirectory);
|
||||
string destinationRoot = Property.Models.Stateless.IResult.GetResultsGroupDirectory(configuration, "Z) Moved");
|
||||
List<int> preloadIds = GetPreloadIds(destinationRoot);
|
||||
Dictionary<int, List<MappingFromItem?>> idToCollection = GetIdToCollection(appSettings, argZero, configuration, argZeroIsConfigurationRootDirectory, containers, destinationRoot, preloadIds);
|
||||
int duplicates = (from l in idToCollection where l.Value.Count > 1 select 1).Sum();
|
||||
if (duplicates == 0)
|
||||
log.Information($"Found {duplicates} duplicate file(s)");
|
||||
else
|
||||
QuestionMove(ticks, log, destinationRoot, idToCollection, duplicates);
|
||||
}
|
||||
}
|
||||
|
||||
private static void WriteIndexData(long ticks, Configuration configuration, string searchPattern)
|
||||
{
|
||||
string? alongSideDirectory;
|
||||
string directoryName = Path.GetFileName(configuration.RootDirectory);
|
||||
if (Path.GetPathRoot(configuration.RootDirectory) == configuration.RootDirectory)
|
||||
alongSideDirectory = configuration.RootDirectory;
|
||||
else
|
||||
alongSideDirectory = Path.GetDirectoryName(configuration.RootDirectory);
|
||||
if (alongSideDirectory is null)
|
||||
throw new NullReferenceException(nameof(alongSideDirectory));
|
||||
IEnumerable<(string, string[])> files = Shared.Models.Stateless.Methods.IFileHolder.GetFiles(configuration.RootDirectory, searchPattern);
|
||||
List<FileHolder> fileHolders = Shared.Models.Stateless.Methods.IFileHolder.GetFileHolders(files.ToArray());
|
||||
File.WriteAllLines(Path.Combine(alongSideDirectory, $"{directoryName}-{ticks}.tsv"), fileHolders.Select(l => l.FullName));
|
||||
string json = JsonSerializer.Serialize(fileHolders);
|
||||
File.WriteAllText(Path.Combine(alongSideDirectory, $"{directoryName}-{ticks}.json"), json);
|
||||
}
|
||||
|
||||
private static void Move(ILogger log, long ticks, string destinationRoot, List<(FileHolder ImageFileHolder, string Destination)> collection)
|
||||
{
|
||||
StringBuilder stringBuilder = new();
|
||||
foreach ((FileHolder fileHolder, string destination) in collection)
|
||||
{
|
||||
_ = stringBuilder.AppendLine(fileHolder.FullName);
|
||||
_ = stringBuilder.AppendLine(destination);
|
||||
}
|
||||
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(Path.Combine(destinationRoot, $"{ticks}file(s).lsv"), stringBuilder.ToString(), updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null);
|
||||
log.Information($"Ready to move {collection.Count} file(s)?");
|
||||
for (int y = 0; y < int.MaxValue; y++)
|
||||
{
|
||||
log.Information("Press \"Y\" key to move file(s) or close console to not move files");
|
||||
if (System.Console.ReadKey().Key == ConsoleKey.Y)
|
||||
break;
|
||||
}
|
||||
log.Information(". . .");
|
||||
foreach ((FileHolder fileHolder, string destination) in collection)
|
||||
{
|
||||
try
|
||||
{ File.Move(fileHolder.FullName, destination); }
|
||||
catch (Exception exception)
|
||||
{ log.Error(exception, $"Failed to move <{fileHolder.FullName}>"); }
|
||||
}
|
||||
log.Information($"{collection.Count} file(s) moved");
|
||||
for (int y = 0; y < int.MaxValue; y++)
|
||||
{
|
||||
log.Information("Press \"Y\" key to move file(s) back or close console to leave them moved");
|
||||
if (System.Console.ReadKey().Key != ConsoleKey.Y)
|
||||
continue;
|
||||
log.Information(". . .");
|
||||
foreach ((FileHolder fileHolder, string destination) in collection)
|
||||
{
|
||||
if (!File.Exists(destination))
|
||||
continue;
|
||||
if (File.Exists(fileHolder.FullName))
|
||||
continue;
|
||||
try
|
||||
{ File.Move(destination, fileHolder.FullName); }
|
||||
catch (Exception exception)
|
||||
{ log.Error(exception, $"Failed to move <{destination}>"); }
|
||||
}
|
||||
}
|
||||
log.Information(". . .");
|
||||
}
|
||||
|
||||
private static void QuestionMove(long ticks, ILogger log, string destinationRoot, Dictionary<int, List<MappingFromItem?>> idToCollection, int duplicates)
|
||||
{
|
||||
int[] ids = (from l in idToCollection orderby l.Key where l.Value.Any(m => m is not null) select l.Key).ToArray();
|
||||
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(Path.Combine(destinationRoot, $"{ticks}-id(s).lsv"), string.Join(Environment.NewLine, ids), updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null);
|
||||
string json = JsonSerializer.Serialize(idToCollection, new JsonSerializerOptions { WriteIndented = true });
|
||||
_ = Shared.Models.Stateless.Methods.IPath.WriteAllText(Path.Combine(destinationRoot, $"{ticks}.json"), json, updateDateWhenMatches: false, compareBeforeWrite: true, updateToWhenMatches: null);
|
||||
log.Information($"Found {duplicates} duplicate file(s)");
|
||||
for (int y = 0; y < int.MaxValue; y++)
|
||||
{
|
||||
log.Information("Press \"Y\" key to continue or close console to leave them moved");
|
||||
if (System.Console.ReadKey().Key != ConsoleKey.Y)
|
||||
continue;
|
||||
log.Information(". . .");
|
||||
List<(FileHolder ImageFileHolder, string Destination)> collection = GetCollectionAndCreateDirectories(idToCollection);
|
||||
Move(log, ticks, destinationRoot, collection);
|
||||
}
|
||||
}
|
||||
|
||||
private static Container[] GetContainers(AppSettings appSettings, long ticks, string argZero, Configuration configuration, bool argZeroIsConfigurationRootDirectory)
|
||||
{
|
||||
int j;
|
||||
int f;
|
||||
int t;
|
||||
Container[] containers;
|
||||
int totalSeconds = (int)Math.Floor(new TimeSpan(DateTime.Now.Ticks - ticks).TotalSeconds);
|
||||
string message = $") Building Container(s) - {totalSeconds} total second(s)";
|
||||
A_Property propertyLogic = new(appSettings.MaxDegreeOfParallelism, configuration, appSettings.OutputExtension, appSettings.Reverse);
|
||||
ProgressBarOptions options = new() { ProgressCharacter = '─', ProgressBarOnBottom = true, DisableBottomPercentage = true };
|
||||
using (ProgressBar progressBar = new(1, message, options))
|
||||
{
|
||||
progressBar.Tick();
|
||||
(j, f, t, containers) = Property.Models.Stateless.Container.GetContainers(configuration, propertyLogic);
|
||||
}
|
||||
if (appSettings.SortContainers)
|
||||
containers = Shared.Models.Stateless.Methods.IContainer.SortContainers(configuration, appSettings.IgnoreRelativePaths, argZeroIsConfigurationRootDirectory, argZero, containers);
|
||||
return containers;
|
||||
}
|
||||
|
||||
private static Dictionary<int, List<MappingFromItem?>> GetIdToCollection(AppSettings appSettings, string argZero, Configuration configuration, bool argZeroIsConfigurationRootDirectory, Container[] containers, string destinationRoot, List<int> preloadIds)
|
||||
{
|
||||
Dictionary<int, List<MappingFromItem?>> results = new();
|
||||
string directory;
|
||||
const int zero = 0;
|
||||
Item[] filteredItems;
|
||||
bool isIgnoreRelativePath;
|
||||
FileHolder resizedFileHolder;
|
||||
DateTime[] containerDateTimes;
|
||||
MappingFromItem? mappingFromItem;
|
||||
List<MappingFromItem?>? collection;
|
||||
const string duplicates = "-Duplicate(s)";
|
||||
if (containers.Any())
|
||||
{
|
||||
foreach (int id in preloadIds)
|
||||
results.Add(id, new() { null });
|
||||
}
|
||||
foreach (Container container in containers)
|
||||
{
|
||||
if (!container.Items.Any())
|
||||
continue;
|
||||
if (!argZeroIsConfigurationRootDirectory && !container.SourceDirectory.StartsWith(argZero))
|
||||
continue;
|
||||
filteredItems = Shared.Models.Stateless.Methods.IContainer.GetFilterItems(configuration, container);
|
||||
if (!filteredItems.Any())
|
||||
continue;
|
||||
isIgnoreRelativePath = appSettings.IgnoreRelativePaths.Any(l => container.SourceDirectory.Contains(l)) && Shared.Models.Stateless.Methods.IContainer.IsIgnoreRelativePath(configuration, appSettings.IgnoreRelativePaths, container.SourceDirectory);
|
||||
containerDateTimes = Shared.Models.Stateless.Methods.IContainer.GetContainerDateTimes(filteredItems);
|
||||
foreach (Item item in filteredItems)
|
||||
{
|
||||
if (item.Property?.Id is null)
|
||||
{
|
||||
if (int.TryParse(item.ImageFileHolder.NameWithoutExtension, out int id))
|
||||
continue;
|
||||
continue;
|
||||
}
|
||||
if (!results.TryGetValue(item.Property.Id.Value, out collection))
|
||||
results.Add(item.Property.Id.Value, new());
|
||||
if (collection is null && !results.TryGetValue(item.Property.Id.Value, out collection))
|
||||
continue;
|
||||
if (collection.Count == 0)
|
||||
directory = $"0{duplicates}";
|
||||
else
|
||||
directory = $"{collection.Count + 1}{duplicates}";
|
||||
if (collection.Count == 1)
|
||||
{
|
||||
mappingFromItem = collection[zero];
|
||||
if (mappingFromItem is not null)
|
||||
{
|
||||
resizedFileHolder = new(mappingFromItem.ResizedFileHolder.FullName.Replace($"0{duplicates}", $"1{duplicates}"));
|
||||
collection[0] = new(mappingFromItem.ContainerDateTimes, mappingFromItem.Id, mappingFromItem.ImageFileHolder, mappingFromItem.IsWrongYear, mappingFromItem.MinimumDateTime, mappingFromItem.RelativePath, resizedFileHolder);
|
||||
}
|
||||
}
|
||||
resizedFileHolder = new(string.Concat(Path.Combine(destinationRoot, directory), item.RelativePath));
|
||||
mappingFromItem = Shared.Models.Stateless.Methods.IMappingFromItem.GetMappingFromItem(containerDateTimes, item, resizedFileHolder);
|
||||
collection.Add(mappingFromItem);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private static List<(FileHolder ImageFileHolder, string Destination)> GetCollectionAndCreateDirectories(Dictionary<int, List<MappingFromItem?>> idToCollection)
|
||||
{
|
||||
List<(FileHolder ImageFileHolder, string Destination)> results = new();
|
||||
List<string> collection = new();
|
||||
foreach (KeyValuePair<int, List<MappingFromItem?>> keyValuePair in idToCollection)
|
||||
{
|
||||
foreach (MappingFromItem? mappingFromItem in keyValuePair.Value)
|
||||
{
|
||||
if (mappingFromItem?.ResizedFileHolder.DirectoryName is null)
|
||||
continue;
|
||||
if (!mappingFromItem.ImageFileHolder.Exists || mappingFromItem.ResizedFileHolder.Exists)
|
||||
continue;
|
||||
collection.Add(mappingFromItem.ResizedFileHolder.DirectoryName);
|
||||
results.Add(new(mappingFromItem.ImageFileHolder, mappingFromItem.ResizedFileHolder.FullName));
|
||||
}
|
||||
}
|
||||
foreach (string directory in collection.Distinct())
|
||||
{
|
||||
if (!Directory.Exists(directory))
|
||||
_ = Directory.CreateDirectory(directory);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private static List<int> GetPreloadIds(string destinationRoot)
|
||||
{
|
||||
List<int> results = new();
|
||||
string[] lines;
|
||||
string preloadDirectory = Path.Combine(destinationRoot, "Preload");
|
||||
if (!Directory.Exists(preloadDirectory))
|
||||
_ = Directory.CreateDirectory(preloadDirectory);
|
||||
string[] files = Directory.GetFiles(preloadDirectory, "*.lsv", SearchOption.TopDirectoryOnly);
|
||||
foreach (string file in files)
|
||||
{
|
||||
lines = File.ReadAllLines(file);
|
||||
foreach (string line in lines)
|
||||
{
|
||||
if (string.IsNullOrEmpty(line) || !int.TryParse(line, out int id) || id == 0)
|
||||
continue;
|
||||
results.Add(id);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
}
|
Reference in New Issue
Block a user