871 lines
42 KiB
C#
871 lines
42 KiB
C#
using DlibDotNet;
|
|
using DlibDotNet.Dnn;
|
|
using System.Drawing;
|
|
using System.Drawing.Imaging;
|
|
using System.Runtime.InteropServices;
|
|
using System.Text;
|
|
using View_by_Distance.FaceRecognitionDotNet.Dlib.Python;
|
|
using View_by_Distance.FaceRecognitionDotNet.Extensions;
|
|
using View_by_Distance.Shared.Models;
|
|
using View_by_Distance.Shared.Models.Stateless;
|
|
|
|
namespace View_by_Distance.FaceRecognitionDotNet;
|
|
|
|
/// <summary>
|
|
/// Provides the method to find and recognize face methods. This class cannot be inherited.
|
|
/// </summary>
|
|
public sealed class FaceRecognition : DisposableObject
|
|
{
|
|
|
|
#region Fields
|
|
|
|
private readonly ShapePredictor _PosePredictor68Point;
|
|
|
|
private readonly ShapePredictor _PosePredictor5Point;
|
|
|
|
private readonly LossMmod _CnnFaceDetector;
|
|
|
|
private readonly LossMetric _FaceEncoder;
|
|
|
|
private readonly FrontalFaceDetector _FaceDetector;
|
|
|
|
#endregion
|
|
|
|
#region Constructors
|
|
|
|
/// <summary>
|
|
/// Initializes a new instance of the <see cref="FaceRecognition"/> class with the directory path that stores model files.
|
|
/// </summary>
|
|
/// <param name="directory">The directory path that stores model files.</param>
|
|
/// <exception cref="FileNotFoundException">The model file is not found.</exception>
|
|
/// <exception cref="DirectoryNotFoundException">The specified directory path is not found.</exception>
|
|
private FaceRecognition(string directory)
|
|
{
|
|
if (!Directory.Exists(directory))
|
|
throw new DirectoryNotFoundException(directory);
|
|
|
|
string? predictor68PointModel = Path.Combine(directory, FaceRecognitionModels.GetPosePredictorModelLocation());
|
|
if (!File.Exists(predictor68PointModel))
|
|
throw new FileNotFoundException(predictor68PointModel);
|
|
|
|
string? predictor5PointModel = Path.Combine(directory, FaceRecognitionModels.GetPosePredictorFivePointModelLocation());
|
|
if (!File.Exists(predictor5PointModel))
|
|
throw new FileNotFoundException(predictor5PointModel);
|
|
|
|
string? cnnFaceDetectionModel = Path.Combine(directory, FaceRecognitionModels.GetCnnFaceDetectorModelLocation());
|
|
if (!File.Exists(cnnFaceDetectionModel))
|
|
throw new FileNotFoundException(cnnFaceDetectionModel);
|
|
|
|
string? faceRecognitionModel = Path.Combine(directory, FaceRecognitionModels.GetFaceRecognitionModelLocation());
|
|
if (!File.Exists(faceRecognitionModel))
|
|
throw new FileNotFoundException(faceRecognitionModel);
|
|
|
|
_FaceDetector?.Dispose();
|
|
_FaceDetector = DlibDotNet.Dlib.GetFrontalFaceDetector();
|
|
|
|
_PosePredictor68Point?.Dispose();
|
|
_PosePredictor68Point = ShapePredictor.Deserialize(predictor68PointModel);
|
|
|
|
_PosePredictor5Point?.Dispose();
|
|
_PosePredictor5Point = ShapePredictor.Deserialize(predictor5PointModel);
|
|
|
|
_CnnFaceDetector?.Dispose();
|
|
_CnnFaceDetector = LossMmod.Deserialize(cnnFaceDetectionModel);
|
|
|
|
_FaceEncoder?.Dispose();
|
|
_FaceEncoder = LossMetric.Deserialize(faceRecognitionModel);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Initializes a new instance of the <see cref="FaceRecognition"/> class with the instance that contains model binary datum.
|
|
/// </summary>
|
|
/// <param name="parameter">The instance that contains model binary datum.</param>
|
|
/// <exception cref="ArgumentNullException"><paramref name="parameter"/> is null.</exception>
|
|
/// <exception cref="NullReferenceException">The model data is null.</exception>
|
|
private FaceRecognition(ModelParameter parameter)
|
|
{
|
|
if (parameter == null)
|
|
throw new ArgumentNullException(nameof(parameter));
|
|
|
|
if (parameter.PosePredictor5FaceLandmarksModel == null)
|
|
throw new NullReferenceException(nameof(parameter.PosePredictor5FaceLandmarksModel));
|
|
|
|
if (parameter.PosePredictor68FaceLandmarksModel == null)
|
|
throw new NullReferenceException(nameof(parameter.PosePredictor68FaceLandmarksModel));
|
|
|
|
if (parameter.CnnFaceDetectorModel == null)
|
|
throw new NullReferenceException(nameof(parameter.CnnFaceDetectorModel));
|
|
|
|
if (parameter.FaceRecognitionModel == null)
|
|
throw new NullReferenceException(nameof(parameter.FaceRecognitionModel));
|
|
|
|
_FaceDetector?.Dispose();
|
|
_FaceDetector = DlibDotNet.Dlib.GetFrontalFaceDetector();
|
|
|
|
_PosePredictor68Point?.Dispose();
|
|
_PosePredictor68Point = ShapePredictor.Deserialize(parameter.PosePredictor68FaceLandmarksModel);
|
|
|
|
_PosePredictor5Point?.Dispose();
|
|
_PosePredictor5Point = ShapePredictor.Deserialize(parameter.PosePredictor5FaceLandmarksModel);
|
|
|
|
_CnnFaceDetector?.Dispose();
|
|
_CnnFaceDetector = LossMmod.Deserialize(parameter.CnnFaceDetectorModel);
|
|
|
|
_FaceEncoder?.Dispose();
|
|
_FaceEncoder = LossMetric.Deserialize(parameter.FaceRecognitionModel);
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Properties
|
|
|
|
/// <summary>
|
|
/// Gets or sets the custom face detector that user defined.
|
|
/// </summary>
|
|
public FaceDetector? CustomFaceDetector { get; set; }
|
|
|
|
/// <summary>
|
|
/// Gets or sets the custom face landmark detector that user defined.
|
|
/// </summary>
|
|
public FaceLandmarkDetector? CustomFaceLandmarkDetector { get; set; }
|
|
|
|
/// <summary>
|
|
/// Gets or sets the character encoding to convert <see cref="System.String"/> to array of <see cref="byte"/> for internal library.
|
|
/// </summary>
|
|
public static Encoding InternalEncoding
|
|
{
|
|
get => DlibDotNet.Dlib.Encoding;
|
|
set => DlibDotNet.Dlib.Encoding = value ?? Encoding.UTF8;
|
|
}
|
|
|
|
#endregion
|
|
|
|
#region Methods
|
|
|
|
/// <summary>
|
|
/// Returns an enumerable collection of array of bounding boxes of human faces in a image using the cnn face detector.
|
|
/// </summary>
|
|
/// <param name="images">An enumerable collection of images.</param>
|
|
/// <param name="numberOfTimesToUpsample">The number of image looking for faces. Higher numbers find smaller faces.</param>
|
|
/// <param name="batchSize">The number of images to include in each GPU processing batch.</param>
|
|
/// <returns>An enumerable collection of array of found face locations.</returns>
|
|
/// <exception cref="ArgumentNullException"><paramref name="images"/> is null.</exception>
|
|
public IEnumerable<Location[]> BatchFaceLocations(IEnumerable<Image> images, int numberOfTimesToUpsample = 1, int batchSize = 128)
|
|
{
|
|
if (images == null)
|
|
throw new ArgumentNullException(nameof(images));
|
|
|
|
List<Location[]>? results = new();
|
|
|
|
Image[]? imagesArray = images.ToArray();
|
|
if (!imagesArray.Any())
|
|
return results;
|
|
|
|
IEnumerable<MModRect>[]? rawDetectionsBatched = RawFaceLocationsBatched(imagesArray, numberOfTimesToUpsample, batchSize).ToArray();
|
|
|
|
Image? image = imagesArray[0];
|
|
for (int index = 0; index < rawDetectionsBatched.Length; index++)
|
|
{
|
|
MModRect[]? faces = rawDetectionsBatched[index].ToArray();
|
|
Location[]? locations = faces.Select(rect => new Location(TrimBound(rect.Rect, image.Width, image.Height), rect.DetectionConfidence)).ToArray();
|
|
foreach (MModRect? face in faces)
|
|
face.Dispose();
|
|
results.Add(locations);
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Compare a known face encoding against a candidate encoding to see if they match.
|
|
/// </summary>
|
|
/// <param name="knownFaceEncoding">A known face encodings.</param>
|
|
/// <param name="faceEncodingToCheck">A single face encoding to compare against a known face encoding.</param>
|
|
/// <param name="tolerance">The distance between faces to consider it a match. Lower is more strict. The default value is 0.6.</param>
|
|
/// <returns>A True/False value indicating which known a face encoding matches the face encoding to check.</returns>
|
|
/// <exception cref="ArgumentNullException"><paramref name="knownFaceEncoding"/> or <paramref name="faceEncodingToCheck"/> is null.</exception>
|
|
/// <exception cref="ObjectDisposedException"><paramref name="knownFaceEncoding"/> or <paramref name="faceEncodingToCheck"/>.</exception>
|
|
public static bool CompareFace(FaceEncoding knownFaceEncoding, FaceEncoding faceEncodingToCheck, double tolerance = 0.6d)
|
|
{
|
|
if (knownFaceEncoding == null)
|
|
throw new ArgumentNullException(nameof(knownFaceEncoding));
|
|
if (faceEncodingToCheck == null)
|
|
throw new ArgumentNullException(nameof(faceEncodingToCheck));
|
|
|
|
knownFaceEncoding.ThrowIfDisposed();
|
|
faceEncodingToCheck.ThrowIfDisposed();
|
|
|
|
return FaceDistance(knownFaceEncoding, faceEncodingToCheck) <= tolerance;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Compare an enumerable collection of face encodings against a candidate encoding to see if they match.
|
|
/// </summary>
|
|
/// <param name="knownFaceEncodings">An enumerable collection of known face encodings.</param>
|
|
/// <param name="faceEncodingToCheck">A single face encoding to compare against the enumerable collection.</param>
|
|
/// <param name="tolerance">The distance between faces to consider it a match. Lower is more strict. The default value is 0.6.</param>
|
|
/// <returns>An enumerable collection of True/False values indicating which known face encodings match the face encoding to check.</returns>
|
|
/// <exception cref="ArgumentNullException"><paramref name="knownFaceEncodings"/> or <paramref name="faceEncodingToCheck"/> is null.</exception>
|
|
/// <exception cref="ObjectDisposedException"><paramref name="faceEncodingToCheck"/> is disposed. Or <paramref name="knownFaceEncodings"/> contains disposed object.</exception>
|
|
public static IEnumerable<bool> CompareFaces(IEnumerable<FaceEncoding> knownFaceEncodings, FaceEncoding faceEncodingToCheck, double tolerance = 0.6d)
|
|
{
|
|
if (knownFaceEncodings == null)
|
|
throw new ArgumentNullException(nameof(knownFaceEncodings));
|
|
if (faceEncodingToCheck == null)
|
|
throw new ArgumentNullException(nameof(faceEncodingToCheck));
|
|
|
|
faceEncodingToCheck.ThrowIfDisposed();
|
|
|
|
FaceEncoding[]? array = knownFaceEncodings.ToArray();
|
|
if (array.Any(encoding => encoding.IsDisposed))
|
|
throw new ObjectDisposedException($"{nameof(knownFaceEncodings)} contains disposed object.");
|
|
|
|
List<bool>? results = new();
|
|
if (array.Length == 0)
|
|
return results;
|
|
|
|
foreach (FaceEncoding? faceEncoding in array)
|
|
results.Add(FaceDistance(faceEncoding, faceEncodingToCheck) <= tolerance);
|
|
|
|
return results;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Create a new instance of the <see cref="FaceRecognition"/> class.
|
|
/// </summary>
|
|
/// <param name="directory">The directory path that stores model files.</param>
|
|
/// <exception cref="FileNotFoundException">The model file is not found.</exception>
|
|
/// <exception cref="DirectoryNotFoundException">The specified directory path is not found.</exception>
|
|
public static FaceRecognition Create(string directory) => new(directory);
|
|
|
|
/// <summary>
|
|
/// Create a new instance of the <see cref="FaceRecognition"/> class.
|
|
/// </summary>
|
|
/// <param name="parameter">The instance that contains model binary datum.</param>
|
|
/// <exception cref="ArgumentNullException"><paramref name="parameter"/> is null.</exception>
|
|
/// <exception cref="NullReferenceException">The model data is null.</exception>
|
|
public static FaceRecognition Create(ModelParameter parameter) => new(parameter);
|
|
|
|
/// <summary>
|
|
/// Crop a specified image with enumerable collection of face locations.
|
|
/// </summary>
|
|
/// <param name="image">The image contains a face.</param>
|
|
/// <param name="locations">The enumerable collection of location rectangle for faces.</param>
|
|
/// <returns></returns>
|
|
/// <exception cref="ArgumentNullException"><paramref name="image"/> or <paramref name="locations"/> is null.</exception>
|
|
/// <exception cref="ObjectDisposedException"><paramref name="image"/> is disposed.</exception>
|
|
public static IEnumerable<Image> CropFaces(Image image, IEnumerable<Location> locations)
|
|
{
|
|
if (image == null)
|
|
throw new ArgumentNullException(nameof(image));
|
|
if (locations == null)
|
|
throw new ArgumentNullException(nameof(locations));
|
|
|
|
image.ThrowIfDisposed();
|
|
|
|
List<Image>? results = new();
|
|
foreach (Location? location in locations)
|
|
{
|
|
DlibDotNet.Rectangle rect = new(location.Left, location.Top, location.Right, location.Bottom);
|
|
DPoint[]? dPoint = new[]
|
|
{
|
|
new DPoint(rect.Left, rect.Top),
|
|
new DPoint(rect.Right, rect.Top),
|
|
new DPoint(rect.Left, rect.Bottom),
|
|
new DPoint(rect.Right, rect.Bottom),
|
|
};
|
|
|
|
int width = (int)rect.Width;
|
|
int height = (int)rect.Height;
|
|
|
|
switch (image.Mode)
|
|
{
|
|
case Mode.Rgb:
|
|
Matrix<RgbPixel>? rgb = image.Matrix as Matrix<RgbPixel>;
|
|
results.Add(new Image(DlibDotNet.Dlib.ExtractImage4Points(rgb, dPoint, width, height),
|
|
Mode.Rgb));
|
|
break;
|
|
case Mode.Greyscale:
|
|
Matrix<byte>? gray = image.Matrix as Matrix<byte>;
|
|
results.Add(new Image(DlibDotNet.Dlib.ExtractImage4Points(gray, dPoint, width, height),
|
|
Mode.Greyscale));
|
|
break;
|
|
}
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Compare a face encoding to a known face encoding and get a euclidean distance for comparison face.
|
|
/// </summary>
|
|
/// <param name="faceEncoding">The face encoding to compare.</param>
|
|
/// <param name="faceToCompare">The face encoding to compare against.</param>
|
|
/// <returns>The euclidean distance for comparison face. If 0, faces are completely equal.</returns>
|
|
/// <exception cref="ArgumentNullException"><paramref name="faceEncoding"/> or <paramref name="faceToCompare"/> is null.</exception>
|
|
/// <exception cref="ObjectDisposedException"><paramref name="faceEncoding"/> or <paramref name="faceToCompare"/> is disposed.</exception>
|
|
public static double FaceDistance(FaceEncoding faceEncoding, FaceEncoding faceToCompare)
|
|
{
|
|
if (faceEncoding == null)
|
|
throw new ArgumentNullException(nameof(faceEncoding));
|
|
if (faceToCompare == null)
|
|
throw new ArgumentNullException(nameof(faceToCompare));
|
|
|
|
faceEncoding.ThrowIfDisposed();
|
|
faceToCompare.ThrowIfDisposed();
|
|
|
|
if (faceEncoding.Encoding.Size == 0)
|
|
return 0;
|
|
|
|
using Matrix<double>? diff = faceEncoding.Encoding - faceToCompare.Encoding;
|
|
return DlibDotNet.Dlib.Length(diff);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Compare an enumerable collection of face encoding to a known face encoding and get an enumerable collection of euclidean distance for comparison face.
|
|
/// </summary>
|
|
/// <param name="faceEncodings">The enumerable collection of face encoding to compare.</param>
|
|
/// <param name="faceToCompare">The face encoding to compare against.</param>
|
|
/// <returns>The enumerable collection of euclidean distance for comparison face. If 0, faces are completely equal.</returns>
|
|
/// <exception cref="ArgumentNullException"><paramref name="faceEncodings"/> or <paramref name="faceToCompare"/> is null.</exception>
|
|
/// <exception cref="ObjectDisposedException"><paramref name="faceToCompare"/> is disposed. Or <paramref name="faceEncodings"/> contains disposed object.</exception>
|
|
public static IEnumerable<double> FaceDistances(IEnumerable<FaceEncoding> faceEncodings, FaceEncoding faceToCompare)
|
|
{
|
|
if (faceEncodings == null)
|
|
throw new ArgumentNullException(nameof(faceEncodings));
|
|
if (faceToCompare == null)
|
|
throw new ArgumentNullException(nameof(faceToCompare));
|
|
|
|
faceToCompare.ThrowIfDisposed();
|
|
|
|
FaceEncoding[]? array = faceEncodings.ToArray();
|
|
if (array.Any(encoding => encoding.IsDisposed))
|
|
throw new ObjectDisposedException($"{nameof(faceEncodings)} contains disposed object.");
|
|
|
|
List<double>? results = new();
|
|
if (array.Length == 0)
|
|
return results;
|
|
|
|
foreach (FaceEncoding? faceEncoding in array)
|
|
using (Matrix<double>? diff = faceEncoding.Encoding - faceToCompare.Encoding)
|
|
results.Add(DlibDotNet.Dlib.Length(diff));
|
|
|
|
return results;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns an enumerable collection of face feature data corresponds to all faces in specified image.
|
|
/// </summary>
|
|
/// <param name="image">The image contains faces. The image can contain multiple faces.</param>
|
|
/// <param name="knownFaceLocation">The enumerable collection of location rectangle for faces. If specified null, method will find face locations.</param>
|
|
/// <param name="numJitters">The number of times to re-sample the face when calculating encoding.</param>
|
|
/// <param name="predictorModel">The dimension of vector which be returned from detector.</param>
|
|
/// <param name="model">The model of face detector to detect in image. If <paramref name="knownFaceLocation"/> is not null, this value is ignored.</param>
|
|
/// <returns>An enumerable collection of face feature data corresponds to all faces in specified image.</returns>
|
|
/// <exception cref="ArgumentNullException"><paramref name="image"/> is null.</exception>
|
|
/// <exception cref="InvalidOperationException"><paramref name="knownFaceLocation"/> contains no elements.</exception>
|
|
/// <exception cref="ObjectDisposedException"><paramref name="image"/> or this object or custom face landmark detector is disposed.</exception>
|
|
/// <exception cref="NotSupportedException"><see cref="PredictorModel.Custom"/> is not supported.</exception>
|
|
public IEnumerable<FaceEncoding> FaceEncodings(Image image,
|
|
IEnumerable<Location>? knownFaceLocation = null,
|
|
int numJitters = 1,
|
|
PredictorModel predictorModel = PredictorModel.Small,
|
|
Model model = Model.Hog)
|
|
{
|
|
if (image == null)
|
|
throw new ArgumentNullException(nameof(image));
|
|
if (predictorModel == PredictorModel.Custom)
|
|
throw new NotSupportedException("FaceRecognition.PredictorModel.Custom is not supported.");
|
|
|
|
if (knownFaceLocation != null && !knownFaceLocation.Any())
|
|
throw new InvalidOperationException($"{nameof(knownFaceLocation)} contains no elements.");
|
|
|
|
image.ThrowIfDisposed();
|
|
ThrowIfDisposed();
|
|
|
|
IEnumerable<FullObjectDetection>? rawLandmarks = RawFaceLandmarks(image, knownFaceLocation, predictorModel, model);
|
|
|
|
List<FaceEncoding>? results = new();
|
|
foreach (FullObjectDetection? landmark in rawLandmarks)
|
|
{
|
|
FaceEncoding? ret = new(FaceRecognitionModelV1.ComputeFaceDescriptor(_FaceEncoder, image, landmark, numJitters));
|
|
landmark.Dispose();
|
|
results.Add(ret);
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns an enumerable collection of dictionary of face parts locations (eyes, nose, etc) for each face in the image.
|
|
/// </summary>
|
|
/// <param name="faceImage">The image contains faces. The image can contain multiple faces.</param>
|
|
/// <param name="faceLocations">The enumerable collection of location rectangle for faces. If specified null, method will find face locations.</param>
|
|
/// <param name="predictorModel">The dimension of vector which be returned from detector.</param>
|
|
/// <param name="model">The model of face detector to detect in image. If <paramref name="faceLocations"/> is not null, this value is ignored.</param>
|
|
/// <returns>An enumerable collection of dictionary of face parts locations (eyes, nose, etc).</returns>
|
|
/// <exception cref="ArgumentNullException"><paramref name="faceImage"/> is null.</exception>
|
|
/// <exception cref="InvalidOperationException"><paramref name="faceLocations"/> contains no elements.</exception>
|
|
/// <exception cref="ObjectDisposedException"><paramref name="faceImage"/> or this object or custom face landmark detector is disposed.</exception>
|
|
/// <exception cref="NotSupportedException">The custom face landmark detector is not ready.</exception>
|
|
public IEnumerable<IDictionary<FacePart, IEnumerable<FacePoint>>> FaceLandmark(Image faceImage,
|
|
IEnumerable<Location>? faceLocations = null,
|
|
PredictorModel predictorModel = PredictorModel.Large,
|
|
Model model = Model.Hog)
|
|
{
|
|
if (faceImage == null)
|
|
throw new ArgumentNullException(nameof(faceImage));
|
|
|
|
if (faceLocations != null && !faceLocations.Any())
|
|
throw new InvalidOperationException($"{nameof(faceLocations)} contains no elements.");
|
|
|
|
faceImage.ThrowIfDisposed();
|
|
ThrowIfDisposed();
|
|
|
|
if (predictorModel == PredictorModel.Custom)
|
|
{
|
|
if (CustomFaceLandmarkDetector == null)
|
|
throw new NotSupportedException("The custom face landmark detector is not ready.");
|
|
|
|
if (CustomFaceLandmarkDetector.IsDisposed)
|
|
throw new ObjectDisposedException($"{nameof(CustomFaceLandmarkDetector)}", "The custom face landmark detector is disposed.");
|
|
}
|
|
|
|
FullObjectDetection[]? landmarks = RawFaceLandmarks(faceImage, faceLocations, predictorModel, model).ToArray();
|
|
IEnumerable<FacePoint[]>? landmarkTuples = landmarks.Select(landmark => Enumerable.Range(0, (int)landmark.Parts)
|
|
.Select(index => new FacePoint(index, landmark.GetPart((uint)index).X, landmark.GetPart((uint)index).Y)).ToArray());
|
|
|
|
List<Dictionary<FacePart, IEnumerable<FacePoint>>>? results = new();
|
|
|
|
try
|
|
{
|
|
|
|
// For a definition of each point index, see https://cdn-images-1.medium.com/max/1600/1*AbEg31EgkbXSQehuNJBlWg.png
|
|
switch (predictorModel)
|
|
{
|
|
case PredictorModel.Large:
|
|
results.AddRange(landmarkTuples.Select(landmarkTuple => new Dictionary<FacePart, IEnumerable<FacePoint>>
|
|
{
|
|
{ FacePart.Chin, Enumerable.Range(0,17).Select(i => landmarkTuple[i]).ToArray() },
|
|
{ FacePart.LeftEyebrow, Enumerable.Range(17,5).Select(i => landmarkTuple[i]).ToArray() },
|
|
{ FacePart.RightEyebrow, Enumerable.Range(22,5).Select(i => landmarkTuple[i]).ToArray() },
|
|
{ FacePart.NoseBridge, Enumerable.Range(27,5).Select(i => landmarkTuple[i]).ToArray() },
|
|
{ FacePart.NoseTip, Enumerable.Range(31,5).Select(i => landmarkTuple[i]).ToArray() },
|
|
{ FacePart.LeftEye, Enumerable.Range(36,6).Select(i => landmarkTuple[i]).ToArray() },
|
|
{ FacePart.RightEye, Enumerable.Range(42,6).Select(i => landmarkTuple[i]).ToArray() },
|
|
{ FacePart.TopLip, Enumerable.Range(48,7).Select(i => landmarkTuple[i])
|
|
.Concat( new [] { landmarkTuple[64] })
|
|
.Concat( new [] { landmarkTuple[63] })
|
|
.Concat( new [] { landmarkTuple[62] })
|
|
.Concat( new [] { landmarkTuple[61] })
|
|
.Concat( new [] { landmarkTuple[60] }) },
|
|
{ FacePart.BottomLip, Enumerable.Range(54,6).Select(i => landmarkTuple[i])
|
|
.Concat( new [] { landmarkTuple[48] })
|
|
.Concat( new [] { landmarkTuple[60] })
|
|
.Concat( new [] { landmarkTuple[67] })
|
|
.Concat( new [] { landmarkTuple[66] })
|
|
.Concat( new [] { landmarkTuple[65] })
|
|
.Concat( new [] { landmarkTuple[64] }) }
|
|
}));
|
|
break;
|
|
case PredictorModel.Small:
|
|
results.AddRange(landmarkTuples.Select(landmarkTuple => new Dictionary<FacePart, IEnumerable<FacePoint>>
|
|
{
|
|
{ FacePart.NoseTip, Enumerable.Range(4,1).Select(i => landmarkTuple[i]).ToArray() },
|
|
{ FacePart.LeftEye, Enumerable.Range(2,2).Select(i => landmarkTuple[i]).ToArray() },
|
|
{ FacePart.RightEye, Enumerable.Range(0,2).Select(i => landmarkTuple[i]).ToArray() }
|
|
}));
|
|
break;
|
|
case PredictorModel.Custom:
|
|
if (CustomFaceLandmarkDetector is null)
|
|
throw new Exception($"{nameof(CustomFaceLandmarkDetector)} is null");
|
|
results.AddRange(CustomFaceLandmarkDetector.GetLandmarks(landmarkTuples));
|
|
break;
|
|
default:
|
|
throw new ArgumentOutOfRangeException(nameof(predictorModel), predictorModel, null);
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
foreach (FullObjectDetection? landmark in landmarks)
|
|
landmark.Dispose();
|
|
}
|
|
|
|
return results.ToArray();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Returns an enumerable collection of face location correspond to all faces in specified image.
|
|
/// </summary>
|
|
/// <param name="image">The image contains faces. The image can contain multiple faces.</param>
|
|
/// <param name="numberOfTimesToUpsample">The number of times to up-sample the image when finding faces.</param>
|
|
/// <param name="model">The model of face detector to detect in image.</param>
|
|
/// <returns>An enumerable collection of face location correspond to all faces in specified image.</returns>
|
|
/// <exception cref="ArgumentNullException"><paramref name="image"/> is null.</exception>
|
|
/// <exception cref="ObjectDisposedException"><paramref name="image"/> or this object is disposed.</exception>
|
|
public IEnumerable<Location> FaceLocations(Image image, int numberOfTimesToUpsample = 1, Model model = Model.Hog)
|
|
{
|
|
if (image == null)
|
|
throw new ArgumentNullException(nameof(image));
|
|
|
|
image.ThrowIfDisposed();
|
|
ThrowIfDisposed();
|
|
|
|
List<Location>? results = new();
|
|
foreach (MModRect? face in RawFaceLocations(image, numberOfTimesToUpsample, model))
|
|
{
|
|
Location? ret = TrimBound(face.Rect, image.Width, image.Height);
|
|
double confidence = face.DetectionConfidence;
|
|
face.Dispose();
|
|
results.Add(new Location(ret, confidence));
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates an <see cref="FaceEncoding"/> from the <see cref="double"/> array.
|
|
/// </summary>
|
|
/// <param name="encoding">The <see cref="double"/> array contains face encoding data.</param>
|
|
/// <returns>The <see cref="FaceEncoding"/> this method creates.</returns>
|
|
/// <exception cref="ArgumentNullException"><paramref name="encoding"/> is null.</exception>
|
|
/// <exception cref="ArgumentOutOfRangeException"><paramref name="encoding"/> must be 128.</exception>
|
|
public static FaceEncoding LoadFaceEncoding(double[] encoding)
|
|
{
|
|
if (encoding == null)
|
|
throw new ArgumentNullException(nameof(encoding));
|
|
if (encoding.Length != 128)
|
|
{
|
|
string message = $"{nameof(encoding)}.{nameof(encoding.Length)} must be 128.";
|
|
throw new ArgumentOutOfRangeException(message);
|
|
}
|
|
#pragma warning disable
|
|
Matrix<double>? matrix = Matrix<double>.CreateTemplateParameterizeMatrix(0, 1);
|
|
#pragma warning restore
|
|
matrix.SetSize(128);
|
|
matrix.Assign(encoding);
|
|
return new FaceEncoding(matrix);
|
|
}
|
|
|
|
#pragma warning disable CA1416
|
|
|
|
/// <summary>
|
|
/// Creates an <see cref="Image"/> from the specified existing bitmap image.
|
|
/// </summary>
|
|
/// <param name="bitmap">The <see cref="Bitmap"/> from which to create the new <see cref="Image"/>.</param>
|
|
/// <returns>The <see cref="Image"/> this method creates.</returns>
|
|
/// <exception cref="ArgumentNullException"><paramref name="bitmap"/> is null.</exception>
|
|
/// <exception cref="ArgumentOutOfRangeException">The specified <see cref="PixelFormat"/> is not supported.</exception>
|
|
public static Image? LoadImage(Bitmap bitmap)
|
|
{
|
|
int width = bitmap.Width;
|
|
int height = bitmap.Height;
|
|
System.Drawing.Rectangle rect = new(0, 0, width, height);
|
|
PixelFormat format = bitmap.PixelFormat;
|
|
|
|
Mode mode;
|
|
int srcChannel;
|
|
int dstChannel;
|
|
switch (format)
|
|
{
|
|
case PixelFormat.Format8bppIndexed:
|
|
mode = Mode.Greyscale;
|
|
srcChannel = 1;
|
|
dstChannel = 1;
|
|
break;
|
|
case PixelFormat.Format24bppRgb:
|
|
mode = Mode.Rgb;
|
|
srcChannel = 3;
|
|
dstChannel = 3;
|
|
break;
|
|
case PixelFormat.Format32bppRgb:
|
|
case PixelFormat.Format32bppArgb:
|
|
mode = Mode.Rgb;
|
|
srcChannel = 4;
|
|
dstChannel = 3;
|
|
break;
|
|
default:
|
|
throw new ArgumentOutOfRangeException($"{nameof(bitmap)}", $"The specified {nameof(PixelFormat)} is not supported.");
|
|
}
|
|
|
|
BitmapData? data = null;
|
|
|
|
try
|
|
{
|
|
data = bitmap.LockBits(rect, ImageLockMode.ReadOnly, format);
|
|
|
|
unsafe
|
|
{
|
|
byte[]? array = new byte[width * height * dstChannel];
|
|
fixed (byte* pArray = &array[0])
|
|
{
|
|
byte* dst = pArray;
|
|
|
|
switch (srcChannel)
|
|
{
|
|
case 1:
|
|
{
|
|
IntPtr src = data.Scan0;
|
|
int stride = data.Stride;
|
|
|
|
for (int h = 0; h < height; h++)
|
|
Marshal.Copy(IntPtr.Add(src, h * stride), array, h * width, width * dstChannel);
|
|
}
|
|
break;
|
|
case 3:
|
|
case 4:
|
|
{
|
|
byte* src = (byte*)data.Scan0;
|
|
int stride = data.Stride;
|
|
|
|
for (int h = 0; h < height; h++)
|
|
{
|
|
int srcOffset = h * stride;
|
|
int dstOffset = h * width * dstChannel;
|
|
|
|
for (int w = 0; w < width; w++)
|
|
{
|
|
// BGR order to RGB order
|
|
dst[dstOffset + w * dstChannel + 0] = src[srcOffset + w * srcChannel + 2];
|
|
dst[dstOffset + w * dstChannel + 1] = src[srcOffset + w * srcChannel + 1];
|
|
dst[dstOffset + w * dstChannel + 2] = src[srcOffset + w * srcChannel + 0];
|
|
}
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
|
|
IntPtr ptr = (IntPtr)pArray;
|
|
switch (mode)
|
|
{
|
|
case Mode.Rgb:
|
|
return new Image(new Matrix<RgbPixel>(ptr, height, width, width * 3), Mode.Rgb);
|
|
case Mode.Greyscale:
|
|
return new Image(new Matrix<byte>(ptr, height, width, width), Mode.Greyscale);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
if (data != null)
|
|
bitmap.UnlockBits(data);
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
#pragma warning restore CA1416
|
|
|
|
/// <summary>
|
|
/// Creates an <see cref="Image"/> from the <see cref="byte"/> array.
|
|
/// </summary>
|
|
/// <param name="array">The <see cref="byte"/> array contains image data.</param>
|
|
/// <param name="row">The number of rows in a image data.</param>
|
|
/// <param name="column">The number of columns in a image data.</param>
|
|
/// <param name="stride">The stride width in bytes.</param>
|
|
/// <param name="mode">A image color mode.</param>
|
|
/// <returns>The <see cref="Image"/> this method creates.</returns>
|
|
/// <exception cref="ArgumentNullException"><paramref name="array"/> is null.</exception>
|
|
/// <exception cref="ArgumentOutOfRangeException"><paramref name="row"/> is less than 0.</exception>
|
|
/// <exception cref="ArgumentOutOfRangeException"><paramref name="column"/> is less than 0.</exception>
|
|
/// <exception cref="ArgumentOutOfRangeException"><paramref name="stride"/> is less than 0.</exception>
|
|
/// <exception cref="ArgumentOutOfRangeException"><paramref name="stride"/> is less than <paramref name="column"/>.</exception>
|
|
/// <exception cref="ArgumentOutOfRangeException"><paramref name="row"/> x <paramref name="stride"/> is less than <see cref="Array.Length"/>.</exception>
|
|
public static Image? LoadImage(byte[] array, int row, int column, int stride, Mode mode)
|
|
{
|
|
if (array == null)
|
|
throw new ArgumentNullException(nameof(array));
|
|
if (row < 0)
|
|
throw new ArgumentOutOfRangeException($"{nameof(row)}", $"{nameof(row)} is less than 0.");
|
|
if (column < 0)
|
|
throw new ArgumentOutOfRangeException($"{nameof(column)}", $"{nameof(column)} is less than 0.");
|
|
if (stride < 0)
|
|
throw new ArgumentOutOfRangeException($"{nameof(stride)}", $"{nameof(stride)} is less than 0.");
|
|
if (stride < column)
|
|
throw new ArgumentOutOfRangeException($"{nameof(stride)}", $"{nameof(stride)} is less than {nameof(column)}.");
|
|
int min = row * stride;
|
|
if (!(array.Length >= min))
|
|
throw new ArgumentOutOfRangeException("", $"{nameof(row)} x {nameof(stride)} is less than {nameof(Array)}.{nameof(Array.Length)}.");
|
|
|
|
unsafe
|
|
{
|
|
fixed (byte* p = &array[0])
|
|
{
|
|
IntPtr ptr = (IntPtr)p;
|
|
switch (mode)
|
|
{
|
|
case Mode.Rgb:
|
|
return new Image(new Matrix<RgbPixel>(ptr, row, column, stride), Mode.Rgb);
|
|
case Mode.Greyscale:
|
|
return new Image(new Matrix<byte>(ptr, row, column, stride), Mode.Greyscale);
|
|
}
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates an <see cref="Image"/> from the unmanaged memory pointer indicates <see cref="byte"/> array image data.
|
|
/// </summary>
|
|
/// <param name="array">The unmanaged memory pointer indicates <see cref="byte"/> array image data.</param>
|
|
/// <param name="row">The number of rows in a image data.</param>
|
|
/// <param name="column">The number of columns in a image data.</param>
|
|
/// <param name="stride">The stride width in bytes.</param>
|
|
/// <param name="mode">A image color mode.</param>
|
|
/// <returns>The <see cref="Image"/> this method creates.</returns>
|
|
/// <exception cref="ArgumentException"><paramref name="array"/> is <see cref="IntPtr.Zero"/>.</exception>
|
|
/// <exception cref="ArgumentOutOfRangeException"><paramref name="row"/> is less than 0.</exception>
|
|
/// <exception cref="ArgumentOutOfRangeException"><paramref name="column"/> is less than 0.</exception>
|
|
/// <exception cref="ArgumentOutOfRangeException"><paramref name="stride"/> is less than 0.</exception>
|
|
/// <exception cref="ArgumentOutOfRangeException"><paramref name="stride"/> is less than <paramref name="column"/>.</exception>
|
|
public static Image? LoadImage(IntPtr array, int row, int column, int stride, Mode mode)
|
|
{
|
|
if (array == IntPtr.Zero)
|
|
throw new ArgumentException($"{nameof(array)} is {nameof(IntPtr)}.{nameof(IntPtr.Zero)}", nameof(array));
|
|
if (row < 0)
|
|
throw new ArgumentOutOfRangeException($"{nameof(row)}", $"{nameof(row)} is less than 0.");
|
|
if (column < 0)
|
|
throw new ArgumentOutOfRangeException($"{nameof(column)}", $"{nameof(column)} is less than 0.");
|
|
if (stride < 0)
|
|
throw new ArgumentOutOfRangeException($"{nameof(stride)}", $"{nameof(stride)} is less than 0.");
|
|
if (stride < column)
|
|
throw new ArgumentOutOfRangeException($"{nameof(stride)}", $"{nameof(stride)} is less than {nameof(column)}.");
|
|
|
|
return mode switch
|
|
{
|
|
Mode.Rgb => new Image(new Matrix<RgbPixel>(array, row, column, stride), mode),
|
|
Mode.Greyscale => new Image(new Matrix<byte>(array, row, column, stride), mode),
|
|
_ => null,
|
|
};
|
|
}
|
|
|
|
/// <summary>
|
|
/// Creates an <see cref="Image"/> from the specified path.
|
|
/// </summary>
|
|
/// <param name="file">A string that contains the path of the file from which to create the <see cref="Image"/>.</param>
|
|
/// <param name="mode">A image color mode.</param>
|
|
/// <returns>The <see cref="Image"/> this method creates.</returns>
|
|
/// <exception cref="FileNotFoundException">The specified path does not exist.</exception>
|
|
public static Image? LoadImageFile(string file, Mode mode = Mode.Rgb)
|
|
{
|
|
if (!File.Exists(file))
|
|
throw new FileNotFoundException(file);
|
|
|
|
return mode switch
|
|
{
|
|
Mode.Rgb => new Image(DlibDotNet.Dlib.LoadImageAsMatrix<RgbPixel>(file), mode),
|
|
Mode.Greyscale => new Image(DlibDotNet.Dlib.LoadImageAsMatrix<byte>(file), mode),
|
|
_ => null,
|
|
};
|
|
}
|
|
|
|
#region Helpers
|
|
|
|
private IEnumerable<FullObjectDetection> RawFaceLandmarks(Image faceImage,
|
|
IEnumerable<Location>? faceLocations = null,
|
|
PredictorModel predictorModel = PredictorModel.Large,
|
|
Model model = Model.Hog)
|
|
{
|
|
IEnumerable<Location> rects;
|
|
|
|
if (faceLocations == null)
|
|
{
|
|
List<Location>? list = new();
|
|
IEnumerable<MModRect>? tmp = RawFaceLocations(faceImage, 1, model);
|
|
foreach (MModRect? mModRect in tmp)
|
|
{
|
|
list.Add(new Location(mModRect.DetectionConfidence, mModRect.Rect.Bottom, mModRect.Rect.Left, mModRect.Rect.Right, mModRect.Rect.Top));
|
|
mModRect.Dispose();
|
|
}
|
|
|
|
rects = list;
|
|
}
|
|
else
|
|
{
|
|
rects = faceLocations;
|
|
}
|
|
|
|
List<FullObjectDetection>? results = new();
|
|
if (predictorModel == PredictorModel.Custom)
|
|
{
|
|
if (CustomFaceLandmarkDetector is null)
|
|
throw new Exception($"{nameof(CustomFaceLandmarkDetector)} is null");
|
|
foreach (Location? rect in rects)
|
|
{
|
|
FullObjectDetection? ret = CustomFaceLandmarkDetector.Detect(faceImage, rect);
|
|
results.Add(ret);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
ShapePredictor? posePredictor = _PosePredictor68Point;
|
|
switch (predictorModel)
|
|
{
|
|
case PredictorModel.Small:
|
|
posePredictor = _PosePredictor5Point;
|
|
break;
|
|
}
|
|
|
|
foreach (Location? rect in rects)
|
|
{
|
|
FullObjectDetection? ret = posePredictor.Detect(faceImage.Matrix, new DlibDotNet.Rectangle(rect.Left, rect.Top, rect.Right, rect.Bottom));
|
|
results.Add(ret);
|
|
}
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
private IEnumerable<MModRect> RawFaceLocations(Image faceImage, int numberOfTimesToUpsample = 1, Model model = Model.Hog)
|
|
{
|
|
switch (model)
|
|
{
|
|
case Model.Custom:
|
|
if (CustomFaceDetector == null)
|
|
throw new NotSupportedException("The custom face detector is not ready.");
|
|
return CustomFaceDetector.Detect(faceImage, numberOfTimesToUpsample).Select(rect => new MModRect
|
|
{
|
|
Rect = new DlibDotNet.Rectangle(rect.Left, rect.Top, rect.Right, rect.Bottom),
|
|
DetectionConfidence = rect.Confidence
|
|
});
|
|
case Model.Cnn:
|
|
return CnnFaceDetectionModelV1.Detect(_CnnFaceDetector, faceImage, numberOfTimesToUpsample);
|
|
default:
|
|
IEnumerable<Tuple<DlibDotNet.Rectangle, double>>? locations = SimpleObjectDetector.RunDetectorWithUpscale2(_FaceDetector, faceImage, (uint)numberOfTimesToUpsample);
|
|
return locations.Select(tuple => new MModRect { Rect = tuple.Item1, DetectionConfidence = tuple.Item2 });
|
|
}
|
|
}
|
|
|
|
private IEnumerable<IEnumerable<MModRect>> RawFaceLocationsBatched(IEnumerable<Image> faceImages, int numberOfTimesToUpsample = 1, int batchSize = 128) => CnnFaceDetectionModelV1.DetectMulti(_CnnFaceDetector, faceImages, numberOfTimesToUpsample, batchSize);
|
|
|
|
private static Location TrimBound(DlibDotNet.Rectangle location, int width, int height) => new(Math.Max(location.Left, 0), Math.Max(location.Top, 0), Math.Min(location.Right, width), Math.Min(location.Bottom, height));
|
|
|
|
#endregion
|
|
|
|
#endregion
|
|
|
|
#region Methods
|
|
|
|
#region Overrides
|
|
|
|
/// <summary>
|
|
/// Releases all unmanaged resources.
|
|
/// </summary>
|
|
protected override void DisposeUnmanaged()
|
|
{
|
|
base.DisposeUnmanaged();
|
|
|
|
_PosePredictor68Point?.Dispose();
|
|
_PosePredictor5Point?.Dispose();
|
|
_CnnFaceDetector?.Dispose();
|
|
_FaceEncoder?.Dispose();
|
|
_FaceDetector?.Dispose();
|
|
}
|
|
|
|
#endregion
|
|
|
|
#endregion
|
|
|
|
} |