phares3757/view-by-distance-mklink-console
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added Class Library FaceRecognitionDotNet

Browse Source
This commit is contained in:
Mike Phares
2022-07-30 16:43:23 -07:00
parent f642c5669a
commit 2ebec0b7a9
45 changed files with 2398 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
FaceRecognitionDotNet/DisposableObject.cs Normal file
View File

@ -0,0 +1,128 @@
namespace View_by_Distance.FaceRecognitionDotNet;
/// <summary>
/// Represents a class which has managed or unmanaged resources.
/// </summary>
public abstract class DisposableObject : IDisposable
{
#region Properties
/// <summary>
/// Gets a value indicating whether this instance has been disposed.
/// </summary>
/// <returns>true if this instance has been disposed; otherwise, false.</returns>
public bool IsDisposed
{
get;
private set;
/* Unmerged change from project 'FaceRecognitionotNet(netstandard2.0)'
Before:
/// If this object is disposed, then <see cref="System.ObjectDisposedException"/> is thrown.
After:
/// If this object is disposed, then <see cref="ObjectDisposedException"/> is thrown.
*/
}
#endregion
#region Methods
/// <summary>
/// If this object is disposed, then <see cref="ObjectDisposedException"/> is thrown.
/// </summary>
public void ThrowIfDisposed()
{
if (IsDisposed)
throw new ObjectDisposedException(GetType().FullName);
}
internal void ThrowIfDisposed(string objectName)
{
if (IsDisposed)
throw new ObjectDisposedException(objectName);
}
#region Overrides
/// <summary>
/// Releases all managed resources.
/// </summary>
protected virtual void DisposeManaged()
{
}
/// <summary>
/// Releases all unmanaged resources.
/// </summary>
protected virtual void DisposeUnmanaged()
{
}
#endregion
#endregion
#region IDisposable Members
/// <summary>
/// Releases all resources used by this <see cref="DisposableObject"/>.
/// </summary>
public void Dispose()
{
GC.SuppressFinalize(this);
/* Unmerged change from project 'FaceRecognitionotNet(netstandard2.0)'
Before:
Dispose(true);
After:
Dispose(true);
*/
Dispose(true);
}
/// <summary>
/// Releases all resources used by this <see cref="DisposableObject"/>.
/// </summary>
/// <param name="disposing">Indicate value whether <see cref="IDisposable.Dispose"/> method was called.</param>
private void Dispose(bool disposing)
{
if (IsDisposed)
{
return;
/* Unmerged change from project 'FaceRecognitionotNet(netstandard2.0)'
Before:
IsDisposed = true;
After:
IsDisposed = true;
*/
}
IsDisposed = true;
if (disposing)
/* Unmerged change from project 'FaceRecognitionotNet(netstandard2.0)'
Before:
DisposeManaged();
After:
DisposeManaged();
*/
DisposeManaged();
/* Unmerged change from project 'FaceRecognitionotNet(netstandard2.0)'
Before:
DisposeUnmanaged();
After:
DisposeUnmanaged();
*/
DisposeUnmanaged();
}
#endregion
}

111
FaceRecognitionDotNet/Dlib/Python/CnnFaceDetectionModelV1.cs Normal file
View File

@ -0,0 +1,111 @@
using DlibDotNet;
using DlibDotNet.Dnn;
using View_by_Distance.Shared.Models.Stateless;
namespace View_by_Distance.FaceRecognitionDotNet.Dlib.Python;
internal sealed class CnnFaceDetectionModelV1
{
#region Methods
public static IEnumerable<MModRect> Detect(LossMmod net, Image image, int upsampleNumTimes)
{
using PyramidDown? pyr = new(2);
List<MModRect>? rects = new();
// Copy the data into dlib based objects
using Matrix<RgbPixel>? matrix = new();
Mode type = image.Mode;
switch (type)
{
case Mode.Greyscale:
case Mode.Rgb:
DlibDotNet.Dlib.AssignImage(image.Matrix, matrix);
break;
default:
throw new NotSupportedException("Unsupported image type, must be 8bit gray or RGB image.");
}
// Upsampling the image will allow us to detect smaller faces but will cause the
// program to use more RAM and run longer.
int levels = upsampleNumTimes;
while (levels > 0)
{
levels--;
DlibDotNet.Dlib.PyramidUp<PyramidDown>(matrix, 2);
}
OutputLabels<IEnumerable<MModRect>>? dets = net.Operator(matrix);
// Scale the detection locations back to the original image size
// if the image was upscaled.
foreach (MModRect? d in dets.First())
{
DRectangle drect = pyr.RectDown(new DRectangle(d.Rect), (uint)upsampleNumTimes);
d.Rect = new Rectangle((int)drect.Left, (int)drect.Top, (int)drect.Right, (int)drect.Bottom);
rects.Add(d);
}
return rects;
}
public static IEnumerable<IEnumerable<MModRect>> DetectMulti(LossMmod net, IEnumerable<Image> images, int upsampleNumTimes, int batchSize = 128)
{
List<Matrix<RgbPixel>>? destImages = new();
List<IEnumerable<MModRect>>? allRects = new();
try
{
using PyramidDown? pyr = new(2);
// Copy the data into dlib based objects
foreach (Image? image in images)
{
Matrix<RgbPixel>? matrix = new();
Mode type = image.Mode;
switch (type)
{
case Mode.Greyscale:
case Mode.Rgb:
DlibDotNet.Dlib.AssignImage(image.Matrix, matrix);
break;
default:
throw new NotSupportedException("Unsupported image type, must be 8bit gray or RGB image.");
}
for (int i = 0; i < upsampleNumTimes; i++)
DlibDotNet.Dlib.PyramidUp(matrix);
destImages.Add(matrix);
}
for (int i = 1; i < destImages.Count; i++)
if (destImages[i - 1].Columns != destImages[i].Columns || destImages[i - 1].Rows != destImages[i].Rows)
throw new ArgumentException("Images in list must all have the same dimensions.");
OutputLabels<IEnumerable<MModRect>>? dets = net.Operator(destImages, (ulong)batchSize);
foreach (IEnumerable<MModRect>? det in dets)
{
List<MModRect>? rects = new();
foreach (MModRect? d in det)
{
DRectangle drect = pyr.RectDown(new DRectangle(d.Rect), (uint)upsampleNumTimes);
d.Rect = new Rectangle((int)drect.Left, (int)drect.Top, (int)drect.Right, (int)drect.Bottom);
rects.Add(d);
}
allRects.Add(rects);
}
}
finally
{
foreach (Matrix<RgbPixel>? matrix in destImages)
matrix.Dispose();
}
return allRects;
}
#endregion
}

129
FaceRecognitionDotNet/Dlib/Python/FaceRecognitionModelV1.cs Normal file
View File

@ -0,0 +1,129 @@
using DlibDotNet;
using DlibDotNet.Dnn;
namespace View_by_Distance.FaceRecognitionDotNet.Dlib.Python;
internal sealed class FaceRecognitionModelV1
{
#region Methods
public static Matrix<double> ComputeFaceDescriptor(LossMetric net, Image img, FullObjectDetection face, int numJitters)
{
FullObjectDetection[]? faces = new[] { face };
return ComputeFaceDescriptors(net, img, faces, numJitters).First();
}
public static IEnumerable<Matrix<double>> ComputeFaceDescriptors(LossMetric net, Image img, IEnumerable<FullObjectDetection> faces, int numJitters)
{
Image[]? batchImage = new[] { img };
IEnumerable<FullObjectDetection>[]? batchFaces = new[] { faces };
return BatchComputeFaceDescriptors(net, batchImage, batchFaces, numJitters).First();
}
public static IEnumerable<IEnumerable<Matrix<double>>> BatchComputeFaceDescriptors(LossMetric net,
IList<Image> batchImages,
IList<IEnumerable<FullObjectDetection>> batchFaces,
int numJitters)
{
if (batchImages.Count != batchFaces.Count)
throw new ArgumentException("The array of images and the array of array of locations must be of the same size");
foreach (IEnumerable<FullObjectDetection>? faces in batchFaces)
foreach (FullObjectDetection? f in faces)
{
if (f.Parts is not 68 and not 5)
throw new ArgumentException("The full_object_detection must use the iBUG 300W 68 point face landmark style or dlib's 5 point style.");
}
List<Array<Matrix<RgbPixel>>>? faceChipsArray = new(batchImages.Count);
List<Matrix<RgbPixel>>? faceChips = new();
for (int i = 0; i < batchImages.Count; ++i)
{
IEnumerable<FullObjectDetection>? faces = batchFaces[i];
Image? img = batchImages[i];
List<ChipDetails>? dets = new(faces.Count());
foreach (FullObjectDetection? f in faces)
dets.Add(DlibDotNet.Dlib.GetFaceChipDetails(f, 150, 0.25));
Array<Matrix<RgbPixel>>? thisImageFaceChips = DlibDotNet.Dlib.ExtractImageChips<RgbPixel>(img.Matrix, dets);
foreach (Matrix<RgbPixel>? chip in thisImageFaceChips)
faceChips.Add(chip);
faceChipsArray.Add(thisImageFaceChips);
foreach (ChipDetails? det in dets)
det.Dispose();
}
List<List<Matrix<double>>>? faceDescriptors = new();
for (int i = 0, count = batchImages.Count; i < count; i++)
faceDescriptors.Add(new List<Matrix<double>>());
if (numJitters <= 1)
{
// extract descriptors and convert from float vectors to double vectors
OutputLabels<Matrix<float>>? descriptors = net.Operator(faceChips, 16);
int index = 0;
Matrix<float>[]? list = descriptors.Select(matrix => matrix).ToArray();
for (int i = 0; i < batchFaces.Count; ++i)
for (int j = 0; j < batchFaces[i].Count(); ++j)
faceDescriptors[i].Add(DlibDotNet.Dlib.MatrixCast<double>(list[index++]));
if (index != list.Length)
throw new ApplicationException();
}
else
{
// extract descriptors and convert from float vectors to double vectors
int index = 0;
for (int i = 0; i < batchFaces.Count; ++i)
for (int j = 0; j < batchFaces[i].Count(); ++j)
{
Matrix<RgbPixel>[]? tmp = JitterImage(faceChips[index++], numJitters).ToArray();
using (OutputLabels<Matrix<float>>? tmp2 = net.Operator(tmp, 16))
using (MatrixOp? mat = DlibDotNet.Dlib.Mat(tmp2))
{
Matrix<double>? r = DlibDotNet.Dlib.Mean<double>(mat);
faceDescriptors[i].Add(r);
}
foreach (Matrix<RgbPixel>? matrix in tmp)
matrix.Dispose();
}
if (index != faceChips.Count)
throw new ApplicationException();
}
if (faceChipsArray.Any())
{
foreach (Array<Matrix<RgbPixel>>? array in faceChipsArray)
{
foreach (Matrix<RgbPixel>? faceChip in array)
faceChip.Dispose();
array.Dispose();
}
}
return faceDescriptors;
}
#region Helpers
private static readonly Rand _Rand = new();
private static IEnumerable<Matrix<RgbPixel>> JitterImage(Matrix<RgbPixel> img, int numJitters)
{
List<Matrix<RgbPixel>>? crops = new();
for (int i = 0; i < numJitters; ++i)
crops.Add(DlibDotNet.Dlib.JitterImage(img, _Rand));
return crops;
}
#endregion
#endregion
}

169
FaceRecognitionDotNet/Dlib/Python/SimpleObjectDetector.cs Normal file
View File

@ -0,0 +1,169 @@
using DlibDotNet;
using View_by_Distance.Shared.Models.Stateless;
namespace View_by_Distance.FaceRecognitionDotNet.Dlib.Python;
internal sealed class SimpleObjectDetector
{
#region Methods
public static IEnumerable<Rectangle> RunDetectorWithUpscale1(FrontalFaceDetector detector,
Image img,
uint upsamplingAmount,
double adjustThreshold,
List<double> detectionConfidences,
List<ulong> weightIndices)
{
List<Rectangle>? rectangles = new();
if (img.Mode == Mode.Greyscale)
{
Matrix<byte>? greyscaleMatrix = img.Matrix as Matrix<byte>;
if (upsamplingAmount == 0)
{
detector.Operator(greyscaleMatrix, out IEnumerable<RectDetection>? rectDetections, adjustThreshold);
RectDetection[]? dets = rectDetections.ToArray();
SplitRectDetections(dets, rectangles, detectionConfidences, weightIndices);
foreach (RectDetection? rectDetection in dets)
rectDetection.Dispose();
}
else
{
using PyramidDown? pyr = new(2);
Matrix<byte>? temp = null;
try
{
DlibDotNet.Dlib.PyramidUp(greyscaleMatrix, pyr, out temp);
uint levels = upsamplingAmount - 1;
while (levels > 0)
{
levels--;
DlibDotNet.Dlib.PyramidUp(temp);
}
detector.Operator(temp, out IEnumerable<RectDetection>? rectDetections, adjustThreshold);
RectDetection[]? dets = rectDetections.ToArray();
foreach (RectDetection? t in dets)
t.Rect = pyr.RectDown(t.Rect, upsamplingAmount);
SplitRectDetections(dets, rectangles, detectionConfidences, weightIndices);
foreach (RectDetection? rectDetection in dets)
rectDetection.Dispose();
}
finally
{
temp?.Dispose();
}
}
return rectangles;
}
else
{
Matrix<RgbPixel>? rgbMatrix = img.Matrix as Matrix<RgbPixel>;
if (upsamplingAmount == 0)
{
detector.Operator(rgbMatrix, out IEnumerable<RectDetection>? rectDetections, adjustThreshold);
RectDetection[]? dets = rectDetections.ToArray();
SplitRectDetections(dets, rectangles, detectionConfidences, weightIndices);
foreach (RectDetection? rectDetection in dets)
rectDetection.Dispose();
}
else
{
using PyramidDown? pyr = new(2);
Matrix<RgbPixel>? temp = null;
try
{
DlibDotNet.Dlib.PyramidUp(rgbMatrix, pyr, out temp);
uint levels = upsamplingAmount - 1;
while (levels > 0)
{
levels--;
DlibDotNet.Dlib.PyramidUp(temp);
}
detector.Operator(temp, out IEnumerable<RectDetection>? rectDetections, adjustThreshold);
RectDetection[]? dets = rectDetections.ToArray();
foreach (RectDetection? t in dets)
t.Rect = pyr.RectDown(t.Rect, upsamplingAmount);
SplitRectDetections(dets, rectangles, detectionConfidences, weightIndices);
foreach (RectDetection? rectDetection in dets)
rectDetection.Dispose();
}
finally
{
temp?.Dispose();
}
}
return rectangles;
}
}
public static IEnumerable<Tuple<Rectangle, double>> RunDetectorWithUpscale2(FrontalFaceDetector detector,
Image image,
uint upsamplingAmount)
{
if (detector == null)
throw new ArgumentNullException(nameof(detector));
if (image == null)
throw new ArgumentNullException(nameof(image));
detector.ThrowIfDisposed();
image.ThrowIfDisposed();
List<double>? detectionConfidences = new();
List<ulong>? weightIndices = new();
const double adjustThreshold = 0.0;
Rectangle[]? rects = RunDetectorWithUpscale1(detector,
image,
upsamplingAmount,
adjustThreshold,
detectionConfidences,
weightIndices).ToArray();
int index = 0;
foreach (Rectangle rect in rects)
yield return new Tuple<Rectangle, double>(rect, detectionConfidences[index++]);
}
#region Helpers
private static void SplitRectDetections(RectDetection[] rectDetections,
List<Rectangle> rectangles,
List<double> detectionConfidences,
List<ulong> weightIndices)
{
rectangles.Clear();
detectionConfidences.Clear();
weightIndices.Clear();
foreach (RectDetection? rectDetection in rectDetections)
{
rectangles.Add(rectDetection.Rect);
detectionConfidences.Add(rectDetection.DetectionConfidence);
weightIndices.Add(rectDetection.WeightIndex);
}
}
#endregion
#endregion
}

26
FaceRecognitionDotNet/Extensions/FaceDetector.cs Normal file
View File

@ -0,0 +1,26 @@
using DlibDotNet;
using View_by_Distance.Shared.Models;
namespace View_by_Distance.FaceRecognitionDotNet.Extensions;
/// <summary>
/// An abstract base class that provides functionality to detect face locations from image.
/// </summary>
public abstract class FaceDetector : DisposableObject
{
#region Methods
internal IEnumerable<Location> Detect(Image image, int numberOfTimesToUpsample) => RawDetect(image.Matrix, numberOfTimesToUpsample);
/// <summary>
/// Returns an enumerable collection of face location correspond to all faces in specified image.
/// </summary>
/// <param name="matrix">The matrix contains a face.</param>
/// <param name="numberOfTimesToUpsample">The number of times to up-sample the image when finding faces.</param>
/// <returns>An enumerable collection of face location correspond to all faces.</returns>
protected abstract IEnumerable<Location> RawDetect(MatrixBase matrix, int numberOfTimesToUpsample);
#endregion
}

36
FaceRecognitionDotNet/Extensions/FaceLandmarkDetector.cs Normal file
View File

@ -0,0 +1,36 @@
using DlibDotNet;
using View_by_Distance.Shared.Models;
using View_by_Distance.Shared.Models.Stateless;
namespace View_by_Distance.FaceRecognitionDotNet.Extensions;
/// <summary>
/// An abstract base class that provides functionality to detect face parts locations from face image.
/// </summary>
public abstract class FaceLandmarkDetector : DisposableObject
{
#region Methods
internal FullObjectDetection Detect(Image image, Location location) => RawDetect(image.Matrix, location);
internal IEnumerable<Dictionary<FacePart, IEnumerable<FacePoint>>> GetLandmarks(IEnumerable<FacePoint[]> landmarkTuples) => RawGetLandmarks(landmarkTuples);
/// <summary>
/// Returns an object contains information of face parts corresponds to specified location in specified image.
/// </summary>
/// <param name="matrix">The matrix contains a face.</param>
/// <param name="location">The location rectangle for a face.</param>
/// <returns>An object contains information of face parts.</returns>
protected abstract FullObjectDetection RawDetect(MatrixBase matrix, Location location);
/// <summary>
/// Returns an enumerable collection of dictionary of face parts locations (eyes, nose, etc).
/// </summary>
/// <param name="landmarkTuples">The enumerable collection of face parts location.</param>
/// <returns>An enumerable collection of dictionary of face parts locations (eyes, nose, etc).</returns>
protected abstract IEnumerable<Dictionary<FacePart, IEnumerable<FacePoint>>> RawGetLandmarks(IEnumerable<FacePoint[]> landmarkTuples);
#endregion
}

105
FaceRecognitionDotNet/FaceEncoding.cs Normal file
View File

@ -0,0 +1,105 @@
using DlibDotNet;
using System.Runtime.Serialization;
namespace View_by_Distance.FaceRecognitionDotNet;
/// <summary>
/// Represents a feature data of face. This class cannot be inherited.
/// </summary>
[Serializable]
public sealed class FaceEncoding : DisposableObject, ISerializable
{
#region Fields
[NonSerialized]
private readonly Matrix<double> _Encoding;
#endregion
#region Constructors
internal FaceEncoding(Matrix<double> encoding) => _Encoding = encoding;
private FaceEncoding(SerializationInfo info, StreamingContext context)
{
if (info == null)
throw new ArgumentNullException(nameof(info));
double[]? array = info.GetValue(nameof(_Encoding), typeof(double[])) as double[];
int? row = (int?)info.GetValue(nameof(_Encoding.Rows), typeof(int));
int? column = (int?)info.GetValue(nameof(_Encoding.Columns), typeof(int));
if (row is null)
throw new Exception($"{nameof(row)} is null");
if (column is null)
throw new Exception($"{nameof(column)} is null");
_Encoding = new Matrix<double>(array, row.Value, column.Value);
}
#endregion
#region Properties
internal Matrix<double> Encoding => _Encoding;
/// <summary>
/// Gets the size of feature data.
/// </summary>
/// <exception cref="ObjectDisposedException">This object is disposed.</exception>
public int Size
{
get
{
ThrowIfDisposed();
return _Encoding.Size;
}
}
#endregion
#region Methods
/// <summary>
/// Gets a feature data of face as raw format.
/// </summary>
/// <returns>A <see cref="double"/> array that represents a feature data.</returns>
/// <remarks><see cref="FaceEncoding"/> class supports serialization. This method is for interoperability between FaceRecognitionotNet and dlib.</remarks>
/// <exception cref="ObjectDisposedException">This object is disposed.</exception>
public double[] GetRawEncoding()
{
ThrowIfDisposed();
return _Encoding.ToArray();
}
#region Overrides
/// <summary>
/// Releases all unmanaged resources.
/// </summary>
protected override void DisposeUnmanaged()
{
base.DisposeUnmanaged();
_Encoding?.Dispose();
}
#endregion
#endregion
#region ISerializable Members
/// <summary>
/// Populates a <see cref="SerializationInfo"/> with the data needed to serialize the target object.
/// </summary>
/// <param name="info">The <see cref="SerializationInfo"/> to populate with data.</param>
/// <param name="context">The destination (see <see cref="StreamingContext"/>) for this serialization.</param>
public void GetObjectData(SerializationInfo info, StreamingContext context)
{
info.AddValue(nameof(_Encoding), _Encoding.ToArray());
info.AddValue(nameof(_Encoding.Rows), _Encoding.Rows);
info.AddValue(nameof(_Encoding.Columns), _Encoding.Columns);
}
#endregion
}

871
FaceRecognitionDotNet/FaceRecognition.cs Normal file
View File

@ -0,0 +1,871 @@
using DlibDotNet;
using DlibDotNet.Dnn;
using System.Drawing;
using System.Drawing.Imaging;
using System.Runtime.InteropServices;
using System.Text;
using View_by_Distance.FaceRecognitionDotNet.Dlib.Python;
using View_by_Distance.FaceRecognitionDotNet.Extensions;
using View_by_Distance.Shared.Models;
using View_by_Distance.Shared.Models.Stateless;
namespace View_by_Distance.FaceRecognitionDotNet;
/// <summary>
/// Provides the method to find and recognize face methods. This class cannot be inherited.
/// </summary>
public sealed class FaceRecognition : DisposableObject
{
#region Fields
private readonly ShapePredictor _PosePredictor68Point;
private readonly ShapePredictor _PosePredictor5Point;
private readonly LossMmod _CnnFaceDetector;
private readonly LossMetric _FaceEncoder;
private readonly FrontalFaceDetector _FaceDetector;
#endregion
#region Constructors
/// <summary>
/// Initializes a new instance of the <see cref="FaceRecognition"/> class with the directory path that stores model files.
/// </summary>
/// <param name="directory">The directory path that stores model files.</param>
/// <exception cref="FileNotFoundException">The model file is not found.</exception>
/// <exception cref="DirectoryNotFoundException">The specified directory path is not found.</exception>
private FaceRecognition(string directory)
{
if (!Directory.Exists(directory))
throw new DirectoryNotFoundException(directory);
string? predictor68PointModel = Path.Combine(directory, FaceRecognitionModels.GetPosePredictorModelLocation());
if (!File.Exists(predictor68PointModel))
throw new FileNotFoundException(predictor68PointModel);
string? predictor5PointModel = Path.Combine(directory, FaceRecognitionModels.GetPosePredictorFivePointModelLocation());
if (!File.Exists(predictor5PointModel))
throw new FileNotFoundException(predictor5PointModel);
string? cnnFaceDetectionModel = Path.Combine(directory, FaceRecognitionModels.GetCnnFaceDetectorModelLocation());
if (!File.Exists(cnnFaceDetectionModel))
throw new FileNotFoundException(cnnFaceDetectionModel);
string? faceRecognitionModel = Path.Combine(directory, FaceRecognitionModels.GetFaceRecognitionModelLocation());
if (!File.Exists(faceRecognitionModel))
throw new FileNotFoundException(faceRecognitionModel);
_FaceDetector?.Dispose();
_FaceDetector = DlibDotNet.Dlib.GetFrontalFaceDetector();
_PosePredictor68Point?.Dispose();
_PosePredictor68Point = ShapePredictor.Deserialize(predictor68PointModel);
_PosePredictor5Point?.Dispose();
_PosePredictor5Point = ShapePredictor.Deserialize(predictor5PointModel);
_CnnFaceDetector?.Dispose();
_CnnFaceDetector = LossMmod.Deserialize(cnnFaceDetectionModel);
_FaceEncoder?.Dispose();
_FaceEncoder = LossMetric.Deserialize(faceRecognitionModel);
}
/// <summary>
/// Initializes a new instance of the <see cref="FaceRecognition"/> class with the instance that contains model binary datum.
/// </summary>
/// <param name="parameter">The instance that contains model binary datum.</param>
/// <exception cref="ArgumentNullException"><paramref name="parameter"/> is null.</exception>
/// <exception cref="NullReferenceException">The model data is null.</exception>
private FaceRecognition(ModelParameter parameter)
{
if (parameter == null)
throw new ArgumentNullException(nameof(parameter));
if (parameter.PosePredictor5FaceLandmarksModel == null)
throw new NullReferenceException(nameof(parameter.PosePredictor5FaceLandmarksModel));
if (parameter.PosePredictor68FaceLandmarksModel == null)
throw new NullReferenceException(nameof(parameter.PosePredictor68FaceLandmarksModel));
if (parameter.CnnFaceDetectorModel == null)
throw new NullReferenceException(nameof(parameter.CnnFaceDetectorModel));
if (parameter.FaceRecognitionModel == null)
throw new NullReferenceException(nameof(parameter.FaceRecognitionModel));
_FaceDetector?.Dispose();
_FaceDetector = DlibDotNet.Dlib.GetFrontalFaceDetector();
_PosePredictor68Point?.Dispose();
_PosePredictor68Point = ShapePredictor.Deserialize(parameter.PosePredictor68FaceLandmarksModel);
_PosePredictor5Point?.Dispose();
_PosePredictor5Point = ShapePredictor.Deserialize(parameter.PosePredictor5FaceLandmarksModel);
_CnnFaceDetector?.Dispose();
_CnnFaceDetector = LossMmod.Deserialize(parameter.CnnFaceDetectorModel);
_FaceEncoder?.Dispose();
_FaceEncoder = LossMetric.Deserialize(parameter.FaceRecognitionModel);
}
#endregion
#region Properties
/// <summary>
/// Gets or sets the custom face detector that user defined.
/// </summary>
public FaceDetector? CustomFaceDetector { get; set; }
/// <summary>
/// Gets or sets the custom face landmark detector that user defined.
/// </summary>
public FaceLandmarkDetector? CustomFaceLandmarkDetector { get; set; }
/// <summary>
/// Gets or sets the character encoding to convert <see cref="System.String"/> to array of <see cref="byte"/> for internal library.
/// </summary>
public static Encoding InternalEncoding
{
get => DlibDotNet.Dlib.Encoding;
set => DlibDotNet.Dlib.Encoding = value ?? Encoding.UTF8;
}
#endregion
#region Methods
/// <summary>
/// Returns an enumerable collection of array of bounding boxes of human faces in a image using the cnn face detector.
/// </summary>
/// <param name="images">An enumerable collection of images.</param>
/// <param name="numberOfTimesToUpsample">The number of image looking for faces. Higher numbers find smaller faces.</param>
/// <param name="batchSize">The number of images to include in each GPU processing batch.</param>
/// <returns>An enumerable collection of array of found face locations.</returns>
/// <exception cref="ArgumentNullException"><paramref name="images"/> is null.</exception>
public IEnumerable<Location[]> BatchFaceLocations(IEnumerable<Image> images, int numberOfTimesToUpsample = 1, int batchSize = 128)
{
if (images == null)
throw new ArgumentNullException(nameof(images));
List<Location[]>? results = new();
Image[]? imagesArray = images.ToArray();
if (!imagesArray.Any())
return results;
IEnumerable<MModRect>[]? rawDetectionsBatched = RawFaceLocationsBatched(imagesArray, numberOfTimesToUpsample, batchSize).ToArray();
Image? image = imagesArray[0];
for (int index = 0; index < rawDetectionsBatched.Length; index++)
{
MModRect[]? faces = rawDetectionsBatched[index].ToArray();
Location[]? locations = faces.Select(rect => new Location(TrimBound(rect.Rect, image.Width, image.Height), rect.DetectionConfidence)).ToArray();
foreach (MModRect? face in faces)
face.Dispose();
results.Add(locations);
}
return results;
}
/// <summary>
/// Compare a known face encoding against a candidate encoding to see if they match.
/// </summary>
/// <param name="knownFaceEncoding">A known face encodings.</param>
/// <param name="faceEncodingToCheck">A single face encoding to compare against a known face encoding.</param>
/// <param name="tolerance">The distance between faces to consider it a match. Lower is more strict. The default value is 0.6.</param>
/// <returns>A True/False value indicating which known a face encoding matches the face encoding to check.</returns>
/// <exception cref="ArgumentNullException"><paramref name="knownFaceEncoding"/> or <paramref name="faceEncodingToCheck"/> is null.</exception>
/// <exception cref="ObjectDisposedException"><paramref name="knownFaceEncoding"/> or <paramref name="faceEncodingToCheck"/>.</exception>
public static bool CompareFace(FaceEncoding knownFaceEncoding, FaceEncoding faceEncodingToCheck, double tolerance = 0.6d)
{
if (knownFaceEncoding == null)
throw new ArgumentNullException(nameof(knownFaceEncoding));
if (faceEncodingToCheck == null)
throw new ArgumentNullException(nameof(faceEncodingToCheck));
knownFaceEncoding.ThrowIfDisposed();
faceEncodingToCheck.ThrowIfDisposed();
return FaceDistance(knownFaceEncoding, faceEncodingToCheck) <= tolerance;
}
/// <summary>
/// Compare an enumerable collection of face encodings against a candidate encoding to see if they match.
/// </summary>
/// <param name="knownFaceEncodings">An enumerable collection of known face encodings.</param>
/// <param name="faceEncodingToCheck">A single face encoding to compare against the enumerable collection.</param>
/// <param name="tolerance">The distance between faces to consider it a match. Lower is more strict. The default value is 0.6.</param>
/// <returns>An enumerable collection of True/False values indicating which known face encodings match the face encoding to check.</returns>
/// <exception cref="ArgumentNullException"><paramref name="knownFaceEncodings"/> or <paramref name="faceEncodingToCheck"/> is null.</exception>
/// <exception cref="ObjectDisposedException"><paramref name="faceEncodingToCheck"/> is disposed. Or <paramref name="knownFaceEncodings"/> contains disposed object.</exception>
public static IEnumerable<bool> CompareFaces(IEnumerable<FaceEncoding> knownFaceEncodings, FaceEncoding faceEncodingToCheck, double tolerance = 0.6d)
{
if (knownFaceEncodings == null)
throw new ArgumentNullException(nameof(knownFaceEncodings));
if (faceEncodingToCheck == null)
throw new ArgumentNullException(nameof(faceEncodingToCheck));
faceEncodingToCheck.ThrowIfDisposed();
FaceEncoding[]? array = knownFaceEncodings.ToArray();
if (array.Any(encoding => encoding.IsDisposed))
throw new ObjectDisposedException($"{nameof(knownFaceEncodings)} contains disposed object.");
List<bool>? results = new();
if (array.Length == 0)
return results;
foreach (FaceEncoding? faceEncoding in array)
results.Add(FaceDistance(faceEncoding, faceEncodingToCheck) <= tolerance);
return results;
}
/// <summary>
/// Create a new instance of the <see cref="FaceRecognition"/> class.
/// </summary>
/// <param name="directory">The directory path that stores model files.</param>
/// <exception cref="FileNotFoundException">The model file is not found.</exception>
/// <exception cref="DirectoryNotFoundException">The specified directory path is not found.</exception>
public static FaceRecognition Create(string directory) => new(directory);
/// <summary>
/// Create a new instance of the <see cref="FaceRecognition"/> class.
/// </summary>
/// <param name="parameter">The instance that contains model binary datum.</param>
/// <exception cref="ArgumentNullException"><paramref name="parameter"/> is null.</exception>
/// <exception cref="NullReferenceException">The model data is null.</exception>
public static FaceRecognition Create(ModelParameter parameter) => new(parameter);
/// <summary>
/// Crop a specified image with enumerable collection of face locations.
/// </summary>
/// <param name="image">The image contains a face.</param>
/// <param name="locations">The enumerable collection of location rectangle for faces.</param>
/// <returns></returns>
/// <exception cref="ArgumentNullException"><paramref name="image"/> or <paramref name="locations"/> is null.</exception>
/// <exception cref="ObjectDisposedException"><paramref name="image"/> is disposed.</exception>
public static IEnumerable<Image> CropFaces(Image image, IEnumerable<Location> locations)
{
if (image == null)
throw new ArgumentNullException(nameof(image));
if (locations == null)
throw new ArgumentNullException(nameof(locations));
image.ThrowIfDisposed();
List<Image>? results = new();
foreach (Location? location in locations)
{
DlibDotNet.Rectangle rect = new(location.Left, location.Top, location.Right, location.Bottom);
DPoint[]? dPoint = new[]
{
new DPoint(rect.Left, rect.Top),
new DPoint(rect.Right, rect.Top),
new DPoint(rect.Left, rect.Bottom),
new DPoint(rect.Right, rect.Bottom),
};
int width = (int)rect.Width;
int height = (int)rect.Height;
switch (image.Mode)
{
case Mode.Rgb:
Matrix<RgbPixel>? rgb = image.Matrix as Matrix<RgbPixel>;
results.Add(new Image(DlibDotNet.Dlib.ExtractImage4Points(rgb, dPoint, width, height),
Mode.Rgb));
break;
case Mode.Greyscale:
Matrix<byte>? gray = image.Matrix as Matrix<byte>;
results.Add(new Image(DlibDotNet.Dlib.ExtractImage4Points(gray, dPoint, width, height),
Mode.Greyscale));
break;
}
}
return results;
}
/// <summary>
/// Compare a face encoding to a known face encoding and get a euclidean distance for comparison face.
/// </summary>
/// <param name="faceEncoding">The face encoding to compare.</param>
/// <param name="faceToCompare">The face encoding to compare against.</param>
/// <returns>The euclidean distance for comparison face. If 0, faces are completely equal.</returns>
/// <exception cref="ArgumentNullException"><paramref name="faceEncoding"/> or <paramref name="faceToCompare"/> is null.</exception>
/// <exception cref="ObjectDisposedException"><paramref name="faceEncoding"/> or <paramref name="faceToCompare"/> is disposed.</exception>
public static double FaceDistance(FaceEncoding faceEncoding, FaceEncoding faceToCompare)
{
if (faceEncoding == null)
throw new ArgumentNullException(nameof(faceEncoding));
if (faceToCompare == null)
throw new ArgumentNullException(nameof(faceToCompare));
faceEncoding.ThrowIfDisposed();
faceToCompare.ThrowIfDisposed();
if (faceEncoding.Encoding.Size == 0)
return 0;
using Matrix<double>? diff = faceEncoding.Encoding - faceToCompare.Encoding;
return DlibDotNet.Dlib.Length(diff);
}
/// <summary>
/// Compare an enumerable collection of face encoding to a known face encoding and get an enumerable collection of euclidean distance for comparison face.
/// </summary>
/// <param name="faceEncodings">The enumerable collection of face encoding to compare.</param>
/// <param name="faceToCompare">The face encoding to compare against.</param>
/// <returns>The enumerable collection of euclidean distance for comparison face. If 0, faces are completely equal.</returns>
/// <exception cref="ArgumentNullException"><paramref name="faceEncodings"/> or <paramref name="faceToCompare"/> is null.</exception>
/// <exception cref="ObjectDisposedException"><paramref name="faceToCompare"/> is disposed. Or <paramref name="faceEncodings"/> contains disposed object.</exception>
public static IEnumerable<double> FaceDistances(IEnumerable<FaceEncoding> faceEncodings, FaceEncoding faceToCompare)
{
if (faceEncodings == null)
throw new ArgumentNullException(nameof(faceEncodings));
if (faceToCompare == null)
throw new ArgumentNullException(nameof(faceToCompare));
faceToCompare.ThrowIfDisposed();
FaceEncoding[]? array = faceEncodings.ToArray();
if (array.Any(encoding => encoding.IsDisposed))
throw new ObjectDisposedException($"{nameof(faceEncodings)} contains disposed object.");
List<double>? results = new();
if (array.Length == 0)
return results;
foreach (FaceEncoding? faceEncoding in array)
using (Matrix<double>? diff = faceEncoding.Encoding - faceToCompare.Encoding)
results.Add(DlibDotNet.Dlib.Length(diff));
return results;
}
/// <summary>
/// Returns an enumerable collection of face feature data corresponds to all faces in specified image.
/// </summary>
/// <param name="image">The image contains faces. The image can contain multiple faces.</param>
/// <param name="knownFaceLocation">The enumerable collection of location rectangle for faces. If specified null, method will find face locations.</param>
/// <param name="numJitters">The number of times to re-sample the face when calculating encoding.</param>
/// <param name="predictorModel">The dimension of vector which be returned from detector.</param>
/// <param name="model">The model of face detector to detect in image. If <paramref name="knownFaceLocation"/> is not null, this value is ignored.</param>
/// <returns>An enumerable collection of face feature data corresponds to all faces in specified image.</returns>
/// <exception cref="ArgumentNullException"><paramref name="image"/> is null.</exception>
/// <exception cref="InvalidOperationException"><paramref name="knownFaceLocation"/> contains no elements.</exception>
/// <exception cref="ObjectDisposedException"><paramref name="image"/> or this object or custom face landmark detector is disposed.</exception>
/// <exception cref="NotSupportedException"><see cref="PredictorModel.Custom"/> is not supported.</exception>
public IEnumerable<FaceEncoding> FaceEncodings(Image image,
IEnumerable<Location>? knownFaceLocation = null,
int numJitters = 1,
PredictorModel predictorModel = PredictorModel.Small,
Model model = Model.Hog)
{
if (image == null)
throw new ArgumentNullException(nameof(image));
if (predictorModel == PredictorModel.Custom)
throw new NotSupportedException("FaceRecognition.PredictorModel.Custom is not supported.");
if (knownFaceLocation != null && !knownFaceLocation.Any())
throw new InvalidOperationException($"{nameof(knownFaceLocation)} contains no elements.");
image.ThrowIfDisposed();
ThrowIfDisposed();
IEnumerable<FullObjectDetection>? rawLandmarks = RawFaceLandmarks(image, knownFaceLocation, predictorModel, model);
List<FaceEncoding>? results = new();
foreach (FullObjectDetection? landmark in rawLandmarks)
{
FaceEncoding? ret = new(FaceRecognitionModelV1.ComputeFaceDescriptor(_FaceEncoder, image, landmark, numJitters));
landmark.Dispose();
results.Add(ret);
}
return results;
}
/// <summary>
/// Returns an enumerable collection of dictionary of face parts locations (eyes, nose, etc) for each face in the image.
/// </summary>
/// <param name="faceImage">The image contains faces. The image can contain multiple faces.</param>
/// <param name="faceLocations">The enumerable collection of location rectangle for faces. If specified null, method will find face locations.</param>
/// <param name="predictorModel">The dimension of vector which be returned from detector.</param>
/// <param name="model">The model of face detector to detect in image. If <paramref name="faceLocations"/> is not null, this value is ignored.</param>
/// <returns>An enumerable collection of dictionary of face parts locations (eyes, nose, etc).</returns>
/// <exception cref="ArgumentNullException"><paramref name="faceImage"/> is null.</exception>
/// <exception cref="InvalidOperationException"><paramref name="faceLocations"/> contains no elements.</exception>
/// <exception cref="ObjectDisposedException"><paramref name="faceImage"/> or this object or custom face landmark detector is disposed.</exception>
/// <exception cref="NotSupportedException">The custom face landmark detector is not ready.</exception>
public IEnumerable<IDictionary<FacePart, IEnumerable<FacePoint>>> FaceLandmark(Image faceImage,
IEnumerable<Location>? faceLocations = null,
PredictorModel predictorModel = PredictorModel.Large,
Model model = Model.Hog)
{
if (faceImage == null)
throw new ArgumentNullException(nameof(faceImage));
if (faceLocations != null && !faceLocations.Any())
throw new InvalidOperationException($"{nameof(faceLocations)} contains no elements.");
faceImage.ThrowIfDisposed();
ThrowIfDisposed();
if (predictorModel == PredictorModel.Custom)
{
if (CustomFaceLandmarkDetector == null)
throw new NotSupportedException("The custom face landmark detector is not ready.");
if (CustomFaceLandmarkDetector.IsDisposed)
throw new ObjectDisposedException($"{nameof(CustomFaceLandmarkDetector)}", "The custom face landmark detector is disposed.");
}
FullObjectDetection[]? landmarks = RawFaceLandmarks(faceImage, faceLocations, predictorModel, model).ToArray();
IEnumerable<FacePoint[]>? landmarkTuples = landmarks.Select(landmark => Enumerable.Range(0, (int)landmark.Parts)
.Select(index => new FacePoint(index, landmark.GetPart((uint)index).X, landmark.GetPart((uint)index).Y)).ToArray());
List<Dictionary<FacePart, IEnumerable<FacePoint>>>? results = new();
try
{
// For a definition of each point index, see https://cdn-images-1.medium.com/max/1600/1*AbEg31EgkbXSQehuNJBlWg.png
switch (predictorModel)
{
case PredictorModel.Large:
results.AddRange(landmarkTuples.Select(landmarkTuple => new Dictionary<FacePart, IEnumerable<FacePoint>>
{
{ FacePart.Chin, Enumerable.Range(0,17).Select(i => landmarkTuple[i]).ToArray() },
{ FacePart.LeftEyebrow, Enumerable.Range(17,5).Select(i => landmarkTuple[i]).ToArray() },
{ FacePart.RightEyebrow, Enumerable.Range(22,5).Select(i => landmarkTuple[i]).ToArray() },
{ FacePart.NoseBridge, Enumerable.Range(27,5).Select(i => landmarkTuple[i]).ToArray() },
{ FacePart.NoseTip, Enumerable.Range(31,5).Select(i => landmarkTuple[i]).ToArray() },
{ FacePart.LeftEye, Enumerable.Range(36,6).Select(i => landmarkTuple[i]).ToArray() },
{ FacePart.RightEye, Enumerable.Range(42,6).Select(i => landmarkTuple[i]).ToArray() },
{ FacePart.TopLip, Enumerable.Range(48,7).Select(i => landmarkTuple[i])
.Concat( new [] { landmarkTuple[64] })
.Concat( new [] { landmarkTuple[63] })
.Concat( new [] { landmarkTuple[62] })
.Concat( new [] { landmarkTuple[61] })
.Concat( new [] { landmarkTuple[60] }) },
{ FacePart.BottomLip, Enumerable.Range(54,6).Select(i => landmarkTuple[i])
.Concat( new [] { landmarkTuple[48] })
.Concat( new [] { landmarkTuple[60] })
.Concat( new [] { landmarkTuple[67] })
.Concat( new [] { landmarkTuple[66] })
.Concat( new [] { landmarkTuple[65] })
.Concat( new [] { landmarkTuple[64] }) }
}));
break;
case PredictorModel.Small:
results.AddRange(landmarkTuples.Select(landmarkTuple => new Dictionary<FacePart, IEnumerable<FacePoint>>
{
{ FacePart.NoseTip, Enumerable.Range(4,1).Select(i => landmarkTuple[i]).ToArray() },
{ FacePart.LeftEye, Enumerable.Range(2,2).Select(i => landmarkTuple[i]).ToArray() },
{ FacePart.RightEye, Enumerable.Range(0,2).Select(i => landmarkTuple[i]).ToArray() }
}));
break;
case PredictorModel.Custom:
if (CustomFaceLandmarkDetector is null)
throw new Exception($"{nameof(CustomFaceLandmarkDetector)} is null");
results.AddRange(CustomFaceLandmarkDetector.GetLandmarks(landmarkTuples));
break;
default:
throw new ArgumentOutOfRangeException(nameof(predictorModel), predictorModel, null);
}
}
finally
{
foreach (FullObjectDetection? landmark in landmarks)
landmark.Dispose();
}
return results.ToArray();
}
/// <summary>
/// Returns an enumerable collection of face location correspond to all faces in specified image.
/// </summary>
/// <param name="image">The image contains faces. The image can contain multiple faces.</param>
/// <param name="numberOfTimesToUpsample">The number of times to up-sample the image when finding faces.</param>
/// <param name="model">The model of face detector to detect in image.</param>
/// <returns>An enumerable collection of face location correspond to all faces in specified image.</returns>
/// <exception cref="ArgumentNullException"><paramref name="image"/> is null.</exception>
/// <exception cref="ObjectDisposedException"><paramref name="image"/> or this object is disposed.</exception>
public IEnumerable<Location> FaceLocations(Image image, int numberOfTimesToUpsample = 1, Model model = Model.Hog)
{
if (image == null)
throw new ArgumentNullException(nameof(image));
image.ThrowIfDisposed();
ThrowIfDisposed();
List<Location>? results = new();
foreach (MModRect? face in RawFaceLocations(image, numberOfTimesToUpsample, model))
{
Location? ret = TrimBound(face.Rect, image.Width, image.Height);
double confidence = face.DetectionConfidence;
face.Dispose();
results.Add(new Location(ret, confidence));
}
return results;
}
/// <summary>
/// Creates an <see cref="FaceEncoding"/> from the <see cref="double"/> array.
/// </summary>
/// <param name="encoding">The <see cref="double"/> array contains face encoding data.</param>
/// <returns>The <see cref="FaceEncoding"/> this method creates.</returns>
/// <exception cref="ArgumentNullException"><paramref name="encoding"/> is null.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="encoding"/> must be 128.</exception>
public static FaceEncoding LoadFaceEncoding(double[] encoding)
{
if (encoding == null)
throw new ArgumentNullException(nameof(encoding));
if (encoding.Length != 128)
{
string message = $"{nameof(encoding)}.{nameof(encoding.Length)} must be 128.";
throw new ArgumentOutOfRangeException(message);
}
#pragma warning disable
Matrix<double>? matrix = Matrix<double>.CreateTemplateParameterizeMatrix(0, 1);
#pragma warning restore
matrix.SetSize(128);
matrix.Assign(encoding);
return new FaceEncoding(matrix);
}
#pragma warning disable CA1416
/// <summary>
/// Creates an <see cref="Image"/> from the specified existing bitmap image.
/// </summary>
/// <param name="bitmap">The <see cref="Bitmap"/> from which to create the new <see cref="Image"/>.</param>
/// <returns>The <see cref="Image"/> this method creates.</returns>
/// <exception cref="ArgumentNullException"><paramref name="bitmap"/> is null.</exception>
/// <exception cref="ArgumentOutOfRangeException">The specified <see cref="PixelFormat"/> is not supported.</exception>
public static Image? LoadImage(Bitmap bitmap)
{
int width = bitmap.Width;
int height = bitmap.Height;
System.Drawing.Rectangle rect = new(0, 0, width, height);
PixelFormat format = bitmap.PixelFormat;
Mode mode;
int srcChannel;
int dstChannel;
switch (format)
{
case PixelFormat.Format8bppIndexed:
mode = Mode.Greyscale;
srcChannel = 1;
dstChannel = 1;
break;
case PixelFormat.Format24bppRgb:
mode = Mode.Rgb;
srcChannel = 3;
dstChannel = 3;
break;
case PixelFormat.Format32bppRgb:
case PixelFormat.Format32bppArgb:
mode = Mode.Rgb;
srcChannel = 4;
dstChannel = 3;
break;
default:
throw new ArgumentOutOfRangeException($"{nameof(bitmap)}", $"The specified {nameof(PixelFormat)} is not supported.");
}
BitmapData? data = null;
try
{
data = bitmap.LockBits(rect, ImageLockMode.ReadOnly, format);
unsafe
{
byte[]? array = new byte[width * height * dstChannel];
fixed (byte* pArray = &array[0])
{
byte* dst = pArray;
switch (srcChannel)
{
case 1:
{
IntPtr src = data.Scan0;
int stride = data.Stride;
for (int h = 0; h < height; h++)
Marshal.Copy(IntPtr.Add(src, h * stride), array, h * width, width * dstChannel);
}
break;
case 3:
case 4:
{
byte* src = (byte*)data.Scan0;
int stride = data.Stride;
for (int h = 0; h < height; h++)
{
int srcOffset = h * stride;
int dstOffset = h * width * dstChannel;
for (int w = 0; w < width; w++)
{
// BGR order to RGB order
dst[dstOffset + w * dstChannel + 0] = src[srcOffset + w * srcChannel + 2];
dst[dstOffset + w * dstChannel + 1] = src[srcOffset + w * srcChannel + 1];
dst[dstOffset + w * dstChannel + 2] = src[srcOffset + w * srcChannel + 0];
}
}
}
break;
}
IntPtr ptr = (IntPtr)pArray;
switch (mode)
{
case Mode.Rgb:
return new Image(new Matrix<RgbPixel>(ptr, height, width, width * 3), Mode.Rgb);
case Mode.Greyscale:
return new Image(new Matrix<byte>(ptr, height, width, width), Mode.Greyscale);
}
}
}
}
finally
{
if (data != null)
bitmap.UnlockBits(data);
}
return null;
}
#pragma warning restore CA1416
/// <summary>
/// Creates an <see cref="Image"/> from the <see cref="byte"/> array.
/// </summary>
/// <param name="array">The <see cref="byte"/> array contains image data.</param>
/// <param name="row">The number of rows in a image data.</param>
/// <param name="column">The number of columns in a image data.</param>
/// <param name="stride">The stride width in bytes.</param>
/// <param name="mode">A image color mode.</param>
/// <returns>The <see cref="Image"/> this method creates.</returns>
/// <exception cref="ArgumentNullException"><paramref name="array"/> is null.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="row"/> is less than 0.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="column"/> is less than 0.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="stride"/> is less than 0.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="stride"/> is less than <paramref name="column"/>.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="row"/> x <paramref name="stride"/> is less than <see cref="Array.Length"/>.</exception>
public static Image? LoadImage(byte[] array, int row, int column, int stride, Mode mode)
{
if (array == null)
throw new ArgumentNullException(nameof(array));
if (row < 0)
throw new ArgumentOutOfRangeException($"{nameof(row)}", $"{nameof(row)} is less than 0.");
if (column < 0)
throw new ArgumentOutOfRangeException($"{nameof(column)}", $"{nameof(column)} is less than 0.");
if (stride < 0)
throw new ArgumentOutOfRangeException($"{nameof(stride)}", $"{nameof(stride)} is less than 0.");
if (stride < column)
throw new ArgumentOutOfRangeException($"{nameof(stride)}", $"{nameof(stride)} is less than {nameof(column)}.");
int min = row * stride;
if (!(array.Length >= min))
throw new ArgumentOutOfRangeException("", $"{nameof(row)} x {nameof(stride)} is less than {nameof(Array)}.{nameof(Array.Length)}.");
unsafe
{
fixed (byte* p = &array[0])
{
IntPtr ptr = (IntPtr)p;
switch (mode)
{
case Mode.Rgb:
return new Image(new Matrix<RgbPixel>(ptr, row, column, stride), Mode.Rgb);
case Mode.Greyscale:
return new Image(new Matrix<byte>(ptr, row, column, stride), Mode.Greyscale);
}
}
}
return null;
}
/// <summary>
/// Creates an <see cref="Image"/> from the unmanaged memory pointer indicates <see cref="byte"/> array image data.
/// </summary>
/// <param name="array">The unmanaged memory pointer indicates <see cref="byte"/> array image data.</param>
/// <param name="row">The number of rows in a image data.</param>
/// <param name="column">The number of columns in a image data.</param>
/// <param name="stride">The stride width in bytes.</param>
/// <param name="mode">A image color mode.</param>
/// <returns>The <see cref="Image"/> this method creates.</returns>
/// <exception cref="ArgumentException"><paramref name="array"/> is <see cref="IntPtr.Zero"/>.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="row"/> is less than 0.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="column"/> is less than 0.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="stride"/> is less than 0.</exception>
/// <exception cref="ArgumentOutOfRangeException"><paramref name="stride"/> is less than <paramref name="column"/>.</exception>
public static Image? LoadImage(IntPtr array, int row, int column, int stride, Mode mode)
{
if (array == IntPtr.Zero)
throw new ArgumentException($"{nameof(array)} is {nameof(IntPtr)}.{nameof(IntPtr.Zero)}", nameof(array));
if (row < 0)
throw new ArgumentOutOfRangeException($"{nameof(row)}", $"{nameof(row)} is less than 0.");
if (column < 0)
throw new ArgumentOutOfRangeException($"{nameof(column)}", $"{nameof(column)} is less than 0.");
if (stride < 0)
throw new ArgumentOutOfRangeException($"{nameof(stride)}", $"{nameof(stride)} is less than 0.");
if (stride < column)
throw new ArgumentOutOfRangeException($"{nameof(stride)}", $"{nameof(stride)} is less than {nameof(column)}.");
return mode switch
{
Mode.Rgb => new Image(new Matrix<RgbPixel>(array, row, column, stride), mode),
Mode.Greyscale => new Image(new Matrix<byte>(array, row, column, stride), mode),
_ => null,
};
}
/// <summary>
/// Creates an <see cref="Image"/> from the specified path.
/// </summary>
/// <param name="file">A string that contains the path of the file from which to create the <see cref="Image"/>.</param>
/// <param name="mode">A image color mode.</param>
/// <returns>The <see cref="Image"/> this method creates.</returns>
/// <exception cref="FileNotFoundException">The specified path does not exist.</exception>
public static Image? LoadImageFile(string file, Mode mode = Mode.Rgb)
{
if (!File.Exists(file))
throw new FileNotFoundException(file);
return mode switch
{
Mode.Rgb => new Image(DlibDotNet.Dlib.LoadImageAsMatrix<RgbPixel>(file), mode),
Mode.Greyscale => new Image(DlibDotNet.Dlib.LoadImageAsMatrix<byte>(file), mode),
_ => null,
};
}
#region Helpers
private IEnumerable<FullObjectDetection> RawFaceLandmarks(Image faceImage,
IEnumerable<Location>? faceLocations = null,
PredictorModel predictorModel = PredictorModel.Large,
Model model = Model.Hog)
{
IEnumerable<Location> rects;
if (faceLocations == null)
{
List<Location>? list = new();
IEnumerable<MModRect>? tmp = RawFaceLocations(faceImage, 1, model);
foreach (MModRect? mModRect in tmp)
{
list.Add(new Location(mModRect.DetectionConfidence, mModRect.Rect.Bottom, mModRect.Rect.Left, mModRect.Rect.Right, mModRect.Rect.Top));
mModRect.Dispose();
}
rects = list;
}
else
{
rects = faceLocations;
}
List<FullObjectDetection>? results = new();
if (predictorModel == PredictorModel.Custom)
{
if (CustomFaceLandmarkDetector is null)
throw new Exception($"{nameof(CustomFaceLandmarkDetector)} is null");
foreach (Location? rect in rects)
{
FullObjectDetection? ret = CustomFaceLandmarkDetector.Detect(faceImage, rect);
results.Add(ret);
}
}
else
{
ShapePredictor? posePredictor = _PosePredictor68Point;
switch (predictorModel)
{
case PredictorModel.Small:
posePredictor = _PosePredictor5Point;
break;
}
foreach (Location? rect in rects)
{
FullObjectDetection? ret = posePredictor.Detect(faceImage.Matrix, new DlibDotNet.Rectangle(rect.Left, rect.Top, rect.Right, rect.Bottom));
results.Add(ret);
}
}
return results;
}
private IEnumerable<MModRect> RawFaceLocations(Image faceImage, int numberOfTimesToUpsample = 1, Model model = Model.Hog)
{
switch (model)
{
case Model.Custom:
if (CustomFaceDetector == null)
throw new NotSupportedException("The custom face detector is not ready.");
return CustomFaceDetector.Detect(faceImage, numberOfTimesToUpsample).Select(rect => new MModRect
{
Rect = new DlibDotNet.Rectangle(rect.Left, rect.Top, rect.Right, rect.Bottom),
DetectionConfidence = rect.Confidence
});
case Model.Cnn:
return CnnFaceDetectionModelV1.Detect(_CnnFaceDetector, faceImage, numberOfTimesToUpsample);
default:
IEnumerable<Tuple<DlibDotNet.Rectangle, double>>? locations = SimpleObjectDetector.RunDetectorWithUpscale2(_FaceDetector, faceImage, (uint)numberOfTimesToUpsample);
return locations.Select(tuple => new MModRect { Rect = tuple.Item1, DetectionConfidence = tuple.Item2 });
}
}
private IEnumerable<IEnumerable<MModRect>> RawFaceLocationsBatched(IEnumerable<Image> faceImages, int numberOfTimesToUpsample = 1, int batchSize = 128) => CnnFaceDetectionModelV1.DetectMulti(_CnnFaceDetector, faceImages, numberOfTimesToUpsample, batchSize);
private static Location TrimBound(DlibDotNet.Rectangle location, int width, int height) => new(Math.Max(location.Left, 0), Math.Max(location.Top, 0), Math.Min(location.Right, width), Math.Min(location.Bottom, height));
#endregion
#endregion
#region Methods
#region Overrides
/// <summary>
/// Releases all unmanaged resources.
/// </summary>
protected override void DisposeUnmanaged()
{
base.DisposeUnmanaged();
_PosePredictor68Point?.Dispose();
_PosePredictor5Point?.Dispose();
_CnnFaceDetector?.Dispose();
_FaceEncoder?.Dispose();
_FaceDetector?.Dispose();
}
#endregion
#endregion
}

52
FaceRecognitionDotNet/FaceRecognitionDotNet.csproj Normal file
View File

@ -0,0 +1,52 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<ImplicitUsings>enable</ImplicitUsings>
<LangVersion>10.0</LangVersion>
<Nullable>enable</Nullable>
<OutputType>library</OutputType>
<RuntimeIdentifier>win-x64</RuntimeIdentifier>
<TargetFramework>net6.0</TargetFramework>
</PropertyGroup>
<PropertyGroup>
<PackageId>Phares.View.by.Distance.FaceRecognitionDotNet</PackageId>
<GeneratePackageOnBuild>false</GeneratePackageOnBuild>
<Version>5.0.402.104</Version>
<Authors>Mike Phares</Authors>
<Company>Phares</Company>
<IncludeSymbols>true</IncludeSymbols>
<SymbolPackageFormat>snupkg</SymbolPackageFormat>
</PropertyGroup>
<PropertyGroup>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>
<PropertyGroup>
<IsWindows Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Windows)))' == 'true'">true</IsWindows>
<IsOSX Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::OSX)))' == 'true'">true</IsOSX>
<IsLinux Condition="'$([System.Runtime.InteropServices.RuntimeInformation]::IsOSPlatform($([System.Runtime.InteropServices.OSPlatform]::Linux)))' == 'true'">true</IsLinux>
</PropertyGroup>
<PropertyGroup Condition="'$(IsWindows)'=='true'">
<DefineConstants>Windows</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$(IsOSX)'=='true'">
<DefineConstants>OSX</DefineConstants>
</PropertyGroup>
<PropertyGroup Condition="'$(IsLinux)'=='true'">
<DefineConstants>Linux</DefineConstants>
</PropertyGroup>
<ItemGroup Condition="'$(RuntimeIdentifier)' == 'browser-wasm'">
<SupportedPlatform Include="browser" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="DlibDotNet" Version="19.21.0.20220724" />
<!--PackageReference Include="configuration.MKL" Version="19.21.0.20210302" /-->
<!--PackageReference Include="DlibDotNet-WithCUDA" Version="19.17.0.20190429" /-->
<!--PackageReference Include="configuration.CUDA92" Version="19.21.0.20210302" /-->
<!--PackageReference Include="configuration.CUDA102" Version="19.21.0.20210302" /-->
<!--PackageReference Include="configuration.CUDA110" Version="19.21.0.20210302" /-->
<!--PackageReference Include="configuration.CUDA111" Version="19.21.0.20210302" /-->
<!--PackageReference Include="configuration.CUDA112" Version="19.21.0.20210302" /-->
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Shared\View-by-Distance.Shared.csproj" />
</ItemGroup>
</Project>

22
FaceRecognitionDotNet/FaceRecognitionModels.cs Normal file
View File

@ -0,0 +1,22 @@
namespace View_by_Distance.FaceRecognitionDotNet;
internal sealed class FaceRecognitionModels
{
public static string GetPosePredictorModelLocation() => "shape_predictor_68_face_landmarks.dat";
public static string GetPosePredictorFivePointModelLocation() => "shape_predictor_5_face_landmarks.dat";
public static string GetFaceRecognitionModelLocation() => "dlib_face_recognition_resnet_model_v1.dat";
public static string GetCnnFaceDetectorModelLocation() => "mmod_human_face_detector.dat";
public static string GetPosePredictor194PointModelLocation() => "helen-dataset.dat";
public static string GetAgeNetworkModelLocation() => "adience-age-network.dat";
public static string GetGenderNetworkModelLocation() => "utkface-gender-network.dat";
public static string GetEmotionNetworkModelLocation() => "corrective-reannotation-of-fer-ck-kdef-emotion-network_test_best.dat";
}

127
FaceRecognitionDotNet/Image.cs Normal file
View File

@ -0,0 +1,127 @@
using DlibDotNet;
using DlibDotNet.Extensions;
using System.Drawing;
using View_by_Distance.Shared.Models.Stateless;
namespace View_by_Distance.FaceRecognitionDotNet;
/// <summary>
/// Represents a image data. This class cannot be inherited.
/// </summary>
public sealed class Image : DisposableObject
{
#region Fields
#endregion
#region Constructors
internal Image(MatrixBase matrix, Mode mode)
{
Matrix = matrix;
Mode = mode;
}
#endregion
#region Properties
/// <summary>
/// Gets the height of the image.
/// </summary>
/// <exception cref="ObjectDisposedException">This object is disposed.</exception>
public int Height
{
get
{
ThrowIfDisposed();
return Matrix.Rows;
}
}
internal MatrixBase Matrix { get; private set; }
internal Mode Mode { get; }
/// <summary>
/// Gets the width of the image.
/// </summary>
/// <exception cref="ObjectDisposedException">This object is disposed.</exception>
public int Width
{
get
{
ThrowIfDisposed();
return Matrix.Columns;
}
}
#endregion
#region Methods
/// <summary>
/// Saves this <see cref="Image"/> to the specified file.
/// </summary>
/// <param name="filename">A string that contains the name of the file to which to save this <see cref="Image"/>.</param>
/// <param name="format">The <see cref="ImageFormat"/> for this <see cref="Image"/>.</param>
/// <exception cref="ArgumentNullException"><paramref name="filename"/> is null.</exception>
/// <exception cref="ObjectDisposedException">This object is disposed.</exception>
public void Save(string filename, ImageFormat format)
{
if (filename == null)
throw new ArgumentNullException(nameof(filename));
ThrowIfDisposed();
string? directory = Path.GetDirectoryName(filename);
if (!Directory.Exists(directory) && !string.IsNullOrWhiteSpace(directory))
_ = Directory.CreateDirectory(directory);
switch (format)
{
case ImageFormat.Bmp:
DlibDotNet.Dlib.SaveBmp(Matrix, filename);
break;
case ImageFormat.Jpeg:
DlibDotNet.Dlib.SaveJpeg(Matrix, filename);
break;
case ImageFormat.Png:
DlibDotNet.Dlib.SavePng(Matrix, filename);
break;
}
}
/// <summary>
/// Converts this <see cref="Image"/> to a GDI+ <see cref="Bitmap"/>.
/// </summary>
/// <returns>A <see cref="Bitmap"/> that represents the converted <see cref="Image"/>.</returns>
/// <exception cref="ObjectDisposedException">This object is disposed.</exception>
/// <exception cref="NotSupportedException">A Greyscale image is not supported.</exception>
public Bitmap ToBitmap()
{
ThrowIfDisposed();
if (Mode == Mode.Greyscale)
throw new NotSupportedException();
return ((Matrix<RgbPixel>)Matrix).ToBitmap();
}
#region Overrides
/// <summary>
/// Releases all unmanaged resources.
/// </summary>
protected override void DisposeUnmanaged()
{
base.DisposeUnmanaged();
Matrix?.Dispose();
}
#endregion
#endregion
}

49
FaceRecognitionDotNet/ModelParameter.cs Normal file
View File

@ -0,0 +1,49 @@
namespace View_by_Distance.FaceRecognitionDotNet;
/// <summary>
/// Describes the model binary datum. This class cannot be inherited.
/// </summary>
public sealed class ModelParameter
{
#region Properties
/// <summary>
/// Gets or sets the binary data of model for 68 points face landmarks.
/// </summary>
public byte[]? PosePredictor68FaceLandmarksModel
{
get;
set;
}
/// <summary>
/// Gets or sets the binary data of model for 5 points face landmarks.
/// </summary>
public byte[]? PosePredictor5FaceLandmarksModel
{
get;
set;
}
/// <summary>
/// Gets or sets the binary data of model for face encoding.
/// </summary>
public byte[]? FaceRecognitionModel
{
get;
set;
}
/// <summary>
/// Gets or sets the binary data of model for face detector by using CNN.
/// </summary>
public byte[]? CnnFaceDetectorModel
{
get;
set;
}
#endregion
}

114
FaceRecognitionDotNet/Point.cs Normal file
View File

@ -0,0 +1,114 @@
namespace View_by_Distance.FaceRecognitionDotNet;
/// <summary>
/// Represents an ordered pair of integer x- and y-coordinates that defines a point in a two-dimensional plane.
/// </summary>
public struct Point : IEquatable<Point>
{
#region Constructors
/// <summary>
/// Initializes a new instance of the <see cref="Point"/> structure with the specified coordinates.
/// </summary>
/// <param name="x">The horizontal position of the point.</param>
/// <param name="y">The vertical position of the point.</param>
public Point(int x, int y)
{
X = x;
Y = y;
}
internal Point(DlibDotNet.Point point)
{
X = point.X;
Y = point.Y;
}
#endregion
#region Properties
/// <summary>
/// Gets the x-coordinate of this <see cref="Point"/>.
/// </summary>
public int X
{
get;
}
/// <summary>
/// Gets the y-coordinate of this <see cref="Point"/>.
/// </summary>
public int Y
{
get;
}
#endregion
#region Methods
/// <summary>
/// Compares two <see cref="Point"/> structures for equality.
/// </summary>
/// <param name="other">The point to compare to this instance.</param>
/// <returns><code>true</code> if both <see cref="Point"/> structures contain the same <see cref="X"/> and <see cref="Y"/> values; otherwise, <code>false</code>.</returns>
public bool Equals(Point other)
{
return X == other.X &&
Y == other.Y;
}
#region overrides
/// <summary>
/// Determines whether the specified <see cref="Object"/> is a <see cref="Point"/> and whether it contains the same coordinates as this <see cref="Point"/>.
/// </summary>
/// <param name="obj">The <see cref="Object"/> to compare.</param>
/// <returns><code>true</code> if <paramref name="obj"/> is a <see cref="Point"/> and contains the same <see cref="X"/> and <see cref="Y"/> values as this <see cref="Point"/>; otherwise, <code>false</code>.</returns>
public override bool Equals(object? obj) => obj is Point point && Equals(point);
/// <summary>
/// Returns the hash code for this <see cref="Point"/>.
/// </summary>
/// <returns>The hash code for this <see cref="Point"/> structure.</returns>
public override int GetHashCode()
{
int hashCode = 1861411795;
hashCode = hashCode * -1521134295 + X.GetHashCode();
hashCode = hashCode * -1521134295 + Y.GetHashCode();
return hashCode;
}
/// <summary>
/// Compares two <see cref="Point"/> structures for equality.
/// </summary>
/// <param name="point1">The first <see cref="Point"/> structure to compare.</param>
/// <param name="point2">The second <see cref="Point"/> structure to compare.</param>
/// <returns><code>true</code> if both the <see cref="X"/> and <see cref="Y"/> coordinates of <paramref name="point1"/> and <paramref name="point2"/> are equal; otherwise, <code>false</code>.</returns>
public static bool operator ==(Point point1, Point point2) => point1.Equals(point2);
/// <summary>
/// Compares two <see cref="Point"/> structures for inequality.
/// </summary>
/// <param name="point1">The first <see cref="Point"/> structure to compare.</param>
/// <param name="point2">The second <see cref="Point"/> structure to compare.</param>
/// <returns><code>true</code> if <paramref name="point1"/> and <paramref name="point2"/> have different <see cref="X"/> or <see cref="Y"/> coordinates; <code>false</code> if <paramref name="point1"/> and <paramref name="point2"/> have the same <see cref="X"/> and <see cref="Y"/> coordinates.</returns>
/* Unmerged change from project 'FaceRecognition(netstandard2.0)'
Before:
public static bool operator !=(Point point1, Point point2)
{
return !(point1 == point2);
}
After:
public static bool operator !=(Point point1, Point point2) => !(point1 == point2);
*/
public static bool operator !=(Point point1, Point point2) => !(point1 == point2);
#endregion
#endregion
}