Feature/annotator

server/ERNI.PhotoDatabase.Annotator/.gitattributes

@@ -0,0 +1 @@
+assets/Model/yolov4.onnx filter=lfs diff=lfs merge=lfs -text

server/ERNI.PhotoDatabase.Annotator/AnnotationPredictor.cs

@@ -0,0 +1,87 @@
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Drawing.Drawing2D;
+using System.Linq;
+using Microsoft.ML;
+using ERNI.PhotoDatabase.Annotator.YoloParser;
+using ERNI.PhotoDatabase.Annotator.Utils;
+using System.Drawing.Imaging;
+
+namespace ERNI.PhotoDatabase.Annotator
+{
+    public class AnnotationPredictor
+    {
+        public (string[], byte[]) MakePrediction(Bitmap bmp)
+        {
+            MLContext mlContext = new MLContext();
+            List<string> tags = new List<string>();
+            var bmpWithBoxes = bmp.Clone(new RectangleF(0, 0, bmp.Width, bmp.Height), bmp.PixelFormat);
+            try
+            {
+                var modelScorer = new OnnxModelScorer(FileUtils.ModelFilePath, mlContext);
+                var prediction = modelScorer.Score(bmp);
+
+                YoloOutputParser parser = new YoloOutputParser();
+                var boxes = parser.ParseOutputs(prediction);
+
+                IList<YoloBoundingBox> detectedObjects = parser.NonMaxSuppression(boxes, 10, .5F); ;
+                DrawBoundingBox(ref bmpWithBoxes, detectedObjects);
+
+                tags = detectedObjects.Select(_ => _.Label).Distinct().ToList();
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine(ex.ToString());
+            }
+
+            return (tags.ToArray(), bmpWithBoxes.ToByteArray(ImageFormat.Jpeg));
+        }
+
+        private static void DrawBoundingBox(ref Bitmap image, 
+                                            IList<YoloBoundingBox> filteredBoundingBoxes)
+        {
+            var originalImageHeight = image.Height;
+            var originalImageWidth = image.Width;
+
+            foreach (var box in filteredBoundingBoxes)
+            {
+                var x = (uint)Math.Max(box.Dimensions.X1, 0);
+                var y = (uint)Math.Max(box.Dimensions.Y1, 0);
+                var width = (uint)Math.Min(originalImageWidth - x, box.Dimensions.Width);
+                var height = (uint)Math.Min(originalImageHeight - y, box.Dimensions.Height);
+
+                x = (uint)originalImageWidth * x / Yolov4ModelSettings.ImageSettings.imageWidth;
+                y = (uint)originalImageHeight * y / Yolov4ModelSettings.ImageSettings.imageHeight;
+                width = (uint)originalImageWidth * width / Yolov4ModelSettings.ImageSettings.imageWidth;
+                height = (uint)originalImageHeight * height / Yolov4ModelSettings.ImageSettings.imageHeight;
+
+                string text = $"{box.Label} ({box.Confidence * 100:0}%)";
+
+                using (Graphics thumbnailGraphic = Graphics.FromImage(image))
+                {
+                    thumbnailGraphic.CompositingQuality = CompositingQuality.HighQuality;
+                    thumbnailGraphic.SmoothingMode = SmoothingMode.HighQuality;
+                    thumbnailGraphic.InterpolationMode = InterpolationMode.HighQualityBicubic;
+
+                    // Define Text Options
+                    Font drawFont = new Font("Arial", 12, FontStyle.Bold);
+                    SizeF size = thumbnailGraphic.MeasureString(text, drawFont);
+                    SolidBrush fontBrush = new SolidBrush(Color.Black);
+                    Point atPoint = new Point((int)x, (int)y - (int)size.Height - 1);
+
+                    // Define BoundingBox options
+                    Pen pen = new Pen(box.BoxColor, 3.2f);
+                    SolidBrush colorBrush = new SolidBrush(box.BoxColor);
+
+                    thumbnailGraphic.FillRectangle(colorBrush, (int)x, (int)(y - size.Height - 1), (int)size.Width, (int)size.Height);
+
+                    thumbnailGraphic.DrawString(text, drawFont, fontBrush, atPoint);
+
+                    // Draw bounding box on image
+                    thumbnailGraphic.DrawRectangle(pen, x, y, width, height);
+                }
+            }
+        }
+    }
+}

server/ERNI.PhotoDatabase.Annotator/DataStructures/ImageNetPrediction.cs

@@ -0,0 +1,25 @@
+using Microsoft.ML.Data;
+
+namespace ERNI.PhotoDatabase.Annotator.DataStructures
+{
+    public class ImageNetPrediction
+    {
+        [VectorType(1, 52, 52, 3, 85)]
+        [ColumnName(Yolov4ModelSettings.Output_1)]
+        public float[] Output_1;
+
+        [VectorType(1, 26, 26, 3, 85)]
+        [ColumnName(Yolov4ModelSettings.Output_2)]
+        public float[] Output_2;
+
+        [VectorType(1, 52, 52, 3, 85)]
+        [ColumnName(Yolov4ModelSettings.Output_3)]
+        public float[] Output_3;
+
+        [ColumnName("width")]
+        public float ImageWidth { get; set; }
+
+        [ColumnName("height")]
+        public float ImageHeight { get; set; }
+    }
+}

server/ERNI.PhotoDatabase.Annotator/DataStructures/InputPicture.cs

@@ -0,0 +1,19 @@
+using Microsoft.ML.Data;
+using Microsoft.ML.Transforms.Image;
+using System.Drawing;
+
+namespace ERNI.PhotoDatabase.Annotator.DataStructures
+{
+    public class InputPicture
+    {
+        [ColumnName("bitmap")]
+        [ImageType(Yolov4ModelSettings.ImageSettings.imageHeight, Yolov4ModelSettings.ImageSettings.imageWidth)]
+        public Bitmap Image { get; set; }
+
+        [ColumnName("width")]
+        public float ImageWidth => Image.Width;
+
+        [ColumnName("height")]
+        public float ImageHeight => Image.Height;
+    }
+}

server/ERNI.PhotoDatabase.Annotator/ERNI.PhotoDatabase.Annotator.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.2</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.ML" Version="1.5.2" />
+    <PackageReference Include="Microsoft.ML.ImageAnalytics" Version="1.5.2" />
+    <PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.5.2" />
+    <PackageReference Include="Microsoft.ML.OnnxTransformer" Version="1.5.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ERNI.PhotoDatabase.DataAccess\ERNI.PhotoDatabase.DataAccess.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="assets\Model\yolov4.onnx">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+</Project>

server/ERNI.PhotoDatabase.Annotator/OnnxModelScorer.cs

@@ -0,0 +1,57 @@
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.ML;
+
+using ERNI.PhotoDatabase.Annotator.DataStructures;
+using System.Drawing;
+
+namespace ERNI.PhotoDatabase.Annotator
+{
+    public class OnnxModelScorer
+    {
+        private readonly string modelLocation;
+        private readonly MLContext mlContext;
+
+        public OnnxModelScorer(string modelLocation, MLContext mlContext)
+        {
+            this.modelLocation = modelLocation;
+            this.mlContext = mlContext;
+        }
+
+        private ITransformer LoadModel(string modelLocation)
+        {
+            var data = mlContext.Data.LoadFromEnumerable(new List<InputPicture>());
+
+            var pipeline = mlContext.Transforms.ResizeImages(outputColumnName: Yolov4ModelSettings.Input,
+                                                            imageWidth: Yolov4ModelSettings.ImageSettings.imageWidth,
+                                                            imageHeight: Yolov4ModelSettings.ImageSettings.imageHeight,
+                                                            inputColumnName: "bitmap",
+                                                            resizing: Microsoft.ML.Transforms.Image.ImageResizingEstimator.ResizingKind.IsoPad)
+                .Append(mlContext.Transforms.ExtractPixels(outputColumnName: Yolov4ModelSettings.Input,
+                                                            scaleImage: 1f / 255f, interleavePixelColors: true))
+                .Append(mlContext.Transforms.ApplyOnnxModel(modelFile: modelLocation,
+                                                            outputColumnNames: Yolov4ModelSettings.ModelOutputNames,
+                                                            inputColumnNames: Yolov4ModelSettings.ModelInputName,
+                                                            shapeDictionary: Yolov4ModelSettings.Shape));
+
+            var model = pipeline.Fit(data);
+
+            return model;
+        }
+
+        private ImageNetPrediction PredictDataUsingModel(Bitmap picture, ITransformer model)
+        {
+            var predictionEngine = mlContext.Model.CreatePredictionEngine<InputPicture, ImageNetPrediction>(model);
+            var prediction = predictionEngine.Predict(new InputPicture { Image = picture });
+
+            return prediction;
+        }
+
+        public ImageNetPrediction Score(Bitmap picture)
+        {
+            var model = LoadModel(modelLocation);
+
+            return PredictDataUsingModel(picture, model);
+        }
+    }
+}

server/ERNI.PhotoDatabase.Annotator/PhotoAnnotator.cs

@@ -0,0 +1,23 @@
+using System.Drawing;
+using System.IO;
+
+namespace ERNI.PhotoDatabase.Annotator
+{
+    public class PhotoAnnotator
+    {
+        public (string[], byte[]) AnnotatePhoto(byte[] photoData)
+        {
+            Bitmap bmp;
+            string[] tags;
+            byte[] pictureData;
+
+            using (var ms = new MemoryStream(photoData))
+            {
+                bmp = new Bitmap(ms);
+                var predictor = new AnnotationPredictor();
+                (tags, pictureData) = predictor.MakePrediction(bmp);
+            }
+            return (tags, pictureData);
+        }
+    }
+}

server/ERNI.PhotoDatabase.Annotator/Utils/Extensions.cs

@@ -0,0 +1,18 @@
+using System.Drawing;
+using System.Drawing.Imaging;
+using System.IO;
+
+namespace ERNI.PhotoDatabase.Annotator.Utils
+{
+    public static class Extensions
+    {
+        public static byte[] ToByteArray(this Image image, ImageFormat format)
+        {
+            using (MemoryStream ms = new MemoryStream())
+            {
+                image.Save(ms, format);
+                return ms.ToArray();
+            }
+        }
+    }
+}

server/ERNI.PhotoDatabase.Annotator/Utils/FileUtils.cs

@@ -0,0 +1,21 @@
+using System.IO;
+
+namespace ERNI.PhotoDatabase.Annotator.Utils
+{
+    public class FileUtils
+    {
+        public static string AssetsRelativePath = @"../../../../ERNI.PhotoDatabase.Annotator/assets";
+        public static string AssetsPath = GetAbsolutePath(AssetsRelativePath);
+        public static string ModelFilePath = Path.Combine(AssetsPath, "Model", "yolov4.onnx");
+
+        public static string GetAbsolutePath(string relativePath)
+        {
+            FileInfo _dataRoot = new FileInfo(typeof(AnnotationPredictor).Assembly.Location);
+            string assemblyFolderPath = _dataRoot.Directory.FullName;
+
+            string fullPath = Path.Combine(assemblyFolderPath, relativePath);
+
+            return fullPath;
+        }
+    }
+}

server/ERNI.PhotoDatabase.Annotator/YoloParser/DimensionsBase.cs

@@ -0,0 +1,20 @@
+namespace ERNI.PhotoDatabase.Annotator.YoloParser
+{
+    public class Dimensions
+    {
+        public float X1 { get; set; }
+        public float Y1 { get; set; }
+        public float X2 { get; set; }
+        public float Y2 { get; set; }
+
+        public float Width 
+        {
+            get => X2 - X1;
+        }
+
+        public float Height
+        {
+            get => Y2 - Y1;
+        }
+    }
+}

server/ERNI.PhotoDatabase.Annotator/YoloParser/YoloBoundingBox.cs

@@ -0,0 +1,20 @@
+using System.Drawing;
+
+namespace ERNI.PhotoDatabase.Annotator.YoloParser
+{
+    public class YoloBoundingBox
+    {
+        public Dimensions Dimensions { get; set; }
+
+        public string Label { get; set; }
+
+        public float Confidence { get; set; }
+
+        public RectangleF Rect
+        {
+            get { return new RectangleF(Dimensions.X1, Dimensions.Y1, Dimensions.Width, Dimensions.Height); }
+        }
+
+        public Color BoxColor { get; set; }
+    }
+}