Merge pull request #185 from Microsoft/dev

1.5 Release Dev

.gitignore

@@ -29,6 +29,7 @@ build/Release
 # Dependency directories
 node_modules
 jspm_packages
+data_sample
 
 # Optional npm cache directory
 .npm

README.md

@@ -1,16 +1,16 @@
-# VoTT: Visual Object Tagging Tool
+# VoTT: Visual Object Tagging Tool 1.5 
 
 This tool provides end to end support for generating datasets and validating object detection models from video and image assets.
 
 ### End to End Object Detection Pipeline:
-![Pipeline: tag video, export tags to CNTK, train model, run model on a new video, validate model suggestions and fix errors, return to export tags](media/detectioninabox.jpg)
+![Pipeline: tag video, export tags to format of choice, train model, run model on a new video, validate model suggestions and fix errors, return to export tags](media/detectioninabox.jpg)
 
 The tool supports the following **features**:
 
 - The ability to tag and annotate Image Directories or Stand alone videos.
 - Computer-assisted tagging and tracking of objects in videos using the [Camshift tracking algorithm](http://opencv.jp/opencv-1.0.0_org/docs/papers/camshift.pdf).
-- Exporting tags and assets to CNTK , Tensorflow (PascalVOC) or YOLO format for training an object detection model.
-- Running and validating a trained CNTK object detection model on new videos to generate stronger models.
+- Exporting tags and assets to Custom Vision Service CNTK , Tensorflow (PascalVOC) or YOLO format for training an object detection model.
+- Use Active Learning with trained object detection models (locally or remotely) on new videos to generate stronger models.
 
 ## Table of Contents
 
@@ -24,12 +24,24 @@ The tool supports the following **features**:
 ---
 ## Installation
 
-### Installing the Visual Object Tagging Tool
+### Installing the Visual Object Tagging Tool Binary
 
  1. Download and extract the app [release package](https://github.com/CatalystCode/CNTK-Object-Detection-Video-Tagging-Tool/releases)
 
  2. Run the app by launching the "VOTT" executable which will be located inside the unzipped folder.
 
+ ### Installing the Visual Object Tagging Tool npm
+
+ 1. Clone this directory 
+
+ 2. Install Node.js
+
+ 3. Run the following command in the terminal.
+    ```
+        npm install
+        npm start
+    ```
+
  ### Installing CNTK with the FRCNN Prerequisites for Reviewing Model
 
 *Please note that installation of **CNTK and FASTER-RCNN dependencies** are **optional for tagging** and are **only required for CNTK model review and training**.*
@@ -78,9 +90,6 @@ The tool supports the following **features**:
     ![](media/4_Tagging_Job.jpg)
 
     **Tagging**: click and drag a bounding box around the desired area, then move or resize the region until it fits the object
-     - Selected regions appear as red ![red](https://placehold.it/15/f03c15/000000?text=+) and unselected regions will appear as blue ![#1589F0](https://placehold.it/15/1589F0/000000?text=+).
-     - Assign a tag to a region by clicking on it and selecting the desired tag from the labeling toolbar at the bottom of the tagging control
-     - Click the ![cleartags](media/cleartags.png) button to clear all tags on a given frame
 
     **Navigation**: users can navigate between video frames by using the ![prev-nxt](media/prev-next.png) buttons, the left/right arrow keys, or the video skip bar
      - Tags are auto-saved each time a frame is changed
@@ -104,7 +113,7 @@ The tool supports the following **features**:
       - *Last Tagged Region*: exports frames up until the last frame containing tags
       - *Last Visited Frame*: exports frames up until the last frame that the user explicitly visited
       - *Last Frame*: exports all video frames<br>
-    - **Output directory**: directory path for exporting training data<br>
+    - **Training Path/Key**: directory path for exporting training data or the custom vision service training key<br>
 
 ---
 
@@ -160,7 +169,16 @@ The tool supports the following **features**:
     - **Output directory**: directory path for exporting training data<br>
 
 ---
-## Reviewing and Improving an Object Detection Model
+## Active Learning and Improving an Object Detection Model
+
+There are two options to run a model for active learning within VoTT one is to use the local reviewer interface and the other is to provide an endpoint. Providing an endpoint reduces the amount of manual configuration work necessary to get your model running so this is the prefered method but for certain scenarios using the interface might still be relevant. Below is an example of both scenarios. 
+
+### Remote Active learning using Docker 
+
+1. Set up your own remote model endpoint locally or on Azure with docker [CNTK Example](https://github.com/User1m/vott-reviewer-ext)
+2. Paste the review service endpoint to review your model.
+
+### Local Active Learning CNTK Example
 
  1. Train model with [Object Detection using FasterRCNN](https://docs.microsoft.com/en-us/cognitive-toolkit/object-detection-using-faster-r-cnn#run-faster-r-cnn-on-your-own-data)<br> 
  2. Since CNTK does not embed the names of the classes in the model, on default, the module returns non descriptive names for the classes, e.g. "class_1", "class_2". To resolve this copy the class_map.txt file generated by the FASTER-RCNN tutorial to the same directory that contains your model.   
@@ -184,7 +202,7 @@ In the latest release we provide support for [Export and Review formats](https:/
 - Altenative Tracking algorithms such as optical flow.
 - Classification Labeling Support
 - Segmentation Annoatation support.
-
+- Zoom in and out
 -----------
 
 ## How to Contribute

main.js

@@ -50,9 +50,9 @@ function createWindow () {
     menu.items[p].submenu.items[3].enabled = true;
     menu.items[p+1].submenu.items[0].enabled = true;
     menu.items[p+1].submenu.items[1].enabled = true;
+    // menu.items[p+1].submenu.items[2].enabled = true;
   });
 
-
   // do this independently for each object
   ipcMain.on('show-popup', function(event, arg) { 
       let popup = new BrowserWindow({
@@ -81,7 +81,16 @@ function createWindow () {
             slashes: true
           }));
           break;
-
+
+        case "review-endpoint":
+          popup.setSize(359, 150);
+          popup.loadURL(url.format({
+            pathname: path.join(__dirname, 'src/public/html/review-endpoint-configuration.html'),
+            protocol: 'file:',
+            slashes: true
+          }));
+          break;
+
         default: return; 
       }
 
@@ -100,6 +109,10 @@ function createWindow () {
     mainWindow.send('review-model', arg);
   });
 
+  ipcMain.on('review-model-endpoint', (event, arg) => {
+    mainWindow.send('review-model-endpoint', arg);
+  });
+
   mainWindow.on('ready-to-show', function() {
       mainWindow.show();
       mainWindow.focus();
@@ -152,8 +165,8 @@ function createWindow () {
           click () { mainWindow.webContents.send('export'); }
         },
         {
-          label: 'Review Detection Model',
-          accelerator: 'CmdOrCtrl+R',
+          label: 'Active Learning',
+          accelerator: 'CmdOrCtrl+A',
           enabled: false,
           click () { mainWindow.webContents.send('review'); }
         }
@@ -175,7 +188,7 @@ function createWindow () {
       label: 'Debug',
       submenu: [
         {
-          label: 'Toggle Developer Tools',
+          label: 'Developer Console',
           accelerator: 'CmdOrCtrl+D',
           click () { mainWindow.webContents.toggleDevTools(); }
         },

src/index.js

@@ -37,7 +37,7 @@ ipcRenderer.on('saveVideo', (event, message) => {
 ipcRenderer.on('export', (event, message) => {
    var args = {
      type : "export",
-     supportedFormats : detection.detectionAlgorithmManager.getAvailbleAlgorthims(),
+     supportedFormats : detection.detectionAlgorithmManager.getAvailbleExporters(),
      assetFolder : assetFolder
    };
 
@@ -57,28 +57,57 @@ ipcRenderer.on('export-tags', (event, exportConfig) => {
 ipcRenderer.on('review', (event, message) => {
     var args = {
       type: 'review',
-      supportedFormats : detection.detectionAlgorithmManager.getAvailbleAlgorthims(),
+      supportedFormats : detection.detectionAlgorithmManager.getAvailbleReviewers(),
       assetFolder : assetFolder
     };
     ipcRenderer.send('show-popup', args);
 });
 
+ipcRenderer.on('reviewEndpoint', (event, message) => {
+  var args = {
+    type: 'review-endpoint',
+  };
+  ipcRenderer.send('show-popup', args);
+});
+
 ipcRenderer.on('review-model', (event, reviewModelConfig) => {
   var modelLocation = reviewModelConfig.modelPath;
   if (fs.existsSync(modelLocation)) {
     addLoader();
-    detection.review( videotagging.imagelist, reviewModelConfig.modelFormat, modelLocation, reviewModelConfig.output, () => {
+    var colors_back = videotagging.optionalTags.colors;
+    videotagging.optionalTags.colors = null;
+
+    detection.review(videotagging.imagelist, reviewModelConfig.modelFormat, modelLocation, reviewModelConfig.output, (err) => {
+        if (err){
+          alert(`An error occured with Local Active Learning \n Please check the debug console for more information.`);
+          videotagging.optionalTags.colors = colors_back;
+        }
         if(!videotagging.imagelist){
           videotagging.video.oncanplay = updateVisitedFrames;
         }      
         $(".loader").remove();        
     });
-  } else {
+  }
+   else {
       alert(`No model found! Please make sure you put your model in the following directory: ${modelLocation}`)
   }
 
 });
 
+ipcRenderer.on('review-model-endpoint', (event, reviewModelConfig) => {
+    addLoader();
+    detection.reviewEndpoint( videotagging.imagelist, reviewModelConfig.endpoint, (err) => {
+      if (err){
+        alert(`An error occured with Remote Active Learning \n Please check the debug console for more information.`);
+      }
+      if(!videotagging.imagelist){
+        videotagging.video.oncanplay = updateVisitedFrames;
+      }      
+      $(".loader").remove();        
+    });    
+});
+
+
 ipcRenderer.on('toggleTracking', (event, message) => {
   if (trackingEnabled) {
     trackingExtension.stopTracking();
@@ -232,25 +261,27 @@ function openPath(pathName, isDir) {
     function loadTagger (e) {
       if(framerate.validity.valid && inputtags.validity.valid) {
         $('.bootstrap-tagsinput').last().removeClass( "invalid" );
-
+             
         videotagging = document.getElementById('video-tagging'); //find out why jquery doesn't play nice with polymer
         videotagging.regiontype = $('#regiontype').val();
         videotagging.multiregions = 1;
         videotagging.regionsize = $('#regionsize').val();
         videotagging.inputtagsarray = $('#inputtags').val().replace(/\s/g,'').split(',');
         videotagging.video.currentTime = 0;
         videotagging.framerate = $('#framerate').val();
+        videotagging.src = ''; // ensures reload if user opens same video 
 
-        if (config) {
-          videotagging.inputframes = config.frames;
+        if (config) {  
+          if (config.tag_colors){
+            videotagging.optionalTags.colors = config.tag_colors;
+          }
+            videotagging.inputframes = config.frames;
           visitedFrames = new Set(config.visitedFrames);
         } else {
           videotagging.inputframes = {};
            visitedFrames =  (isDir) ? new Set([0]) : new Set();
         } 
 
-        videotagging.src = ''; // ensures reload if user opens same video 
-
         if (isDir){
             $('title').text(`Image Tagging Job: ${path.dirname(pathName)}`); //set title indicator
 
@@ -332,6 +363,7 @@ function save() {
       "suggestiontype": $('#suggestiontype').val(),
       "scd": document.getElementById("scd").checked,
       "visitedFrames": Array.from(visitedFrames),
+      "tag_colors" : videotagging.optionalTags.colors,
     };
     //if nothing changed don't save
     if (saveState === JSON.stringify(saveObject) ) {

src/lib/detection_algorithm_manager/main.js

@@ -5,25 +5,34 @@ const detection_algorithms_path = path.join(__dirname,'../detection_algorithms')
 const detection_algorithms_dirs = fs.readdirSync(detection_algorithms_path)
                                     .filter(file => (fs.statSync(path.join(detection_algorithms_path, file)).isDirectory()));
 
-var detection_modules = {};
+var review_modules = {};
+var export_modules = {};
 
 detection_algorithms_dirs.forEach((dir) => {
     var dm = require(path.join(detection_algorithms_path, dir));
     if (dm.displayName) {
-        detection_modules[dm.displayName] = dm;
+        if (dm.Reviewer){
+            review_modules[dm.displayName] = dm;
+        }
+        if (dm.Exporter){
+            export_modules[dm.displayName] = dm;
+        }
     }
 });
 
 function DetectionAlgorithmManager() {
     var self = this;
     //this returns a list of the availble detection modules
-    this.getAvailbleAlgorthims = function getAvailbleAlgorthims() {
-        return Object.keys(detection_modules);
+    this.getAvailbleReviewers = function getAvailbleReviewers() {
+        return Object.keys(review_modules);
+    },
+    this.getAvailbleExporters = function getAvailbleExporters() {
+        return Object.keys(export_modules);
     },
 
     //Set the  exporter to the specified detection module
     this.initExporter = function(algorithm, exportDirPath, classes, posFramesCount, frameWidth, frameHeight, testSplit, cb) {
-         var exporter = new detection_modules[algorithm].Exporter(exportDirPath, classes, posFramesCount, frameWidth, frameHeight, testSplit);
+         var exporter = new export_modules[algorithm].Exporter(exportDirPath, classes, posFramesCount, frameWidth, frameHeight, testSplit);
          exporter.init().then(()=> {
              return cb(null, exporter.exportFrame);         
          }).catch((err) =>{
@@ -32,7 +41,7 @@ function DetectionAlgorithmManager() {
     },
 
     this.initReviewer = function (algorithm, modelPath, cb) {
-        var reviewer = new detection_modules[algorithm].Reviewer(modelPath);
+        var reviewer = new review_modules[algorithm].Reviewer(modelPath);
         return cb(reviewer.reviewImagesFolder);
     }
 

src/lib/detection_algorithms/cntkfastercnn/reviewer.js

@@ -55,14 +55,18 @@ function Reviewer(modelPath) {
 	//      }
     // }
     this.reviewImagesFolder = function reviewImagesFolder(imagesFolderPath) {
-		var model = new cntkModel.CNTKFRCNNModel({cntkModelPath : self.modelPath, cntkPath: cntkConfig.cntkPath, anacondaPath: cntkConfig.anacondaPath, verbose : true});
 		return new Promise((resolve, reject) => { 
+			try {
+				var model = new cntkModel.CNTKFRCNNModel({cntkModelPath : self.modelPath, cntkPath: cntkConfig.cntkPath, anacondaPath: cntkConfig.anacondaPath, verbose : true})
+			} catch(err) {
+				console.info(err);				
+				reject(err);
+			}
 			model.evaluateDirectory(imagesFolderPath, (err, res) => {
 				if (err) {
 					console.info(err);
-					reject();
+					reject(err);
 				}
-				console.log(res);
 				resolve(res);
 			});
 		});