[MERGE SECOND!] Ridiculous implementation of the Show and Tell model

.gitattributes

@@ -0,0 +1,7 @@
+src/main/resources/model filter=lfs diff=lfs merge=lfs -text
+src/main/resources/model/variables/variables.index filter=lfs diff=lfs merge=lfs -text
+src/main/resources/model/variables/variables.data-00000-of-00001 filter=lfs diff=lfs merge=lfs -text
+src/main/resources/model/saved_model.pbtxt filter=lfs diff=lfs merge=lfs -text
+src/main/resources/full_model/ckpt5.data-00000-of-00001 filter=lfs diff=lfs merge=lfs -text
+src/main/resources/full_model/ckpt5.index filter=lfs diff=lfs merge=lfs -text
+src/main/resources/full_model/ckpt5.meta filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -144,3 +144,6 @@ bh_unicode_properties.cache
 # Sublime-github package stores a github token in this file
 # https://packagecontrol.io/packages/sublime-github
 GitHub.sublime-settings
+
+src/main/resources/full_model
+src/main/resources/__*

README.md

@@ -2,25 +2,77 @@
 
 A native caption generation application using the Show and Tell model.
 
-![A screenshot of the MVP app in action! It shows a giraffe with the caption "This is definitely an image of something".](doc/mvp_screenshot.png)
+![A screenshot of the TF app in action! It shows a giraffe with the caption "a giraffe standing in the middle of a forest".](doc/v3.0.0_screenshot.png)
 
 ## Quick Start
 
-This app requires [Maven](https://maven.apache.org/install.html) for setup.
+[![Demonstration of installing, building, and running the application.](https://img.youtube.com/vi/tB1WtMTo-Yo/0.jpg)](https://www.youtube.com/watch?v=tB1WtMTo-Yo)
 
-First, build the app:
+This application works best on OS X, but it is compatible with multiple
+platforms. In order to run the installation scripts on any system, several
+dependencies are required:
+
+**Required Dependencies (all systems):**
+
+* [JDK 8](https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html)
+* [Maven](https://maven.apache.org/)
+* [Tensorflow-compatible Python](https://www.python.org/) (we recommend 3.6 or 2.7)
+* [Bazel](https://bazel.build/)
+
+### OS X (Recommended)
+
+**Required OS X Dependencies:**
+
+* [Homebrew package manager](https://brew.sh/)
+
+This application was built and tested primarily on OS X machines, so we
+recommend OS X for the smoothest build/run process.
+
+To install and build the application, run:
+
+```bash
+bin/install.osx.sh && bin/build.sh
+```
+
+To start the application, run:
+
+```bash
+bin/run.sh
+```
+
+### Ubuntu
+
+**Required Ubuntu Dependencies:**
+
+* `apt` package manager
+* sudo access (for dependency installation)
+
+The installation process for Ubuntu is similar for OS X.
+
+To install and build the application, run:
 
 ```bash
-mvn package
+bin/install.ubuntu.sh && bin/build.sh
 ```
 
-Next, launch the app with Maven:
+To start the application, run:
 
 ```bash
-mvn exec:java
+bin/run.sh
 ```
 
-Voilà, the app is running!
+### Other Operating System (e.g. Windows, Mint, Debian, etc.)
+
+We don't provide built-in support for other operating systems, but that doesn't
+mean that the application won't work! Using `bin/install.osx.sh` and
+`bin/build.sh` as a guide, install the dependencies and build the project
+manually.
+
+After that, you can start the application with:
+
+```bash
+bin/run.sh
+```
 
 ## Validation and Testing
 

bin/build.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+mvn clean
+mvn compile
+
+cd src/main/python
+bazel clean
+bazel build -c opt //im2txt:run_inference
+cd ../../..

bin/install.osx.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+echo "This script assumes that you have the Homebrew package manager installed."
+echo "If you don't, please either install it (https://brew.sh/) or manually"
+echo "install Bazel (https://bazel.build/) and Maven (https://maven.apache.org/)."
+echo ""
+echo "If you want to use a python virtual environment, stop this script and"
+echo "activate it now."
+echo ""
+
+echo "=========================="
+echo "Downloading model files..."
+echo "=========================="
+curl -L https://www.dropbox.com/s/vz7ydmtpnyb8cs1/full_model.zip?dl=0 > \
+  src/main/resources/full_model.zip
+unzip src/main/resources/full_model.zip -d src/main/resources/
+
+echo "============================="
+echo "Installing bazel and maven..."
+echo "============================="
+brew install bazel
+brew install maven
+
+echo ""
+echo "============================="
+echo "Installing python packages..."
+echo "============================="
+pip install tensorflow
+pip install numpy
+pip install nltk
+
+echo ""
+echo "============================"
+echo "Installing java packages..."
+echo "============================"
+mvn validate
+
+echo ""
+echo "=========================="
+echo "Verifying installations..."
+echo "=========================="
+bazel version
+mvn --version
+pip --version

bin/install.ubuntu.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+echo "This script assumes that you have the apt-get package manager installed."
+echo "If you don't, please either install it or manually install the dependencies"
+echo "Bazel (https://bazel.build/) and Maven (https://maven.apache.org/)."
+echo ""
+echo "If you want to use a python virtual environment, stop this script and"
+echo "activate it now."
+echo ""
+
+echo "============================="
+echo "Installing bazel and maven..."
+echo "============================="
+sudo apt-get update -y
+sudo apt-get upgrade -y
+# Bazel
+sudo apt-get install pkg-config zip g++ zlib1g-dev unzip python
+chmod +x bazel-0.17.2-installer-linux-x86_64.sh
+./bazel-0.17.2-installer-linux-x86_64.sh --user
+export PATH="$PATH:$HOME/bin"
+
+# Maven
+cd /opt/
+wget http://www-eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
+sudo tar -xvzf apache-maven-3.3.9-bin.tar.gz
+rm apache-maven-3.3.9-bin.tar.gz
+sudo mv apache-maven-3.3.9 maven
+# env vars
+sudo echo "export M2_HOME=/opt/maven" > /etc/profile.d/mavenenv.sh
+sudo echo "export PATH=\${M2_HOME}/bin:\${PATH}" > /etc/profile.d/mavenenv.sh
+sudo chmod +x /etc/profile.d/mavenenv.sh
+sudo source /etc/profile.d/mavenenv.sh
+
+echo ""
+echo "============================="
+echo "Installing python packages..."
+echo "============================="
+pip install tensorflow
+pip install numpy
+pip install nltk
+
+echo ""
+echo "============================"
+echo "Installing java packages..."
+echo "============================"
+mvn validate
+
+echo ""
+echo "=========================="
+echo "Verifying installations..."
+echo "=========================="
+bazel version
+mvn --version
+pip --version

bin/run.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+mvn exec:java

pom.xml

@@ -6,7 +6,7 @@
 
   <groupId>com.cmsc495.groupfantastic</groupId>
   <artifactId>what-does-it-mean</artifactId>
-  <version>1.0.0</version>
+  <version>3.0.0</version>
   <packaging>jar</packaging>
 
 
@@ -105,6 +105,11 @@
       <artifactId>tensorflow</artifactId>
       <version>1.10.0</version>
     </dependency>
+    <dependency>
+      <groupId>org.tensorflow</groupId>
+      <artifactId>proto</artifactId>
+      <version>1.10.0</version>
+    </dependency>
 
     <!-- Testing -->
     <dependency>

src/main/java/CaptionGenerator.java

@@ -10,84 +10,66 @@
 
 import java.awt.image.BufferedImage;
 import java.io.BufferedReader;
-import java.io.FileReader;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import java.io.InputStreamReader;
 
 /**
  * Generates a caption from a stored image.
  */
 class CaptionGenerator {
-  private String[] captions;
 
   /**
    * Constructs a new CaptionGenerator with canned captions.
    */
   CaptionGenerator() {
-    String captionsPath = "./src/main/resources/canned_captions.txt";
-    this.captions = readLines(captionsPath);
-  }
-
-  /**
-   * Reads a file and returns the lines, one line per array entry.
-   *
-   * @param filename input file path
-   * @return lines in file
-   */
-  private String[] readLines(String filename) {
-    List<String> lines = new ArrayList<>();
-    String line;
-
-    // open the file and read in lines
-    try (BufferedReader br = new BufferedReader(new FileReader(filename))) {
-      while ((line = br.readLine()) != null) {
-        lines.add(line);
-      }
-    } catch (IOException err) {
-      err.printStackTrace();
-    }
-
-    return lines.toArray(new String[0]);
   }
 
   /**
    * Returns a relevant, readable caption describing a given image.
    *
    * @return image caption
    */
-  String generateCaption(BufferedImage image) {
-    if (image == null) {
-      throw new IllegalArgumentException("Input must be a valid image.");
-    }
+  String generateCaption(String filename) throws IOException {
+    //    if (image == null) {
+    //      throw new IllegalArgumentException("Input must be a valid image.");
+    //    }
 
-    // choose a caption (pre-Tensorflow)
-    int digest = hash(image);
-    int idx = Math.abs(digest % captions.length);
+    String checkpointPath = "src/main/resources/full_model/ckpt5";
+    String vocabFile = "src/main/resources/word_counts.txt";
+    String imageFile = filename
+        .replace("\\", "\\\\")
+        .replace(" ", "\\ ");
 
-    return captions[idx];
-  }
+    String str;
+    String cmd = String.format("src/main/python/bazel-bin/im2txt/run_inference \\\n"
+            + "  --checkpoint_path=%s \\\n"
+            + "  --vocab_file=%s \\\n"
+            + "  --input_files=%s",
+        checkpointPath, vocabFile, imageFile);
+    StringBuilder caption = new StringBuilder();
+    System.out.println("Captioning image with the following command:");
+    System.out.println(cmd);
 
-  /**
-   * Naive hash to identify the image content.
-   *
-   * <p>This is done by simply summing up the pixel values in the image. This
-   * is definitely not a secure method, but it will return the same value if two
-   * images are identical and (likely) return different values for different
-   * images.
-   *
-   * <p>All this code will be deleted once we have Tensorflow integration.
-   *
-   * @param image image input
-   * @return integer digest
-   */
-  private int hash(BufferedImage image) {
-    int sum = 0;
-    for (int i = 0; i < image.getWidth(); i++) {
-      for (int j = 0; j < image.getHeight(); j++) {
-        sum += image.getRGB(i,j);
+    try {
+      Process proc = Runtime.getRuntime().exec(cmd);
+
+      // read the output from the command
+      BufferedReader stdInput = new BufferedReader(new InputStreamReader(proc.getInputStream()));
+      System.out.println("Here is the standard output of the command:\n");
+      while ((str = stdInput.readLine()) != null) {
+        caption.append(str);
+        System.out.println(str);
       }
+
+      BufferedReader stdError = new BufferedReader(new InputStreamReader(proc.getErrorStream()));
+      System.out.println("Here is the error output of the command:\n");
+      while ((str = stdError.readLine()) != null) {
+        System.out.println(str);
+      }
+
+    } catch (IOException ex) {
+      ex.printStackTrace();
     }
-    return Math.abs(sum);
+    return caption.toString();
   }
 }

src/main/java/GuiController.java

@@ -19,6 +19,7 @@
 
 
 public class GuiController {
+  private String imageFilename = "";
 
   @FXML
   public Button btnBrowse;
@@ -82,6 +83,7 @@ public void handleBtnBrowse() {
 
     try {
       Image image = new Image(String.valueOf(file.toURI().toURL()));
+      imageFilename = file.getPath();
       imageView.setImage(image);
       imageView.setPreserveRatio(true);
       imageView.setFitHeight(300);
@@ -101,12 +103,12 @@ public void handleBtnBrowse() {
   /**
    * Handles the Caption button action.
    */
-  public void handleBtnCaption() {
+  public void handleBtnCaption() throws IOException {
     //Instantiate CaptionGenerator
     CaptionGenerator cg = new CaptionGenerator();
     //Convert JavaFX Image to BufferedImage
     BufferedImage img = SwingFXUtils.fromFXImage(imageView.getImage(), null);
     //Run captioning method and set label text
-    caption.setText(cg.generateCaption(img));
+    caption.setText(cg.generateCaption(imageFilename));
   }
 }