diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..85f47da --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +gaze_tracking/trained_models/* filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7db61e6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,104 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..37519b5 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Antoine Lamé + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..5ec1280 --- /dev/null +++ b/README.md @@ -0,0 +1,160 @@ +# Gaze Tracking + +![made-with-python](https://img.shields.io/badge/Made%20with-Python-1f425f.svg) +![Open Source Love](https://badges.frapsoft.com/os/v1/open-source.svg?v=103) +![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg) + +This is a Python (2 and 3) library that provides a **webcam-based eye tracking system**. It gives you the exact position of the pupils and the gaze's direction, in real time. + +[![Demo](https://i.imgur.com/WNqgQkO.gif)](https://youtu.be/YEZMk1P0-yw) + +## Installation + +Clone this project: + +``` +git clone https://github.com/antoinelame/GazeTracking.git +``` + +GazeTracking requires these dependencies: + +* NumPy +* OpenCV 3.4 +* Dlib + +To install them: + +``` +pip install -r requirements.txt +``` + +> The Dlib library has four primary prerequisites: Boost, Boost.Python, CMake and X11/XQuartx. If you doesn't have them, you can [read this article](https://www.pyimagesearch.com/2017/03/27/how-to-install-dlib/) to know how to easily install them. + +> OpenCV 4 is not supported yet, make sure to install version 3.4 + +Run the demo: + +``` +python example.py +``` + +## Simple Demo + +```python +import cv2 +from gaze_tracking import GazeTracking + + +gaze = GazeTracking() + +while True: + gaze.refresh() + + frame = gaze.main_frame(True) + text = "" + + if gaze.is_right(): + text = "Looking right" + elif gaze.is_left(): + text = "Looking left" + elif gaze.is_center(): + text = "Looking center" + + cv2.putText(frame, text, (60, 60), cv2.FONT_HERSHEY_DUPLEX, 2, (255, 0, 0), 2) + cv2.imshow("Demo", frame) + + if cv2.waitKey(1) == 27: + break +``` + +## Documentation + +In the following examples, ```gaze``` refers to an instance of the ```GazeTracking``` class. + +### Refresh the frame + +```python +gaze.refresh() +``` + +Captures a new frame with the webcam and analyzes it. + +### Position of the left pupil + +```python +gaze.pupil_left_coords() +``` + +Returns the coordinates (x,y) of the left pupil. + +### Position of the right pupil + +```python +gaze.pupil_right_coords() +``` + +Returns the coordinates (x,y) of the right pupil. + +### Looking to the left + +```python +gaze.is_left() +``` + +Returns `True` if the user is looking to the left. + +### Looking to the right + +```python +gaze.is_right() +``` + +Returns `True` if the user is looking to the right. + +### Looking at the center + +```python +gaze.is_center() +``` + +Returns `True` if the user is looking at the center. + +### Horizontal direction of the gaze + +```python +ratio = gaze.horizontal_ratio() +``` + +Returns a number between 0.0 and 1.0 that indicates the horizontal direction of the gaze. The extreme right is 0.0, the center is 0.5 and the extreme left is 1.0. + +### Vertical direction of the gaze + +```python +ratio = gaze.vertical_ratio() +``` + +Returns a number between 0.0 and 1.0 that indicates the vertical direction of the gaze. The extreme top is 0.0, the center is 0.5 and the extreme bottom is 1.0. + +### Blinking + +```python +gaze.is_blinking() +``` + +Returns `True` is the user's eyes are closed. + +### Webcam frame + +```python +# Without pupils highlighting +frame = gaze.main_frame(False) + +# With pupils highlighting +frame = gaze.main_frame(True) +``` + +Returns the main frame from the webcam. + +## Licensing + +This project is released by Antoine Lamé under the terms of the MIT Open Source License. View LICENSE for more information. \ No newline at end of file diff --git a/example.py b/example.py new file mode 100644 index 0000000..4c7f34b --- /dev/null +++ b/example.py @@ -0,0 +1,36 @@ +""" +Demonstration of the GazeTracking library. +Check the README.md for complete documentation. +""" + +import cv2 +from gaze_tracking import GazeTracking + +gaze = GazeTracking() + +while True: + gaze.refresh() + + frame = gaze.main_frame(True) + text = "" + + if gaze.is_blinking(): + text = "Blinking" + elif gaze.is_right(): + text = "Looking right" + elif gaze.is_left(): + text = "Looking left" + elif gaze.is_center(): + text = "Looking center" + + cv2.putText(frame, text, (90, 60), cv2.FONT_HERSHEY_DUPLEX, 1.6, (147, 58, 31), 2) + + left_pupil = gaze.pupil_left_coords() + right_pupil = gaze.pupil_right_coords() + cv2.putText(frame, "Left pupil: " + str(left_pupil), (90, 130), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) + cv2.putText(frame, "Right pupil: " + str(right_pupil), (90, 165), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) + + cv2.imshow("Demo", frame) + + if cv2.waitKey(1) == 27: + break diff --git a/gaze_tracking/__init__.py b/gaze_tracking/__init__.py new file mode 100644 index 0000000..24e327e --- /dev/null +++ b/gaze_tracking/__init__.py @@ -0,0 +1 @@ +from .gaze_tracking import GazeTracking diff --git a/gaze_tracking/eyes.py b/gaze_tracking/eyes.py new file mode 100644 index 0000000..cc623e0 --- /dev/null +++ b/gaze_tracking/eyes.py @@ -0,0 +1,106 @@ +import os +import math +import numpy as np +import cv2 +import dlib + + +class EyesDetector(object): + """ + This class detects the position of the eyes of a face, + and creates two new frames to isolate each eye. + """ + + LEFT_EYE_POINTS = [36, 37, 38, 39, 40, 41] + RIGHT_EYE_POINTS = [42, 43, 44, 45, 46, 47] + + def __init__(self): + self.frame_left = None + self.frame_left_origin = None + self.frame_right = None + self.frame_right_origin = None + self.blinking = None + self._face_detector = dlib.get_frontal_face_detector() + cwd = os.path.abspath(os.path.dirname(__file__)) + model_path = os.path.abspath(os.path.join(cwd, "trained_models/shape_predictor_68_face_landmarks.dat")) + self._predictor = dlib.shape_predictor(model_path) + + @staticmethod + def _middle_point(p1, p2): + """Returns the middle point (x,y) between two points + + Arguments: + p1 (dlib.point): First point + p2 (dlib.point): Second point + """ + x = int((p1.x + p2.x) / 2) + y = int((p1.y + p2.y) / 2) + return (x, y) + + @staticmethod + def isolate_eye(frame, landmarks, points): + """Isolate an eye, to have a frame without other part of the face. + + Arguments: + frame (numpy.ndarray): Frame containing the face + landmarks (dlib.full_object_detection): Facial landmarks for the face region + points (list): Points of an eye (from the 68 Multi-PIE landmarks) + + Returns: + A tuple with the eye frame and its origin + """ + region = np.array([(landmarks.part(point).x, landmarks.part(point).y) for point in points]) + region = region.astype(np.int32) + + # Applying a mask to get only the eye + height, width = frame.shape[:2] + black_frame = np.zeros((height, width), np.uint8) + mask = np.full((height, width), 255, np.uint8) + cv2.fillPoly(mask, [region], (0, 0, 0)) + eye = cv2.bitwise_not(black_frame, frame.copy(), mask=mask) + + # Cropping on the eye + margin = 5 + min_x = np.min(region[:, 0]) - margin + max_x = np.max(region[:, 0]) + margin + min_y = np.min(region[:, 1]) - margin + max_y = np.max(region[:, 1]) + margin + roi = eye[min_y:max_y, min_x:max_x] + + return (roi, (min_x, min_y)) + + def blinking_ratio(self, landmarks, points): + """Calculates a ratio that can indicate whether an eye is closed or not. + It's the division of the width of the eye, by its height. + + Arguments: + landmarks (dlib.full_object_detection): Facial landmarks for the face region + points (list): Points of an eye (from the 68 Multi-PIE landmarks) + """ + left = (landmarks.part(points[0]).x, landmarks.part(points[0]).y) + right = (landmarks.part(points[3]).x, landmarks.part(points[3]).y) + top = self._middle_point(landmarks.part(points[1]), landmarks.part(points[2])) + bottom = self._middle_point(landmarks.part(points[5]), landmarks.part(points[4])) + + eye_width = math.hypot((left[0] - right[0]), (left[1] - right[1])) + eye_height = math.hypot((top[0] - bottom[0]), (top[1] - bottom[1])) + + return eye_width / eye_height + + def process(self, frame): + """Run eyes detection""" + frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + faces = self._face_detector(frame_gray) + + try: + landmarks = self._predictor(frame, faces[0]) + + self.frame_left, self.frame_left_origin = self.isolate_eye(frame_gray, landmarks, self.LEFT_EYE_POINTS) + self.frame_right, self.frame_right_origin = self.isolate_eye(frame_gray, landmarks, self.RIGHT_EYE_POINTS) + + blinking_left = self.blinking_ratio(landmarks, self.LEFT_EYE_POINTS) + blinking_right = self.blinking_ratio(landmarks, self.RIGHT_EYE_POINTS) + self.blinking = (blinking_left + blinking_right) / 2 + + except IndexError: + pass diff --git a/gaze_tracking/gaze_tracking.py b/gaze_tracking/gaze_tracking.py new file mode 100644 index 0000000..4bc41e3 --- /dev/null +++ b/gaze_tracking/gaze_tracking.py @@ -0,0 +1,90 @@ +import cv2 +from .eyes import EyesDetector +from .pupil import PupilDetector + + +class GazeTracking(object): + """ + This class tracks the user's gaze. + It provides useful information like the position of the eyes + and the pupil and allows to know if the eyes are open or closed + """ + + def __init__(self): + self.capture = cv2.VideoCapture(0) + self.frame = None + self.eyes = EyesDetector() + self.pupil_left = PupilDetector() + self.pupil_right = PupilDetector() + + def refresh(self): + """Captures a new frame with the webcam and analyzes it.""" + _, self.frame = self.capture.read() + self.eyes.process(self.frame) + self.pupil_left.process(self.eyes.frame_left) + self.pupil_right.process(self.eyes.frame_right) + + def pupil_left_coords(self): + """Returns the coordinates of the left pupil""" + x = self.eyes.frame_left_origin[0] + self.pupil_left.x + y = self.eyes.frame_left_origin[1] + self.pupil_left.y + return (x, y) + + def pupil_right_coords(self): + """Returns the coordinates of the right pupil""" + x = self.eyes.frame_right_origin[0] + self.pupil_right.x + y = self.eyes.frame_right_origin[1] + self.pupil_right.y + return (x, y) + + def horizontal_ratio(self): + """Returns a number between 0.0 and 1.0 that indicates the + horizontal direction of the gaze. The extreme right is 0.0, + the center is 0.5 and the extreme left is 1.0 + """ + pupil_right = self.pupil_right.x / (self.pupil_right.center[0] * 2 - 10) + pupil_left = self.pupil_left.x / (self.pupil_left.center[0] * 2 - 10) + return (pupil_right + pupil_left) / 2 + + def vertical_ratio(self): + """Returns a number between 0.0 and 1.0 that indicates the + vertical direction of the gaze. The extreme top is 0.0, + the center is 0.5 and the extreme bottom is 1.0 + """ + pupil_right = self.pupil_right.y / (self.pupil_right.center[1] * 2 - 10) + pupil_left = self.pupil_left.y / (self.pupil_left.center[1] * 2 - 10) + return (pupil_right + pupil_left) / 2 + + def is_right(self): + """Returns true is the user is looking to the right""" + return self.horizontal_ratio() <= 0.35 + + def is_left(self): + """Returns true is the user is looking to the left""" + return self.horizontal_ratio() >= 0.65 + + def is_center(self): + """Returns true is the user is looking to the center""" + return self.is_right() is not True and self.is_left() is not True + + def is_blinking(self): + """Returns true if the user closes his eyes""" + return self.eyes.blinking > 3.8 + + def main_frame(self, highlighting=False): + """Returns the main frame from the webcam + + Parameters: + - highlighting (bool): Highlights pupils + """ + frame = self.frame.copy() + + if highlighting: + color = (0, 255, 0) + x_left, y_left = self.pupil_left_coords() + x_right, y_right = self.pupil_right_coords() + cv2.line(frame, (x_left - 5, y_left), (x_left + 5, y_left), color) + cv2.line(frame, (x_left, y_left - 5), (x_left, y_left + 5), color) + cv2.line(frame, (x_right - 5, y_right), (x_right + 5, y_right), color) + cv2.line(frame, (x_right, y_right - 5), (x_right, y_right + 5), color) + + return frame diff --git a/gaze_tracking/pupil.py b/gaze_tracking/pupil.py new file mode 100644 index 0000000..bddf692 --- /dev/null +++ b/gaze_tracking/pupil.py @@ -0,0 +1,49 @@ +import numpy as np +import cv2 + + +class PupilDetector(object): + """ + This class detects the iris of an eye and estimates + the position of the pupil + """ + + def __init__(self): + self.modified_frame = None + self.center = None + self.x = None + self.y = None + + @staticmethod + def image_processing(eye_frame): + """Performs operations on the eye frame to isolate the iris + + Arguments: + eye_frame (numpy.ndarray): Frame containing an eye and nothing else + + Returns: + A frame with a single element representing the iris + """ + kernel = np.ones((3, 3), np.uint8) + new_frame = cv2.bilateralFilter(eye_frame, 10, 15, 15) + new_frame = cv2.threshold(new_frame, 20, 255, cv2.THRESH_BINARY)[1] + new_frame = cv2.erode(new_frame, kernel, iterations=3) + new_frame = cv2.dilate(new_frame, kernel, iterations=2) + return new_frame + + def process(self, frame): + """Run iris detection and pupil estimation""" + self.modified_frame = self.image_processing(frame) + + height, width = self.modified_frame.shape[:2] + self.center = (width / 2, height / 2) + + _, contours, _ = cv2.findContours(self.modified_frame, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE) + contours = sorted(contours, key=cv2.contourArea) + + try: + moments = cv2.moments(contours[-2]) + self.x = int(moments['m10'] / moments['m00']) + self.y = int(moments['m01'] / moments['m00']) + except IndexError: + pass diff --git a/gaze_tracking/trained_models/shape_predictor_68_face_landmarks.dat b/gaze_tracking/trained_models/shape_predictor_68_face_landmarks.dat new file mode 100644 index 0000000..1e5da4f --- /dev/null +++ b/gaze_tracking/trained_models/shape_predictor_68_face_landmarks.dat @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbdc2cb80eb9aa7a758672cbfdda32ba6300efe9b6e6c7a299ff7e736b11b92f +size 99693937 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..dfca1cf --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +numpy == 1.16.1 +opencv_python == 3.4.5.20 +dlib == 19.16.0