Image timestamp support (#88)

* Support for capture images timestamps * Support for capture images timestamps * Add more changes
etiennedub · Dec 14, 2020 · a4849a2 · a4849a2
1 parent c7a9878
commit a4849a2
Show file tree

Hide file tree

Showing 9 changed files with 86 additions and 15 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -70,7 +70,7 @@ jobs:
           pip install -e .
       - name: Run tests
         run: |
-          make test-no-hardware
+          make test-ci
       - name: Coverage
         uses: codecov/codecov-action@v1
         with:

diff --git a/Makefile b/Makefile
@@ -36,4 +36,8 @@ test-hardware:
 	pytest  -m "device" $(TESTS)
 
 test-no-hardware:
-	pytest -m "not device" $(TESTS)
+	pytest -m "not device" $(TESTS)
+
+test-ci:
+	pytest -m "not device and not opengl" $(TESTS)
+
diff --git a/pyk4a/capture.py b/pyk4a/capture.py
@@ -24,8 +24,11 @@ def __init__(
         self._color_format = color_format
 
         self._color: Optional[np.ndarray] = None
+        self._color_timestamp_usec: int = 0
         self._depth: Optional[np.ndarray] = None
+        self._depth_timestamp_usec: int = 0
         self._ir: Optional[np.ndarray] = None
+        self._ir_timestamp_usec: int = 0
         self._depth_point_cloud: Optional[np.ndarray] = None
         self._transformed_depth: Optional[np.ndarray] = None
         self._transformed_depth_point_cloud: Optional[np.ndarray] = None
@@ -35,21 +38,46 @@ def __init__(
     @property
     def color(self) -> Optional[np.ndarray]:
         if self._color is None:
-            self._color = k4a_module.capture_get_color_image(self._capture_handle, self.thread_safe)
+            self._color, self._color_timestamp_usec = k4a_module.capture_get_color_image(
+                self._capture_handle, self.thread_safe
+            )
         return self._color
 
+    @property
+    def color_timestamp_usec(self) -> int:
+        """Device timestamp for color image. Not equal host machine timestamp!"""
+        if self._color is None:
+            self.color
+        return self._color_timestamp_usec
+
     @property
     def depth(self) -> Optional[np.ndarray]:
         if self._depth is None:
-            self._depth = k4a_module.capture_get_depth_image(self._capture_handle, self.thread_safe)
+            self._depth, self._depth_timestamp_usec = k4a_module.capture_get_depth_image(
+                self._capture_handle, self.thread_safe
+            )
         return self._depth
 
+    @property
+    def depth_timestamp_usec(self) -> int:
+        """Device timestamp for depth image. Not equal host machine timestamp!. Like as equal IR image timestamp"""
+        if self._depth is None:
+            self.depth
+        return self._depth_timestamp_usec
+
     @property
     def ir(self) -> Optional[np.ndarray]:
+        """Device timestamp for IR image. Not equal host machine timestamp!. Like as equal depth image timestamp"""
         if self._ir is None:
-            self._ir = k4a_module.capture_get_ir_image(self._capture_handle, self.thread_safe)
+            self._ir, self._ir_timestamp_usec = k4a_module.capture_get_ir_image(self._capture_handle, self.thread_safe)
         return self._ir
 
+    @property
+    def ir_timestamp_usec(self) -> int:
+        if self._ir is None:
+            self.ir
+        return self._ir_timestamp_usec
+
     @property
     def transformed_depth(self) -> Optional[np.ndarray]:
         if self._transformed_depth is None and self.depth is not None:

diff --git a/pyk4a/pyk4a.cpp b/pyk4a/pyk4a.cpp
@@ -793,6 +793,7 @@ extern "C" {
         k4a_capture_t* capture_handle;
         PyObject *capsule;
         int thread_safe;
+        uint64_t device_timestamp_usec = 0;
         PyThreadState *thread_state;
         k4a_result_t res = K4A_RESULT_FAILED;
 
@@ -802,7 +803,7 @@ extern "C" {
         k4a_image_t* image = (k4a_image_t*) malloc(sizeof(k4a_image_t));
         if (image == NULL) {
             fprintf(stderr, "Cannot allocate memory");
-            return Py_BuildValue("");
+            return Py_BuildValue("NK", Py_None, device_timestamp_usec);
         }
 
         thread_state = _gil_release(thread_safe);
@@ -815,18 +816,20 @@ extern "C" {
         }
 
         if (K4A_RESULT_SUCCEEDED == res) {
-            return PyArray_Return(np_image);
+            device_timestamp_usec = k4a_image_get_device_timestamp_usec(*image);
+            return Py_BuildValue("NK", np_image, device_timestamp_usec);
         }
         else {
             free(image);
-            return Py_BuildValue("");
+            return Py_BuildValue("NK", Py_None, device_timestamp_usec);
         }
     }
 
     static PyObject* capture_get_depth_image(PyObject* self, PyObject* args){
         k4a_capture_t* capture_handle;
         PyObject *capsule;
         int thread_safe;
+        uint64_t device_timestamp_usec = 0;
         PyThreadState *thread_state;
         k4a_result_t res = K4A_RESULT_FAILED;
 
@@ -836,7 +839,7 @@ extern "C" {
         k4a_image_t* image = (k4a_image_t*) malloc(sizeof(k4a_image_t));
         if (image == NULL) {
             fprintf(stderr, "Cannot allocate memory");
-            return Py_BuildValue("");
+            return Py_BuildValue("NK", Py_None, device_timestamp_usec);
         }
 
         thread_state = _gil_release(thread_safe);
@@ -849,18 +852,20 @@ extern "C" {
         }
 
         if (K4A_RESULT_SUCCEEDED == res) {
-            return PyArray_Return(np_image);
+            device_timestamp_usec = k4a_image_get_device_timestamp_usec(*image);
+            return Py_BuildValue("NK", np_image, device_timestamp_usec);
         }
         else {
             free(image);
-            return Py_BuildValue("");
+            return Py_BuildValue("NK", Py_None, device_timestamp_usec);
         }
     }
 
     static PyObject* capture_get_ir_image(PyObject* self, PyObject* args){
         k4a_capture_t* capture_handle;
         PyObject *capsule;
         int thread_safe;
+        uint64_t device_timestamp_usec = 0;
         PyThreadState *thread_state;
         k4a_result_t res = K4A_RESULT_FAILED;
 
@@ -870,7 +875,7 @@ extern "C" {
         k4a_image_t* image = (k4a_image_t*) malloc(sizeof(k4a_image_t));
         if (image == NULL) {
             fprintf(stderr, "Cannot allocate memory");
-            return Py_BuildValue("");
+            return Py_BuildValue("NK", Py_None, device_timestamp_usec);
         }
 
         thread_state = _gil_release(thread_safe);
@@ -883,11 +888,12 @@ extern "C" {
         }
 
         if (K4A_RESULT_SUCCEEDED == res) {
-            return PyArray_Return(np_image);
+            device_timestamp_usec = k4a_image_get_device_timestamp_usec(*image);
+            return Py_BuildValue("NK", np_image, device_timestamp_usec);
         }
         else {
             free(image);
-            return Py_BuildValue("");
+            return Py_BuildValue("NK", Py_None, device_timestamp_usec);
         }
     }
 
@@ -975,6 +981,7 @@ extern "C" {
         return Py_BuildValue("II(fff)", res, valid, target_point3d_mm.xyz.x, target_point3d_mm.xyz.y, target_point3d_mm.xyz.z);
     }
 
+
     static PyObject* playback_open(PyObject* self, PyObject *args) {
         int thread_safe;
         PyThreadState *thread_state;

diff --git a/pyproject.toml b/pyproject.toml
@@ -6,6 +6,7 @@ line-length = 120
 
 [tool.pytest.ini_options]
 markers = [
-        "device: Tests require connected real device"
+        "device: Tests require connected real device",
+        "opengl: Tests require opengl for GPU accelerated depth engine software"
 ]
 addopts = "--cov=pyk4a --cov-report=xml --verbose"
diff --git a/tests/functional/test_calibration.py b/tests/functional/test_calibration.py
@@ -26,6 +26,7 @@ def test_from_raw(calibration_raw):
         assert calibration
 
     @staticmethod
+    @pytest.mark.opengl
     def test_creating_transfromation_handle(calibration: Calibration):
         transformation = calibration.transformation_handle
         assert transformation is not None
diff --git a/tests/functional/test_capture.py b/tests/functional/test_capture.py
@@ -7,6 +7,7 @@
 class TestCapture:
     @staticmethod
     @pytest.mark.device
+    @pytest.mark.opengl
     def test_device_capture_images(device: PyK4A):
         device.open()
         device._start_cameras()

diff --git a/tests/functional/test_device.py b/tests/functional/test_device.py
@@ -60,13 +60,15 @@ def test_get_calibration(device: PyK4A):
 class TestCameras:
     @staticmethod
     @pytest.mark.device
+    @pytest.mark.opengl
     def test_start_stop_cameras(device: PyK4A):
         device.open()
         device._start_cameras()
         device._stop_cameras()
 
     @staticmethod
     @pytest.mark.device
+    @pytest.mark.opengl
     def test_capture(device: PyK4A):
         device.open()
         device._start_cameras()
@@ -77,6 +79,7 @@ def test_capture(device: PyK4A):
 class TestIMU:
     @staticmethod
     @pytest.mark.device
+    @pytest.mark.opengl
     def test_start_stop_imu(device: PyK4A):
         device.open()
         device._start_cameras()  # imu will not work without cameras
@@ -86,6 +89,7 @@ def test_start_stop_imu(device: PyK4A):
 
     @staticmethod
     @pytest.mark.device
+    @pytest.mark.opengl
     def test_get_imu_sample(device: PyK4A):
         device.open()
         device._start_cameras()

diff --git a/tests/functional/test_playback.py b/tests/functional/test_playback.py
@@ -103,3 +103,28 @@ def test_seek_by_device_time(playback: PyK4APlayback):
         playback.seek(1, origin=SeekOrigin.DEVICE_TIME)  # TODO add correct timestamp from datablock here
         capture = playback.get_next_capture()
         assert capture.color is not None
+
+
+class TestGetCapture:
+    @staticmethod
+    def test_get_next_capture(playback: PyK4APlayback):
+        playback.open()
+        capture = playback.get_next_capture()
+        assert capture is not None
+        assert capture.depth is not None
+        assert capture.color is not None
+        assert capture.depth_timestamp_usec == 800222
+        assert capture.color_timestamp_usec == 800222
+        assert capture.ir_timestamp_usec == 800222
+
+    @staticmethod
+    def test_get_previouse_capture(playback: PyK4APlayback):
+        playback.open()
+        playback.seek(0, origin=SeekOrigin.END)
+        capture = playback.get_previouse_capture()
+        assert capture is not None
+        assert capture.depth is not None
+        assert capture.color is not None
+        assert capture.depth_timestamp_usec == 800222
+        assert capture.color_timestamp_usec == 800222
+        assert capture.ir_timestamp_usec == 800222