gh: debug egl runner

ubi-agni · Dec 5, 2024 · e44ebef · e44ebef
1 parent 7fcd1a1
commit e44ebef
Show file tree

Hide file tree

Showing 2 changed files with 56 additions and 11 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -23,19 +23,37 @@ jobs:
         distro: [noetic, one]
         render_backend: [GLFW, OSMESA, EGL, DISABLE]
         mujoco: [3.2.0]
-        include:
+        # include:
+        #   - distro: noetic
+        #     mujoco: 3.2.0
+        #     env:
+        #       CATKIN_LINT: true
+        #       CCOV: true
+        #   - distro: one
+        #     mujoco: 3.2.0
+        #     env:
+        #       CLANG_TIDY: pedantic
+        exclude:
           - distro: noetic
-            mujoco: 3.2.0
-            env:
-              CATKIN_LINT: true
-              CCOV: true
+            render_backend: GLFW
+          - distro: noetic
+            render_backend: OSMESA
+          - distro: noetic
+            render_backend: EGL
+          - distro: noetic
+            render_backend: DISABLE
+          - distro: one
+            render_backend: GLFW
+          - distro: one
+            render_backend: OSMESA
           - distro: one
-            mujoco: 3.2.0
-            env:
-              CLANG_TIDY: pedantic
+            render_backend: DISABLE
 
     env:
       BUILDER: colcon
+      EGL_DEVICE_ID: 0
+      NVIDIA_DRIVER_CAPABILITIES: compute,graphics,utility,video
+      EGL_LOG_LEVEL: debug
       MUJOCO_DIR: /vol/mujoco/mujoco-${{ matrix.mujoco }} # for self-hosted runners
       DOCKER_RUN_OPTS: -e MUJOCO_DIR=/root/mujoco/${{ matrix.mujoco }} -e DISPLAY=:99.0
       DOCKER_IMAGE: ubiagni/mujoco_ros:${{ matrix.distro }}-ci
@@ -47,13 +65,14 @@ jobs:
       CLANG_TIDY_ARGS: --fix --fix-errors --format-style=file
       UPSTREAM_WORKSPACE: .github/workflows/upstream.rosinstall
       # Install and start xvfb for GLFW
-      AFTER_INIT: ${{ matrix.render_backend == 'GLFW' && 'apt-get install -qy xvfb; Xvfb :99 &' || '' }}
+      # eglinfo
+      AFTER_INIT: ${{ matrix.render_backend == 'GLFW' && 'apt-get install -qy xvfb; Xvfb :99 &' || matrix.render_backend == 'EGL' && 'nvidia-smi'  || '' }}
       # Pull any updates to the upstream workspace
       AFTER_SETUP_UPSTREAM_WORKSPACE: vcs pull $BASEDIR/upstream_ws/src
       AFTER_SETUP_DOWNSTREAM_WORKSPACE: vcs pull $BASEDIR/downstream_ws/src
       # Clear ccache stats before and log the stats after the build
       AFTER_SETUP_CCACHE: ccache --zero-stats --max-size=10.0G
-      AFTER_BUILD_TARGET_WORKSPACE: ccache --show-stats
+      AFTER_BUILD_TARGET_WORKSPACE: ccache --show-stats; nvidia-smi; ldd $BASEDIR/target_ws/install/mujoco_ros/lib/libmujoco_ros.so;
       AFTER_BUILD_DOWNSTREAM_WORKSPACE: ccache --show-stats
       # Compile CCOV with Debug. Enable -Werror.
       TARGET_CMAKE_ARGS: >

diff --git a/mujoco_ros/src/offscreen_rendering.cpp b/mujoco_ros/src/offscreen_rendering.cpp
@@ -42,6 +42,7 @@
 
 #if RENDER_BACKEND == EGL_BACKEND
 #include <EGL/egl.h>
+#include <EGL/eglext.h>
 #endif
 
 namespace mujoco_ros {
@@ -183,6 +184,30 @@ void MujocoEnv::initializeRenderResources()
 bool MujocoEnv::InitGL()
 {
 	ROS_DEBUG("Initializing EGL...");
+
+	EGLDeviceEXT egl_devices[4];
+	EGLint num_devices;
+
+	// Get devices
+	PFNEGLQUERYDEVICESEXTPROC eglQueryDevicesEXT =
+	    reinterpret_cast<PFNEGLQUERYDEVICESEXTPROC>(eglGetProcAddress("eglQueryDevicesEXT"));
+	eglQueryDevicesEXT(4, egl_devices, &num_devices);
+	ROS_WARN_STREAM("Found " << num_devices << " EGL devices");
+
+	PFNEGLQUERYDEVICESTRINGEXTPROC eglQueryDeviceStringEXT =
+	    reinterpret_cast<PFNEGLQUERYDEVICESTRINGEXTPROC>(eglGetProcAddress("eglQueryDeviceStringEXT"));
+	const char *extensions;
+	int choose_device = 0;
+	for (int i = 0; i < num_devices; i++) {
+		extensions = eglQueryDeviceStringEXT(egl_devices[i], EGL_EXTENSIONS);
+		ROS_DEBUG_STREAM("Device " << i << " has extensions: " << extensions);
+		if (strstr(extensions, "EGL_NV_device_cuda")) {
+			ROS_WARN_STREAM("Choosing device " << i << " for CUDA support");
+			choose_device = i;
+			break;
+		}
+	}
+
 	// clang-format off
 	const EGLint config[] = {
 		EGL_RED_SIZE,		   8,
@@ -198,7 +223,8 @@ bool MujocoEnv::InitGL()
 	// clang-format on
 
 	// Get Display
-	EGLDisplay display = eglGetDisplay(EGL_DEFAULT_DISPLAY);
+	// EGLDisplay display = eglGetDisplay(EGL_DEFAULT_DISPLAY);
+	EGLDisplay display = eglGetPlatformDisplay(EGL_PLATFORM_DEVICE_EXT, egl_devices[0], nullptr);
 	if (display == EGL_NO_DISPLAY) {
 		ROS_ERROR_STREAM("Failed to get EGL display. Error type: " << eglGetError());
 		return false;