Split build and test parallel steps (#1453)

This speeds up e2e pipeline time since all three builds take around 10-11m now and tests take about 5m, making those two phases faster.
Kaggle · Dec 5, 2024 · c9bf1dd · c9bf1dd
1 parent adffc5d
commit c9bf1dd
Showing 1 changed file with 55 additions and 54 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -61,22 +61,6 @@ pipeline {
                 '''
               }
             }
-            stage('Test CPU Image') {
-              options {
-                timeout(time: 15, unit: 'MINUTES')
-              }
-              steps {
-                retry(2) {
-                  sh '''#!/bin/bash
-                    set -exo pipefail
-
-                    date
-                    docker pull gcr.io/kaggle-images/python:${PRETEST_TAG}
-                    ./test --image gcr.io/kaggle-images/python:${PRETEST_TAG}
-                  '''
-                }
-              }
-            }
             stage('Diff CPU image') {
               steps {
                 sh '''#!/bin/bash
@@ -113,44 +97,6 @@ pipeline {
                 '''
               }
             }
-            stage('Test GPU Image') {
-              stages {
-                stage('Test on P100') {
-                  agent { label 'ephemeral-linux-gpu' }
-                  options {
-                    timeout(time: 40, unit: 'MINUTES')
-                  }
-                  steps {
-                    retry(2) {
-                      sh '''#!/bin/bash
-                        set -exo pipefail
-
-                        date
-                        docker pull gcr.io/kaggle-private-byod/python:${PRETEST_TAG}
-                        ./test --gpu --image gcr.io/kaggle-private-byod/python:${PRETEST_TAG}
-                      '''
-                    }
-                  }
-                }
-                stage('Test on T4x2') {
-                  agent { label 'ephemeral-linux-gpu-t4x2' }
-                  options {
-                    timeout(time: 60, unit: 'MINUTES')
-                  }
-                  steps {
-                    retry(2) {
-                      sh '''#!/bin/bash
-                        set -exo pipefail
-
-                        date
-                        docker pull gcr.io/kaggle-private-byod/python:${PRETEST_TAG}
-                        ./test --gpu --image gcr.io/kaggle-private-byod/python:${PRETEST_TAG}
-                      '''
-                    }
-                  }
-                }
-              }
-            }
             stage('Diff GPU Image') {
               steps {
                 sh '''#!/bin/bash
@@ -193,6 +139,61 @@ pipeline {
       }
     }
 
+    stage('Test') {
+      parallel {
+        stage('Test CPU Image') {
+          options {
+            timeout(time: 15, unit: 'MINUTES')
+          }
+          steps {
+            retry(2) {
+              sh '''#!/bin/bash
+                set -exo pipefail
+
+                date
+                docker pull gcr.io/kaggle-images/python:${PRETEST_TAG}
+                ./test --image gcr.io/kaggle-images/python:${PRETEST_TAG}
+              '''
+            }
+          }
+        }
+        stage('Test on P100') {
+          agent { label 'ephemeral-linux-gpu' }
+          options {
+            timeout(time: 40, unit: 'MINUTES')
+          }
+          steps {
+            retry(2) {
+              sh '''#!/bin/bash
+                set -exo pipefail
+
+                date
+                docker pull gcr.io/kaggle-private-byod/python:${PRETEST_TAG}
+                ./test --gpu --image gcr.io/kaggle-private-byod/python:${PRETEST_TAG}
+              '''
+            }
+          }
+        }
+        stage('Test on T4x2') {
+          agent { label 'ephemeral-linux-gpu-t4x2' }
+          options {
+            timeout(time: 60, unit: 'MINUTES')
+          }
+          steps {
+            retry(2) {
+              sh '''#!/bin/bash
+                set -exo pipefail
+
+                date
+                docker pull gcr.io/kaggle-private-byod/python:${PRETEST_TAG}
+                ./test --gpu --image gcr.io/kaggle-private-byod/python:${PRETEST_TAG}
+              '''
+            }
+          }
+        }
+      }
+    }
+
     stage('Label CPU/GPU Staging Images') {
       steps {
         sh '''#!/bin/bash