Tracking errors during build (dotnet#79)

rr-wfm · Jul 29, 2020 · e2b8f3c · e2b8f3c
1 parent ce7383b
commit e2b8f3c
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 11 deletions.
diff --git a/src/Microsoft.Crank.Agent/Startup.cs b/src/Microsoft.Crank.Agent/Startup.cs
@@ -638,6 +638,18 @@ private static async Task ProcessJobs(string hostname, string dockerHostname, Ca
                                             cts.Cancel();
                                             await buildAndRunTask;
                                         }
+
+                                        if (buildAndRunTask.IsFaulted)
+                                        {
+                                            Log.WriteLine($"An unexpected error occurred while building the job. {buildAndRunTask.Exception}");
+                                            job.Error = $"An unexpected error occurred while building the job: {buildAndRunTask.Exception.Message}";
+
+                                            Log.WriteLine($"{job.State} -> Failed");
+                                            job.State = JobState.Failed;
+
+                                            cts.Cancel();
+                                            await buildAndRunTask;
+                                        }
                                     }
 
                                     if (job.State != JobState.Failed)
@@ -2789,7 +2801,7 @@ private static string PatchOrCreateGlobalJson(Job job, string benchmarkedApp, st
 
                     var globalJson = "{ \"sdk\": { \"version\": \"" + sdkVersion + "\" } }";
                     File.WriteAllText(Path.Combine(benchmarkedApp, "global.json"), globalJson);
-                }
+                    }
                 else
                 {
                     // File found, we need to update it

diff --git a/src/Microsoft.Crank.Controller/JobConnection.cs b/src/Microsoft.Crank.Controller/JobConnection.cs
@@ -104,13 +104,7 @@ string jobName
 
             while (true)
             {
-                Log.Verbose($"GET {_serverJobUri}...");
-                response = await _httpClient.GetAsync(_serverJobUri);
-                responseContent = await response.Content.ReadAsStringAsync();
-
-                response.EnsureSuccessStatusCode();
-
-                Job = JsonConvert.DeserializeObject<Job>(responseContent);
+                Job = await GetJobAsync();
 
                 #region Ensure the job is valid
 
@@ -318,8 +312,6 @@ string jobName
                 var previouState = currentState;
                 currentState = await GetStateAsync();
 
-                // Job = JsonConvert.DeserializeObject<Job>(responseContent);
-
                 if (currentState == JobState.Running)
                 {
                     if (previouState != JobState.Running)
@@ -334,6 +326,9 @@ string jobName
                 {
                     Log.Write($"Job failed on benchmark server, stopping...");
 
+                    // Refreshing Job state to display the error
+                    Job = await GetJobAsync();
+
                     Log.Write(Job.Error, notime: true, error: true);
 
                     // Returning will also send a Delete message to the server
@@ -870,6 +865,18 @@ public async Task<IEnumerable<JobView>> GetQueueAsync()
             Log.Verbose($"{(int)response.StatusCode} {response.StatusCode} {responseContent}");
 
             return JsonConvert.DeserializeObject<JobView[]>(responseContent);
-        }        
+        }      
+
+        public async Task<Job> GetJobAsync()
+        {
+            Log.Verbose($"GET {_serverJobUri}...");
+
+            var response = await _httpClient.GetAsync(_serverJobUri);
+            var responseContent = await response.Content.ReadAsStringAsync();
+
+            response.EnsureSuccessStatusCode();
+
+            return JsonConvert.DeserializeObject<Job>(responseContent);
+        }  
     }
 }