diff --git a/src/Microsoft.Crank.Agent/Startup.cs b/src/Microsoft.Crank.Agent/Startup.cs index 8465de127..cba418a0e 100644 --- a/src/Microsoft.Crank.Agent/Startup.cs +++ b/src/Microsoft.Crank.Agent/Startup.cs @@ -638,6 +638,18 @@ private static async Task ProcessJobs(string hostname, string dockerHostname, Ca cts.Cancel(); await buildAndRunTask; } + + if (buildAndRunTask.IsFaulted) + { + Log.WriteLine($"An unexpected error occurred while building the job. {buildAndRunTask.Exception}"); + job.Error = $"An unexpected error occurred while building the job: {buildAndRunTask.Exception.Message}"; + + Log.WriteLine($"{job.State} -> Failed"); + job.State = JobState.Failed; + + cts.Cancel(); + await buildAndRunTask; + } } if (job.State != JobState.Failed) @@ -2789,7 +2801,7 @@ private static string PatchOrCreateGlobalJson(Job job, string benchmarkedApp, st var globalJson = "{ \"sdk\": { \"version\": \"" + sdkVersion + "\" } }"; File.WriteAllText(Path.Combine(benchmarkedApp, "global.json"), globalJson); - } + } else { // File found, we need to update it diff --git a/src/Microsoft.Crank.Controller/JobConnection.cs b/src/Microsoft.Crank.Controller/JobConnection.cs index 602caedb6..7ff8ff707 100644 --- a/src/Microsoft.Crank.Controller/JobConnection.cs +++ b/src/Microsoft.Crank.Controller/JobConnection.cs @@ -104,13 +104,7 @@ string jobName while (true) { - Log.Verbose($"GET {_serverJobUri}..."); - response = await _httpClient.GetAsync(_serverJobUri); - responseContent = await response.Content.ReadAsStringAsync(); - - response.EnsureSuccessStatusCode(); - - Job = JsonConvert.DeserializeObject(responseContent); + Job = await GetJobAsync(); #region Ensure the job is valid @@ -318,8 +312,6 @@ string jobName var previouState = currentState; currentState = await GetStateAsync(); - // Job = JsonConvert.DeserializeObject(responseContent); - if (currentState == JobState.Running) { if (previouState != JobState.Running) @@ -334,6 +326,9 @@ string jobName { Log.Write($"Job failed on benchmark server, stopping..."); + // Refreshing Job state to display the error + Job = await GetJobAsync(); + Log.Write(Job.Error, notime: true, error: true); // Returning will also send a Delete message to the server @@ -870,6 +865,18 @@ public async Task> GetQueueAsync() Log.Verbose($"{(int)response.StatusCode} {response.StatusCode} {responseContent}"); return JsonConvert.DeserializeObject(responseContent); - } + } + + public async Task GetJobAsync() + { + Log.Verbose($"GET {_serverJobUri}..."); + + var response = await _httpClient.GetAsync(_serverJobUri); + var responseContent = await response.Content.ReadAsStringAsync(); + + response.EnsureSuccessStatusCode(); + + return JsonConvert.DeserializeObject(responseContent); + } } }