Debugging

.buildkite/Manifest.toml

@@ -1,6 +1,6 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.10.6"
+julia_version = "1.10.7"
 manifest_format = "2.0"
 project_hash = "fc780a0dc833e209a503e8cdbd13bf068bf6b381"
 
@@ -264,9 +264,11 @@ weakdeps = ["CUDA", "MPI"]
 
 [[deps.ClimaCore]]
 deps = ["Adapt", "BandedMatrices", "BlockArrays", "ClimaComms", "CubedSphere", "DataStructures", "ForwardDiff", "GaussQuadrature", "GilbertCurves", "HDF5", "InteractiveUtils", "IntervalSets", "KrylovKit", "LinearAlgebra", "MultiBroadcastFusion", "NVTX", "PkgVersion", "RecursiveArrayTools", "RootSolvers", "SparseArrays", "StaticArrays", "Statistics", "Unrolled"]
-git-tree-sha1 = "7632911d7b09fb3e067984626b07b3eb1113e55e"
+git-tree-sha1 = "f92a814407e6eb7a20748c67ac3fb42eff604dd8"
+repo-rev = "main"
+repo-url = "https://github.com/CliMA/ClimaCore.jl.git"
 uuid = "d414da3d-4745-48bb-8d80-42e94e092884"
-version = "0.14.20"
+version = "0.14.22"
 weakdeps = ["CUDA", "Krylov"]
 
     [deps.ClimaCore.extensions]

Project.toml

@@ -5,6 +5,7 @@ version = "0.7.38"
 
 [deps]
 ClimaComms = "3a4d1b5c-c61d-41fd-a00a-5873ba7a1b0d"
+ClimaCore = "d414da3d-4745-48bb-8d80-42e94e092884"
 Colors = "5ae59095-9a9b-59fe-a467-6f913c188581"
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
@@ -20,18 +21,18 @@ StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
-StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
+StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 
 [extensions]
 ClimaTimeSteppersBenchmarkToolsExt = ["CUDA", "BenchmarkTools", "OrderedCollections", "StatsBase", "PrettyTables"]
 
 [compat]
 Aqua = "0.8"
 BenchmarkTools = "1"
+CUDA = "3, 4, 5"
 ClimaComms = "0.4, 0.5, 0.6"
 ClimaCore = "0.10, 0.11, 0.12, 0.13, 0.14"
-CUDA = "3, 4, 5"
 Colors = "0.12, 0.13"
 DataStructures = "0.18"
 DiffEqBase = "6"
@@ -66,4 +67,4 @@ SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Aqua","ClimaCore","Distributions","Krylov", "MPI","ODEConvergenceTester","PrettyTables","Random","SafeTestsets","Test"]
+test = ["Aqua", "ClimaCore", "Distributions", "Krylov", "MPI", "ODEConvergenceTester", "PrettyTables", "Random", "SafeTestsets", "Test"]

docs/Manifest.toml

@@ -1,6 +1,6 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.10.6"
+julia_version = "1.10.7"
 manifest_format = "2.0"
 project_hash = "1c1340e84980c5448f8e36bd3cc295c2eeba04c5"
 
@@ -219,9 +219,11 @@ version = "0.6.4"
 
 [[deps.ClimaCore]]
 deps = ["Adapt", "BandedMatrices", "BlockArrays", "ClimaComms", "CubedSphere", "DataStructures", "ForwardDiff", "GaussQuadrature", "GilbertCurves", "HDF5", "InteractiveUtils", "IntervalSets", "KrylovKit", "LinearAlgebra", "MultiBroadcastFusion", "NVTX", "PkgVersion", "RecursiveArrayTools", "RootSolvers", "SparseArrays", "StaticArrays", "Statistics", "Unrolled"]
-git-tree-sha1 = "7632911d7b09fb3e067984626b07b3eb1113e55e"
+git-tree-sha1 = "f92a814407e6eb7a20748c67ac3fb42eff604dd8"
+repo-rev = "main"
+repo-url = "https://github.com/CliMA/ClimaCore.jl.git"
 uuid = "d414da3d-4745-48bb-8d80-42e94e092884"
-version = "0.14.20"
+version = "0.14.22"
 
     [deps.ClimaCore.extensions]
     ClimaCoreCUDAExt = "CUDA"

docs/src/dev/compute_convergence.jl

@@ -120,9 +120,12 @@ function export_convergence_results(alg_name, test_problem, num_steps; kwargs...
     out_dict[string(test_name)][string(alg_name)] = Dict()
     out_dict[string(test_name)]["args"] = (alg_name, test_problem, num_steps)
     out_dict[string(test_name)]["kwargs"] = kwargs
-    compute_convergence!(out_dict, alg_name, test_problem, num_steps; kwargs...)
-    Logging.with_logger(Logging.NullLogger()) do
-        JLD2.save_object("convergence_$(alg_name)_$(test_problem.test_name).jld2", out_dict)
+    try
+        compute_convergence!(out_dict, alg_name, test_problem, num_steps; kwargs...)
+        Logging.with_logger(Logging.NullLogger()) do
+            JLD2.save_object("convergence_$(alg_name)_$(test_problem.test_name).jld2", out_dict)
+        end
+    catch
     end
 end
 

docs/src/dev/maybe_error.jl

@@ -0,0 +1,20 @@
+import ArgParse
+function parse_commandline()
+    s = ArgParse.ArgParseSettings()
+    ArgParse.@add_arg_table s begin
+        "--alg"
+        help = "Algorithm to test convergence"
+        arg_type = String
+        default = "ARS343"
+    end
+    parsed_args = ArgParse.parse_args(ARGS, s)
+    return parsed_args
+end
+
+import ClimaTimeSteppers as CTS
+parsed_args = parse_commandline()
+alg_name = getproperty(CTS, Symbol(parsed_args["alg"]))()
+
+if isfile("output/$alg_name/data_with_nans.dat")
+    error("Found NaNs!")
+end

docs/src/dev/report_gen_alg.jl

@@ -62,13 +62,36 @@ include(joinpath(@__DIR__, "compute_convergence.jl"))
 include(joinpath(pkgdir(ClimaTimeSteppers), "test", "problems.jl"))
 import ClimaTimeSteppers as CTS
 all_subtypes(::Type{T}) where {T} = isabstracttype(T) ? vcat(all_subtypes.(subtypes(T))...) : [T]
+_global_parsed_args = parse_commandline()
+_global_alg_name = getproperty(CTS, Symbol(_global_parsed_args["alg"]))()
+nans_exported = false
+import ClimaCore
+ClimaCore.DataLayouts.call_post_op_callback() = true
+function ClimaCore.DataLayouts.post_op_callback(data, args...; kwargs...)
+    if any(isnan, parent(data))
+        global nans_exported
+        if !nans_exported
+            @show count(isnan, parent(data))
+            @show length(parent(data))
+            @show size(parent(data))
+            aname = nameof(typeof(_global_alg_name))
+            file = "output/$_global_alg_name/data_with_nans.dat"
+            mkpath(dirname(file))
+            open(file, "w") do io
+                println(io, parent(data))
+            end
+            nans_exported = true
+        end
+    end
+end
 
 let # Convergence
-    parsed_args = parse_commandline()
-    alg_name = getproperty(CTS, Symbol(parsed_args["alg"]))()
     # NOTE: Some imperfections in the convergence order for SSPKnoth are to be
     # expected because we are not using the exact Jacobian
 
+    parsed_args = parse_commandline()
+    alg_name = getproperty(CTS, Symbol(parsed_args["alg"]))()
+
     export_convergence_results(alg_name, ark_analytic_nonlin_test_cts(Float64), 200)
     export_convergence_results(alg_name, ark_analytic_sys_test_cts(Float64), 400)
 

test/problems.jl

@@ -510,7 +510,7 @@ function climacore_2Dheat_test_cts(::Type{FT}) where {FT}
         dss_tendency && Spaces.weighted_dss!(tendency.u)
         _FT = Spaces.undertype(axes(state.u))
         set_boundaries!(tendency.u, _FT(0))
-        if t ≤ t_end*0.01
+        if t ≤ t_end * 0.01
             e0 = extrema(state.u)
             e1 = extrema(divgradu)
             @show e0, e1, count(isnan, parent(state.u)), length(parent(state.u))
@@ -519,12 +519,12 @@ function climacore_2Dheat_test_cts(::Type{FT}) where {FT}
     end
 
     function dss!(state, _, t)
-        if t ≤ t_end*0.01
+        if t ≤ t_end * 0.01
             e0_dss = extrema(state.u)
             @show e0_dss, count(isnan, parent(state.u)), length(parent(state.u))
         end
         dss_tendency || Spaces.weighted_dss!(state.u)
-        if t ≤ t_end*0.01
+        if t ≤ t_end * 0.01
             e1_dss = extrema(state.u)
             @show e1_dss, count(isnan, parent(state.u)), length(parent(state.u))
         end