Skip to content

Commit

Permalink
refactor driver and benchmark toml format
Browse files Browse the repository at this point in the history
  • Loading branch information
@johnbowen42
johnbowen42 committed Jan 31, 2025
1 parent 7611aa8 commit ffb1085
Show file tree
Hide file tree
Showing 3 changed files with 620 additions and 472 deletions.
126 changes: 90 additions & 36 deletions benchmarks.toml
View file
Original file line number Diff line number Diff line change
@@ -1,65 +1,119 @@
[adam]
[adam.nvidia]
aot = "benchmarks/hecbench/cuda/adam"
proteus = "benchmarks/hecbench/cuda/adam"
jitify = "benchmarks/hecbench/cuda-jitify/adam"
[adam.amd]
aot = "benchmarks/hecbench/hip/adam"
proteus = "benchmarks/hecbench/hip/adam"
[adam.nvidia.aot]
path = "benchmarks/hecbench/cuda/adam"
exe = "adam-aot.x"
[adam.nvidia.proteus]
path = "benchmarks/hecbench/cuda/adam"
exe = "adam-proteus.x"
[adam.nvidia.jitify]
path = "benchmarks/hecbench/cuda-jitify/adam"
exe = "adam-jitify.x"
[adam.amd.aot]
path = "benchmarks/hecbench/hip/adam"
exe = "adam-aot.x"
[adam.amd.proteus]
path = "benchmarks/hecbench/hip/adam"
exe = "adam-proteus.x"
[adam.inputs]
default = "160000 1600 1000"

[feynman-kac]
[feynman-kac.nvidia]
aot = "benchmarks/hecbench/cuda/feynman-kac"
proteus = "benchmarks/hecbench/cuda/feynman-kac"
jitify = "benchmarks/hecbench/cuda-jitify/feynman-kac"
[feynman-kac.amd]
aot = "benchmarks/hecbench/hip/feynman-kac"
proteus = "benchmarks/hecbench/hip/feynman-kac"
[feynman-kac.nvidia.aot]
path = "benchmarks/hecbench/cuda/feynman-kac"
exe = "kac-aot.x"
[feynman-kac.nvidia.proteus]
path = "benchmarks/hecbench/cuda/feynman-kac"
exe = "kac-proteus.x"
[feynman-kac.nvidia.jitify]
path = "benchmarks/hecbench/cuda-jitify/feynman-kac"
exe = "kac-jitify.x"
[feynman-kac.amd.aot]
path = "benchmarks/hecbench/hip/feynman-kac"
exe = "kac-aot.x"
[feynman-kac.amd.proteus]
path = "benchmarks/hecbench/hip/feynman-kac"
exe = "kac-proteus.x"
[feynman-kac.inputs]
default = "1"

[lulesh]
[lulesh.nvidia]
aot = "benchmarks/hecbench/cuda/LULESH"
proteus = "benchmarks/hecbench/cuda/LULESH"
jitify = "benchmarks/hecbench/cuda-jitify/LULESH"
[lulesh.amd]
aot = "benchmarks/hecbench/hip/LULESH"
proteus = "benchmarks/hecbench/hip/LULESH"
[lulesh.nvidia.aot]
path = "benchmarks/hecbench/cuda/LULESH"
exe = "LULESH-aot.x"
[lulesh.nvidia.proteus]
path = "benchmarks/hecbench/cuda/LULESH"
exe = "LULESH-proteus.x"
[lulesh.nvidia.jitify]
path = "benchmarks/hecbench/cuda-jitify/LULESH"
exe = "LULESH-jitify.x"
[lulesh.amd.aot]
path = "benchmarks/hecbench/hip/LULESH"
exe = "LULESH-aot.x"
[lulesh.amd.proteus]
path = "benchmarks/hecbench/hip/LULESH"
exe = "LULESH-proteus.x"
[lulesh.inputs]
default = "-s 128 -i 1000"

[rsbench]
[rsbench.nvidia]
aot = "benchmarks/hecbench/cuda/rsbench"
proteus = "benchmarks/hecbench/cuda/rsbench"
jitify = "benchmarks/hecbench/cuda-jitify/rsbench"
[rsbench.amd]
aot = "benchmarks/hecbench/hip/rsbench"
proteus = "benchmarks/hecbench/hip/rsbench"
[rsbench.nvidia.aot]
path = "benchmarks/hecbench/cuda/rsbench"
exe = "rsbench-aot.x"
[rsbench.nvidia.proteus]
path = "benchmarks/hecbench/cuda/rsbench"
exe = "rsbench-proteus.x"
[rsbench.nvidia.jitify]
path = "benchmarks/hecbench/cuda-jitify/rsbench"
exe = "rsbench-jitify.x"
[rsbench.amd.aot]
path = "benchmarks/hecbench/hip/rsbench"
exe = "rsbench-aot.x"
[rsbench.amd.proteus]
path = "benchmarks/hecbench/hip/rsbench"
exe = "rsbench-proteus.x"
[rsbench.inputs]
default = "-s large -m event"

[sw4ck]
[sw4ck.nvidia]
aot = "benchmarks/hecbench/cuda/sw4ck"
proteus = "benchmarks/hecbench/cuda/sw4ck"
jitify = "benchmarks/hecbench/cuda-jitify/sw4ck"
[sw4ck.amd]
aot = "benchmarks/hecbench/hip/sw4ck"
proteus = "benchmarks/hecbench/hip/sw4ck"
[sw4ck.nvidia.aot]
path = "benchmarks/hecbench/cuda/sw4ck"
exe = "sw4ck-aot.x"
[sw4ck.nvidia.proteus]
path = "benchmarks/hecbench/cuda/sw4ck"
exe = "sw4ck-proteus.x"
[sw4ck.nvidia.jitify]
path = "benchmarks/hecbench/cuda-jitify/sw4ck"
exe = "sw4ck-jitify.x"
[sw4ck.amd.aot]
path = "benchmarks/hecbench/hip/sw4ck"
exe = "sw4ck-aot.x"
[sw4ck.amd.proteus]
path = "benchmarks/hecbench/hip/sw4ck"
exe = "sw4ck-proteus.x"
[sw4ck.inputs]
default = "sw4ck.in 100"

[wsm5]
[wsm5.nvidia]
aot = "benchmarks/hecbench/cuda/wsm5"
proteus = "benchmarks/hecbench/cuda/wsm5"
jitify = "benchmarks/hecbench/cuda-jitify/wsm5"
[wsm5.amd]
aot = "benchmarks/hecbench/hip/wsm5"
proteus = "benchmarks/hecbench/hip/wsm5"
[wsm5.nvidia.aot]
path = "benchmarks/hecbench/cuda/wsm5"
exe = "wsm5-aot.x"
[wsm5.nvidia.proteus]
path = "benchmarks/hecbench/cuda/wsm5"
exe = "wsm5-proteus.x"
[wsm5.nvidia.jitify]
path = "benchmarks/hecbench/cuda-jitify/wsm5"
exe = "wsm5-jitify.x"
[wsm5.amd.aot]
path = "benchmarks/hecbench/hip/wsm5"
exe = "wsm5-aot.x"
[wsm5.amd.proteus]
path = "benchmarks/hecbench/hip/wsm5"
exe = "wsm5-proteus.x"
[wsm5.inputs]
default = "10"
44 changes: 30 additions & 14 deletions driver.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,18 +142,21 @@ def get_hash(x):


class Executor:
def __init__(self, benchmark, path, exemode, build_command, inputs, cc, proteus_path, env_configs):
def __init__(self, benchmark, path, executable_name, exemode, build_command, inputs, cc, proteus_path, env_configs, build_once):
self.benchmark = benchmark
self.path = path
self.exemode = exemod
self.executable_name = executable_name
self.exemode = exemode
# the build command is meant to be a full bash command to build the benchmark, eg
# `cmake -DCMAKE_BUILD_TYPE=Debug --build` or `make benchmark`
# If none is provided, it will default to `make`
self.build_command = 'make' if build_command == None else build_command
self.build_command = 'make' if build_command == None else build_command
self.inputs = inputs
self.cc = cc
self.proteus_path = proteus_path
self.env_configs = env_configs
self.build_once = build_once
self.built = False

def __str__(self):
return f"{self.benchmark} {self.path} {self.exemode}"
Expand Down Expand Up @@ -185,7 +188,7 @@ def clean(self):
cmd = "make clean"
self.execute_command(cmd)

def build(self, cmd, do_jit):
def build(self, do_jit):
os.chdir(self.path)
env = os.environ.copy()
env["ENABLE_PROTEUS"] = "yes" if do_jit else "no"
Expand All @@ -194,16 +197,16 @@ def build(self, cmd, do_jit):
t1 = time.perf_counter()
print(
"Build command",
cmd,
self.build_command,
"CC=" + env["CC"],
"PROTEUS_PATH=" + env["PROTEUS_PATH"],
"ENABLE_PROTEUS=" + env["ENABLE_PROTEUS"],
)
self.execute_command(cmd, env=env)
self.execute_command(self.build_command, env=env)
t2 = time.perf_counter()
return t2 - t1

def build_and_run(self, build_command, reps, profiler=None):
def build_and_run(self, reps, profiler=None):
os.chdir(self.path)

results = pd.DataFrame()
Expand All @@ -214,12 +217,10 @@ def build_and_run(self, build_command, reps, profiler=None):
or self.exemode == "jitify"
), "Expected aot or proteus or jitify for exemode"

exe = f"{self.benchmark}-{self.exemode}.x"
self.clean()
print("BUILD", self.path, "type", self.exemode)

ctime = self.build(build_command, self.exemode != "aot")
exe_size = Path(f"{self.path}/{exe}").stat().st_size
ctime = self.build(self.exemode != "aot")
exe_size = Path(f"{self.path}/{self.executable_name}").stat().st_size
print("=> BUILT")

for repeat in range(0, reps):
Expand All @@ -228,7 +229,7 @@ def build_and_run(self, build_command, reps, profiler=None):
cmd_env = os.environ.copy()
for k, v in env.items():
cmd_env[k] = v
cmd = f"./{exe} {args}"
cmd = f"./{self.executable_name} {args}"

set_launch_bounds = (
False if env["ENV_PROTEUS_SET_LAUNCH_BOUNDS"] == "0" else True
Expand Down Expand Up @@ -464,19 +465,34 @@ def main():
env_configs = JitifyConfig().get_env_configs()
else:
raise Exception(f"Invalid exemode {args.exemode}")

proteus_install = args.proteus_path
assert os.path.exists(proteus_install), f"Error: Proteus install path '{proteus_install}' does not exist!"
for env in env_configs:
env["PROTEUS_INSTALL_PATH"] = proteus_install
experiments = []
build_command = None
build_once = False
# custom toml wide level build command specified
if "build" in benchmark_configs:
build_command = benchmark_configs["build"][args.machine]
build_once = True

for benchmark in args.bench if args.bench else benchmark_configs:
if benchmark == "build":
continue
config = benchmark_configs[benchmark]
experiments.append(
Executor(
benchmark,
Path.cwd() / Path(config[args.machine][args.exemode]),
Path.cwd() / Path(config[args.machine][args.exemode]["path"]),
Path(config[args.machine][args.exemode]["exe"]),
args.exemode,
build_command,
config["inputs"],
args.compiler,
args.proteus_path,
env_configs,
build_once
)
)

Expand Down
Loading

0 comments on commit ffb1085

Please sign in to comment.