+{% block login_container %}
+
+
+{% endblock login %}
+
+{% endblock %}
+
+{% block script %}
+{{ super() }}
+
+{% endblock %}
diff --git a/2024-RIKEN-AWS/JupyterNotebook/docker/start.sh b/2024-RIKEN-AWS/JupyterNotebook/docker/start.sh
new file mode 100755
index 0000000..bad19ab
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/docker/start.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+/usr/bin/flux start --test-size=4 /usr/local/bin/jupyter-lab --ip=0.0.0.0
diff --git a/2024-RIKEN-AWS/JupyterNotebook/flux-tree/flux-tree b/2024-RIKEN-AWS/JupyterNotebook/flux-tree/flux-tree
new file mode 100644
index 0000000..f12a9b8
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/flux-tree/flux-tree
@@ -0,0 +1,847 @@
+#! /bin/bash
+
+##############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, LICENSE)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+set -o errexit
+set -o pipefail
+#set -o xtrace
+
+FT_TOPO='1' # store arg to --topology (e.g., 2x2)
+FT_QUEUE='default' # store arg to --queue-policy (e.g., fcfs:easy)
+FT_PARAMS='default' # store arg to --queue-params
+FT_MATCH='default' # store arg to --match-policy (e.g., low:high)
+FT_PERF_OUT='%^+_no' # store perf-out filename given by --perf-out
+FT_PERF_FORMAT='{treeid:<15s} {elapse:>20f} {begin:>20f} {end:>20f} {match:>15f} '\
+'{njobs:>15d} {my_nodes:>5d} {my_cores:>4d} {my_gpus:>4d}'
+ # store perf-out with format given by --perf-format
+FT_FXLOGS='%^+_no' # dir in which flux logs are produced
+FT_FXRDIR='%^+_no' # flux rundir attribute to pass
+FT_LEAF='no' # is this a leaf instance (given by --leaf)?
+FT_G_PREFIX='tree' # store hierarchical path given by --prefix
+FT_DRY_RUN='no' # --dry-run for testing?
+FT_INTER='no' # is an option for internal levels given?
+FT_MAX_EXIT_CODE=0 # maximum exit code detected
+FT_MAX_FLUX_JOBID=0 # maximum flux jobid that reports max exit code
+FT_MAX_TREE_ID="" # FLUX_TREE_ID that reports max exit code
+FT_MAX_JOBSCRIPT_IX="" # FLUX_TREE_JOBSCRIPT_INDEX reporting max code
+ # --prefix is an internal-use-only option
+FT_JOB_NAME='%^+_no' # job name to use when submitting children
+ORIG_FLUXION_QMANAGER_OPTIONS='' #
+ORIG_FLUXION_RESOURCE_OPTIONS='' # to apply and unapply FLUXION_RESOURCE options
+ORIG_FLUXION_QMANAGER_RC_NOOP='' # module load options.
+ORIG_FLUXION_RESOURCE_RC_NOOP='' #
+
+declare -i FT_NJOBS=1 # store num of jobs to run, given by --njobs
+declare -i FT_NNODES=1 # store num of nodes assigned, given by --nnodes
+declare -i FT_NCORES=1 # store num of cores per node (--ncores-per-node)
+declare -i FT_NGPUS=0 # store num of gpus per node (--ngpus-per-node)
+declare -r top_prefix='tree' # prefix name to identify the top Flux instance
+declare -r t_delim='x' # topology delimiter
+declare -r p_delim=':' # match policy delimiter
+declare -r perf_format='%-15s %20s %20s %20s %15s %15s %5s %4s %4s'
+declare -a FT_CL=() # save the jobscript command into an array
+declare -A mp_policies=( # make sure to update this when match
+ [low]=1 # policies are updated.
+ [high]=1
+ [locality]=1
+ [variation]=1
+ [default]=1
+)
+declare -A qp_policies=( # make sure to update this when
+ [fcfs]=1 # queuing policies are updated.
+ [easy]=1
+ [hybrid]=1
+ [conservative]=1
+ [default]=1
+)
+declare -A q_params=( # make sure to update this when
+ [queue-depth]=1 # queuing parameters are updated.
+ [reservation-depth]=1
+ [default]=1
+)
+declare -a jobids # array to store a set of submitted job IDs
+
+declare -r long_opts='help,leaf,flux-logs:,flux-rundir:,nnodes:'\
+',ncores-per-node:,ngpus-per-node:,topology:,queue-policy:,queue-params:'\
+',match-policy:,njobs:,perf-out:,perf-format:,prefix:,job-name:,dry-run'
+declare -r short_opts='hlf:r:N:c:g:T:Q:P:M:J:o:X:d'
+declare -r prog=${0##*/}
+declare -r usage="
+Usage: ${prog} [OPTIONS] -- Jobscript\n\
+\n\
+Create a Flux instance hierarchy according to the specified\n\
+policies and schedule/run the specified number\n\
+of Jobscripts at the last level of this hierarchy.\n\
+\n\
+If --topology=2x4 and --njobs=32 are given, for instance,\n\
+2 Flux instances will be spawned from within the current instance,\n\
+each of which will in turn spawn 4 child Flux instances, totaling\n\
+8 instances at the last level of this hierarchy.\n\
+Once this is done, 4 jobs (of Jobscripts) will be scheduled\n\
+and executed at each of these 8 last-level Flux instances.\n\
+\n\
+The resources specified by --nnodes (total number of nodes) and\n\
+--ncores-per-node (total number of cores per node)\n\
+are recursively divided such that each sibling Flux instance\n\
+will be assigned to an equal split of the resources of their\n\
+parent instance. In addition, --ngpus-per-node can be given,\n\
+in which case the given GPU count will also be split.\n\
+If not given, it is assumed that there is no GPU on nodes.\n\
+\n\
+Jobscript is expected to submit one or more programs through\n\
+the flux-job submit command or its variants.\n\
+Jobscript is passed with five environment variables:\n\
+FLUX_TREE_ID, FLUX_TREE_JOBSCRIPT_INDEX, FLUX_TREE_NNODES,\n\
+FLUX_TREE_NCORES_PER_NODE and FLUX_TREE_NGPUS_PER_NODE.\n\
+FLUX_TREE_ID is an ID string uniquely identifying the hierarchical\n\
+path of the Flux instance on which Jobscript is being executed.\n\
+FLUX_TREE_JOBSCRIPT_INDEX is the integer ID of each jobscript\n\
+invocation local to the Flux instance. It starts from 1 and\n\
+sequentially increases.\n\
+FLUX_TREE_NNODES is the number nodes assigned to the instance.\n\
+FLUX_TREE_NCORES_PER_NODE is the number of cores per node\n\
+assigned to the instance.\n\
+FLUX_TREE_NGPUS_PER_NODE is the number of GPUs per node\n\
+assigned to the instance.\n\
+\n\
+If --queue-policy (additionally --queue-params) and/or\n\
+--match-policy are given, each level of this hierarchy will\n\
+be set to the specified queuing and matching policies and\n\
+parameters. Otherwise, all levels will be configured\n\
+to be used either the default policies or policies specified\n\
+through the FLUXION_RESOURCE_OPTIONS and/or FLUXION_QMANAGER_OPTIONS\n\
+environment variables.\n\
+\n\
+If any one of Jobscripts returns a non-zero exit code, flux-tree\n\
+detects the script invocation exited with the highest code and print\n\
+both that exit code and the outputs printed from executing the script.\n\
+In this case, FLUX_TREE_ID and FLUX_TREE_JOBSCRIPT_INDEX are also\n\
+reported in the from of \${FLUX_TREE_ID}@index[\${FLUX_TREE_JOBSCRIPT_INDEX}]\n\
+\n\
+Options:\n\
+ -h, --help Display this message\n\
+ -l, --leaf Leaf instance. Directly submit jobs\n\
+ to enclosing Flux instance. Mutually-exclusive\n\
+ with internal tree-node options like -T.\n\
+ (default=${FT_LEAF})\n\
+ -f, --flux-logs=DIR Dump Flux logs for all instances into DIR\n\
+ -r, --flux-rundir=DIR Set the rundir attribute of each Flux tree instance\n\
+ into a subdirectory within DIR. The content\n\
+ stores will be redirected to them as well\n\
+ -N, --nnodes=NNODES Total num of nodes to use\n\
+ (default=${FT_NNODES})\n\
+ -c, --ncores-per-node=NCORES Total num of cores per node to use\n\
+ (default=${FT_NCORES})\n\
+ -g, --ngpus-per-node=NGPUS Total num of gpus per node to use\n\
+ (default=${FT_NGPUS})\n\
+ -T, --topology=HPOLICY Topology of Flux instance hierarchy:\n\
+ e.g., 2x2 (default=${FT_TOPO})\n\
+ -Q, --queue-policy=QPOLICY Queuing policy for each level of\n\
+ the hierarchy: e.g., easy:fcfs\n\
+ -P, --queue-params=QPARAMS Queuing parameters for each level of\n\
+ the hierarchy: e.g.,\n\
+ queue-depth=5:reservation-depth=5\n\
+ -M, --match-policy=MPOLICY Match policy for each level of\n\
+ the hierarchy: e.g., low:high\n\
+ -J, --njobs=NJOBS Total num of Jobscripts to run\n\
+ (default=${FT_NJOBS})\n\
+ -o, --perf-out=FILENAME Dump the performance data into\n\
+ the given file (default: don't print)\n\
+ --perf-format=FORMAT Dump the performance data with the given\n\
+ format. Uses the python format\n\
+ specification mini-language.\n\
+ Example: \"{treeid:<15s},{elapse:>20f}\"\n\
+ --job-name=NAME Name to use when submitting child jobs\n\
+ -- Stop parsing options after this\n\
+"
+
+die() { echo -e "${prog}:" "$@"; exit 1; }
+warn() { echo -e "${prog}: warning:" "$@"; }
+dr_print() { echo -e "${prog}: dry-run:" "$@"; }
+
+#
+# Roll up the performance records for each Flux instance to the KVS
+# guest namespace of the parent Flux instance or print them out if top level.
+#
+rollup() {
+ local prefix="${1}"
+ local blurb="${2}"
+ local out="${3}"
+ local num_children="${4}"
+ local format="${5}"
+
+ if [[ "${prefix}" == "${top_prefix}" && "${out}" != "%^+_no" ]]; then
+ flux tree-helper --perf-out="${out}" --perf-format="${format}" \
+ ${num_children} "tree-perf" "${FT_JOB_NAME}" <<< "${blurb}"
+ else
+ flux tree-helper ${num_children} "tree-perf" "${FT_JOB_NAME}" \
+ <<< "${blurb}"
+ fi
+}
+
+
+#
+# Return a JSON string out of the performance data passed.
+#
+jsonify() {
+ local prefix="${1}"
+ local njobs="${2}"
+ local nnodes="${3}"
+ local ncores="${4}"
+ local ngpus="${5}"
+ local begin="${6}"
+ local end="${7}"
+ local avg=0
+ local avail="no"
+ local el_match=0
+
+ # Print resource match time only for internal study
+ # flux-resource isn't a public command
+ if [[ "x${FT_DRY_RUN}" = "xno" ]]
+ then
+ flux ion-resource -h > /dev/null 2>&1 && avail="yes"
+ fi
+
+ if [[ "${avail}" = "yes" ]]
+ then
+ avg=$(flux ion-resource stat | grep "Avg" | awk '{print $4}')
+ el_match=$(awk "BEGIN {print ${avg}*${njobs}*1000000.0}")
+ fi
+
+ local elapse=0
+ elapse=$(awk "BEGIN {print ${end} - ${begin}}")
+ echo "{\"treeid\":\"${prefix}\",\"njobs\":${njobs},\"my_nodes\":${nnodes},\
+\"my_cores\":${ncores},\"my_gpus\":${ngpus},\"perf\":{\"begin\":${begin},\
+\"end\":${end},\"elapse\":${elapse},\"match\":${el_match}}}"
+}
+
+
+#
+# Fetch the next topology parameter that will be passed to
+# the next-level Flux instances. E.g., If the current level topology
+# is 2x3x4, the topology handled at the next level will be 3x4.
+#
+next_topo() {
+ local topo="${1}"
+ local nx=''
+ local nfields=0
+ nfields=$(echo "${topo}" | awk -F"${t_delim}" '{print NF}')
+ # Remove the first topo parameter
+ [[ ${nfields} -gt 1 ]] && nx="${topo#*${t_delim}}"
+ echo "${nx}"
+}
+
+
+#
+# Fetch the next policy parameter that will be passed to
+# the next-level Flux instances. E.g., If the current policy parameter
+# is high:low:locality, the policies handled at the next level
+# will be low:locality.
+#
+next_policy_or_param() {
+ local policy_or_param="${1}"
+ local nx=""
+ local nfields=0
+ nfields=$(echo "${policy_or_param}" | awk -F"${p_delim}" '{print NF}')
+ [[ ${nfields} -gt 1 ]] && nx="${policy_or_param#*${p_delim}}"
+ echo "${nx}"
+}
+
+
+#
+# Check if the given queuing policy is valid
+#
+qpolicy_check() {
+ local policy=${1%%${p_delim}*}
+ [[ "x${policy}" = "x" ]] && return 1
+ [[ "${qp_policies["${policy}"]:-missing}" = "missing" ]] && return 1
+ return 0
+}
+
+
+#
+# Check if the given match policy is valid
+#
+mpolicy_check() {
+ local policy=${1%%${p_delim}*}
+ [[ "x${policy}" = "x" ]] && return 1
+ [[ "${mp_policies["${policy}"]:-missing}" = "missing" ]] && return 1
+ return 0
+}
+
+
+#
+# Check if the given queue param is valid
+#
+qparams_check() {
+ local param=''
+ param=$(echo "${1}" | awk -F"${p_delim}" '{print $1}')
+ param=${1%%${p_delim}*}
+ local final_param=''
+ final_param=${param##*,}
+
+ for i in $(seq 1 10)
+ do
+ local token1=${param%%,*}
+ local token2=${token1%=*}
+ [[ "x${token2}" = "x" ]] && return 1
+ [[ "${q_params["${token2}"]:-missing}" = "missing" ]] && return 1
+ [[ "x${token1}" = "x${final_param}" ]] && break
+ param=${param#*,}
+ done
+ return 0
+}
+
+
+#
+# Calculate the number of jobs to execute based on the number of Flux instances
+# being used at a level and the rank of the instance amongst its siblings.
+#
+get_my_njobs(){
+ local njobs="${1}"
+ local size="${2}" # rank starts from 1
+ local rank="${3}"
+ echo $(( njobs / size + (size + njobs % size)/(size + rank) ))
+}
+
+
+#
+# Calculate the total number of cores that will be assigned to a child
+# Flux instance based on the total number of nodes and cores per node
+# assigned to the current Flux instance as well as the size and rank parameter.
+#
+get_my_cores(){
+ local nnodes="${1}"
+ local ncores="${2}"
+ local size="${3}"
+ local rank="${4}"
+ local t_cores=$(( nnodes * ncores ))
+ echo $(( t_cores / size + (size + t_cores % size) / (size + rank) ))
+}
+
+
+#
+# Calculate the total number of GPUs that will be assigned to a child
+# Flux instance based on the total number of nodes and GPUs per node
+# assigned to the current Flux instance as well as the size and rank parameter.
+#
+get_my_gpus(){
+ local nnodes="${1}"
+ local ngpus="${2}"
+ local size="${3}"
+ local rank="${4}"
+ local t_gpus=$(( nnodes * ngpus ))
+ echo $(( t_gpus / size + (size + t_gpus % size) / (size + rank) ))
+}
+
+
+#
+# Adjust the number of Flux instances to spawn at the next level
+# if the amount of resources managed by the parent instance is small.
+#
+get_effective_size(){
+ local ncores="${1}"
+ local ngpus="${2}"
+ local size="${3}"
+ [[ ${ngpus} -ne 0 && ${ngpus} -lt ${size} ]] && size=${ngpus}
+ [[ ${ncores} -lt ${size} ]] && size=${ncores}
+ echo "${size}"
+}
+
+
+#
+# Calculate the total number of nodes that will be assigned to a child
+# Flux instance based on the total number of cores per node as well as
+# the total number of cores assigned to this child instance. Returns
+# minimum num of nodes required.
+#
+get_my_nodes(){
+ local ncores="${1}"
+ local m_cores="${2}"
+ echo $(( m_cores / ncores + (ncores + m_cores % ncores) / (ncores + 1 )))
+}
+
+
+#
+# Apply all of the policies for the target Flux instance
+# by setting environment variables.
+#
+apply_policies() {
+ local queue_policy="${1%%${p_delim}*}"
+ local queue_param="${2%%${p_delim}*}"
+ local match_policy="${3%%${p_delim}*}"
+
+ ORIG_FLUXION_QMANAGER_OPTIONS=${FLUXION_QMANAGER_OPTIONS:-none}
+ ORIG_FLUXION_RESOURCE_OPTIONS=${FLUXION_RESOURCE_OPTIONS:-none}
+ ORIG_FLUXION_QMANAGER_RC_NOOP=${FLUXION_QMANAGER_RC_NOOP:-none}
+ ORIG_FLUXION_RESOURCE_RC_NOOP=${FLUXION_RESOURCE_RC_NOOP:-none}
+ unset FLUXION_QMANAGER_RC_NOOP
+ unset FLUXION_RESOURCE_RC_NOOP
+
+ if [[ "${queue_policy}" != "default" ]]
+ then
+ export FLUXION_QMANAGER_OPTIONS="queue-policy=${queue_policy}"
+ fi
+ if [[ "${queue_param}" != "default" ]]
+ then
+ local qo="${FLUXION_QMANAGER_OPTIONS}"
+ export FLUXION_QMANAGER_OPTIONS="${qo:+${qo},}queue-params=${queue_param}"
+ fi
+ if [[ "${match_policy}" != "default" ]]
+ then
+ export FLUXION_RESOURCE_OPTIONS="hwloc-allowlist=node,core,gpu \
+policy=${match_policy}"
+ fi
+ if [[ "x${FT_DRY_RUN}" = "xyes" ]]
+ then
+ dr_print "FLUXION_QMANAGER_OPTIONS:${FLUXION_QMANAGER_OPTIONS}"
+ dr_print "FLUXION_RESOURCE_OPTIONS:${FLUXION_RESOURCE_OPTIONS}"
+ fi
+}
+
+
+#
+# Undo all of the policies set for the target Flux instance
+# by unsetting environment variables.
+#
+unapply_policies() {
+ unset FLUXION_QMANAGER_OPTIONS
+ unset FLUXION_RESOURCE_OPTIONS
+
+ if [ "${ORIG_FLUXION_QMANAGER_OPTIONS}" != "none" ]
+ then
+ export FLUXION_QMANAGER_OPTIONS="${ORIG_FLUXION_QMANAGER_OPTIONS}"
+ fi
+ if [ "${ORIG_FLUXION_RESOURCE_OPTIONS}" != "none" ]
+ then
+ export FLUXION_RESOURCE_OPTIONS="${ORIG_FLUXION_RESOURCE_OPTIONS}"
+ fi
+ if [ "${ORIG_FLUXION_QMANAGER_RC_NOOP}" != "none" ]
+ then
+ export FLUXION_QMANAGER_RC_NOOP="${ORIG_FLUXION_QMANAGER_RC_NOOP}"
+ fi
+ if [ "${ORIG_FLUXION_RESOURCE_RC_NOOP}" != "none" ]
+ then
+ export FLUXION_RESOURCE_RC_NOOP="${ORIG_FLUXION_RESOURCE_RC_NOOP}"
+ fi
+ if [[ "x${FT_DRY_RUN}" = "xyes" ]]
+ then
+ dr_print "FLUXION_QMANAGER_OPTIONS:${FLUXION_QMANAGER_OPTIONS}"
+ dr_print "FLUXION_RESOURCE_OPTIONS:${FLUXION_RESOURCE_OPTIONS}"
+ dr_print "FLUXION_QMANAGER_RC_NOOP:${FLUXION_QMANAGER_RC_NOOP}"
+ dr_print "FLUXION_RESOURCE_RC_NOOP:${FLUXION_RESOURCE_RC_NOOP}"
+ fi
+}
+
+
+
+################################################################################
+# #
+# Handle Leaf or Internal Flux Instances #
+# #
+################################################################################
+
+#
+# Execute the script. Export a predefined set of
+# environment variables and execute the given jobscript.
+#
+execute() {
+ local prefix="${1}"
+ local nnodes="${2}"
+ local ncores="${3}"
+ local ngpus="${4}"
+ local njobs="${5}"
+ local rc=0
+
+ for job in $(seq 1 "${njobs}");
+ do
+ export FLUX_TREE_ID="${prefix}"
+ export FLUX_TREE_JOBSCRIPT_INDEX="${job}"
+ export FLUX_TREE_NNODES="${nnodes}"
+ export FLUX_TREE_NCORES_PER_NODE="${ncores}"
+ export FLUX_TREE_NGPUS_PER_NODE="${ngpus}"
+
+ if [[ "x${FT_DRY_RUN}" = "xyes" ]]
+ then
+ dr_print "FLUX_TREE_ID=${FLUX_TREE_ID}"
+ dr_print "FLUX_TREE_JOBSCRIPT_INDEX=${FLUX_TREE_JOBSCRIPT_INDEX}"
+ dr_print "FLUX_TREE_NCORES_PER_NODE=${FLUX_TREE_NCORES_PER_NODE}"
+ dr_print "FLUX_TREE_NGPUS_PER_NODE=${FLUX_TREE_NGPUS_PER_NODE}"
+ dr_print "FLUX_TREE_NNODES=${FLUX_TREE_NNODES}"
+ dr_print "eval ${FT_CL[@]}"
+ continue
+ else
+ rc=0
+ "${FT_CL[@]}" || rc=$?
+ if [[ ${rc} -gt ${FT_MAX_EXIT_CODE} ]]
+ then
+ FT_MAX_EXIT_CODE=${rc}
+ FT_MAX_TREE_ID="${FLUX_TREE_ID}"
+ FT_MAX_JOBSCRIPT_IX="${FLUX_TREE_JOBSCRIPT_INDEX}"
+ fi
+ fi
+ done
+
+ [[ "x${FT_DRY_RUN}" = "xno" ]] && flux queue drain
+
+ if [[ "x${FT_MAX_TREE_ID}" != "x" ]]
+ then
+ warn "${FT_CL[@]}: exited with exit code (${FT_MAX_EXIT_CODE})"
+ warn "invocation id: ${FT_MAX_TREE_ID}@index[${FT_MAX_JOBSCRIPT_IX}]"
+ warn "output displayed above, if any"
+ fi
+
+ unset FLUX_TREE_ID
+ unset FLUX_TREE_NNODES
+ unset FLUX_TREE_NCORES_PER_NODE
+}
+
+
+#
+# Entry point to execute the job script. When this is invoke,
+# the parent Flux instance has already been started.
+# Measure the elapse time of the job script execution, and
+# dump the performance data.
+#
+leaf() {
+ local prefix="${1}"
+ local nnodes="${2}"
+ local ncores="${3}"
+ local ngpus="${4}"
+ local njobs="${5}"
+ local perfout="${6}"
+ local format="${7}"
+
+ # Begin Time Stamp
+ local B=''
+ B=$(date +%s.%N)
+
+ execute "$@"
+
+ # End Time Stamp
+ local E=''
+ E=$(date +%s.%N)
+
+ local o=''
+
+ o=$(jsonify "${prefix}" "${njobs}" "${nnodes}" "${ncores}" \
+"${ngpus}" "${B}" "${E}")
+ rollup "${prefix}" "${o}" "${perfout}" "0" "${format}"
+}
+
+
+#
+# Roll up exit code from child instances
+#
+rollup_exit_code() {
+ local rc=0
+ for job in "${jobids[@]}"
+ do
+ rc=0
+ flux job status --exception-exit-code=255 ${job} || rc=$?
+ if [[ ${rc} -gt ${FT_MAX_EXIT_CODE} ]]
+ then
+ FT_MAX_EXIT_CODE=${rc}
+ FT_MAX_FLUX_JOBID=${job}
+ fi
+ done
+
+ if [[ "${FT_MAX_FLUX_JOBID}" != "0" ]]
+ then
+ flux job attach ${FT_MAX_FLUX_JOBID} || true
+ fi
+}
+
+#
+# Submit the specified number of Flux instances at the next level of the calling
+# instance. Use flux-tree recursively. Instances that have 0 jobs assigned are
+# not launched.
+#
+submit() {
+ local prefix="${1}"
+ local nx_topo=$(next_topo "${2}")
+ local nx_queue=$(next_policy_or_param "${3}")
+ local nx_q_params=$(next_policy_or_param "${4}")
+ local nx_match=$(next_policy_or_param "${5}")
+ local nnodes="${6}"
+ local ncores="${7}"
+ local ngpus="${8}"
+ local size="${9}"
+ local njobs="${10}"
+ local log="${11}"
+ local rdir="${12}"
+
+ # Flux instance rank-agnostic command-line options for the next level
+ local T="${nx_topo:+--topology=${nx_topo}}"
+ T="${T:---leaf}"
+ local Q="${nx_queue:+--queue-policy=${nx_queue}}"
+ local P="${nx_q_params:+--queue-params=${nx_q_params}}"
+ local M="${nx_match:+--match-policy=${nx_match}}"
+ local F=''
+ [[ "x${log}" != "x%^+_no" ]] && F="--flux-logs=${log}"
+ local R=''
+ [[ "x${rdir}" != "x%^+_no" ]] && R="--flux-rundir=${rdir}"
+ local rank=0
+
+ # Main Loop to Submit the Next-Level Flux Instances
+ size=$(get_effective_size "${ncores}" "${ngpus}" "${size}")
+ apply_policies "${3}" "${4}" "${5}"
+ for rank in $(seq 1 "${size}"); do
+ local my_cores=0
+ my_cores=$(get_my_cores "${nnodes}" "${ncores}" "${size}" "${rank}")
+ local my_gpus=0
+ my_gpus=$(get_my_gpus "${nnodes}" "${ngpus}" "${size}" "${rank}")
+ local my_njobs=0
+ my_njobs=$(get_my_njobs "${njobs}" "${size}" "${rank}")
+
+ [[ "${my_njobs}" -eq 0 ]] && break
+
+ # Flux instance rank-aware command-line options
+ local J="--njobs=${my_njobs}"
+ local o=''
+ if [[ x"${log}" != "x%^+_no" ]]
+ then
+ if [[ "x${FT_DRY_RUN}" != "xyes" ]]
+ then
+ mkdir -p "${log}"
+ fi
+ o="-o,-Slog-filename=${log}/${prefix}.${rank}.log"
+ fi
+ if [[ x"${rdir}" != "x%^+_no" ]]
+ then
+ if [[ "x${FT_DRY_RUN}" != "xyes" ]]
+ then
+ rm -rf "${rdir}/${prefix}.${rank}.pfs"
+ mkdir -p "${rdir}/${prefix}.${rank}.pfs"
+ fi
+ o="${o:+${o} }-o,-Srundir=${rdir}/${prefix}.${rank}.pfs"
+ fi
+ local N=0
+ N=$(get_my_nodes "${ncores}" "${my_cores}")
+ local c=0
+ c=$((my_cores/N + (my_cores + my_cores % N)/(my_cores + 1)))
+ local g=0
+ g=$((my_gpus/N + (my_gpus + my_gpus % N)/(my_gpus + 1)))
+ local G=''
+ [[ ${g} -gt 0 ]] && G="-g ${g}"
+ local X="--prefix=${prefix}.${rank}"
+
+ if [[ "x${FT_DRY_RUN}" = "xyes" ]]
+ then
+ dr_print "Rank=${rank}: N=${N} c=${c} ${G:+g=${G}} ${o:+o=${o}}"
+ dr_print "Rank=${rank}: ${T:+T=${T}}"
+ dr_print "Rank=${rank}: ${Q:+Q=${Q}} ${P:+P=${P}} ${M:+M=${M}}"
+ dr_print "Rank=${rank}: ${X:+X=${X}} ${J:+J=${J}} ${FT_CL:+S=${FT_CL[@]}}"
+ dr_print ""
+ continue
+ fi
+ jobid=$(\
+flux submit --job-name=${FT_JOB_NAME} -N${N} -n${N} -c${c} ${G} \
+ flux start ${o} \
+ flux tree -N${N} -c${c} ${G} ${T} ${Q} ${P} ${M} ${F} ${R} ${X} ${J} \
+ -- "${FT_CL[@]}")
+ jobids["${rank}"]="${jobid}"
+ done
+
+ [[ "x${FT_DRY_RUN}" = "xno" ]] && flux queue drain && rollup_exit_code
+ unapply_policies
+}
+
+
+#
+# Collect the performance record for sibling Flux instances at one level.
+# For each child instance, get the performance record from the guest KVS
+# namespace, which had all of the records gathered for the subtree rooted
+# at this instance, and add that to the current record with its child key.
+#
+coll_perf() {
+ local prefix="${1}"
+ local nnodes="${2}"
+ local ncores="${3}"
+ local ngpus="${4}"
+ local njobs="${5}"
+ local begin="${6}"
+ local end="${7}"
+ local perfout="${8}"
+ local nchildren="${9}"
+ local format="${10}"
+
+ #
+ # Make a JSON string from the performance data
+ #
+ local blurb=''
+ blurb=$(jsonify "${prefix}" "${njobs}" "${nnodes}" "${ncores}" "${ngpus}" "${begin}" "${end}")
+ rollup "${prefix}" "${blurb}" "${perfout}" "${nchildren}" "${format}"
+}
+
+
+#
+# Entry point to submit child Flux instances at the next level from the
+# calling Flux instance. Measure the elapse time of running all of these
+# Flux instances. Collect the performance record for that level at the end.
+#
+internal() {
+ local prefix="${1}"
+ local nnodes="${6}"
+ local ncores="${7}"
+ local ngpus="${8}"
+ local njobs="${10}"
+ local perfout="${13}"
+ local format="${14}"
+
+ # Begin Time Stamp
+ local B=''
+ B=$(date +%s.%N)
+
+ submit "$@"
+
+ # End Time Stamp
+ local E=''
+ E=$(date +%s.%N)
+
+ if [[ "x${FT_DRY_RUN}" = "xyes" ]]; then
+ nchildren=0
+ else
+ nchildren=${#jobids[@]}
+ fi
+ coll_perf "${prefix}" "${nnodes}" "${ncores}" "${ngpus}" \
+"${njobs}" "${B}" "${E}" "${perfout}" "${nchildren}" "${format}"
+}
+
+
+################################################################################
+# #
+# Main #
+# #
+################################################################################
+
+main() {
+ local leaf="${1}" # is this a leaf Flux instance?
+ local prefix="${2}" # id showing hierarchical path of the instance
+ local topo="${3}" # topology shape at the invoked level
+ local queue="${4}" # queuing policies at the invoked level and below
+ local param="${5}" # queue parameters at the invoked level and below
+ local match="${6}" # match policy shape at the invoked level
+ local nnodes="${7}" # num of nodes allocated to this instance
+ local ncores="${8}" # num of cores per node
+ local ngpus="${9}" # num of gpus per node
+ local njobs="${10}" # num of jobs assigned to this Flux instance
+ local flogs="${11}" # flux log output option
+ local frdir="${12}" # flux rundir attribute
+ local out="${13}" # perf output filename
+ local format="${14}" # perf output format
+ local size=0
+
+ if [[ ${leaf} = "yes" ]]
+ then
+ #
+ # flux-tree is invoked for a leaf: all of the internal Flux instances
+ # leading to this leaf have been instantiated and ${script} should
+ # be executed on the last-level Flux instance.
+ #
+ leaf "${prefix}" "${nnodes}" "${ncores}" "${ngpus}" "${njobs}" \
+ "${out}" "${format}"
+ else
+ #
+ # flux-tree is invoked to instantiate ${size} internal Flux instances
+ # at the next level of the calling instance.
+ #
+ size=${topo%%${t_delim}*}
+ internal "${prefix}" "${topo}" "${queue}" "${param}" "${match}" \
+ "${nnodes}" "${ncores}" "${ngpus}" "${size}" "${njobs}" \
+ "${flogs}" "${frdir}" "${out}" "${format}"
+ fi
+
+ exit ${FT_MAX_EXIT_CODE}
+}
+
+
+################################################################################
+# #
+# Commandline Parsing and Validate Options #
+# #
+################################################################################
+
+GETOPTS=$(/usr/bin/getopt -o ${short_opts} -l ${long_opts} -n "${prog}" -- "${@}")
+eval set -- "${GETOPTS}"
+rcopt=$?
+
+while true; do
+ case "${1}" in
+ -h|--help) echo -ne "${usage}"; exit 0 ;;
+ -l|--leaf) FT_LEAF="yes"; shift 1 ;;
+ -d|--dry-run) FT_DRY_RUN="yes"; shift 1 ;;
+ -f|--flux-logs) FT_FXLOGS="${2}"; shift 2 ;;
+ -r|--flux-rundir) FT_FXRDIR="${2}"; shift 2 ;;
+ -N|--nnodes) FT_NNODES=${2}; shift 2 ;;
+ -c|--ncores-per-node) FT_NCORES=${2}; shift 2 ;;
+ -g|--ngpus-per-node) FT_NGPUS=${2}; shift 2 ;;
+ -T|--topology) FT_TOPO="${2}"; FT_INTER="yes"; shift 2 ;;
+ -Q|--queue-policy) FT_QUEUE="${2}"; FT_INTER="yes"; shift 2 ;;
+ -P|--queue-params) FT_PARAMS="${2}"; FT_INTER="yes"; shift 2 ;;
+ -M|--match-policy) FT_MATCH="${2}"; FT_INTER="yes"; shift 2 ;;
+ -J|--njobs) FT_NJOBS=${2}; shift 2 ;;
+ -o|--perf-out) FT_PERF_OUT="${2}"; shift 2 ;;
+ --perf-format) FT_PERF_FORMAT="${2}"; shift 2 ;;
+ -X|--prefix) FT_G_PREFIX="${2}"; shift 2 ;;
+ --job-name) FT_JOB_NAME="${2}"; shift 2 ;;
+ --) shift; break; ;;
+ *) die "Invalid option '${1}'\n${usage}" ;;
+ esac
+done
+
+FT_SCRIPT="${1}"
+FT_CL=( "${@}" )
+
+[[ "$#" -lt 1 || "${rcopt}" -ne 0 ]] && die "${usage}"
+
+[[ ! -x $(which ${FT_SCRIPT}) ]] && die "cannot execute ${FT_SCRIPT}!"
+
+[[ "${FT_NNODES}" -le 0 ]] && die "nnodes must be greater than 0!"
+
+[[ "${FT_NCORES}" -le 0 ]] && die "ncores must be greater than 0!"
+
+[[ "${FT_NGPUS}" -lt 0 ]] && die "incorrect ngpus!"
+
+qpolicy_check "${FT_QUEUE}" || die "invalid queue policy!"
+
+mpolicy_check "${FT_MATCH}" || die "invalid match policy!"
+
+qparams_check "${FT_PARAMS}" || die "invalid queue params!"
+
+if [[ "${FT_INTER}" = "yes" && "${FT_LEAF}" = "yes" ]]
+then
+ die "--leaf must not be used together with internal tree-node options!"
+fi
+
+# if the user did not set a name, then use a partially random string to prevent
+# conflicts with other flux-tree instances during performance data collection
+# via flux-tree-helper
+if [[ "$FT_JOB_NAME" == '%^+_no' ]]; then
+ # code copied from:
+ # https://unix.stackexchange.com/questions/230673/how-to-generate-a-random-string
+ FT_JOB_NAME="flux-tree-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 32)"
+fi
+
+
+################################################################################
+# #
+# Invoke the Main Entry Level #
+# #
+################################################################################
+
+main "${FT_LEAF}" "${FT_G_PREFIX}" "${FT_TOPO}" "${FT_QUEUE}" "${FT_PARAMS}" \
+ "${FT_MATCH}" "${FT_NNODES}" "${FT_NCORES}" "${FT_NGPUS}" "${FT_NJOBS}" \
+ "${FT_FXLOGS}" "${FT_FXRDIR}" "${FT_PERF_OUT}" "${FT_PERF_FORMAT}"
+
+#
+# vi:tabstop=4 shiftwidth=4 expandtab
+#
diff --git a/2024-RIKEN-AWS/JupyterNotebook/flux-tree/flux-tree-helper.py b/2024-RIKEN-AWS/JupyterNotebook/flux-tree/flux-tree-helper.py
new file mode 100644
index 0000000..eba17d5
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/flux-tree/flux-tree-helper.py
@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+
+##############################################################
+# Copyright 2020 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, LICENSE)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+##############################################################
+
+import os
+import sys
+import time
+import json
+import argparse
+import logging
+
+import flux
+import flux.util
+import flux.kvs
+import flux.job
+
+LOGGER = logging.getLogger("flux-tree-helper")
+
+
+def get_child_jobids(flux_handle, num_children, child_name):
+ """
+ Get the jobids of num_children instances. Will repeatedly query the
+ job-info module until num_children jobids are collected, with sleeps
+ inbetween queries.
+ """
+ jobids = set()
+ since = 0.0
+ LOGGER.debug("Getting IDs of inactive children with name == %s", child_name)
+ while True:
+ for job in flux.job.job_list_inactive(
+ flux_handle,
+ max_entries=num_children,
+ since=since,
+ attrs=["t_inactive"],
+ name=child_name,
+ ).get_jobs():
+ jobid = job["id"]
+ since = max(since, job["t_inactive"])
+ jobids.add(jobid)
+ if len(jobids) >= num_children:
+ break
+ LOGGER.debug(
+ "Only %d out of %d children are inactive, sleeping before trying again",
+ len(jobids),
+ num_children,
+ )
+ time.sleep(1)
+ return jobids
+
+
+def get_this_instance_data():
+ data = json.load(sys.stdin)
+ return data
+
+
+def get_child_data(flux_handle, num_children, child_name, kvs_key):
+ child_data = []
+ jobids = get_child_jobids(flux_handle, num_children, child_name)
+ for jobid in jobids:
+ kvs_dir = flux.job.job_kvs_guest(flux_handle, jobid)
+ child_data.append(kvs_dir[kvs_key])
+ return child_data
+
+
+def combine_data(this_instance_data, child_data):
+ this_instance_data["child"] = child_data
+ return this_instance_data
+
+
+class PerfOutputFormat(flux.util.OutputFormat):
+ """
+ Store a parsed version of the program's output format,
+ allowing the fields to iterated without modifiers, building
+ a new format suitable for headers display, etc...
+ """
+
+ # List of legal format fields and their header names
+ headings = dict(
+ treeid="TreeID",
+ elapse="Elapsed(sec)",
+ begin="Begin(Epoch)",
+ end="End(Epoch)",
+ match="Match(usec)",
+ njobs="NJobs",
+ my_nodes="NNodes",
+ my_cores="CPN",
+ my_gpus="GPN",
+ )
+
+ def __init__(self, fmt):
+ """
+ Parse the input format fmt with string.Formatter.
+ Save off the fields and list of format tokens for later use,
+ (converting None to "" in the process)
+
+ Throws an exception if any format fields do not match the allowed
+ list of headings above.
+ """
+ # Support both new and old style OutputFormat constructor:
+ try:
+ super().__init__(fmt, headings=self.headings, prepend="")
+ except TypeError:
+ super().__init__(PerfOutputFormat.headings, fmt)
+
+
+def write_data_to_file(output_filename, output_format, data):
+ def json_traverser(data):
+ fieldnames = PerfOutputFormat.headings.keys()
+ output = {k: v for k, v in data.items() if k in fieldnames}
+ output.update(data["perf"])
+ yield output
+ for child in data["child"]:
+ yield from json_traverser(child)
+
+ formatter = PerfOutputFormat(output_format)
+ with open(output_filename, "w") as outfile:
+ header = formatter.header() + "\n"
+ outfile.write(header)
+ fmt = formatter.get_format() + "\n"
+ for data_row in json_traverser(data):
+ # newline = formatter.format(data_row)
+ newline = fmt.format(**data_row)
+ outfile.write(newline)
+
+
+def write_data_to_parent(flux_handle, kvs_key, data):
+ try:
+ parent_uri = flux_handle.flux_attr_get("parent-uri")
+ except FileNotFoundError:
+ return
+ parent_handle = flux.Flux(parent_uri)
+
+ try:
+ parent_kvs_namespace = flux_handle.flux_attr_get("parent-kvs-namespace").decode(
+ "utf-8"
+ )
+ except FileNotFoundError:
+ return
+ env_name = "FLUX_KVS_NAMESPACE"
+ os.environ[env_name] = parent_kvs_namespace
+
+ flux.kvs.put(parent_handle, kvs_key, data)
+ flux.kvs.commit(parent_handle)
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(
+ prog="flux-tree-helper", formatter_class=flux.util.help_formatter()
+ )
+ parser.add_argument(
+ "num_children",
+ type=int,
+ help="number of children to collect data from. Should be 0 at leaves.",
+ )
+ parser.add_argument(
+ "kvs_key", type=str, help="key to use when propagating data up through the tree"
+ )
+ parser.add_argument(
+ "job_name",
+ type=str,
+ help="name of the child jobs to use when filtering the inactive jobs",
+ )
+ parser.add_argument(
+ "--perf-out",
+ type=str,
+ help="Dump the performance data into the given file. "
+ "Assumed to be given at the root instance.",
+ )
+ parser.add_argument(
+ "--perf-format",
+ type=str,
+ help="Dump the performance data with the given format string.",
+ )
+ return parser.parse_args()
+
+
+@flux.util.CLIMain(LOGGER)
+def main():
+ args = parse_args()
+ flux_handle = None
+ try:
+ flux_handle = flux.Flux()
+ except FileNotFoundError:
+ flux_handle = None
+
+ LOGGER.debug("Getting this instance's data")
+ this_data = get_this_instance_data()
+ if flux_handle is not None and args.num_children > 0:
+ LOGGER.debug("Getting children's data")
+ child_data = get_child_data(
+ flux_handle, args.num_children, args.job_name, args.kvs_key
+ )
+ else:
+ child_data = []
+ LOGGER.debug("Combining data")
+ combined_data = combine_data(this_data, child_data)
+ if flux_handle is not None:
+ LOGGER.debug("Writing data to parent's KVS")
+ write_data_to_parent(flux_handle, args.kvs_key, combined_data)
+ if args.perf_out:
+ LOGGER.debug("Writing data to file")
+ write_data_to_file(args.perf_out, args.perf_format, combined_data)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/2024-RIKEN-AWS/JupyterNotebook/gcp/config.yaml b/2024-RIKEN-AWS/JupyterNotebook/gcp/config.yaml
new file mode 100644
index 0000000..b44626c
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/gcp/config.yaml
@@ -0,0 +1,62 @@
+# A few notes!
+# The hub -> authentic class defaults to "dummy"
+# We shouldn't need any image pull secrets assuming public
+# There is a note about the database being a sqlite pvc
+# (and a TODO for better solution for Kubernetes)
+
+# This is the concurrent spawn limit, likely should be increased (deafults to 64)
+hub:
+ concurrentSpawnLimit: 10
+ config:
+ DummyAuthenticator:
+ password: butter
+ JupyterHub:
+ admin_access: true
+ authenticator_class: dummy
+
+ # This is the image I built based off of jupyterhub/k8s-hub, 3.0.2 at time of writing this
+ image:
+ name: ghcr.io/flux-framework/flux-jupyter-hub
+ tag: "2023"
+ pullPolicy: Always
+
+# https://z2jh.jupyter.org/en/latest/administrator/optimization.html#scaling-up-in-time-user-placeholders
+scheduling:
+ podPriority:
+ enabled: true
+ userPlaceholder:
+ # Specify 3 dummy user pods will be used as placeholders
+ replicas: 3
+
+# This is the "spawn" image
+singleuser:
+ image:
+ name: ghcr.io/flux-framework/flux-jupyter-spawn
+ tag: "2023"
+ pullPolicy: Always
+ cpu:
+ limit: 1
+ memory:
+ limit: '4G'
+ cmd: /entrypoint.sh
+
+ initContainers:
+ - name: init-myservice
+ image: alpine/git
+ command: ["git", "clone", "https://github.com/rse-ops/flux-radiuss-tutorial-2023", "/home/jovyan/flux-tutorial"]
+ volumeMounts:
+ - name: flux-tutorial
+ mountPath: /home/jovyan
+
+ # This is how we get the tutorial files added
+ storage:
+ type: none
+
+ # gitRepo volume is deprecated so we need another way
+ # https://kubernetes.io/docs/concepts/storage/volumes/#gitrepo
+ extraVolumes:
+ - name: flux-tutorial
+ emptyDir: {}
+ extraVolumeMounts:
+ - name: flux-tutorial
+ mountPath: /home/jovyan/
diff --git a/2024-RIKEN-AWS/JupyterNotebook/requirements.txt b/2024-RIKEN-AWS/JupyterNotebook/requirements.txt
new file mode 100644
index 0000000..0d9d99e
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/requirements.txt
@@ -0,0 +1,339 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+# Use the "Run workflow" button at https://github.com/jupyterhub/zero-to-jupyterhub-k8s/actions/workflows/watch-dependencies.yaml
+#
+alembic==1.11.3
+ # via jupyterhub
+anyio==3.7.1
+ # via jupyter-server
+argon2-cffi==23.1.0
+ # via
+ # jupyter-server
+ # nbclassic
+argon2-cffi-bindings==21.2.0
+ # via argon2-cffi
+arrow==1.2.3
+ # via isoduration
+asttokens==2.2.1
+ # via stack-data
+async-generator==1.10
+ # via jupyterhub
+async-lru==2.0.4
+ # via jupyterlab
+attrs==23.1.0
+ # via
+ # jsonschema
+ # referencing
+babel==2.12.1
+ # via jupyterlab-server
+backcall==0.2.0
+ # via ipython
+beautifulsoup4==4.12.2
+ # via nbconvert
+bleach==6.0.0
+ # via nbconvert
+certifi==2023.7.22
+ # via requests
+certipy==0.1.3
+ # via jupyterhub
+cffi==1.15.1
+ # via
+ # argon2-cffi-bindings
+ # cryptography
+charset-normalizer==3.2.0
+ # via requests
+comm==0.1.4
+ # via ipykernel
+cryptography==41.0.3
+ # via pyopenssl
+debugpy==1.6.7.post1
+ # via ipykernel
+decorator==5.1.1
+ # via ipython
+defusedxml==0.7.1
+ # via nbconvert
+executing==1.2.0
+ # via stack-data
+fastjsonschema==2.18.0
+ # via nbformat
+fqdn==1.5.1
+ # via jsonschema
+greenlet==2.0.2
+ # via sqlalchemy
+idna==3.4
+ # via
+ # anyio
+ # jsonschema
+ # requests
+ipykernel==6.25.1
+ # via
+ # jupyterlab
+ # nbclassic
+ipython==8.13.0
+ # via ipykernel
+ipython-genutils==0.2.0
+ # via nbclassic
+isoduration==20.11.0
+ # via jsonschema
+jedi==0.19.0
+ # via ipython
+jinja2==3.1.2
+ # via
+ # jupyter-server
+ # jupyterhub
+ # jupyterlab
+ # jupyterlab-server
+ # nbclassic
+ # nbconvert
+json5==0.9.14
+ # via jupyterlab-server
+jsonpointer==2.4
+ # via jsonschema
+jsonschema[format-nongpl]==4.19.0
+ # via
+ # jupyter-events
+ # jupyter-telemetry
+ # jupyterlab-server
+ # nbformat
+jsonschema-specifications==2023.7.1
+ # via jsonschema
+jupyter-client==8.3.0
+ # via
+ # ipykernel
+ # jupyter-server
+ # nbclassic
+ # nbclient
+jupyter-core==5.3.1
+ # via
+ # ipykernel
+ # jupyter-client
+ # jupyter-server
+ # jupyterlab
+ # nbclassic
+ # nbclient
+ # nbconvert
+ # nbformat
+jupyter-events==0.7.0
+ # via jupyter-server
+jupyter-lsp==2.2.0
+ # via jupyterlab
+jupyter-server==2.7.2
+ # via
+ # jupyter-lsp
+ # jupyterlab
+ # jupyterlab-server
+ # nbclassic
+ # nbgitpuller
+ # notebook-shim
+jupyter-server-terminals==0.4.4
+ # via jupyter-server
+jupyter-telemetry==0.1.0
+ # via jupyterhub
+jupyterhub==4.0.2
+ # via -r requirements.in
+jupyterlab==4.0.5
+ # via -r requirements.in
+jupyterlab-pygments==0.2.2
+ # via nbconvert
+jupyterlab-server==2.24.0
+ # via jupyterlab
+mako==1.2.4
+ # via alembic
+markupsafe==2.1.3
+ # via
+ # jinja2
+ # mako
+ # nbconvert
+matplotlib-inline==0.1.6
+ # via
+ # ipykernel
+ # ipython
+mistune==3.0.1
+ # via nbconvert
+nbclassic==1.0.0
+ # via -r requirements.in
+nbclient==0.8.0
+ # via nbconvert
+nbconvert==7.7.4
+ # via
+ # jupyter-server
+ # nbclassic
+nbformat==5.9.2
+ # via
+ # jupyter-server
+ # nbclassic
+ # nbclient
+ # nbconvert
+nbgitpuller==1.2.0
+ # via -r requirements.in
+nest-asyncio==1.5.7
+ # via
+ # ipykernel
+ # nbclassic
+notebook-shim==0.2.3
+ # via
+ # jupyterlab
+ # nbclassic
+oauthlib==3.2.2
+ # via jupyterhub
+overrides==7.4.0
+ # via jupyter-server
+packaging==23.1
+ # via
+ # ipykernel
+ # jupyter-server
+ # jupyterhub
+ # jupyterlab
+ # jupyterlab-server
+ # nbconvert
+pamela==1.1.0
+ # via jupyterhub
+pandocfilters==1.5.0
+ # via nbconvert
+parso==0.8.3
+ # via jedi
+pexpect==4.8.0
+ # via ipython
+pickleshare==0.7.5
+ # via ipython
+platformdirs==3.10.0
+ # via jupyter-core
+prometheus-client==0.17.1
+ # via
+ # jupyter-server
+ # jupyterhub
+ # nbclassic
+prompt-toolkit==3.0.39
+ # via ipython
+psutil==5.9.5
+ # via ipykernel
+ptyprocess==0.7.0
+ # via
+ # pexpect
+ # terminado
+pure-eval==0.2.2
+ # via stack-data
+pycparser==2.21
+ # via cffi
+pygments==2.16.1
+ # via
+ # ipython
+ # nbconvert
+pyopenssl==23.2.0
+ # via certipy
+python-dateutil==2.8.2
+ # via
+ # arrow
+ # jupyter-client
+ # jupyterhub
+python-json-logger==2.0.7
+ # via
+ # jupyter-events
+ # jupyter-telemetry
+pyyaml==6.0.1
+ # via jupyter-events
+pyzmq==25.1.1
+ # via
+ # ipykernel
+ # jupyter-client
+ # jupyter-server
+ # nbclassic
+referencing==0.30.2
+ # via
+ # jsonschema
+ # jsonschema-specifications
+ # jupyter-events
+requests==2.31.0
+ # via
+ # jupyterhub
+ # jupyterlab-server
+rfc3339-validator==0.1.4
+ # via
+ # jsonschema
+ # jupyter-events
+rfc3986-validator==0.1.1
+ # via
+ # jsonschema
+ # jupyter-events
+rpds-py==0.9.2
+ # via
+ # jsonschema
+ # referencing
+ruamel-yaml==0.17.32
+ # via jupyter-telemetry
+ruamel-yaml-clib==0.2.7
+ # via ruamel-yaml
+send2trash==1.8.2
+ # via
+ # jupyter-server
+ # nbclassic
+six==1.16.0
+ # via
+ # asttokens
+ # bleach
+ # python-dateutil
+ # rfc3339-validator
+sniffio==1.3.0
+ # via anyio
+soupsieve==2.4.1
+ # via beautifulsoup4
+sqlalchemy==2.0.20
+ # via
+ # alembic
+ # jupyterhub
+stack-data==0.6.2
+ # via ipython
+terminado==0.17.1
+ # via
+ # jupyter-server
+ # jupyter-server-terminals
+ # nbclassic
+tinycss2==1.2.1
+ # via nbconvert
+tornado==6.3.3
+ # via
+ # ipykernel
+ # jupyter-client
+ # jupyter-server
+ # jupyterhub
+ # jupyterlab
+ # nbclassic
+ # nbgitpuller
+ # terminado
+traitlets==5.9.0
+ # via
+ # comm
+ # ipykernel
+ # ipython
+ # jupyter-client
+ # jupyter-core
+ # jupyter-events
+ # jupyter-server
+ # jupyter-telemetry
+ # jupyterhub
+ # jupyterlab
+ # matplotlib-inline
+ # nbclassic
+ # nbclient
+ # nbconvert
+ # nbformat
+typing-extensions==4.7.1
+ # via
+ # alembic
+ # sqlalchemy
+uri-template==1.3.0
+ # via jsonschema
+urllib3==2.0.4
+ # via requests
+wcwidth==0.2.6
+ # via prompt-toolkit
+webcolors==1.13
+ # via jsonschema
+webencodings==0.5.1
+ # via
+ # bleach
+ # tinycss2
+websocket-client==1.6.1
+ # via jupyter-server
diff --git a/2024-RIKEN-AWS/JupyterNotebook/tutorial/.gitignore b/2024-RIKEN-AWS/JupyterNotebook/tutorial/.gitignore
new file mode 100644
index 0000000..3ebb2aa
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/tutorial/.gitignore
@@ -0,0 +1,2 @@
+flux*.out
+.ipynb_checkpoints
diff --git a/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/.github/workflows/main.yml b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/.github/workflows/main.yml
new file mode 100644
index 0000000..5d301e8
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/.github/workflows/main.yml
@@ -0,0 +1,33 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+on: [pull_request]
+jobs:
+ check-pr:
+ name: check formatting
+ runs-on: ubuntu-latest
+
+ strategy:
+ matrix:
+ python-version: [3.6, 3.7, 3.8]
+
+ steps:
+ - uses: actions/checkout@v2
+ with:
+ ref: ${{ github.event.pull_request.head.sha }}
+ fetch-depth: 0
+ - run: git fetch origin master
+ - uses: flux-framework/pr-validator@master
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v1
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Lint with flake8
+ run: |
+ pip install flake8
+ pip install black
+ # stop the build if there are Python syntax errors or undefined names
+ flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+ # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+ flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+ black .
diff --git a/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/.mergify.yml b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/.mergify.yml
new file mode 100644
index 0000000..65c6341
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/.mergify.yml
@@ -0,0 +1,18 @@
+pull_request_rules:
+ - name: rebase and merge when passing all checks
+ conditions:
+ - base=master
+ - status-success="check formatting (3.6)"
+ - status-success="check formatting (3.7)"
+ - status-success="check formatting (3.8)"
+ - label="merge-when-passing"
+ - label!="work-in-progress"
+ - "approved-reviews-by=@flux-framework/core"
+ - "#approved-reviews-by>0"
+ - "#changes-requested-reviews-by=0"
+ - -title~=^\[*[Ww][Ii][Pp]
+ actions:
+ merge:
+ method: merge
+ strict: smart
+ strict_method: rebase
diff --git a/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/Makefile b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/Makefile
new file mode 100644
index 0000000..f219905
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/Makefile
@@ -0,0 +1,25 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS ?=
+SPHINXBUILD ?= sphinx-build
+SOURCEDIR = .
+BUILDDIR = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+ @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile check spelling $(SCHEMA_DIRS)
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+ @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+check: spelling
+
+spelling:
+ @$(SPHINXBUILD) -W -b spelling "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/README.md b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/README.md
new file mode 100644
index 0000000..9208b12
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/README.md
@@ -0,0 +1,73 @@
+**WARNING**
+
+This repository has been archived. It is no longer maintained and it is
+likely the examples do not work or are no longer good or suggested
+examples.
+
+Please look elswhere for examples.
+
+**Flux Workflow Examples**
+
+The examples contained here demonstrate and explain some simple use-cases with Flux,
+and make use of Flux's command-line interface (CLI), Flux's C library,
+and the Python and Lua bindings to the C library.
+
+**Requirements**
+
+The examples assume that you have installed:
+
+1. A recent version of Flux
+
+2. Python 3.6+
+
+3. Lua 5.1+
+
+**_1. [CLI: Job Submission](https://github.com/flux-framework/flux-workflow-examples/tree/master/job-submit-cli)_**
+
+Launch a flux instance and schedule/launch compute and io-forwarding jobs on
+separate nodes using the CLI
+
+**_2. [Python: Job Submission](https://github.com/flux-framework/flux-workflow-examples/tree/master/job-submit-api)_**
+
+Schedule/launch compute and io-forwarding jobs on separate nodes using the Python bindings
+
+**_3. [Python: Job Submit/Wait](https://github.com/flux-framework/flux-workflow-examples/tree/master/job-submit-wait)_**
+
+Submit jobs and wait for them to complete using the Flux Python bindings
+
+**_4. [Python: Asynchronous Bulk Job Submission](https://github.com/flux-framework/flux-workflow-examples/tree/master/async-bulk-job-submit)_**
+
+Asynchronously submit jobspec files from a directory and wait for them to complete in any order
+
+**_5. [Python: Tracking Job Status and Events](https://github.com/flux-framework/flux-workflow-examples/tree/master/job-status-control)_**
+
+Submit job bundles, get event updates, and wait until all jobs complete
+
+**_6. [Python: Job Cancellation](https://github.com/flux-framework/flux-workflow-examples/tree/master/job-cancel)_**
+
+Cancel a running job
+
+**_7. [Lua: Use Events](https://github.com/flux-framework/flux-workflow-examples/tree/master/synchronize-events)_**
+
+Use events to synchronize compute and io-forwarding jobs running on separate
+nodes
+
+**_8. [Python: Simple KVS Example](https://github.com/flux-framework/flux-workflow-examples/tree/master/kvs-python-bindings)_**
+
+Use KVS Python interfaces to store user data into KVS
+
+**_9. [CLI/Lua: Job Ensemble Submitted with a New Flux Instance](https://github.com/flux-framework/flux-workflow-examples/tree/master/job-ensemble)_**
+
+Submit job bundles, print live job events, and exit when all jobs are complete
+
+**_10. [CLI: Hierarchical Launching](https://github.com/flux-framework/flux-workflow-examples/tree/master/hierarchical-launching)_**
+
+Launch a large number of sleep 0 jobs
+
+**_11. [C/Lua: Use a Flux Comms Module](https://github.com/flux-framework/flux-workflow-examples/tree/master/comms-module)_**
+
+Use a Flux Comms Module to communicate with job elements
+
+**_12. [C/Python: A Data Conduit Strategy](https://github.com/flux-framework/flux-workflow-examples/tree/master/data-conduit)_**
+
+Attach to a job that receives OS time data from compute jobs
diff --git a/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/async-bulk-job-submit/README.md b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/async-bulk-job-submit/README.md
new file mode 100644
index 0000000..719af07
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/async-bulk-job-submit/README.md
@@ -0,0 +1,99 @@
+## Python Asynchronous Bulk Job Submission
+
+Parts (a) and (b) demonstrate different implementations of the same basic use-case---submitting
+large numbers of jobs to Flux. For simplicity, in these examples all of the jobs are identical.
+
+In part (a), we use the `flux.job.submit_async` and `flux.job.wait` functions to submit jobs and wait for them.
+In part (b), we use the `FluxExecutor` class, which offers a higher-level interface. It is important to note that
+these two different implementations deal with very different kinds of futures.
+The executor's futures fulfill in the background and callbacks added to the futures may
+be invoked by different threads; the `submit_async` futures do not fulfill in the background, callbacks are always
+invoked by the same thread that added them, and sharing the futures among threads is not supported.
+
+### Setup - Downloading the Files
+
+If you haven't already, download the files and change your working directory:
+
+```
+$ git clone https://github.com/flux-framework/flux-workflow-examples.git
+$ cd flux-workflow-examples/async-bulk-job-submit
+```
+
+### Part (a) - Using `submit_async`
+
+#### Description: Asynchronously submit jobspec files from a directory and wait for them to complete in any order
+
+1. Allocate three nodes from a resource manager:
+
+`salloc -N3 -ppdebug`
+
+2. Make a **jobs** directory:
+
+`mkdir jobs`
+
+3. Launch a Flux instance on the current allocation by running `flux start` once per node, redirecting log messages to the file `out` in the current directory:
+
+`srun --pty --mpi=none -N3 flux start -o,-S,log-filename=out`
+
+4. Store the jobspec of a `sleep 0` job in the **jobs** directory:
+
+`flux mini run --dry-run -n1 sleep 0 > jobs/0.json`
+
+5. Copy the jobspec of **job0** 1024 times to create a directory of 1025 `sleep 0` jobs:
+
+``for i in `seq 1 1024`; do cp jobs/0.json jobs/${i}.json; done``
+
+6. Run the **bulksubmit.py** script and pass all jobspec in the **jobs** directory as an argument with a shell glob `jobs/*.json`:
+
+`./bulksubmit.py jobs/*.json`
+
+```
+bulksubmit: Starting...
+bulksubmit: submitted 1025 jobs in 3.04s. 337.09job/s
+bulksubmit: First job finished in about 3.089s
+|ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ| 100.0% (29.4 job/s)
+bulksubmit: Ran 1025 jobs in 34.9s. 29.4 job/s
+```
+
+### Notes to Part (a)
+
+- `h = flux.Flux()` creates a new Flux handle which can be used to connect to and interact with a Flux instance.
+
+- `job_submit_async(h, jobspec.read(), waitable=True).then(submit_cb)` submits a jobspec, returning a future which will be fulfilled when the submission of this job is complete.
+
+`.then(submit_cb)`, called on the returned future, will cause our callback `submit_cb()` to be invoked when the submission of this job is complete and a jobid is available. To process job submission RPC responses and invoke callbacks, the flux reactor for handle `h` must be run:
+
+```python
+if h.reactor_run() < 0:
+οΏΌ h.fatal_error("reactor start failed")
+```
+
+The reactor will return automatically when there are no more outstanding RPC responses, i.e., all jobs have been submitted.
+
+- `job.wait(h)` waits for any job submitted with the `FLUX_JOB_WAITABLE` flag to transition to the **INACTIVE** state.
+
+
+### Part (b) - Using FluxExecutor
+
+#### Description: Asynchronously submit a single command repeatedly
+
+If continuing from part (a), skip to step 3.
+
+1. Allocate three nodes from a resource manager:
+
+`salloc -N3 -ppdebug`
+
+2. Launch a Flux instance on the current allocation by running `flux start` once per node, redirecting log messages to the file `out` in the current directory:
+
+`srun --pty --mpi=none -N3 flux start -o,-S,log-filename=out`
+
+3. Run the **bulksubmit_executor.py** script and pass the command (`/bin/sleep 0` in this example) and the number of times to run it (default is 100):
+
+`./bulksubmit_executor.py -n200 /bin/sleep 0`
+
+```
+bulksubmit_executor: submitted 200 jobs in 0.45s. 441.15job/s
+bulksubmit_executor: First job finished in about 1.035s
+|ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ| 100.0% (24.9 job/s)
+bulksubmit_executor: Ran 200 jobs in 8.2s. 24.4 job/s
+```
diff --git a/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/async-bulk-job-submit/bulksubmit.py b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/async-bulk-job-submit/bulksubmit.py
new file mode 100755
index 0000000..c1a2e9a
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/async-bulk-job-submit/bulksubmit.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+
+import time
+import sys
+import flux
+
+from flux import job
+from flux import constants
+
+t0 = time.time()
+jobs = []
+label = "bulksubmit"
+
+# open connection to broker
+h = flux.Flux()
+
+
+def log(s):
+ print(label + ": " + s)
+
+
+def progress(fraction, length=72, suffix=""):
+ fill = int(round(length * fraction))
+ bar = "\u2588" * fill + "-" * (length - fill)
+ s = "\r|{0}| {1:.1f}% {2}".format(bar, 100 * fraction, suffix)
+ sys.stdout.write(s)
+ if fraction == 1.0:
+ sys.stdout.write("\n")
+
+
+def submit_cb(f):
+ jobs.append(job.submit_get_id(f))
+
+
+# asynchronously submit jobspec files from a directory
+log("Starting...")
+for file in sys.argv[1:]:
+ with open(file) as jobspec:
+ job.submit_async(h, jobspec.read(), waitable=True).then(submit_cb)
+
+if h.reactor_run() < 0:
+ h.fatal_error("reactor start failed")
+
+total = len(jobs)
+dt = time.time() - t0
+jps = len(jobs) / dt
+log("submitted {0} jobs in {1:.2f}s. {2:.2f}job/s".format(total, dt, jps))
+
+count = 0
+while count < total:
+ # wait for jobs to complete in any order
+ job.wait(h)
+ count = count + 1
+ if count == 1:
+ log("First job finished in about {0:.3f}s".format(time.time() - t0))
+ suffix = "({0:.1f} job/s)".format(count / (time.time() - t0))
+ progress(count / total, length=58, suffix=suffix)
+
+dt = time.time() - t0
+log("Ran {0} jobs in {1:.1f}s. {2:.1f} job/s".format(total, dt, total / dt))
+
+# vi: ts=4 sw=4 expandtab
diff --git a/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/async-bulk-job-submit/bulksubmit_executor.py b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/async-bulk-job-submit/bulksubmit_executor.py
new file mode 100755
index 0000000..5280863
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/async-bulk-job-submit/bulksubmit_executor.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+import time
+import sys
+import argparse
+import concurrent.futures as cf
+
+from flux.job import FluxExecutor, JobspecV1
+
+
+def log(label, s):
+ print(label + ": " + s)
+
+
+def progress(fraction, length=72, suffix=""):
+ fill = int(round(length * fraction))
+ bar = "\u2588" * fill + "-" * (length - fill)
+ s = f"\r|{bar}| {100 * fraction:.1f}% {suffix}"
+ sys.stdout.write(s)
+ if fraction == 1.0:
+ sys.stdout.write("\n")
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description="Submit a command repeatedly using FluxExecutor"
+ )
+ parser.add_argument(
+ "-n",
+ "--njobs",
+ type=int,
+ metavar="N",
+ help="Set the total number of jobs to run",
+ default=100,
+ )
+ parser.add_argument("command", nargs=argparse.REMAINDER)
+ args = parser.parse_args()
+ if not args.command:
+ args.command = ["true"]
+ t0 = time.perf_counter()
+ label = "bulksubmit_executor"
+ with FluxExecutor() as executor:
+ compute_jobspec = JobspecV1.from_command(args.command)
+ futures = [executor.submit(compute_jobspec) for _ in range(args.njobs)]
+ # wait for the jobid for each job, as a proxy for the job being submitted
+ for fut in futures:
+ fut.jobid()
+ # all jobs submitted - print timings
+ dt = time.perf_counter() - t0
+ jps = args.njobs / dt
+ log(label, f"submitted {args.njobs} jobs in {dt:.2f}s. {jps:.2f}job/s")
+ # wait for jobs to complete
+ for i, _ in enumerate(cf.as_completed(futures)):
+ if i == 0:
+ log(
+ label,
+ f"First job finished in about {time.perf_counter() - t0:.3f}s",
+ )
+ jps = (i + 1) / (time.perf_counter() - t0)
+ progress((i + 1) / args.njobs, length=58, suffix=f"({jps:.1f} job/s)")
+ # print time summary
+ dt = time.perf_counter() - t0
+ log(label, f"Ran {args.njobs} jobs in {dt:.1f}s. {args.njobs / dt:.1f} job/s")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/comms-module/Makefile b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/comms-module/Makefile
new file mode 100644
index 0000000..ccc018d
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/comms-module/Makefile
@@ -0,0 +1,19 @@
+all: capp.so ioapp.so
+
+FLUX_CORE_LIBS = $(shell pkg-config --libs flux-core)
+FLUX_CORE_INCLUDES = $(shell pkg-config --cflags flux-core)
+
+ioapp.so: ioapp.o
+ gcc -Wl,--no-undefined --disable-static -shared -export-dynamic $^ -o $@ $(FLUX_CORE_LIBS)
+
+ioapp.o: app.c
+ gcc $(FLUX_CORE_INCLUDES) $^ -DIO_SERVICE=1 -fPIC -c -o $@
+
+capp.so: capp.o
+ gcc -Wl,--no-undefined --disable-static -shared -export-dynamic $^ -o $@ $(FLUX_CORE_LIBS)
+
+capp.o: app.c
+ gcc $(FLUX_CORE_INCLUDES) $^ -DCOMP_SERVICE=1 -fPIC -c -o $@
+
+clean:
+ rm *.o *.so
diff --git a/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/comms-module/README.md b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/comms-module/README.md
new file mode 100644
index 0000000..3acdc5c
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/tutorial/flux-workflow-examples/comms-module/README.md
@@ -0,0 +1,36 @@
+### Using a Flux Comms Module
+
+#### Description: Use a Flux comms module to communicate with job elements
+
+##### Setup
+
+If you haven't already, download the files and change your working directory:
+
+```
+$ git clone https://github.com/flux-framework/flux-workflow-examples.git
+$ cd flux-workflow-examples/comms-module
+```
+
+##### Execution
+
+1. `salloc -N3 -ppdebug`
+
+2. Point to `flux-core`'s `pkgconfig` directory:
+
+| Shell | Command |
+| ----- | ---------- |
+| tcsh | `setenv PKG_CONFIG_PATH
+
+ 
+
+
+{% endblock login_container %}
+
+
+
+
+
+ Flux Tutorial running on AWS
+After logging in, flux-tutorial --> notebook and double-click flux.ipynb to get started
+\n",
+ "![](\"dyad/dyad_example1.svg\")
\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "427c8a90-3d00-403d-825c-7e24d2117512",
+ "metadata": {},
+ "source": [
+ "Before running the DYAD-enabled applications, there are two things we must do:\n",
+ "1. Setup a namespace in the Flux KVS to be used by DYAD\n",
+ "2. Load DYAD's Flux module\n",
+ "\n",
+ "To begin, set the `kvs_namespace` variable to the namespace you wish to use for DYAD. This namespace can be any string value you want."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e2190770-fe49-4343-b2c8-eb625eb980d2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "kvs_namespace = \"dyad_test\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e116b785-5bdb-441b-9171-47e0b27a6e7d",
+ "metadata": {},
+ "source": [
+ "Next, create the namespace by running `flux kvs namespace create`. The cell below also runs `flux kvs namespace list` to allow you to verify that the namespace was created successfully."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a41c3f13-7b04-4ec5-9c5d-c5033d4977ca",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux kvs namespace create {kvs_namespace}\n",
+ "!flux kvs namespace list"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "0840b124-f805-432e-9764-1b167df39f64",
+ "metadata": {},
+ "source": [
+ "The next step is to load DYAD's Flux module. This module is the component of DYAD that actually sends files from producer to consumer.\n",
+ "\n",
+ "To start this step, set `dyad_module` below to the path to the DYAD module (i.e., `dyad.so`). For this demo, DYAD has already been installed under the `/usr` prefix, so the path to the DYAD module should be `/usr/lib/dyad.so`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cdb1dfc4-1f8a-434d-8be8-626382d124c6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dyad_module = \"/usr/lib/dyad.so\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6c2260bc",
+ "metadata": {},
+ "source": [
+ "Next, choose the communication backend for DYAD to use. This backend is used by DYAD's data transport layer (DTL) component to move data from producer to consumer. Currently, valid values are:\n",
+ "* `UCX`: use Unified Communication X for data movement\n",
+ "* `FLUX_RPC`: use Flux Remote Procedure Call (RPC) feature for data movement"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "46b38519",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dtl_mode = \"UCX\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "aab31cd8-5034-4450-bb1b-7b299fc5be86",
+ "metadata": {},
+ "source": [
+ "Finally, load the DYAD module by running `flux module load` on each broker. We load the module onto each broker because, normally, we would not know exactly which brokers the producer and consumer would be running on.\n",
+ "\n",
+ "When being loaded, the DYAD module takes a single command-line argument: the producer-managed directory. The module uses this directory to determine the path to any files it needs to transfer to consumers."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "edc65fdb-f746-46f6-81df-17602fd94acc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux exec -r all flux module load {dyad_module} {producer_managed_directory} {dtl_mode}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a71e4d07-f17e-4416-8f8d-0e36137b461a",
+ "metadata": {},
+ "source": [
+ "After loading the module, we can double check it has been loaded by running the cell below."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7061d3af-5d12-4edb-aa9e-6a678798ef14",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux exec -r all flux module list | grep dyad"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "efcaef56-02e4-43fd-af28-2b6689db19e6",
+ "metadata": {},
+ "source": [
+ "Now, we will generate the shell commands that we will use to run the producer and consumer applications. These commands can be broken down into three pieces.\n",
+ "\n",
+ "First, the commands will set the `LD_PRELOAD` environment variable if running the C version of the producer or consumer. We set `LD_PRELOAD` because DYAD's C API uses the preload trick to intercept the `open`, `close`, `fopen`, and `fclose` functions.\n",
+ "\n",
+ "Second, the commands set a couple of environment variables to configure DYAD. The environment variables used in this example are:\n",
+ "* `DYAD_KVS_NAMESPACE`: specifies the Flux KVS namespace to use with DYAD\n",
+ "* `DYAD_DTL_MODE`: sets the communication backend to use for data movement\n",
+ "* `DYAD_PATH_PRODUCER`: sets the producer-managed path\n",
+ "* `DYAD_PATH_CONSUMER`: sets the consumer-managed path\n",
+ "\n",
+ "Finally, the rest of the commands are the invocation of the applications themselves.\n",
+ "\n",
+ "Run the following 2 cells to generate and see the commands for the producer and consumer."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c6def81f-f17a-4601-9572-b00d2959287f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "producer_launch_cmd = \"{preload} DYAD_KVS_NAMESPACE={kvs_namespace} DYAD_DTL_MODE={dtl_mode} \\\n",
+ "DYAD_PATH_PRODUCER={producer_managed_directory} flux exec -r 0 \\\n",
+ "{producer_program} {num_files_transfered} {producer_managed_directory}\".format(\n",
+ " preload=\"LD_PRELOAD=\\\"/usr/lib/dyad_wrapper.so\\\"\" if producer_program.split(\"/\")[-1].strip().startswith(\"c_\") else \"\",\n",
+ " kvs_namespace=kvs_namespace,\n",
+ " dtl_mode=dtl_mode,\n",
+ " producer_managed_directory=producer_managed_directory,\n",
+ " producer_program=producer_program,\n",
+ " num_files_transfered=num_files_transfered,\n",
+ ")\n",
+ "print(producer_launch_cmd)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "42d3007b-d6b4-40b9-a4a1-153aef536c90",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "consumer_launch_cmd = \"{preload} DYAD_KVS_NAMESPACE={kvs_namespace} DYAD_DTL_MODE={dtl_mode} \\\n",
+ "DYAD_PATH_CONSUMER={consumer_managed_directory} flux exec -r 1 \\\n",
+ "{consumer_program} {num_files_transfered} {consumer_managed_directory}\".format(\n",
+ " preload=\"LD_PRELOAD=\\\"/usr/lib/dyad_wrapper.so\\\"\" if producer_program.split(\"/\")[-1].strip().startswith(\"c_\") else \"\",\n",
+ " kvs_namespace=kvs_namespace,\n",
+ " dtl_mode=dtl_mode,\n",
+ " consumer_managed_directory=consumer_managed_directory,\n",
+ " consumer_program=consumer_program,\n",
+ " num_files_transfered=num_files_transfered,\n",
+ ")\n",
+ "print(consumer_launch_cmd)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7f51f9ea-c48c-4b75-a780-92059a1c7c61",
+ "metadata": {},
+ "source": [
+ "Finally, we will run the producer and consumer applications. Thanks to DYAD's fine-grained, per-file synchronization features, the order in which we launch the applications does not matter. In this example, we will run the consumer first to illustrate DYAD's synchronization features.\n",
+ "\n",
+ "Run the cell below to run the consumer. You will see that the consumer will immediately begin waiting for data to be made available."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9b2d62ba-d772-40a9-ab54-cef5695fc869",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!{consumer_launch_cmd}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c413a90d-7429-4cad-bb98-fdaf8bbe5644",
+ "metadata": {},
+ "source": [
+ "Now that the consumer is running, we will run the producer. However, Jupyter will not let us launch the producer from within this notebook for as long as the consumer is running. To get around this, we will use the Jupyter Lab terminal.\n",
+ "\n",
+ "First, copy the producer command from above. Then, from the top of the file explorer on the left, click the plus (`+`) button. In the new Jupyter Lab tab that opens, click on \"Terminal\" (in the \"Other\" category) to launch the Jupyter Lab terminal. Finally, paste the producer command into the terminal, and run it.\n",
+ "\n",
+ "We know that the applications ran successfully if the consumer outputs \"OK\" for each file it checks."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c03e13e5-f8f8-4e33-bd5a-9432309dc2e8",
+ "metadata": {},
+ "source": [
+ "To see that the files were transfered, we can check the contents of the producer-managed and consumer-managed directories. If everything worked correctly, we will see the same files in both directories.\n",
+ "\n",
+ "Run the next two cells to check the contents of these directories."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "3b4e23bc-2f13-4bc3-bf58-7a2dc838d47c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux exec -r 0 ls -lah {producer_managed_directory}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "55ee61b5-1df8-4036-8709-23dec67de7d6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux exec -r 1 ls -lah {consumer_managed_directory}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1a5ca339-d930-48d5-89f6-519b01a92fc6",
+ "metadata": {},
+ "source": [
+ "Before moving onto the next example, we need to remove the KVS namespace and unload the DYAD module. We cannot just reuse the namspace and module from this example for two reasons.\n",
+ "\n",
+ "First, the keys in the KVS that DYAD uses are based on the paths to the files *relative to the producer- and consumer-managed directories.* Since we are using the same applications for the next example, these relative paths will be the same, which means the keys will already be present in the KVS. This can interfere with the synchronization of the consumer.\n",
+ "\n",
+ "Second, the DYAD module currently tracks only a single directory at a time. We will be using a different directory for the next example, so we will need to startup the DYAD module from scratch to track this new directory.\n",
+ "\n",
+ "Run the next two cells to unload the DYAD module and remove the KVS namespace."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0bcea21e-f2e2-488f-bd95-8534f78c70b6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux exec -r all flux module unload dyad"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d3197840-01e4-4479-8d9b-440e03155ca9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux kvs namespace remove {kvs_namespace}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4f437928-c3b6-4ff7-8d78-7ed31b09cda0",
+ "metadata": {},
+ "source": [
+ "Run this cell to verify that the DYAD module and KVS namespace are no longer present."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d537769c-6566-48cf-a150-20d40150f059",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!echo \"Modules Post-Cleanup\"\n",
+ "!echo \"====================\"\n",
+ "!flux module list\n",
+ "!echo \"\"\n",
+ "!echo \"KVS Namespaces Post-Cleanup\"\n",
+ "!echo \"===========================\"\n",
+ "!flux kvs namespace list"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "de9f7143-f22a-4eea-911e-582f6c90e529",
+ "metadata": {},
+ "source": [
+ "## Example 2\n",
+ "\n",
+ "In the second example, we will show how DYAD can help workflows even if data is in shared storage (e.g., parallel file system) by still providing built-in and transparent fine-grained synchronization.\n",
+ "\n",
+ "The figure below illustrates the data movement that will happen in this example."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "90ed7911-f507-4a69-a6f9-59185887a097",
+ "metadata": {},
+ "source": [
+ "\n",
+ "\n",
+ "![](\"dyad/dyad_example2.svg\")
\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1c13f27e-551c-4841-8f58-825844d88cd9",
+ "metadata": {},
+ "source": [
+ "To start, we must setup the Flux KVS namespace and DYAD module again. \n",
+ "\n",
+ "Run the cells below to setup the Flux KVS namespace and the DYAD module."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7a9bcc65-0780-4652-87b3-a0b942dd48b2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux kvs namespace create {kvs_namespace}\n",
+ "!flux kvs namespace list"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c43fcd2a-9291-407c-a313-f9be8a85cf4d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux exec -r all flux module load {dyad_module} {shared_managed_directory} {dtl_mode}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d8f3db83-8b97-440a-8b5f-f7cf76656928",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux exec -r all flux module list | grep dyad"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4b4a1949-2454-4e5b-aeba-ed420e42e620",
+ "metadata": {},
+ "source": [
+ "Next, we will generate the shell commands that we will use to run the producer and consumer applications. The only differences between these commands and the ones in Example 1 are as follows:\n",
+ "* The `DYAD_PATH_PRODUCER`, `DYAD_PATH_CONSUMER`, and second command-line argument to the applications all have the same value (i.e., the value of `shared_managed_directory` from the top of the notebook).\n",
+ "* The `DYAD_SHARED_STORAGE` environment variable is provided and set to 1. This tells DYAD to only perform fine-grained synchronization, rather than both synchronization and file transfer.\n",
+ "\n",
+ "Run the next two cells to generate the commands."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "37fa6ba6-375e-4aba-9253-f5e37cc701b9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "producer_launch_cmd = \"{preload} DYAD_KVS_NAMESPACE={kvs_namespace} DYAD_DTL_MODE={dtl_mode} \\\n",
+ "DYAD_PATH_PRODUCER={producer_managed_directory} DYAD_SHARED_STORAGE=1 \\\n",
+ "flux exec -r 0 \\\n",
+ "{producer_program} {num_files_transfered} {producer_managed_directory}\".format(\n",
+ " preload=\"LD_PRELOAD=\\\"/usr/lib/dyad_wrapper.so\\\"\" if producer_program.split(\"/\")[-1].strip().startswith(\"c_\") else \"\",\n",
+ " kvs_namespace=kvs_namespace,\n",
+ " dtl_mode=dtl_mode,\n",
+ " producer_managed_directory=shared_managed_directory,\n",
+ " producer_program=producer_program,\n",
+ " num_files_transfered=num_files_transfered,\n",
+ ")\n",
+ "print(producer_launch_cmd)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "02f13978-be62-4330-a1d7-3574c1d09573",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "consumer_launch_cmd = \"{preload} DYAD_KVS_NAMESPACE={kvs_namespace} DYAD_DTL_MODE={dtl_mode} \\\n",
+ "DYAD_PATH_CONSUMER={consumer_managed_directory} DYAD_SHARED_STORAGE=1 \\\n",
+ "flux exec -r 1 \\\n",
+ "{consumer_program} {num_files_transfered} {consumer_managed_directory}\".format(\n",
+ " preload=\"LD_PRELOAD=\\\"/usr/lib/dyad_wrapper.so\\\"\" if producer_program.split(\"/\")[-1].strip().startswith(\"c_\") else \"\",\n",
+ " kvs_namespace=kvs_namespace,\n",
+ " dtl_mode=dtl_mode,\n",
+ " consumer_managed_directory=shared_managed_directory,\n",
+ " consumer_program=consumer_program,\n",
+ " num_files_transfered=num_files_transfered,\n",
+ ")\n",
+ "print(consumer_launch_cmd)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c11fa139-026c-4b8d-8b64-3b73ba4c1ab8",
+ "metadata": {},
+ "source": [
+ "Finally, we will run the producer and consumer applications. To show how DYAD provides fine-grained synchronization even to shared storage workflows (e.g., workflows that use the parallel file system for data movement), we will run the consumer first.\n",
+ "\n",
+ "Run the cell below to run the consumer. The consumer will immediately begin waiting for data to be made available in shared storage."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ce54e2b4-d5fb-4451-bdf0-143450892292",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!{consumer_launch_cmd}"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ae80ba91-149e-46b6-b1ac-3742626b0664",
+ "metadata": {},
+ "source": [
+ "Now that the consumer is running, we will run the producer. Just like Example 1, we will run the producer by copying the producer command from above and running it in the Jupyter Lab terminal.\n",
+ "\n",
+ "As with Example 1, we know that the applications ran successfully if the consumer outputs \"OK\" for each file it checks."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "eb92651e-ca2d-4c88-bb3f-95aef77d3938",
+ "metadata": {},
+ "source": [
+ "Finally, we need to remove the KVS namespace and unload the DYAD module.\n",
+ "\n",
+ "Run the next two cells to do this.\n",
+ "\n",
+ "Run the final code cell to verify that the DYAD module and KVS namespace are no longer present."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a8be46d2-e138-4849-9b51-6a02542f0bdd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux exec -r all flux module unload dyad"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "de0bec06-3c6b-4644-b6a3-db4183bc3d46",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux kvs namespace remove {kvs_namespace}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "16dbbcc2-aea2-4ba7-9410-4c21c5d0858f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!echo \"Modules Post-Cleanup\"\n",
+ "!echo \"====================\"\n",
+ "!flux module list\n",
+ "!echo \"\"\n",
+ "!echo \"KVS Namespaces Post-Cleanup\"\n",
+ "!echo \"===========================\"\n",
+ "!flux kvs namespace list"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "81d7d87f-1e09-42c8-b165-8902551f6847",
+ "metadata": {},
+ "source": [
+ "# This concludes the notebook tutorial for DYAD.\n",
+ "\n",
+ "## If you are interested in learning more about DYAD, check out our [ReadTheDocs page](https://dyad.readthedocs.io/en/latest/), our [GitHub repository](https://github.com/flux-framework/dyad), and our [short paper](https://dyad.readthedocs.io/en/latest/_downloads/27090817b034a89b76e5538e148fea9e/ShortPaper_2022_eScience_LLNL.pdf) and [poster](https://dyad.readthedocs.io/en/latest/_downloads/1f11761622683662c33fe0086d1d7ad2/Poster_2022_eScience_LLNL.pdf) from eScience 2022.\n",
+ "\n",
+ "## If you are interested in working with us, please reach out to Jae-Seung Yeom (yeom2@llnl.gov) or Ian Lumsden (ilumsden@vols.utk.edu)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "71d04206-343f-4407-880c-d67e659656d6",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/2024-RIKEN-AWS/JupyterNotebook/tutorial/notebook/dyad/dyad_example1.svg b/2024-RIKEN-AWS/JupyterNotebook/tutorial/notebook/dyad/dyad_example1.svg
new file mode 100644
index 0000000..e24636a
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/tutorial/notebook/dyad/dyad_example1.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/2024-RIKEN-AWS/JupyterNotebook/tutorial/notebook/dyad/dyad_example2.svg b/2024-RIKEN-AWS/JupyterNotebook/tutorial/notebook/dyad/dyad_example2.svg
new file mode 100644
index 0000000..2539695
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/tutorial/notebook/dyad/dyad_example2.svg
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/2024-RIKEN-AWS/JupyterNotebook/tutorial/notebook/flux.ipynb b/2024-RIKEN-AWS/JupyterNotebook/tutorial/notebook/flux.ipynb
new file mode 100644
index 0000000..9fc4c85
--- /dev/null
+++ b/2024-RIKEN-AWS/JupyterNotebook/tutorial/notebook/flux.ipynb
@@ -0,0 +1,1651 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "2507d149-dcab-458a-a554-37388e0ee13a",
+ "metadata": {
+ "tags": []
+ },
+ "source": [
+ "\n",
+ "\n",
+ "![](\"Flux-logo.svg\")
\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "40e867ba-f689-4301-bb60-9a448556bb84",
+ "metadata": {
+ "tags": []
+ },
+ "source": [
+ "# Welcome to the Flux Tutorial\n",
+ "\n",
+ "> What is Flux Framework? π€οΈ\n",
+ " \n",
+ "Flux is a flexible framework for resource management, built for your site. The framework consists of a suite of projects, tools, and libraries which may be used to build site-custom resource managers for High Performance Computing centers. Flux is a next-generation resource manager and scheduler with many transformative capabilities like hierarchical scheduling and resource management (you can think of it as \"fractal scheduling\") and directed-graph based resource representations.\n",
+ "\n",
+ "> I'm ready! How do I do this tutorial? ποΈ\n",
+ "\n",
+ "To step through examples in this notebook you need to execute cells. To run a cell, press Shift+Enter on your keyboard. If you prefer, you can also paste the shell commands in the JupyterLab terminal and execute them there.\n",
+ "Let's get started! To provide some brief, added background on Flux and a bit more motivation for our tutorial, \"Shift+Enter\" the cell below to watch our YouTube video!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d71ecd22-8552-4b4d-9bc4-61d86f8d33fe",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "%%html\n",
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "15e82c38-8465-49ac-ae2b-b0bb56a79ec9",
+ "metadata": {
+ "tags": []
+ },
+ "source": [
+ "# Getting started with Flux\n",
+ "\n",
+ "The code and examples that this tutorial is based on can be found at [flux-framework/Tutorials](https://github.com/flux-framework/Tutorials/tree/master/2023-RADIUSS-AWS). You can also find the examples one level up in the flux-workflow-examples directory in this JupyterLab instance.\n",
+ "\n",
+ "## Resources\n",
+ "\n",
+ "> Looking for other resources? We got you covered! π€οΈ\n",
+ "\n",
+ " - [https://flux-framework.org/](https://flux-framework.org/) Flux Framework portal for projects, releases, and publication.\n",
+ " - [Flux Documentation](https://flux-framework.readthedocs.io/en/latest/).\n",
+ " - [Flux Framework Cheat Sheet](https://flux-framework.org/cheat-sheet/)\n",
+ " - [Flux Glossary of Terms](https://flux-framework.readthedocs.io/en/latest/glossary.html)\n",
+ " - [Flux Comics](https://flux-framework.readthedocs.io/en/latest/comics/fluxonomicon.html) come and meet FluxBird - the pink bird who knows things!\n",
+ " - [Flux Learning Guide](https://flux-framework.readthedocs.io/en/latest/guides/learning_guide.html) learn about what Flux does, how it works, and real research applications \n",
+ " - [Getting Started with Flux and Go](https://converged-computing.github.io/flux-go/)\n",
+ " - [Getting Started with Flux in C](https://converged-computing.github.io/flux-c-examples/) *looking for contributors*\n",
+ "\n",
+ "To read the Flux manpages and get help, run `flux help`. To get documentation on a subcommand, run, e.g. `flux help config`. Here is an example of running `flux help` right from the notebook. Yes, did you know we are running in a Flux Instance right now?"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c7d616de-70cd-4090-bd43-ffacb5ade1f6",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "!flux help"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ae33fef6-278c-4996-8534-fd15e548b338",
+ "metadata": {
+ "tags": []
+ },
+ "source": [
+ "Did you know you can also get help for a specific command? For example, let's run, e.g. `flux help jobs` to get information on a sub-command:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "2e54f640-283a-4523-8dde-9617fd6ef0c5",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "# We have commented this out because the output is huge! Feel free to uncomment (remove the #) and run the command\n",
+ "#!flux help jobs"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "17e435d6-0927-4966-a4d7-47a128c94158",
+ "metadata": {
+ "tags": []
+ },
+ "source": [
+ "### You can run any of the commands and examples that follow in the JupyterLab terminal. You can find the terminal in the JupyterLab launcher.\n",
+ "If you do `File -> New -> Terminal` you can open a raw terminal to play with Flux. You'll see a prompt like this: \n",
+ "\n",
+ "`Ζ(s=4,d=0) fluxuser@6e0f43fd90eb:~$`\n",
+ "\n",
+ "`s=4` indicates the number of running Flux brokers, `d=0` indicates the Flux hierarchy depth. `@6e0f43fd90eb` references the host, which is a Docker container for our tutorial."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "70e3df1d-32c9-4996-b6f7-2fa85f4c02ad",
+ "metadata": {
+ "tags": []
+ },
+ "source": [
+ "# Creating Flux Instances\n",
+ "\n",
+ "A Flux instance is a fully functional set of services which manage compute resources under its domain with the capability to launch jobs on those resources. A Flux instance may be running as the default resource manager on a cluster, a job in a resource manager such as Slurm, LSF, or Flux itself, or as a test instance launched locally.\n",
+ "\n",
+ "When run as a job in another resource manager, Flux is started like an MPI program, e.g., under Slurm we might run `srun [OPTIONS] flux start [SCRIPT]`. Flux is unique in that a test instance which mimics a multi-node instance can be started locally with simply `flux start --test-size=N`. This offers users to a way to learn and test interfaces and commands without access to an HPC cluster.\n",
+ "\n",
+ "To start a Flux session with 4 brokers in your container, run:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d568de50-f9e0-452f-8364-e52853013d83",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux start --test-size=4 flux getattr size"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e693f2d9-651f-4f58-bf53-62528caa83d9",
+ "metadata": {},
+ "source": [
+ "The output indicates the number of brokers started successfully."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "eda1a33c-9f9e-4ba0-a013-e97601f79e41",
+ "metadata": {},
+ "source": [
+ "## Flux uptime\n",
+ "Flux provides an `uptime` utility to display properties of the Flux instance such as state of the current instance, how long it has been running, its size and if scheduling is disabled or stopped. The output shows how long the instance has been up, the instance owner, the instance depth (depth in the Flux hierarchy), and the size of the instance (number of brokers)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6057ce25-d1b3-4cc6-b26a-4b05a1639616",
+ "metadata": {
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "!flux uptime"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "dee2d6af-43fa-490e-88e9-10f13e660125",
+ "metadata": {
+ "tags": []
+ },
+ "source": [
+ "# Submitting Jobs to Flux\n",
+ "## Submission CLI\n",
+ "### `flux`: the Job Submission Tool\n",
+ "\n",
+ "To submit jobs to Flux, you can use the `flux` `submit`, `run`, `bulksubmit`, `batch`, and `alloc` commands. The `flux submit` command submits a job to Flux and prints out the jobid. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8a5e7d41-1d8d-426c-8198-0ad4a57e7d04",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux submit hostname"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a7e4c25e-3ca8-4277-bb70-a0e94bcd223b",
+ "metadata": {},
+ "source": [
+ "`submit` supports common options like `--nnodes`, `--ntasks`, and `--cores-per-task`. There are short option equivalents (`-N`, `-n`, and `-c`, respectively) of these options as well. `--cores-per-task=1` is the default."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "571d8c3d-b24a-415e-b9ac-f58b99a7e92c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux submit -N1 -n2 sleep inf"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "cc2bddee-f454-4674-80d4-4a39c5f1bee2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Let's peek at the help for flux submit!\n",
+ "!flux submit --help | head -n 15"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "ac798095",
+ "metadata": {},
+ "source": [
+ "The `flux run` command submits a job to Flux (similar to `flux submit`) but then attaches to the job with `flux job attach`, printing the job's stdout/stderr to the terminal and exiting with the same exit code as the job:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "52d26496-dd1f-44f7-bb10-8a9b4b8c9c80",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux run hostname"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "53357a9d-11d8-4c2d-87d8-c30ae38d01ba",
+ "metadata": {},
+ "source": [
+ "The output from the previous command is the hostname (a container ID string in this case). If the job exits with a non-zero exit code this will be reported by `flux job attach` (occurs implicitly with `flux run`). For example, execute the following:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fa40cb98-a138-4771-a7ef-f1860dddf7db",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux run /bin/false"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6b2b5c3f-e24a-45a8-a10c-e10bfdbb7b87",
+ "metadata": {},
+ "source": [
+ "A job submitted with `run` can be canceled with two rapid `Cltr-C`s in succession, or a user can detach from the job with `Ctrl-C Ctrl-Z`. The user can then re-attach to the job by using `flux job attach JOBID`."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "81e5213d",
+ "metadata": {},
+ "source": [
+ "`flux submit` and `flux run` also support many other useful flags:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "02032748",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux run -n4 --label-io --time-limit=5s --env-remove=LD_LIBRARY_PATH hostname"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f52bb357-a7ce-458d-9c3f-4d664eca4fbd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Uncomment and run this help command if you want to see all the flags for flux run\n",
+ "# !flux run --help"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "91e9ed6c",
+ "metadata": {},
+ "source": [
+ "The `flux bulksubmit` command enqueues jobs based on a set of inputs which are substituted on the command line, similar to `xargs` and the GNU `parallel` utility, except the jobs have access to the resources of an entire Flux instance instead of only the local system."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f0e82702",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux bulksubmit --watch --wait echo {} ::: foo bar baz"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "392a8056-1661-4b76-9ca3-5e536c687e82",
+ "metadata": {},
+ "source": [
+ "The `--cc` option to `submit` makes repeated submission even easier via, `flux submit --cc=IDSET`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0ea1962b-1831-4bd2-8dab-c61fd710df9c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux submit --cc=1-10 --watch hostname"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "27ca3706-8bb4-4fd6-a37c-e6135fb05604",
+ "metadata": {},
+ "source": [
+ "Try it in the JupyterLab terminal with a progress bar and jobs/s rate report: `flux submit --cc=1-100 --watch --progress --jps hostname`\n",
+ "\n",
+ "Note that `--wait` is implied by `--watch`."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "4c5a18ff-8d6a-47e9-a164-931ed1275ef4",
+ "metadata": {},
+ "source": [
+ "Of course, Flux can launch more than just single-node, single-core jobs. We can submit multiple heterogeneous jobs and Flux will co-schedule the jobs while also ensuring no oversubscription of resources (e.g., cores).\n",
+ "\n",
+ "Note: in this tutorial, we cannot assume that the host you are running on has multiple cores, thus the examples below only vary the number of nodes per job. Varying the `cores-per-task` is also possible on Flux when the underlying hardware supports it (e.g., a multi-core node)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "brazilian-former",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux submit --nodes=2 --ntasks=2 --cores-per-task=1 --job-name simulation sleep inf\n",
+ "!flux submit --nodes=1 --ntasks=1 --cores-per-task=1 --job-name analysis sleep inf"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "641f446c-b2e8-40d8-b6bd-eb6b9dba3c71",
+ "metadata": {},
+ "source": [
+ "### `flux watch` to watch jobs\n",
+ "\n",
+ "Wouldn't it be cool to submit a job and then watch it? Well, yeah! We can do this now with flux watch. Let's run a fun example, and then watch the output. We have sleeps in here interspersed with echos only to show you the live action! π₯οΈ\n",
+ "Also note a nice trick - you can always use `flux job last` to get the last JOBID.\n",
+ "Here is an example (not runnable, as notebooks don't support environment variables) for getting and saving a job id:\n",
+ "\n",
+ "```bash\n",
+ "flux submit hostname\n",
+ "JOBID=$(flux job last)\n",
+ "```\n",
+ "\n",
+ "And then you could use the variable `$JOBID` in your subsequent script or interactions with Flux! So what makes `flux watch` different from `flux job attach`? Aside from the fact that `flux watch` is read-only, `flux watch` can watch many (or even all (`flux watch --all`) jobs at once!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5ad231c2-4cdb-4d18-afc2-7cb3a74759c2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux submit ../flux-workflow-examples/job-watch/job-watch.sh\n",
+ "!flux watch $(flux job last)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3f8c2af2",
+ "metadata": {},
+ "source": [
+ "### Listing job properties with `flux jobs`\n",
+ "\n",
+ "We can now list the jobs in the queue with `flux jobs` and we should see both jobs that we just submitted. Jobs that are instances are colored blue in output, red jobs are failed jobs, and green jobs are those that completed successfully. Note that the JupyterLab notebook may not display these colors. You will be able to see them in the terminal."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "institutional-vocabulary",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux jobs"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "77ca4277",
+ "metadata": {},
+ "source": [
+ "Since those jobs won't ever exit (and we didn't specify a timelimit), let's cancel them all now and free up the resources."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "46dd8ec8-6c64-4d8d-9a00-949f5f58c07b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# This was previously flux cancelall -f\n",
+ "!flux cancel --all\n",
+ "!flux jobs"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "544aa0a9",
+ "metadata": {},
+ "source": [
+ "We can use the `flux batch` command to easily created nested flux instances. When `flux batch` is invoked, Flux will automatically create a nested instance that spans the resources allocated to the job, and then Flux runs the batch script passed to `flux batch` on rank 0 of the nested instance. \"Rank\" refers to the rank of the Tree-Based Overlay Network (TBON) used by the Flux brokers: https://flux-framework.readthedocs.io/projects/flux-core/en/latest/man1/flux-broker.html\n",
+ "\n",
+ "While a batch script is expected to launch parallel jobs using `flux run` or `flux submit` at this level, nothing prevents the script from further batching other sub-batch-jobs using the `flux batch` interface, if desired.\n",
+ "\n",
+ "Note: Flux also provides a `flux alloc` which is an interactive version of `flux batch`, but demonstrating that in a Jupyter notebook is difficult due to the lack of pseudo-terminal."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "blank-carpet",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux batch --nslots=2 --cores-per-slot=1 --nodes=2 ./sleep_batch.sh\n",
+ "!flux batch --nslots=2 --cores-per-slot=1 --nodes=2 ./sleep_batch.sh"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "da98bfa1",
+ "metadata": {},
+ "source": [
+ "The contents of `sleep_batch.sh`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "381a3f6c-0da1-4923-801f-486ca5226d3c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from IPython.display import Code\n",
+ "Code(filename='sleep_batch.sh', language='bash')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "edff8993-3c39-4f46-939d-4c8be5739fbc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Here we are submitting a job that generates output, and asking to write it to /tmp/cheese.txt\n",
+ "!flux submit --out /tmp/cheese.txt echo \"Sweet dreams ποΈ are made of cheese, who am I to diss a brie? π§οΈ\"\n",
+ "\n",
+ "# This will show us JOBIDs\n",
+ "!flux jobs\n",
+ "\n",
+ "# We can even see jobs in sub-instances with \"-R\" (for recursive)\n",
+ "!flux jobs -R\n",
+ "\n",
+ "# You could copy a JOBID from above and paste it in the line below to examine the job's resources and output\n",
+ "# or get the last jobid with \"flux job last\" (this is what we will do here)\n",
+ "# JOBID=\"ΖFoRYVpt7\"\n",
+ "\n",
+ "# Note here we are using flux job last to see the last one\n",
+ "# The \"R\" here asks for the resource spec\n",
+ "!flux job info $(flux job last) R\n",
+ "\n",
+ "# When we attach it will direct us to our output file\n",
+ "!flux job attach $(flux job last)\n",
+ "\n",
+ "# And we can look at the output file to see our expected output!\n",
+ "from IPython.display import Code\n",
+ "Code(filename='/tmp/cheese.txt', language='text')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f4e525e2-6c89-4c14-9fae-d87a0d4fc574",
+ "metadata": {},
+ "source": [
+ "To list all completed jobs, run `flux jobs -a`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "df8a8b7c-f475-4a51-8bc6-9983dc9d78ab",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux jobs -a"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3e415ecc-f451-4909-a2bf-351a639cd7fa",
+ "metadata": {},
+ "source": [
+ "To restrict the output to failed (i.e., jobs that exit with nonzero exit code, time out, or are canceled or killed) jobs, run:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "032597d2-4b02-47ea-a5e5-915313cdd7f9",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux jobs -f failed"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "04b405b1-219f-489c-abfc-e2983e82124a",
+ "metadata": {},
+ "source": [
+ "# The Flux Hierarchy\n",
+ "\n",
+ "One feature of the Flux Framework scheduler that is unique is its ability to submit jobs within instances, where an instance can be thought of as a level in a graph. Let's start with a basic image - this is what it might look like to submit to a scheduler that is not graph-based,\n",
+ "where all jobs go to a central job queue or database. Note that our maximum job throughput is one job per second.\n",
+ "\n",
+ "\n",
+ "\n",
+ "The throughput is limited by the workload manager's ability to process a single job. We can improve upon this by simply adding another level, perhaps with three instances. For example, let's say we create a flux allocation or batch that has control of some number of child nodes. We might launch three new instances (each with its own scheduler and queue) at that level two, and all of a sudden, we get a throughput of 1x3, or three jobs per second. \n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "All of a sudden, the throughout can increase exponentially because we are essentially submitting to different schedulers. The example above is not impressive, but our [learning guide](https://flux-framework.readthedocs.io/en/latest/guides/learning_guide.html#fully-hierarchical-resource-management-techniques) (Figure 10) has a beautiful example of how it can scale, done via an actual experiment. We were able to submit 500 jobs/second using only three levels, vs. close to 1 job/second with one level. \n",
+ "\n",
+ "\n",
+ "\n",
+ "And for an interesting detail, you can vary the scheduler algorithm or topology within each sub-instance, meaning that you can do some fairly interesting things with scheduling work, and all without stressing the top level system instance. Next, let's look at a prototype tool called `flux-tree` that you can use to see how this works.\n",
+ "\n",
+ "## Flux tree\n",
+ "\n",
+ "Flux tree is a prototype tool that allows you to easily submit work to different levels of your flux instance, or more specifically, creating a nested hierarchy of jobs that scale out. Let's run the command, look at the output, and talk about it."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "id": "2735b1ca-e761-46be-b509-a86b771628fc",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "TreeID Elapsed(sec) Begin(Epoch) End(Epoch) Match(usec) NJobs NNodes CPN GPN\n",
+ "tree 3.280890 1711671101.237483 1711671104.518378 1574.710000 4 1 4 0\n",
+ "tree.2 1.807340 1711671101.913623 1711671103.720962 690.937000 2 1 2 0\n",
+ "tree.2.2 0.116084 1711671102.753773 1711671102.869857 0.000000 1 1 1 0\n",
+ "tree.2.1 0.113461 1711671102.525129 1711671102.638590 0.000000 1 1 1 0\n",
+ "tree.1 1.823700 1711671101.837330 1711671103.661027 698.328000 2 1 2 0\n",
+ "tree.1.2 0.114873 1711671102.689943 1711671102.804816 0.000000 1 1 1 0\n",
+ "tree.1.1 0.115360 1711671102.447201 1711671102.562560 0.000000 1 1 1 0\n"
+ ]
+ }
+ ],
+ "source": [
+ "!flux tree -T2x2 -J 4 -N 1 -c 4 -o ./tree.out -Q easy:fcfs hostname \n",
+ "! cat ./tree.out"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9d5fe7a0-af54-4c90-be6f-75f50c918dea",
+ "metadata": {},
+ "source": [
+ "In the above, we are running `flux-tree` and looking at the output file. What is happening is that the `flux tree` command is creating a hierarchy of instances. Based on their names you can tell that:\n",
+ "\n",
+ " - `2x2` in the command is the topology\n",
+ " - It says to create two flux instances, and make them each spawn two more.\n",
+ " - `tree` is the root\n",
+ " - `tree.1` is the first instance\n",
+ " - `tree.2` is the second instance\n",
+ " - `tree.1.1` and `tree.1.2` refer to the nested instances under `tree.1`\n",
+ " - `tree.2.1` and `tree.2.2` refer to the nested instances under `tree.2`\n",
+ " \n",
+ "And we provided the command `hostname` to this script, but a more complex example would generate more interested hierarchies,\n",
+ "and with differet functionality for each. Note that although this is just a dummy prototype, you could use `flux-tree` for actual work,\n",
+ "or more likely, you would want to use `flux batch` to submit multiple commands within a single flux instance to take advantage of the same\n",
+ "hierarchy. \n",
+ "\n",
+ "## Flux batch\n",
+ "\n",
+ "Next, let's look at an example that doesn't use `flux tree` but instead uses `flux batch`, which is how you will likely interact with your nested instances. Let's start with a batch script `hello-batch.sh`.\n",
+ "\n",
+ "##### hello-batch.sh\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "id": "e82863e5-b2a1-456b-9ff1-f669b3525fa1",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "#!/bin/bash\n",
+ "\n",
+ "flux submit --flags=waitable -N1 --out /tmp/hello-batch-1.out echo "Hello job 1 from $(hostname) ποΈ"\n",
+ "flux submit --flags=waitable -N1 --out /tmp/hello-batch-2.out echo "Hello job 2 from $(hostname) ποΈ"\n",
+ "flux submit --flags=waitable -N1 --out /tmp/hello-batch-3.out echo "Hello job 3 from $(hostname) ποΈ"\n",
+ "flux submit --flags=waitable -N1 --out /tmp/hello-batch-4.out echo "Hello job 4 from $(hostname) ποΈ"\n",
+ "# Wait for the jobs to finish\n",
+ "flux job wait --all\n",
+ "#!/bin/bash\n",
+ "\n",
+ "flux batch -N1 ./sub_job2.sh\n",
+ "flux queue drain\n",
+ "#!/bin/bash\n",
+ "\n",
+ "flux run -N1 sleep 30\n",
+ "