#!/bin/bash\n",
- "#FLUX: --nodes=2\n",
- "#FLUX: --nslots=2\n",
- "#FLUX: --cores-per-slot=1\n",
- "\n",
- "echo "Starting my batch job"\n",
- "echo "Print the resources allocated to this batch job"\n",
- "flux resource list\n",
- "\n",
- "echo "Use sleep to emulate a parallel program"\n",
- "echo "Run the program at a total of 2 processes each requiring"\n",
- "echo "1 core. These processes are equally spread across 2 nodes."\n",
- "flux run -N 2 -n 2 sleep 30\n",
- "flux run -N 2 -n 2 sleep 30\n",
+ ".output_html .il { color: #666666 } /* Literal.Number.Integer.Long */Sweet dreams 🌚️ are made of cheese, who am I to diss a brie? 🧀️\n",
"\n"
],
"text/latex": [
"\\begin{Verbatim}[commandchars=\\\\\\{\\}]\n",
- "\\PY{c+ch}{\\PYZsh{}!/bin/bash}\n",
- "\\PY{c+c1}{\\PYZsh{}FLUX: \\PYZhy{}\\PYZhy{}nodes=2}\n",
- "\\PY{c+c1}{\\PYZsh{}FLUX: \\PYZhy{}\\PYZhy{}nslots=2}\n",
- "\\PY{c+c1}{\\PYZsh{}FLUX: \\PYZhy{}\\PYZhy{}cores\\PYZhy{}per\\PYZhy{}slot=1}\n",
- "\n",
- "\\PY{n+nb}{echo}\\PY{+w}{ }\\PY{l+s+s2}{\\PYZdq{}Starting my batch job\\PYZdq{}}\n",
- "\\PY{n+nb}{echo}\\PY{+w}{ }\\PY{l+s+s2}{\\PYZdq{}Print the resources allocated to this batch job\\PYZdq{}}\n",
- "flux\\PY{+w}{ }resource\\PY{+w}{ }list\n",
- "\n",
- "\\PY{n+nb}{echo}\\PY{+w}{ }\\PY{l+s+s2}{\\PYZdq{}Use sleep to emulate a parallel program\\PYZdq{}}\n",
- "\\PY{n+nb}{echo}\\PY{+w}{ }\\PY{l+s+s2}{\\PYZdq{}Run the program at a total of 2 processes each requiring\\PYZdq{}}\n",
- "\\PY{n+nb}{echo}\\PY{+w}{ }\\PY{l+s+s2}{\\PYZdq{}1 core. These processes are equally spread across 2 nodes.\\PYZdq{}}\n",
- "flux\\PY{+w}{ }run\\PY{+w}{ }\\PYZhy{}N\\PY{+w}{ }\\PY{l+m}{2}\\PY{+w}{ }\\PYZhy{}n\\PY{+w}{ }\\PY{l+m}{2}\\PY{+w}{ }sleep\\PY{+w}{ }\\PY{l+m}{30}\n",
- "flux\\PY{+w}{ }run\\PY{+w}{ }\\PYZhy{}N\\PY{+w}{ }\\PY{l+m}{2}\\PY{+w}{ }\\PYZhy{}n\\PY{+w}{ }\\PY{l+m}{2}\\PY{+w}{ }sleep\\PY{+w}{ }\\PY{l+m}{30}\n",
+ "Sweet dreams 🌚️ are made of cheese, who am I to diss a brie? 🧀️\n",
"\\end{Verbatim}\n"
],
"text/plain": [
- "#!/bin/bash\n",
- "#FLUX: --nodes=2\n",
- "#FLUX: --nslots=2\n",
- "#FLUX: --cores-per-slot=1\n",
- "\n",
- "echo \"Starting my batch job\"\n",
- "echo \"Print the resources allocated to this batch job\"\n",
- "flux resource list\n",
- "\n",
- "echo \"Use sleep to emulate a parallel program\"\n",
- "echo \"Run the program at a total of 2 processes each requiring\"\n",
- "echo \"1 core. These processes are equally spread across 2 nodes.\"\n",
- "flux run -N 2 -n 2 sleep 30\n",
- "flux run -N 2 -n 2 sleep 30\n"
+ "Sweet dreams 🌚️ are made of cheese, who am I to diss a brie? 🧀️"
]
},
- "execution_count": 15,
+ "execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
+ "# Note here we are using flux job last to see the last job id\n",
+ "# The \"R\" here asks for the resource spec\n",
+ "!flux job info $(flux job last) R\n",
+ "\n",
+ "# When we attach it will direct us to our output file\n",
+ "!flux job attach $(flux job last)\n",
+ "\n",
+ "# And we can look at the output file to see our expected output!\n",
"from IPython.display import Code\n",
- "Code(filename='sleep_batch.sh', language='bash')"
+ "Code(filename='/tmp/cheese.txt', language='text')"
]
},
{
- "cell_type": "code",
- "execution_count": 16,
- "id": "edff8993-3c39-4f46-939d-4c8be5739fbc",
+ "cell_type": "markdown",
+ "id": "f4e525e2-6c89-4c14-9fae-d87a0d4fc574",
"metadata": {},
+ "source": [
+ "We can again see a list all completed jobs with `flux jobs -a`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 26,
+ "id": "df8a8b7c-f475-4a51-8bc6-9983dc9d78ab",
+ "metadata": {
+ "collapsed": true,
+ "jupyter": {
+ "outputs_hidden": true
+ }
+ },
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "ƒU5u1XQcf\n",
" JOBID USER NAME ST NTASKS NNODES TIME INFO\n",
- "\u001b[01;34m ƒThRLkwsm jovyan ./sleep_b+ R 2 2 51.17s 399f5da372b[0,0]\n",
- "\u001b[0;0m\u001b[01;34m ƒThKfdhKD jovyan ./sleep_b+ R 2 2 51.39s 399f5da372b[0,0]\n",
- "\u001b[0;0m JOBID USER NAME ST NTASKS NNODES TIME INFO\n",
- "\u001b[01;34m ƒThRLkwsm jovyan ./sleep_b+ R 2 2 51.34s 399f5da372b[0,0]\n",
- "\u001b[0;0m\u001b[01;34m ƒThKfdhKD jovyan ./sleep_b+ R 2 2 51.56s 399f5da372b[0,0]\n",
- "\u001b[0;0m\n",
- "ƒThRLkwsm:\n",
- " ƒEgNEfjm jovyan sleep R 2 2 20.11s 399f5da372b[0,0]\n",
- "\n",
- "ƒThKfdhKD:\n",
- " ƒEga6ZzX jovyan sleep R 2 2 20.32s 399f5da372b[0,0]\n",
- "{\"version\": 1, \"execution\": {\"R_lite\": [{\"rank\": \"3\", \"children\": {\"core\": \"7\"}}], \"nodelist\": [\"399f5da372b0\"], \"starttime\": 1721424338, \"expiration\": 4875020774}}\n",
- "0: stdout redirected to /tmp/cheese.txt\n",
- "0: stderr redirected to /tmp/cheese.txt\n"
+ "\u001b[01;34m ƒ3Vw6xW9wD jovyan ./sleep_b+ R 2 2 0.998s 8660c254a8e[5,5]\n",
+ "\u001b[0;0m\u001b[01;34m ƒ3Vw1mYfjD jovyan ./sleep_b+ R 2 2 1.202s 8660c254a8e[5,5]\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VwC9Te9D jovyan echo CD 1 1 0.014s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[37m ƒnyvM4Nb jovyan sleep CA 2 1 5.269h 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[37m ƒ3VqVSHr7q jovyan sleep CA 2 1 12.04s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[37m ƒ3VqzNXq8B jovyan analysis CA 1 1 10.91s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[37m ƒ3VqtrJWFh jovyan simulation CA 2 2 11.12s 8660c254a8e[5,5]\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3Vr6FWywV jovyan job-watch+ CD 1 1 10.03s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3Vqogq1L3 jovyan echo CD 1 1 0.015s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3Vqogq1L4 jovyan echo CD 1 1 0.014s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VqhAnAUA jovyan hostname CD 1 1 0.060s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VqhAnAU9 jovyan hostname CD 1 1 0.050s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VqhAnAU8 jovyan hostname CD 1 1 0.047s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VqhAnAU7 jovyan hostname CD 1 1 0.047s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VqadFLKq jovyan echo CD 1 1 0.025s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VqabmM3W jovyan echo CD 1 1 0.025s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VqabmM3V jovyan echo CD 1 1 0.012s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VqNqo3Qs jovyan hostname CD 1 1 0.016s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VqBbUVR9 jovyan hostname CD 4 1 0.017s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ3Vq5LFXNf jovyan false F 1 1 0.037s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VpyWEM83 jovyan hostname CD 1 1 0.013s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VPB8ZEqV jovyan echo CD 1 1 0.012s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3V7Tprhqh jovyan echo CD 1 1 0.060s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3V35oKmEo jovyan echo CD 1 1 0.015s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2mzETcgvB jovyan echo CD 1 1 0.012s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2mnMLCXPd jovyan echo CD 1 1 0.036s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2mfhe5NCX jovyan echo CD 1 1 0.036s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF545FE jovyan sleep CD 1 1 0.077s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF3a5y2 jovyan sleep CD 1 1 0.108s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF3a5y8 jovyan sleep CD 1 1 0.078s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF3a5xx jovyan sleep CD 1 1 0.118s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF3a5y5 jovyan sleep CD 1 1 0.107s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF3a5y7 jovyan sleep CD 1 1 0.078s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF3a5y1 jovyan sleep CD 1 1 0.107s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF3a5xs jovyan sleep CD 1 1 0.118s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF3a5xt jovyan sleep CD 1 1 0.118s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF3a5y3 jovyan sleep CD 1 1 0.107s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF266gd jovyan sleep CD 1 1 0.118s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF3a5y4 jovyan sleep CD 1 1 0.107s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF545FF jovyan sleep CD 1 1 0.076s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF545FG jovyan sleep CD 1 1 0.076s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF3a5xv jovyan sleep CD 1 1 0.117s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF545FH jovyan sleep CD 1 1 0.073s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF3a5xu jovyan sleep CD 1 1 0.117s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF545FD jovyan sleep CD 1 1 0.076s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF3a5xw jovyan sleep CD 1 1 0.093s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF3a5y6 jovyan sleep CD 1 1 0.083s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF3a5xz jovyan sleep CD 1 1 0.083s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF266gc jovyan sleep CD 1 1 0.090s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF266gb jovyan sleep CD 1 1 0.087s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF3a5xy jovyan sleep CD 1 1 0.086s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF266gX jovyan sleep CD 1 1 0.101s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEzc7QK jovyan sleep CD 1 1 0.109s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEzc7QJ jovyan sleep CD 1 1 0.107s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEzc7QD jovyan sleep CD 1 1 0.111s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEzc7QG jovyan sleep CD 1 1 0.085s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF266gY jovyan sleep CD 1 1 0.076s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF266ga jovyan sleep CD 1 1 0.074s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEF266gZ jovyan sleep CD 1 1 0.076s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEzc7QL jovyan sleep CD 1 1 0.074s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEzc7QH jovyan sleep CD 1 1 0.079s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEzc7QF jovyan sleep CD 1 1 0.077s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEzc7QE jovyan sleep CD 1 1 0.053s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEzc7QC jovyan sleep CD 1 1 0.062s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEy887z jovyan sleep CD 1 1 0.079s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEy887y jovyan sleep CD 1 1 0.079s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEzc7QB jovyan sleep CD 1 1 0.078s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEy887w jovyan sleep CD 1 1 0.078s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEy887x jovyan sleep CD 1 1 0.078s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEy887v jovyan sleep CD 1 1 0.088s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEvA9ZN jovyan sleep CD 1 1 0.091s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEy887s jovyan sleep CD 1 1 0.088s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEy887u jovyan sleep CD 1 1 0.076s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEwe8qV jovyan sleep CD 1 1 0.078s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEvA9ZL jovyan sleep CD 1 1 0.077s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEy887r jovyan sleep CD 1 1 0.076s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEy887t jovyan sleep CD 1 1 0.073s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEvA9ZM jovyan sleep CD 1 1 0.075s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEy887q jovyan sleep CD 1 1 0.075s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEwe8qW jovyan sleep CD 1 1 0.075s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEvA9ZG jovyan sleep CD 1 1 0.082s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEvA9ZF jovyan sleep CD 1 1 0.085s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEvA9Z9 jovyan sleep CD 1 1 0.094s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEvA9ZE jovyan sleep CD 1 1 0.085s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEvA9ZB jovyan sleep CD 1 1 0.094s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEvA9ZA jovyan sleep CD 1 1 0.094s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEvA9ZH jovyan sleep CD 1 1 0.079s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEvA9ZK jovyan sleep CD 1 1 0.078s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEvA9ZJ jovyan sleep CD 1 1 0.079s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAH6 jovyan sleep CD 1 1 0.081s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEvA9ZD jovyan sleep CD 1 1 0.069s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEvA9ZC jovyan sleep CD 1 1 0.075s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAH3 jovyan sleep CD 1 1 0.084s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAH5 jovyan sleep CD 1 1 0.072s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAH2 jovyan sleep CD 1 1 0.064s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAH1 jovyan sleep CD 1 1 0.067s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAGw jovyan sleep CD 1 1 0.084s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAGx jovyan sleep CD 1 1 0.084s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAH4 jovyan sleep CD 1 1 0.063s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAGz jovyan sleep CD 1 1 0.083s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAGv jovyan sleep CD 1 1 0.083s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAzd jovyan sleep CD 1 1 0.108s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAGy jovyan sleep CD 1 1 0.058s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAGt jovyan sleep CD 1 1 0.059s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAGu jovyan sleep CD 1 1 0.058s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAzb jovyan sleep CD 1 1 0.108s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAzc jovyan sleep CD 1 1 0.108s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAza jovyan sleep CD 1 1 0.090s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAze jovyan sleep CD 1 1 0.090s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAGq jovyan sleep CD 1 1 0.079s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAGr jovyan sleep CD 1 1 0.079s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAGp jovyan sleep CD 1 1 0.079s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAGs jovyan sleep CD 1 1 0.079s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAzY jovyan sleep CD 1 1 0.088s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAzZ jovyan sleep CD 1 1 0.085s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAzf jovyan sleep CD 1 1 0.084s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAzi jovyan sleep CD 1 1 0.074s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEtgAGo jovyan sleep CD 1 1 0.074s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAzg jovyan sleep CD 1 1 0.080s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAzj jovyan sleep CD 1 1 0.071s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAzW jovyan sleep CD 1 1 0.074s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAzX jovyan sleep CD 1 1 0.074s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEqiBiM jovyan sleep CD 1 1 0.112s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEqiBiJ jovyan sleep CD 1 1 0.111s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAzh jovyan sleep CD 1 1 0.062s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEqiBiK jovyan sleep CD 1 1 0.095s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEqiBiL jovyan sleep CD 1 1 0.095s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEqiBiH jovyan sleep CD 1 1 0.095s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAzT jovyan sleep CD 1 1 0.093s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAzV jovyan sleep CD 1 1 0.092s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEsCAzU jovyan sleep CD 1 1 0.092s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEqiBiG jovyan sleep CD 1 1 0.087s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEqiBiA jovyan sleep CD 1 1 0.103s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEqiBiF jovyan sleep CD 1 1 0.103s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEqiBiD jovyan sleep CD 1 1 0.103s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEqiBiE jovyan sleep CD 1 1 0.070s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEpECRy jovyan sleep CD 1 1 0.069s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEqiBi9 jovyan sleep CD 1 1 0.068s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEqiBi8 jovyan sleep CD 1 1 0.066s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEqiBiB jovyan sleep CD 1 1 0.065s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEqiBiC jovyan sleep CD 1 1 0.064s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEqiBi7 jovyan sleep CD 1 1 0.064s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEpECRu jovyan sleep CD 1 1 0.053s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEpECRw jovyan sleep CD 1 1 0.053s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEpECRt jovyan sleep CD 1 1 0.053s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEpECRx jovyan sleep CD 1 1 0.053s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEpECRr jovyan sleep CD 1 1 0.053s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEpECRv jovyan sleep CD 1 1 0.048s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEpECRs jovyan sleep CD 1 1 0.048s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEpECRq jovyan sleep CD 1 1 0.059s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEpECRp jovyan sleep CD 1 1 0.114s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEnkD9d jovyan sleep CD 1 1 0.123s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEnkD9Y jovyan sleep CD 1 1 0.123s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEnkD9e jovyan sleep CD 1 1 0.123s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEnkD9b jovyan sleep CD 1 1 0.123s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEnkD9W jovyan sleep CD 1 1 0.123s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEpECRm jovyan sleep CD 1 1 0.122s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEnkD9f jovyan sleep CD 1 1 0.122s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEnkD9a jovyan sleep CD 1 1 0.123s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEnkD9Z jovyan sleep CD 1 1 0.123s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEnkD9X jovyan sleep CD 1 1 0.122s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEmGDsC jovyan sleep CD 1 1 0.100s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEnkD9c jovyan sleep CD 1 1 0.099s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEnkD9U jovyan sleep CD 1 1 0.095s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEnkD9T jovyan sleep CD 1 1 0.095s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEnkD9R jovyan sleep CD 1 1 0.095s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEpECRn jovyan sleep CD 1 1 0.093s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEnkD9S jovyan sleep CD 1 1 0.093s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEnkD9V jovyan sleep CD 1 1 0.091s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEpECRo jovyan sleep CD 1 1 0.090s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEQWPvN jovyan sleep CD 1 1 0.105s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEmGDs5 jovyan sleep CD 1 1 0.105s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEmGDs8 jovyan sleep CD 1 1 0.104s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEQWPvQ jovyan sleep CD 1 1 0.105s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEmGDs9 jovyan sleep CD 1 1 0.103s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEmGDs7 jovyan sleep CD 1 1 0.104s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEmGDs6 jovyan sleep CD 1 1 0.104s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEQWPvP jovyan sleep CD 1 1 0.104s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEQWPvR jovyan sleep CD 1 1 0.104s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEQWPvS jovyan sleep CD 1 1 0.104s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEmGDsB jovyan sleep CD 1 1 0.078s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEQWPvK jovyan sleep CD 1 1 0.079s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEQWPvJ jovyan sleep CD 1 1 0.076s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEQWPvT jovyan sleep CD 1 1 0.075s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEQWPvM jovyan sleep CD 1 1 0.076s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEQWPvL jovyan sleep CD 1 1 0.076s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEQWPvU jovyan sleep CD 1 1 0.071s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEmGDsA jovyan sleep CD 1 1 0.069s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEQWPvH jovyan sleep CD 1 1 0.074s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEP2Qe7 jovyan sleep CD 1 1 0.193s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEQWPvG jovyan sleep CD 1 1 0.193s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEP2Qe4 jovyan sleep CD 1 1 0.193s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEP2Qe6 jovyan sleep CD 1 1 0.193s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEQWPvF jovyan sleep CD 1 1 0.193s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEP2Qe5 jovyan sleep CD 1 1 0.193s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEP2Qe8 jovyan sleep CD 1 1 0.193s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMm jovyan sleep CD 1 1 0.193s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMg jovyan sleep CD 1 1 0.193s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMf jovyan sleep CD 1 1 0.193s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMi jovyan sleep CD 1 1 0.193s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEP2Qdy jovyan sleep CD 1 1 0.192s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEP2Qdv jovyan sleep CD 1 1 0.192s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMh jovyan sleep CD 1 1 0.193s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEP2Qe2 jovyan sleep CD 1 1 0.192s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEP2Qe3 jovyan sleep CD 1 1 0.192s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEP2Qdw jovyan sleep CD 1 1 0.192s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEP2Qdz jovyan sleep CD 1 1 0.192s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEP2Qe9 jovyan sleep CD 1 1 0.191s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMp jovyan sleep CD 1 1 0.175s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMj jovyan sleep CD 1 1 0.173s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEP2Qdx jovyan sleep CD 1 1 0.174s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMq jovyan sleep CD 1 1 0.173s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEP2Qe1 jovyan sleep CD 1 1 0.174s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMn jovyan sleep CD 1 1 0.171s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMZ jovyan sleep CD 1 1 0.168s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEL4S5G jovyan sleep CD 1 1 0.168s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEL4S5F jovyan sleep CD 1 1 0.168s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEL4S5E jovyan sleep CD 1 1 0.168s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEP2Qdu jovyan sleep CD 1 1 0.166s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMo jovyan sleep CD 1 1 0.167s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMk jovyan sleep CD 1 1 0.166s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMd jovyan sleep CD 1 1 0.165s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMb jovyan sleep CD 1 1 0.163s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMe jovyan sleep CD 1 1 0.162s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMa jovyan sleep CD 1 1 0.162s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEMYRMc jovyan sleep CD 1 1 0.162s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2eEEL4S5D jovyan sleep CD 1 1 0.032s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ2YnijmLwy jovyan compute.py F 1 1 0.031s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ2YiqfxNdm jovyan compute.py F 1 1 0.012s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ2YYgVHnyV jovyan compute.py F 1 1 0.062s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ2YYE7Ja9d jovyan compute.py F 1 1 0.048s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2Fr5PCm9h jovyan ./sub_job+ CD 1 1 31.58s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒS4xykqnw jovyan echo CD 1 1 0.023s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3wSjr2ik jovyan echo CD 1 1 0.015s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3wSjr2ij jovyan echo CD 1 1 0.013s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3mYvC1Rj jovyan hostname CD 1 1 0.030s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3mYvC1Ri jovyan hostname CD 1 1 0.014s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3mYvC1Rh jovyan hostname CD 1 1 0.014s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3mYvC1Rk jovyan hostname CD 1 1 0.014s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ3cZKNgsB jovyan Hello I a+ F 1 1 0.014s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ3cZKNgsA jovyan Hello I a+ F 1 1 0.014s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ3cZKNgs9 jovyan Hello I a+ F 1 1 0.015s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ3cZKNgsC jovyan Hello I a+ F 1 1 0.012s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VGjwQ2U jovyan echo CD 1 1 0.032s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VGjwQ2T jovyan echo CD 1 1 0.027s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VGjwQ2V jovyan echo CD 1 1 0.024s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VGiTQkA jovyan echo CD 1 1 0.024s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VGiTQkB jovyan echo CD 1 1 0.023s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VGiTQk9 jovyan echo CD 1 1 0.023s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VGiTQk8 jovyan echo CD 1 1 0.023s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VGiTQk7 jovyan echo CD 1 1 0.018s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VGgyRTn jovyan echo CD 1 1 0.018s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ3VGgyRTm jovyan echo CD 1 1 0.018s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2rpm1UiB jovyan hostname CD 1 1 0.015s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2rpm1UiC jovyan hostname CD 1 1 0.014s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2rpm1UiD jovyan hostname CD 1 1 0.014s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒ2rpm1UiE jovyan hostname CD 1 1 0.012s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒxah9Lhg jovyan hostname CD 1 1 0.016s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒxah9Lhf jovyan hostname CD 1 1 0.016s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒxah9Lhd jovyan hostname CD 1 1 0.014s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒxah9Lhe jovyan hostname CD 1 1 0.013s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒp5VSGEC jovyan echo CD 1 1 0.051s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒp5TxGwq jovyan echo CD 1 1 0.047s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒp5VSGEB jovyan echo CD 1 1 0.047s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒm4dLyPD jovyan hostname CD 1 1 0.014s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒioKWajq jovyan hostname CD 4 1 0.015s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒhFVr6U7 jovyan false F 1 1 0.012s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;32m ƒgmdsbJF jovyan hostname CD 1 1 0.013s 8660c254a8e5\n",
+ "\u001b[0;0m"
]
- },
+ }
+ ],
+ "source": [
+ "!flux jobs -a"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3e415ecc-f451-4909-a2bf-351a639cd7fa",
+ "metadata": {},
+ "source": [
+ "To restrict the output to failed (i.e., jobs that exit with nonzero exit code, time out, or are canceled or killed) jobs, run:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "id": "032597d2-4b02-47ea-a5e5-915313cdd7f9",
+ "metadata": {},
+ "outputs": [
{
- "data": {
- "text/html": [
- "Sweet dreams 🌚️ are made of cheese, who am I to diss a brie? 🧀️\n",
- "
\n"
- ],
- "text/latex": [
- "\\begin{Verbatim}[commandchars=\\\\\\{\\}]\n",
- "Sweet dreams 🌚️ are made of cheese, who am I to diss a brie? 🧀️\n",
- "\\end{Verbatim}\n"
- ],
- "text/plain": [
- "Sweet dreams 🌚️ are made of cheese, who am I to diss a brie? 🧀️"
- ]
- },
- "execution_count": 16,
- "metadata": {},
- "output_type": "execute_result"
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ " JOBID USER NAME ST NTASKS NNODES TIME INFO\n",
+ "\u001b[01;31m ƒ3Vq5LFXNf jovyan false F 1 1 0.037s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ2YnijmLwy jovyan compute.py F 1 1 0.031s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ2YiqfxNdm jovyan compute.py F 1 1 0.012s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ2YYgVHnyV jovyan compute.py F 1 1 0.062s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ2YYE7Ja9d jovyan compute.py F 1 1 0.048s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ3cZKNgsB jovyan Hello I a+ F 1 1 0.014s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ3cZKNgsA jovyan Hello I a+ F 1 1 0.014s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ3cZKNgs9 jovyan Hello I a+ F 1 1 0.015s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒ3cZKNgsC jovyan Hello I a+ F 1 1 0.012s 8660c254a8e5\n",
+ "\u001b[0;0m\u001b[01;31m ƒhFVr6U7 jovyan false F 1 1 0.012s 8660c254a8e5\n",
+ "\u001b[0;0m"
+ ]
}
],
"source": [
- "# Here we are submitting a job that generates output, and asking to write it to /tmp/cheese.txt\n",
- "!flux submit --out /tmp/cheese.txt echo \"Sweet dreams 🌚️ are made of cheese, who am I to diss a brie? 🧀️\"\n",
- "\n",
- "# This will show us JOBIDs\n",
- "!flux jobs\n",
- "\n",
- "# We can even see jobs in sub-instances with \"-R\" (for recursive)\n",
- "!flux jobs -R\n",
+ "!flux jobs -f failed"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6bc17bac-2fc4-4418-8939-e930f9929976",
+ "metadata": {},
+ "source": [
+ "### flux submit from within a batch\n",
"\n",
- "# You could copy a JOBID from above and paste it in the line below to examine the job's resources and output\n",
- "# or get the last jobid with \"flux job last\" (this is what we will do here)\n",
- "# JOBID=\"ƒFoRYVpt7\"\n",
+ "Next open up [hello-batch.sh](hello-batch.sh) to see an example of using `flux batch` to submit jobs within the instance, and then wait for them to finish. This script is going to:\n",
"\n",
- "# Note here we are using flux job last to see the last one\n",
- "# The \"R\" here asks for the resource spec\n",
- "!flux job info $(flux job last) R\n",
+ "1. Create a flux instance with the top level resources you specify\n",
+ "2. Submit jobs to the scheduler controlled by the broker of that sub-instance\n",
+ "3. Run the four jobs, with `--flags=waitable` and `flux job wait --all` to wait for the output file\n",
+ "4. Within the batch script, you can add `--wait` or `--flags=waitable` to individual jobs, and use `flux queue drain` to wait for the queue to drain, _or_ `flux job wait --all` to wait for the jobs you flagged to finish. \n",
"\n",
- "# When we attach it will direct us to our output file\n",
- "!flux job attach $(flux job last)\n",
+ "Note that when you submit a batch job, you'll get a job id back for the _batch job_, and usually when you look at the output of that with `flux job attach $jobid` you will see the output file(s) where the internal contents are written. Since we want to print the output file easily to the terminal, we are waiting for the batch job by adding the `--flags=waitable` and then waiting for it. Let's try to run our batch job now."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 28,
+ "id": "72358a03-6f1f-4c5e-91eb-cab71883a232",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "ƒ3VwkUsydR\n",
+ "ƒ3VwkUsydR\n",
+ "Hello job 1 from 8660c254a8e5 💛️\n",
+ "Hello job 2 from 8660c254a8e5 💚️\n",
+ "Hello job 3 from 8660c254a8e5 💙️\n",
+ "Hello job 4 from 8660c254a8e5 💜️\n"
+ ]
+ }
+ ],
+ "source": [
+ "! flux batch --flags=waitable --out /tmp/flux-batch.out -N2 ./hello-batch.sh\n",
+ "! flux job wait\n",
+ "! cat /tmp/hello-batch-1.out\n",
+ "! cat /tmp/hello-batch-2.out\n",
+ "! cat /tmp/hello-batch-3.out\n",
+ "! cat /tmp/hello-batch-4.out"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "75c0ae3f-2813-4ae8-83be-00be3df92a4b",
+ "metadata": {},
+ "source": [
+ "Each of `flux batch` and `flux alloc` hints at creating a Flux instance. How deep can we go into that rabbit hole, perhaps for jobs and workflows with nested logic or more orchestration complexity?\n",
"\n",
- "# And we can look at the output file to see our expected output!\n",
- "from IPython.display import Code\n",
- "Code(filename='/tmp/cheese.txt', language='text')"
+ "![](\"img/flux-batch.jpg\")
"
]
},
{
"cell_type": "markdown",
- "id": "f4e525e2-6c89-4c14-9fae-d87a0d4fc574",
+ "id": "04b405b1-219f-489c-abfc-e2983e82124a",
"metadata": {},
"source": [
- "We can again see a list all completed jobs with `flux jobs -a`:"
+ "### The Flux Hierarchy 🍇️\n",
+ "\n",
+ "One feature of the Flux Framework scheduler that is unique is its ability to submit jobs within instances, where an instance can be thought of as a level in a graph. Let's start with a basic image - this is what it might look like to submit to a scheduler that is not graph-based (left), where all jobs go to a central job queue or database. Note that our maximum job throughput is one job per second. The throughput is limited by the workload manager's ability to process a single job. We can improve upon this by simply adding another level, perhaps with three instances. For example, let's say we create a flux allocation or batch that has control of some number of child nodes. We might launch three new instances (each with its own scheduler and queue, right image) at that level two, and all of a sudden, we get a throughput of 1x3, or three jobs per second.\n",
+ "\n",
+ "\n",
+ "![](\"img/single-submit.png\") \n",
+ " | \n",
+ " \n",
+ "![](\"img/instance-submit.png\") \n",
+ " | \n",
+ "
\n",
- "Description: Allocation for an interactive instance\n",
- "
\n",
- "\n",
- "You might want to request an allocation for a set of resources (an allocation) and then attach to the interactively. This is the goal of flux alloc. Since we can't easily do that in a cell, try opening up the and doing: \n",
- "\n",
- "```bash\n",
- "# Look at the resources you have outside of the allocation\n",
- "flux resource list\n",
- "\n",
- "# Request an allocation with 2 \"nodes\" - a subset of what you have in total\n",
- "flux alloc -N 2\n",
- "\n",
- "# See the resources you are given\n",
- "flux resource list\n",
- "\n",
- "# You can exit from the allocation like this!\n",
- "exit\n",
- "```"
+ "You can also try a more detailed view with `flux pstree -a -X`!"
]
},
{
"cell_type": "markdown",
- "id": "04b405b1-219f-489c-abfc-e2983e82124a",
+ "id": "eda1a33c-9f9e-4ba0-a013-e97601f79e41",
"metadata": {},
"source": [
- "# The Flux Hierarchy 🍇️\n",
- "\n",
- "One feature of the Flux Framework scheduler that is unique is its ability to submit jobs within instances, where an instance can be thought of as a level in a graph. Let's start with a basic image - this is what it might look like to submit to a scheduler that is not graph-based,\n",
- "where all jobs go to a central job queue or database. Note that our maximum job throughput is one job per second.\n",
- "\n",
- "\n",
- "\n",
- "The throughput is limited by the workload manager's ability to process a single job. We can improve upon this by simply adding another level, perhaps with three instances. For example, let's say we create a flux allocation or batch that has control of some number of child nodes. We might launch three new instances (each with its own scheduler and queue) at that level two, and all of a sudden, we get a throughput of 1x3, or three jobs per second. \n",
- "\n",
- "\n",
- "\n",
+ "# Process, Monitoring, and Job Utilities ⚙️\n",
"\n",
- "All of a sudden, the throughout can increase exponentially because we are essentially submitting to different schedulers. The example above is not impressive, but our [learning guide](https://flux-framework.readthedocs.io/en/latest/guides/learning_guide.html#fully-hierarchical-resource-management-techniques) (Figure 10) has a beautiful example of how it can scale, done via an actual experiment. We were able to submit 500 jobs/second using only three levels, vs. close to 1 job/second with one level. \n",
+ "## flux exec 👊️\n",
"\n",
- "\n",
- "\n",
- "And for an interesting detail, you can vary the scheduler algorithm or topology within each sub-instance, meaning that you can do some fairly interesting things with scheduling work, and all without stressing the top level system instance. Next, let's look at a prototype tool called `flux-tree` that you can use to see how this works.\n",
- "\n",
- "## flux tree\n",
+ "\n",
+ "Description: Execute commands across ranks\n",
+ "
\n",
"\n",
- "Flux tree is a prototype tool that allows you to easily submit work to different levels of your flux instance, or more specifically, creating a nested hierarchy of jobs that scale out. Let's run the command, look at the output, and talk about it."
+ "Have you ever wanted a quick way to execute a command to all of your nodes in a flux instance? It might be to create a directory, or otherwise interact with a file. This can be hugely useful in environments where you don't have a shared filesystem, for example. This is a job for flux exec! Here is a toy example to execute the command to every rank (`-r all`) to print."
]
},
{
"cell_type": "code",
- "execution_count": 1,
- "id": "2735b1ca-e761-46be-b509-a86b771628fc",
+ "execution_count": 31,
+ "id": "df8e5a5d-76aa-4151-a25f-4d8f3aa4a738",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "flux-job: task(s) exited with exit code 1\n",
- "flux-job: task(s) exited with exit code 1\n",
- "awk: line 1: syntax error at or near :\n",
- "e8cfed35f636\n",
- "flux-tree-helper: ERROR: Expecting value: line 1 column 160 (char 159)\n",
- "Jul 05 05:20:32.333883 UTC broker.err[0]: rc2.0: flux tree -N1 -c1 --leaf --prefix=tree.1.1 --njobs=1 -- hostname Exited (rc=1) 0.6s\n",
- "awk: line 1: syntax error at or near :\n",
- "flux-tree-helper: ERROR: Expecting value: line 1 column 156 (char 155)\n",
- "Jul 05 05:20:33.523886 UTC broker.err[0]: rc2.0: flux tree -N1 -c2 --topology=2 --queue-policy=fcfs --prefix=tree.1 --njobs=2 -- hostname Exited (rc=1) 2.4s\n",
- "awk: line 1: syntax error at or near :\n",
- "flux-tree-helper: ERROR: Expecting value: line 1 column 155 (char 154)\n",
- "cat: ./tree.out: No such file or directory\n"
+ "Hello from a flux rank!\n",
+ "Hello from a flux rank!\n",
+ "Hello from a flux rank!\n",
+ "Hello from a flux rank!\n"
]
}
],
"source": [
- "!flux tree -T2x2 -J 4 -N 1 -c 4 -o ./tree.out -Q easy:fcfs hostname\n",
- "! cat ./tree.out"
+ "!flux exec -r all echo \"Hello from a flux rank!\""
]
},
{
"cell_type": "markdown",
- "id": "9d5fe7a0-af54-4c90-be6f-75f50c918dea",
+ "id": "768c05fe-461e-4f88-bb3d-c74f9d8bc217",
"metadata": {},
"source": [
- "In the above, we are running `flux-tree` and looking at the output file. What is happening is that the `flux tree` command is creating a hierarchy of instances. Based on their names you can tell that:\n",
- "\n",
- " - `2x2` in the command is the topology\n",
- " - It says to create two flux instances, and make them each spawn two more.\n",
- " - `tree` is the root\n",
- " - `tree.1` is the first instance\n",
- " - `tree.2` is the second instance\n",
- " - `tree.1.1` and `tree.1.2` refer to the nested instances under `tree.1`\n",
- " - `tree.2.1` and `tree.2.2` refer to the nested instances under `tree.2`\n",
- " \n",
- "And we provided the command `hostname` to this script, but a more complex example would generate more interested hierarchies,\n",
- "and with differet functionality for each. Note that although this is just a dummy prototype, you could use `flux-tree` for actual work,\n",
- "or more likely, you would want to use `flux batch` to submit multiple commands within a single flux instance to take advantage of the same\n",
- "hierarchy. \n",
- "\n",
- "## flux batch\n",
- "\n",
- "Let's return to flux batch, but now with our new knowledge about flux instances! Flux tree is actually an experimental command that you won't encounter in the wild. Instead, you will likely interact with your nested flux instances with `flux batch`. Let's start with a batch script `hello-batch.sh`.\n",
- "\n",
- "### hello-batch.sh\n"
+ "You can also use `-x` to exclude ranks. For example, we often do custom actions on the main or \"leader\" rank, and just want to issue commands to the workers."
]
},
{
"cell_type": "code",
- "execution_count": 19,
- "id": "e82863e5-b2a1-456b-9ff1-f669b3525fa1",
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "#!/bin/bash\n",
- "\n",
- "flux submit --flags=waitable -N1 --output=/tmp/hello-batch-1.out echo "Hello job 1 from $(hostname) 💛️"\n",
- "flux submit --flags=waitable -N1 --output=/tmp/hello-batch-2.out echo "Hello job 2 from $(hostname) 💚️"\n",
- "flux submit --flags=waitable -N1 --output=/tmp/hello-batch-3.out echo "Hello job 3 from $(hostname) 💙️"\n",
- "flux submit --flags=waitable -N1 --output=/tmp/hello-batch-4.out echo "Hello job 4 from $(hostname) 💜️"\n",
- "# Wait for the jobs to finish\n",
- "flux job wait --all\n",
- "#!/bin/bash\n",
- "\n",
- "flux batch -N1 ./sub_job2.sh\n",
- "flux queue drain\n",
- "#!/bin/bash\n",
- "\n",
- "flux run -N1 sleep 30\n",
- "\n",
+ "Description: Create file and content archives to access later and between ranks\n",
+ "
\n",
+ "\n",
+ "As Flux is used more in cloud environments, we might find ourselves in a situation where we have a cluster without a shared filesystem. The `flux archive` command helps with this situation. At a high level, `flux archive` allows us to save named pieces of data (e.g., files) to the Flux KVS for later retrieval.\n",
+ "\n",
+ "When using `flux archive`, we first have to create an named archive. In the code below, we will create a text file and then save it into an archive using `flux archive`. Note that, for larger files, you can speed up the creation and extraction of archives by using the `--mmap` flag."
]
},
{
"cell_type": "code",
- "execution_count": 23,
- "id": "2d2b1f0b-e6c2-4583-8068-7c76fa341884",
+ "execution_count": 35,
+ "id": "3928d581-9815-4f7b-98cb-72d6a804813d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!echo \"Sweet dreams 🌚️ are made of cheese, who am I to diss a brie? 🧀️\" > shared-file.txt\n",
+ "!flux archive create --name myarchive --directory $(pwd) shared-file.txt"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1341da82-b8f0-445c-b335-6a10271994d9",
+ "metadata": {},
+ "source": [
+ "When we run this code, we are creating an archive in the leader broker. Now that the archive is created, we will want to extract its contents onto the other nodes of our cluster. To do this, we first need to ensure that the directory that we will extract into exists on those nodes. This can be done using `flux exec`. The `flux exec` command will execute a command on the nodes associated with specified brokers. Let's use `flux exec` to run `mkdir` on all the nodes of our cluster except the leader broker's node."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 36,
+ "id": "2bf40c7b-3ca3-4e4f-b21c-4e843c7562a6",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux exec -r all -x 0 mkdir -p $(pwd)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9913e925-aefc-400e-9ff3-0f541f9c3ed2",
+ "metadata": {},
+ "source": [
+ "The flags provided to `flux exec` do the following:\n",
+ "* `-r all`: run across all brokers in the Flux instance\n",
+ "* `-x 0`: don't runn on broker 0 (i.e., the leader broker)\n",
+ "\n",
+ "Now that the directory has been created on all our nodes, we can extract the archive onto those nodes by combining `flux exec` and `flux archive extract`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "id": "72567af7-aa40-46b7-be43-c9e8124c1c7e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- ".\n",
- "├── ./sub_job1.sh\n",
- "│ └── ./sub_job2.sh\n",
- "│ └── sleep:R\n",
- "├── ./hello-batch.sh:CD\n",
- "├── 2*[./sleep_batch.sh:CD]\n",
- "├── 4*[echo:CD]\n",
- "├── sleep:CA\n",
- "├── simulation:CA\n",
- "├── analysis:CA\n",
- "├── job-watch.sh:CD\n",
- "├── 13*[hostname:CD]\n",
- "└── false:F\n"
+ "flux-archive: shared-file.txt: write: Attempt to overwrite existing file\n",
+ "flux-archive: shared-file.txt: write: Attempt to overwrite existing file\n",
+ "flux-archive: shared-file.txt: write: Attempt to overwrite existing file\n",
+ "[1-3]: Exit 1\n"
]
}
],
"source": [
- "!flux pstree -a"
+ "!flux exec -r all -x 0 flux archive extract --name myarchive --directory $(pwd) shared-file.txt"
]
},
{
"cell_type": "markdown",
- "id": "7724130f-b0db-4ccf-a01e-98907b9a27ca",
+ "id": "8b35f8a6-869b-4f4f-874a-074919dfcc51",
"metadata": {},
"source": [
- "You can also try a more detailed view with `flux pstree -a -X`!"
+ "Finally, when we're done with the archive, we can remove it with `flux archive remove`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 38,
+ "id": "38472bab-c7b9-409b-9058-734527898eb7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!flux archive remove --name myarchive"
]
},
{
"cell_type": "markdown",
- "id": "eda1a33c-9f9e-4ba0-a013-e97601f79e41",
+ "id": "76fa64ca-ebde-4c5f-a505-7ca2b0173f98",
+ "metadata": {},
+ "source": [
+ "Finally, note that `flux archive` was named `flux filemap` in earlier versions of Flux."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "32e110ce-db7a-4066-81f7-7191c1968496",
"metadata": {},
"source": [
- "# Process and Job Utilities ⚙️\n",
+ "## flux uptime\n",
"\n",
- "## Flux uptime\n",
+ "\n",
+ "Description: Show how long this flux instance has been running\n",
+ "
\n",
"\n",
"Did someone say... [uptime](https://youtu.be/SYRlTISvjww?si=zDlvpWbBljUmZw_Q)? ☝️🕑️🕺️\n",
"\n",
@@ -2385,7 +2705,7 @@
},
{
"cell_type": "code",
- "execution_count": 31,
+ "execution_count": 39,
"id": "095f2ac3-145b-4cda-8350-7c281f2b2b45",
"metadata": {},
"outputs": [
@@ -2393,7 +2713,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
- " 22:16:17 run 1.8h, owner jovyan, depth 0, size 4\n"
+ " 00:03:20 run 5.3h, owner jovyan, depth 0, size 4\n"
]
}
],
@@ -2404,60 +2724,287 @@
{
"cell_type": "markdown",
"id": "03e2ae62-3e3b-4c82-a0c7-4c97ff1376d2",
- "metadata": {
- "jp-MarkdownHeadingCollapsed": true
- },
+ "metadata": {},
"source": [
- "## Flux top \n",
+ "## flux top \n",
+ "\n",
+ "\n",
+ "Description: Show a table of real-time Flux processes\n",
+ "
\n",
+ "\n",
"Flux provides a feature-full version of `top` for nested Flux instances and jobs. In the JupyterLab terminal, invoke `flux top` to see the \"sleep\" jobs. If they have already completed you can resubmit them. \n",
"\n",
"We recommend not running `flux top` in the notebook as it is not designed to display output from a command that runs continuously.\n",
"\n",
- "## Flux pstree \n",
- "In analogy to `top`, Flux provides `flux pstree`. Try it out in the or here in the notebook.\n",
+ "## flux pstree \n",
+ "\n",
+ "\n",
+ "Description: Show a flux process tree (and see nesting in instances)\n",
+ "
\n",
"\n",
- "## Flux proxy\n",
+ "In analogy to `top`, Flux provides `flux pstree`. Try it out in the or here in the notebook.\n",
"\n",
- "### flux proxy\n",
+ "## flux proxy\n",
"\n",
- "> To interact with a job hierarchy\n",
+ "\n",
+ "Description: Interact with a job hierarchy\n",
+ "
\n",
"\n",
"Flux proxy is used to route messages to and from a Flux instance. We can use `flux proxy` to connect to a running Flux instance and then submit more nested jobs inside it. You may want to edit `sleep_batch.sh` with the JupyterLab text editor (double click the file in the window on the left) to sleep for `60` or `120` seconds. Then from the run the commands below!"
]
},
{
"cell_type": "markdown",
- "id": "a609b2f8-e24d-40c7-b022-ce02e91a49f8",
+ "id": "523e04b6-7427-4c77-8eb0-b5b8998c6224",
+ "metadata": {},
+ "source": [
+ "## flux queue\n",
+ "\n",
+ "\n",
+ "Description: Interact with and inspect Flux queues\n",
+ "
\n",
+ "\n",
+ "Flux has a command for controlling the queue within the `job-manager`: `flux queue`. This includes disabling job submission, re-enabling it, waiting for the queue to become idle or empty, and checking the queue status:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 40,
+ "id": "800de4eb",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Job submission is disabled: maintenance outage\n",
+ "Job submission is enabled\n",
+ "usage: flux-queue [-h] {status,list,enable,disable,start,stop,drain,idle} ...\n",
+ "\n",
+ "options:\n",
+ " -h, --help show this help message and exit\n",
+ "\n",
+ "subcommands:\n",
+ "\n",
+ " {status,list,enable,disable,start,stop,drain,idle}\n"
+ ]
+ }
+ ],
+ "source": [
+ "!flux queue disable \"maintenance outage\"\n",
+ "!flux queue enable\n",
+ "!flux queue -h"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e958b3ce-9220-48ad-8f3e-f76c8d6a800c",
+ "metadata": {},
+ "source": [
+ "## flux getattr\n",
+ "\n",
+ "\n",
+ "Description: Get attributes about your system and environment\n",
+ "
\n",
+ "\n",
+ "Each Flux instance has a set of attributes that are set at startup that affect the operation of Flux, such as `rank`, `size`, and `local-uri` (the Unix socket usable for communicating with Flux). Many of these attributes can be modified at runtime, such as `log-stderr-level` (1 logs only critical messages to stderr while 7 logs everything, including debug messages). Here is an example set that you might be interested in looking at:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 41,
+ "id": "biblical-generic",
+ "metadata": {
+ "collapsed": true,
+ "jupyter": {
+ "outputs_hidden": true
+ },
+ "scrolled": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0\n",
+ "4\n",
+ "local:///tmp/flux-iwjuLe/local-0\n",
+ "broker.boot-method simple\n",
+ "broker.critical-ranks 0\n",
+ "broker.mapping [[0,1,4,1]]\n",
+ "broker.pid 8\n",
+ "broker.quorum 4\n",
+ "broker.quorum-timeout 1m\n",
+ "broker.rc1_path /etc/flux/rc1\n",
+ "broker.rc3_path /etc/flux/rc3\n",
+ "broker.starttime 1721501121.61\n",
+ "conf.shell_initrc /etc/flux/shell/initrc.lua\n",
+ "conf.shell_pluginpath /usr/lib/flux/shell/plugins\n",
+ "config.path -\n",
+ "content.backing-module content-sqlite\n",
+ "content.hash sha1\n",
+ "hostlist 8660c254a8e[5,5,5,5]\n",
+ "instance-level 0\n",
+ "jobid -\n",
+ "local-uri local:///tmp/flux-iwjuLe/local-0\n",
+ "log-critical-level 2\n",
+ "log-filename -\n",
+ "log-forward-level 7\n",
+ "log-level 7\n",
+ "log-ring-size 1024\n",
+ "log-stderr-level 3\n",
+ "log-stderr-mode leader\n",
+ "parent-kvs-namespace -\n",
+ "parent-uri -\n",
+ "rank 0\n",
+ "rundir /tmp/flux-iwjuLe\n",
+ "security.owner 1000\n",
+ "size 4\n",
+ "statedir -\n",
+ "tbon.child_rcvhwm 0\n",
+ "tbon.connect_timeout 30s\n",
+ "tbon.descendants 3\n",
+ "tbon.endpoint ipc:///tmp/flux-iwjuLe/tbon-0\n",
+ "tbon.level 0\n",
+ "tbon.maxlevel 1\n",
+ "tbon.parent-endpoint -\n",
+ "tbon.prefertcp 0\n",
+ "tbon.tcp_user_timeout 20s\n",
+ "tbon.topo kary:32\n",
+ "tbon.torpid_max 30s\n",
+ "tbon.torpid_min 5s\n",
+ "tbon.zmq_io_threads 1\n",
+ "tbon.zmqdebug 0\n",
+ "version 0.63.0-5-g0ddc3d9e8\n"
+ ]
+ }
+ ],
+ "source": [
+ "!flux getattr rank\n",
+ "!flux getattr size\n",
+ "!flux getattr local-uri\n",
+ "!flux setattr log-stderr-level 3\n",
+ "!flux lsattr -v"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e78568c0-7b78-4a1c-aa7f-40d7ea43620f",
+ "metadata": {},
+ "source": [
+ "## flux module\n",
+ "\n",
+ "\n",
+ "Description: Manage Flux extension modules\n",
+ "
\n",
+ "\n",
+ "Services within a Flux instance are implemented by modules. To query and manage broker modules, use `flux module`. Modules that we have already directly interacted with in this tutorial include `resource` (via `flux resource`), `job-ingest` (via `flux` and the Python API) `job-list` (via `flux jobs`) and `job-manager` (via `flux queue`), and we will interact with the `kvs` module in a few cells. For the most part, services are implemented by modules of the same name (e.g., `kvs` implements the `kvs` service and thus the `kvs.lookup` RPC). In some circumstances, where multiple implementations for a service exist, a module of a different name implements a given service (e.g., in this instance, `sched-fluxion-qmanager` provides the `sched` service and thus `sched.alloc`, but in another instance `sched-simple` might provide the `sched` service)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 42,
+ "id": "spatial-maintenance",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Module Idle S Service\n",
+ "job-exec 2 R \n",
+ "heartbeat 0 R \n",
+ "job-list 2 R \n",
+ "sched-fluxion-qmanager 2 R sched\n",
+ "content-sqlite 1 R content-backing\n",
+ "resource 1 R \n",
+ "job-ingest 2 R \n",
+ "content 1 R \n",
+ "job-info 5 R \n",
+ "kvs-watch 5 R \n",
+ "sched-fluxion-resource 2 R \n",
+ "kvs 1 R \n",
+ "cron idle R \n",
+ "job-manager 0 R \n",
+ "barrier idle R \n",
+ "connector-local 0 R 1000-shell-f3Vw1mYfjD,1000-shell-f3Vw6xW9wD\n"
+ ]
+ }
+ ],
+ "source": [
+ "!flux module list"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a4c7d50b-50d2-4190-a42e-9b13f1f30380",
+ "metadata": {},
+ "source": [
+ "See the [Flux Management Notebook](02_flux_framework.ipynb) for a small tutorial of unloading and reloading the Fluxion (flux scheduler) modules."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5fea958f-e12c-4229-b8a6-e40dcfbd0692",
+ "metadata": {},
+ "source": [
+ "## flux dmesg\n",
+ "\n",
+ "\n",
+ "Description: View Flux system messages\n",
+ "
\n",
+ "\n",
+ "\n",
+ "If you need some additional help debugging your Flux setup, you might be interested in `flux dmesg`, which is akin to the [Linux dmesg](https://man7.org/linux/man-pages/man1/dmesg.1.html) but delivers messages for Flux."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 43,
+ "id": "c34899ba",
"metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u001b[32m2024-07-20T22:56:18.760174Z\u001b[0m \u001b[33mbroker.debug[0]\u001b[0m: \u001b[34mrmmod sched-simple\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.760532Z\u001b[0m \u001b[33mbroker.debug[0]\u001b[0m: \u001b[34mmodule sched-simple exited\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.760597Z\u001b[0m \u001b[33mresource.debug[0]\u001b[0m: \u001b[34maborted 1 resource.acquire(s)\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.760615Z\u001b[0m \u001b[33mjob-manager.debug[0]\u001b[0m: \u001b[34malloc: stop due to disconnect: Success\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.879655Z\u001b[0m \u001b[33mbroker.debug[0]\u001b[0m: \u001b[34minsmod sched-fluxion-resource\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.879925Z\u001b[0m \u001b[33msched-fluxion-resource.info[0]\u001b[0m: version 0.34.0-38-g0fad5268\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.879954Z\u001b[0m \u001b[33msched-fluxion-resource.debug[0]\u001b[0m: \u001b[34mmod_main: resource module starting\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.880319Z\u001b[0m \u001b[33msched-fluxion-resource.warning[0]\u001b[0m: \u001b[1mcreate_reader: allowlist unsupported\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.880472Z\u001b[0m \u001b[33msched-fluxion-resource.debug[0]\u001b[0m: \u001b[34mresource graph datastore loaded with rv1exec reader\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.880477Z\u001b[0m \u001b[33msched-fluxion-resource.info[0]\u001b[0m: populate_resource_db: loaded resources from core's resource.acquire\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.880519Z\u001b[0m \u001b[33msched-fluxion-resource.debug[0]\u001b[0m: \u001b[34mresource status changed (rankset=[all] status=DOWN)\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.880522Z\u001b[0m \u001b[33msched-fluxion-resource.debug[0]\u001b[0m: \u001b[34mresource status changed (rankset=[0-3] status=UP)\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.880523Z\u001b[0m \u001b[33msched-fluxion-resource.debug[0]\u001b[0m: \u001b[34mmod_main: resource graph database loaded\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.998112Z\u001b[0m \u001b[33mbroker.debug[0]\u001b[0m: \u001b[34minsmod sched-fluxion-qmanager\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.998336Z\u001b[0m \u001b[33msched-fluxion-qmanager.info[0]\u001b[0m: version 0.34.0-38-g0fad5268\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.998452Z\u001b[0m \u001b[33msched-fluxion-qmanager.debug[0]\u001b[0m: \u001b[34mservice_register\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.998472Z\u001b[0m \u001b[33msched-fluxion-qmanager.debug[0]\u001b[0m: \u001b[34menforced policy (queue=default): fcfs\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.998477Z\u001b[0m \u001b[33msched-fluxion-qmanager.debug[0]\u001b[0m: \u001b[34meffective queue params (queue=default): queue-depth=4\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.998478Z\u001b[0m \u001b[33msched-fluxion-qmanager.debug[0]\u001b[0m: \u001b[34meffective policy params (queue=default): default\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.998726Z\u001b[0m \u001b[33msched-fluxion-qmanager.debug[0]\u001b[0m: \u001b[34mhandshaking with sched-fluxion-resource completed\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:18.998919Z\u001b[0m \u001b[33mjob-manager.debug[0]\u001b[0m: \u001b[34mscheduler: hello\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:19.000882Z\u001b[0m \u001b[33msched-fluxion-qmanager.debug[0]\u001b[0m: \u001b[34mrequeue success (queue=default id=1750450831360)\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:19.001022Z\u001b[0m \u001b[33mjob-manager.debug[0]\u001b[0m: \u001b[34mscheduler: ready unlimited\u001b[0m\n",
+ "\u001b[32m2024-07-20T22:56:19.001141Z\u001b[0m \u001b[33msched-fluxion-qmanager.debug[0]\u001b[0m: \u001b[34mhandshaking with job-manager completed\u001b[0m\n",
+ "\u001b[32m2024-07-21T00:03:03.486096Z\u001b[0m \u001b[33mkvs.debug[0]\u001b[0m: \u001b[34maggregated 2 transactions (2 ops)\u001b[0m\n",
+ "\u001b[32m2024-07-21T00:03:03.487165Z\u001b[0m \u001b[33mkvs.debug[0]\u001b[0m: \u001b[34maggregated 2 transactions (2 ops)\u001b[0m\n",
+ "\u001b[32m2024-07-21T00:03:03.749285Z\u001b[0m \u001b[33mkvs.debug[0]\u001b[0m: \u001b[34maggregated 3 transactions (3 ops)\u001b[0m\n",
+ "\u001b[32m2024-07-21T00:03:03.752634Z\u001b[0m \u001b[33mkvs.debug[0]\u001b[0m: \u001b[34maggregated 2 transactions (2 ops)\u001b[0m\n",
+ "\u001b[32m2024-07-21T00:03:15.324218Z\u001b[0m \u001b[33mjob-exec.debug[0]\u001b[0m: \u001b[34mexec aborted: id=ƒ3VqVSHr7q\u001b[0m\n",
+ "\u001b[32m2024-07-21T00:03:15.324274Z\u001b[0m \u001b[33mjob-exec.debug[0]\u001b[0m: \u001b[34mexec aborted: id=ƒ3VqtrJWFh\u001b[0m\n",
+ "\u001b[32m2024-07-21T00:03:15.324296Z\u001b[0m \u001b[33mjob-exec.debug[0]\u001b[0m: \u001b[34mexec aborted: id=ƒ3VqzNXq8B\u001b[0m\n",
+ "\u001b[32m2024-07-21T00:03:15.324309Z\u001b[0m \u001b[33mjob-exec.debug[0]\u001b[0m: \u001b[34mexec aborted: id=ƒnyvM4Nb\u001b[0m\n"
+ ]
+ }
+ ],
"source": [
- "```bash\n",
- "# The terminal will start at the root, ensure you are in the right spot!\n",
- "# jovyan - that's you! \n",
- "cd /home/jovyan/flux-radiuss-tutorial-2023/notebook/\n",
- "\n",
- "# Outputs the JOBID\n",
- "flux batch --nslots=2 --cores-per-slot=1 --nodes=2 ./sleep_batch.sh\n",
- "\n",
- "# Put the JOBID into an environment variable\n",
- "JOBID=$(flux job last)\n",
- "\n",
- "# See the flux process tree\n",
- "flux pstree -a\n",
- "\n",
- "# Connect to the Flux instance corresponding to JOBID above\n",
- "flux proxy ${JOBID}\n",
- "\n",
- "# Note the depth is now 1 and the size is 2: we're one level deeper in a Flux hierarchy and we have only 2 brokers now.\n",
- "flux uptime\n",
- "\n",
- "# This instance has 2 \"nodes\" and 2 cores allocated to it\n",
- "flux resource list\n",
- "\n",
- "# Have you used the top command in your terminal? We have one for flux!\n",
- "flux top\n",
- "```\n",
- "\n",
- "`flux top` was pretty cool, right? 😎️"
+ "!flux dmesg"
]
},
{
@@ -2466,12 +3013,16 @@
"metadata": {},
"source": [
"# Python Submission API 🐍️\n",
- "Flux also provides first-class python bindings which can be used to submit jobs programmatically. The following script shows this with the `flux.job.submit()` call:"
+ "Flux also provides first-class python bindings which can be used to submit jobs programmatically. \n",
+ "\n",
+ "### `flux.job.JobspecV1` to create job specifications\n",
+ "\n",
+ "Flux represents work as a standard called the [Jobspec](https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/spec_25.html). While you could write YAML or JSON, it's much easier to use provided Python functions that take high level metadata (command, resources, etc) to generate them. We can then replicate our previous example of submitting multiple heterogeneous jobs using these Python helpers, and testing that Flux co-schedules them."
]
},
{
"cell_type": "code",
- "execution_count": 24,
+ "execution_count": 44,
"id": "third-comment",
"metadata": {},
"outputs": [],
@@ -2485,7 +3036,7 @@
},
{
"cell_type": "code",
- "execution_count": 25,
+ "execution_count": 45,
"id": "selective-uganda",
"metadata": {},
"outputs": [
@@ -2493,16 +3044,23 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "ƒZoXw7Pdq\n"
+ "ƒ3Vyv4d99Z\n"
]
}
],
"source": [
- "f = flux.Flux() # connect to the running Flux instance\n",
+ "# connect to the running Flux instance\n",
+ "f = flux.Flux()\n",
+ "\n",
+ "# Create the Jobspec from a command to run a python script, and specify resources\n",
"compute_jobreq = JobspecV1.from_command(\n",
" command=[\"./compute.py\", \"120\"], num_tasks=1, num_nodes=1, cores_per_task=1\n",
- ") # construct a jobspec\n",
- "compute_jobreq.cwd = os.path.expanduser(\"~/flux-tutorial/flux-workflow-examples/job-submit-api/\") # set the CWD\n",
+ ")\n",
+ "\n",
+ "# This is the \"current working directory\" (cwd)\n",
+ "compute_jobreq.cwd = os.path.expanduser(\"~/flux-tutorial/flux-workflow-examples/job-submit-api/\")\n",
+ "\n",
+ "# When we submit, we get back the job identifier (JobID)\n",
"print(JobID(flux.job.submit(f,compute_jobreq)).f58) # submit and print out the jobid (in f58 format)"
]
},
@@ -2511,12 +3069,14 @@
"id": "0c4b260f-f08a-46ae-ad66-805911a857a7",
"metadata": {},
"source": [
+ "Once we create the job, when we submit it in Python we get back a job identifier or jobid. We can then interact with the Flux handle, a connection to Flux, to get information about that job.\n",
+ "\n",
"### `flux.job.get_job(handle, jobid)` to get job info"
]
},
{
"cell_type": "code",
- "execution_count": 26,
+ "execution_count": 46,
"id": "ed65cb46-8d8a-41f0-bec1-92b9a89e6db2",
"metadata": {},
"outputs": [
@@ -2524,64 +3084,36 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "🎉️ Hooray, we just submitted ƒZrqPNeNb!\n",
- "{\n",
- " \"t_depend\": 1721425098.682836,\n",
- " \"t_run\": 0.0,\n",
- " \"t_cleanup\": 0.0,\n",
- " \"t_inactive\": 0.0,\n",
- " \"duration\": 0.0,\n",
- " \"expiration\": 0.0,\n",
- " \"name\": \"compute.py\",\n",
- " \"cwd\": \"/home/jovyan/flux-tutorial/flux-workflow-examples/job-submit-api/\",\n",
- " \"queue\": \"\",\n",
- " \"project\": \"\",\n",
- " \"bank\": \"\",\n",
- " \"ntasks\": 1,\n",
- " \"ncores\": 1,\n",
- " \"nnodes\": 1,\n",
- " \"priority\": 16,\n",
- " \"ranks\": \"\",\n",
- " \"nodelist\": \"\",\n",
- " \"success\": \"\",\n",
- " \"result\": \"\",\n",
- " \"waitstatus\": \"\",\n",
- " \"id\": 72552617607168,\n",
- " \"t_submit\": 1721425098.6718118,\n",
- " \"t_remaining\": 0.0,\n",
- " \"state\": \"SCHED\",\n",
- " \"username\": \"jovyan\",\n",
- " \"userid\": 1000,\n",
- " \"urgency\": 16,\n",
- " \"runtime\": 0.0,\n",
- " \"status\": \"SCHED\",\n",
- " \"returncode\": \"\",\n",
- " \"dependencies\": [],\n",
- " \"annotations\": {},\n",
- " \"exception\": {\n",
- " \"occurred\": \"\",\n",
- " \"severity\": \"\",\n",
- " \"type\": \"\",\n",
- " \"note\": \"\"\n",
- " }\n",
- "}\n"
+ "🎉️ Hooray, we just submitted ƒ3VyvraktF!\n",
+ "\n",
+ "{\"t_depend\": 1721520202.4164655, \"t_run\": 0.0, \"t_cleanup\": 0.0, \"t_inactive\": 0.0, \"duration\": 0.0, \"expiration\": 0.0, \"name\": \"compute.py\", \"cwd\": \"/home/jovyan/flux-tutorial/flux-workflow-examples/job-submit-api/\", \"queue\": \"\", \"project\": \"\", \"bank\": \"\", \"ntasks\": 1, \"ncores\": 1, \"nnodes\": 1, \"priority\": 16, \"ranks\": \"\", \"nodelist\": \"\", \"success\": \"\", \"result\": \"\", \"waitstatus\": \"\", \"id\": 320116914913280, \"t_submit\": 1721520202.4053006, \"t_remaining\": 0.0, \"state\": \"SCHED\", \"username\": \"jovyan\", \"userid\": 1000, \"urgency\": 16, \"runtime\": 0.0, \"status\": \"SCHED\", \"returncode\": \"\", \"dependencies\": [], \"annotations\": {}, \"exception\": {\"occurred\": \"\", \"severity\": \"\", \"type\": \"\", \"note\": \"\"}}\n"
]
}
],
"source": [
- "# This is a new command to get info about your job from the id!\n",
- "fluxjob = flux.job.submit(f,compute_jobreq)\n",
+ "# Let's submit again to retrieve (and save) the job identifier\n",
+ "fluxjob = flux.job.submit(f, compute_jobreq)\n",
"fluxjobid = JobID(fluxjob.f58)\n",
- "print(f\"🎉️ Hooray, we just submitted {fluxjobid}!\")\n",
+ "print(f\"🎉️ Hooray, we just submitted {fluxjobid}!\\n\")\n",
"\n",
"# Here is how to get your info. The first argument is the flux handle, then the jobid\n",
"jobinfo = flux.job.get_job(f, fluxjobid)\n",
- "print(json.dumps(jobinfo, indent=4))"
+ "print(json.dumps(jobinfo))"
]
},
{
"cell_type": "code",
- "execution_count": 27,
+ "execution_count": 47,
+ "id": "810b72b9-2de2-4b62-9330-252eede22abb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "### `flux jobs`"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 48,
"id": "5d679897-7054-4f96-b340-7f39245aca89",
"metadata": {},
"outputs": [
@@ -2589,8 +3121,12 @@
"name": "stdout",
"output_type": "stream",
"text": [
- " ƒZrqPNeNb jovyan compute.py F 1 1 0.009s 399f5da372b0\n",
- " ƒZoXw7Pdq jovyan compute.py F 1 1 0.011s 399f5da372b0\n"
+ " ƒ3VyvraktF jovyan compute.py F 1 1 0.014s 8660c254a8e5\n",
+ " ƒ3Vyv4d99Z jovyan compute.py F 1 1 0.020s 8660c254a8e5\n",
+ " ƒ2YnijmLwy jovyan compute.py F 1 1 0.031s 8660c254a8e5\n",
+ " ƒ2YiqfxNdm jovyan compute.py F 1 1 0.012s 8660c254a8e5\n",
+ " ƒ2YYgVHnyV jovyan compute.py F 1 1 0.062s 8660c254a8e5\n",
+ " ƒ2YYE7Ja9d jovyan compute.py F 1 1 0.048s 8660c254a8e5\n"
]
}
],
@@ -2608,9 +3144,15 @@
},
{
"cell_type": "code",
- "execution_count": 28,
+ "execution_count": 49,
"id": "efa06478",
- "metadata": {},
+ "metadata": {
+ "collapsed": true,
+ "jupyter": {
+ "outputs_hidden": true
+ },
+ "scrolled": true
+ },
"outputs": [
{
"name": "stdout",
@@ -2663,52 +3205,9 @@
"print(compute_jobreq.dumps(indent=2))"
]
},
- {
- "cell_type": "markdown",
- "id": "73bbc90e",
- "metadata": {},
- "source": [
- "### `flux.job.JobspecV1` to create job specifications\n",
- "\n",
- "Flux represents work as a standard called the [Jobspec](https://flux-framework.readthedocs.io/projects/flux-rfc/en/latest/spec_25.html). While you could write YAML or JSON, it's much easier to use provided Python functions that take high level metadata (command, resources, etc) to generate them. We can then replicate our previous example of submitting multiple heterogeneous jobs using these Python helpers, and testing that Flux co-schedules them."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 40,
- "id": "industrial-privacy",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "ƒKf7Aiz4P\n",
- "ƒKf7rkefh\n"
- ]
- }
- ],
- "source": [
- "# Here we create our job specification from a command\n",
- "compute_jobreq = JobspecV1.from_command(\n",
- " command=[\"./compute.py\", \"120\"], num_tasks=4, num_nodes=2, cores_per_task=2\n",
- ")\n",
- "\n",
- "# This is the \"current working directory\" (cwd)\n",
- "compute_jobreq.cwd = os.path.expanduser(\"~/flux-workflow-examples/job-submit-api/\")\n",
- "print(JobID(flux.job.submit(f, compute_jobreq)))\n",
- "\n",
- "# Here is a second I/O job\n",
- "io_jobreq = JobspecV1.from_command(\n",
- " command=[\"./io-forwarding.py\", \"120\"], num_tasks=1, num_nodes=1, cores_per_task=1\n",
- ")\n",
- "io_jobreq.cwd = os.path.expanduser(\"~/flux-workflow-examples/job-submit-api/\")\n",
- "print(JobID(flux.job.submit(f, io_jobreq)))"
- ]
- },
{
"cell_type": "code",
- "execution_count": 41,
+ "execution_count": 50,
"id": "pregnant-creativity",
"metadata": {},
"outputs": [
@@ -2716,10 +3215,12 @@
"name": "stdout",
"output_type": "stream",
"text": [
- " ƒKf7Aiz4P jovyan compute.py R 4 2 2.727s 7db0bdd6f[967,967]\n",
- " ƒKNequHnF jovyan compute.py F 4 2 0.012s 7db0bdd6f[967,967]\n",
- " ƒKLZWG53M jovyan compute.py F 1 1 0.037s 7db0bdd6f967\n",
- " ƒKKdeYAGo jovyan compute.py F 1 1 0.012s 7db0bdd6f967\n"
+ " ƒ3VyvraktF jovyan compute.py F 1 1 0.014s 8660c254a8e5\n",
+ " ƒ3Vyv4d99Z jovyan compute.py F 1 1 0.020s 8660c254a8e5\n",
+ " ƒ2YnijmLwy jovyan compute.py F 1 1 0.031s 8660c254a8e5\n",
+ " ƒ2YiqfxNdm jovyan compute.py F 1 1 0.012s 8660c254a8e5\n",
+ " ƒ2YYgVHnyV jovyan compute.py F 1 1 0.062s 8660c254a8e5\n",
+ " ƒ2YYE7Ja9d jovyan compute.py F 1 1 0.048s 8660c254a8e5\n"
]
}
],
@@ -2734,7 +3235,7 @@
"source": [
"### `FluxExecutor` for bulk submission\n",
"\n",
- "We can use the FluxExecutor class to submit large numbers of jobs to Flux. This method uses python's `concurrent.futures` interface. Here is an example snippet from `~/flux-workflow-examples/async-bulk-job-submit/bulksubmit_executor.py`:"
+ "We can use the FluxExecutor class to submit large numbers of jobs to Flux. This method uses python's `concurrent.futures` interface. Here is an example snippet from [flux-workflow-examples/async-bulk-job-submit/bulksubmit_executor.py](flux-workflow-examples/async-bulk-job-submit/bulksubmit_executor.py)."
]
},
{
@@ -2744,18 +3245,18 @@
"source": [
"``` python \n",
"with FluxExecutor() as executor:\n",
- " compute_jobspec = JobspecV1.from_command(args.command)\n",
- " futures = [executor.submit(compute_jobspec) for _ in range(args.njobs)]\n",
- " # wait for the jobid for each job, as a proxy for the job being submitted\n",
- " for fut in futures:\n",
- " fut.jobid()\n",
- " # all jobs submitted - print timings\n",
+ " compute_jobspec = JobspecV1.from_command(args.command)\n",
+ " futures = [executor.submit(compute_jobspec) for _ in range(args.njobs)]\n",
+ " # wait for the jobid for each job, as a proxy for the job being submitted\n",
+ " for fut in futures:\n",
+ " fut.jobid()\n",
+ " # all jobs submitted - print timings\n",
"```"
]
},
{
"cell_type": "code",
- "execution_count": 29,
+ "execution_count": 51,
"id": "cleared-lawsuit",
"metadata": {},
"outputs": [
@@ -2763,259 +3264,79 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "bulksubmit_executor: submitted 200 jobs in 0.24s. 831.05job/s\n",
- "bulksubmit_executor: First job finished in about 0.254s\n",
- "|██████████████████████████████████████████████████████████| 100.0% (278.2 job/s)\n",
- "bulksubmit_executor: Ran 200 jobs in 0.9s. 221.8 job/s\n"
+ "bulksubmit_executor: submitted 200 jobs in 0.16s. 1246.61job/s\n",
+ "bulksubmit_executor: First job finished in about 0.172s\n",
+ "|██████████████████████████████████████████████████████████| 100.0% (298.2 job/s)\n",
+ "bulksubmit_executor: Ran 200 jobs in 0.8s. 249.6 job/s\n"
]
}
],
"source": [
- "# Submit a FluxExecutor based script.\n",
+ "# Submit the FluxExecutor based script.\n",
"%run ./flux-workflow-examples/async-bulk-job-submit/bulksubmit_executor.py -n200 /bin/sleep 0"
]
},
{
"cell_type": "markdown",
- "id": "5ee1c49d",
+ "id": "e5e39506-7f89-4be2-880e-fc21cfe33548",
"metadata": {},
"source": [
- "# Deeper Dive into Flux Internals 🧐️\n",
+ "### `flux.event_watch` to watch events\n",
"\n",
- "## flux queue\n",
- "\n",
- "Flux has a command for controlling the queue within the `job-manager`: `flux queue`. This includes disabling job submission, re-enabling it, waiting for the queue to become idle or empty, and checking the queue status:"
+ "If you want to get the output of a job (or more generally, stream events) you can do that as follows. Let's submit a quick job, and then look at the output.\n"
]
},
{
"cell_type": "code",
- "execution_count": 46,
- "id": "800de4eb",
+ "execution_count": 53,
+ "id": "b24124f8-0faf-4e99-83a5-bd983300fda6",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Job submission is disabled: maintenance outage\n",
- "Job submission is enabled\n",
- "usage: flux-queue [-h] {status,list,enable,disable,start,stop,drain,idle} ...\n",
- "\n",
- "optional arguments:\n",
- " -h, --help show this help message and exit\n",
- "\n",
- "subcommands:\n",
- "\n",
- " {status,list,enable,disable,start,stop,drain,idle}\n"
+ "1721520256.00717: header {'version': 1, 'encoding': {'stdout': 'UTF-8', 'stderr': 'UTF-8'}, 'count': {'stdout': 1, 'stderr': 1}, 'options': {}}\n",
+ "1721520256.01083: data {'stream': 'stderr', 'rank': '0', 'eof': True}\n",
+ "1721520256.01085: data {'stream': 'stdout', 'rank': '0', 'data': 'Flux Plumbing 💩️🚽️\\n'}\n",
+ "1721520256.01087: data {'stream': 'stdout', 'rank': '0', 'eof': True}\n"
]
}
],
"source": [
- "!flux queue disable \"maintenance outage\"\n",
- "!flux queue enable\n",
- "!flux queue -h"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "67aa7559",
- "metadata": {},
- "source": [
- "## flux getattr\n",
- "\n",
- "> Get attributes about your system and environment\n",
+ "# Create the Jobspec from a command to run a python script, and specify resources\n",
+ "jobspec = JobspecV1.from_command(\n",
+ " command=[\"echo\", \"Flux Plumbing 💩️🚽️\"], num_tasks=1, num_nodes=1, cores_per_task=1)\n",
+ "jobid = flux.job.submit(f, jobspec)\n",
"\n",
- "Each Flux instance has a set of attributes that are set at startup that affect the operation of Flux, such as `rank`, `size`, and `local-uri` (the Unix socket usable for communicating with Flux). Many of these attributes can be modified at runtime, such as `log-stderr-level` (1 logs only critical messages to stderr while 7 logs everything, including debug messages). Here is an example set that you might be interested in looking at:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 47,
- "id": "biblical-generic",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "0\n",
- "4\n",
- "local:///tmp/flux-rWMT6G/local-0\n",
- "broker.boot-method simple\n",
- "broker.critical-ranks 0-1\n",
- "broker.mapping [[0,1,4,1]]\n",
- "broker.pid 8\n",
- "broker.quorum 4\n",
- "broker.quorum-timeout 1m\n",
- "broker.rc1_path /etc/flux/rc1\n",
- "broker.rc3_path /etc/flux/rc3\n",
- "broker.starttime 1712894811.07\n",
- "conf.shell_initrc /etc/flux/shell/initrc.lua\n",
- "conf.shell_pluginpath /usr/lib/flux/shell/plugins\n",
- "config.path -\n",
- "content.backing-module content-sqlite\n",
- "content.hash sha1\n",
- "hostlist 993a4f[746854,746854,746854,746854]\n",
- "instance-level 0\n",
- "jobid -\n",
- "local-uri local:///tmp/flux-rWMT6G/local-0\n",
- "log-critical-level 2\n",
- "log-filename -\n",
- "log-forward-level 7\n",
- "log-level 7\n",
- "log-ring-size 1024\n",
- "log-stderr-level 3\n",
- "log-stderr-mode leader\n",
- "parent-kvs-namespace -\n",
- "parent-uri -\n",
- "rank 0\n",
- "rundir /tmp/flux-rWMT6G\n",
- "security.owner 1000\n",
- "size 4\n",
- "statedir -\n",
- "tbon.connect_timeout 30s\n",
- "tbon.descendants 3\n",
- "tbon.endpoint ipc:///tmp/flux-rWMT6G/tbon-0\n",
- "tbon.level 0\n",
- "tbon.maxlevel 2\n",
- "tbon.parent-endpoint -\n",
- "tbon.prefertcp 0\n",
- "tbon.tcp_user_timeout 20s\n",
- "tbon.topo kary:2\n",
- "tbon.torpid_max 30s\n",
- "tbon.torpid_min 5s\n",
- "tbon.zmqdebug 0\n",
- "version 0.61.1\n"
- ]
- }
- ],
- "source": [
- "!flux getattr rank\n",
- "!flux getattr size\n",
- "!flux getattr local-uri\n",
- "!flux setattr log-stderr-level 3\n",
- "!flux lsattr -v"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "d74fdfcf",
- "metadata": {},
- "source": [
- "## flux module\n",
+ "# Give some time to run and finish\n",
+ "import time\n",
+ "time.sleep(5)\n",
"\n",
- "Services within a Flux instance are implemented by modules. To query and manage broker modules, use `flux module`. Modules that we have already directly interacted with in this tutorial include `resource` (via `flux resource`), `job-ingest` (via `flux` and the Python API) `job-list` (via `flux jobs`) and `job-manager` (via `flux queue`), and we will interact with the `kvs` module in a few cells. For the most part, services are implemented by modules of the same name (e.g., `kvs` implements the `kvs` service and thus the `kvs.lookup` RPC). In some circumstances, where multiple implementations for a service exist, a module of a different name implements a given service (e.g., in this instance, `sched-fluxion-qmanager` provides the `sched` service and thus `sched.alloc`, but in another instance `sched-simple` might provide the `sched` service)."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 48,
- "id": "spatial-maintenance",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Module Idle S Service\n",
- "job-info 4 R \n",
- "sched-fluxion-resource 4 R \n",
- "heartbeat 1 R \n",
- "job-manager 1 R \n",
- "connector-local 0 R \n",
- "content-sqlite 2 R content-backing\n",
- "kvs 2 R \n",
- "resource 2 R \n",
- "kvs-watch 4 R \n",
- "job-exec 4 R \n",
- "barrier idle R \n",
- "job-list 4 R \n",
- "content 2 R \n",
- "sched-fluxion-qmanager 4 R sched\n",
- "cron idle R \n",
- "job-ingest 4 R \n"
- ]
- }
- ],
- "source": [
- "!flux module list"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "ad7090eb",
- "metadata": {},
- "source": [
- "See the [Flux Management Notebook](02_flux_framework.ipynb) for a small tutorial of unloading and reloading the Fluxion (flux scheduler) modules."
+ "for line in flux.job.event_watch(f, jobid, \"guest.output\"):\n",
+ " print(line)"
]
},
{
"cell_type": "markdown",
- "id": "722c4ecf",
+ "id": "432a6b44-4a37-4b75-9035-ade107def5de",
"metadata": {},
"source": [
- "## flux dmesg\n",
+ "### `flux.job.job_list` to list jobs\n",
"\n",
- "If you need some additional help debugging your Flux setup, you might be interested in `flux dmesg`, which is akin to the [Linux dmesg](https://man7.org/linux/man-pages/man1/dmesg.1.html) but delivers messages for Flux."
+ "Finally, it can be really helpful to get an entire listing of jobs. You can do that as follows. Note that the `job_list` is creating a remote procedure call (rpc) and we call `get` to retrieve the output."
]
},
{
"cell_type": "code",
- "execution_count": 7,
- "id": "c34899ba",
- "metadata": {},
+ "execution_count": null,
+ "id": "b0d109d8-8586-4b91-bbfc-89e523199707",
+ "metadata": {
+ "scrolled": true
+ },
"outputs": [],
"source": [
- "!flux dmesg"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "c3920f9e",
- "metadata": {},
- "source": [
- "## flux exec\n",
- "\n",
- "Flux provides a built-in mechanism for executing commands on nodes without requiring a job or resource allocation: `flux exec`. `flux exec` is typically used by sys admins to execute administrative commands and load/unload modules across multiple ranks simultaneously."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 52,
- "id": "e9507c7b-de5c-4129-9a99-c943614a9ba2",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "2\n"
- ]
- }
- ],
- "source": [
- "!flux exec -r 2 flux getattr rank # only execute on rank 2"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 53,
- "id": "6a9de119-abc4-4917-a339-2010ccc7b9b7",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "0\n",
- "1\n",
- "3\n",
- "2\n"
- ]
- }
- ],
- "source": [
- "!flux exec flux getattr rank # execute on all ranks"
+ "flux.job.job_list(f).get()"
]
},
{
diff --git a/2024-RADIUSS-AWS/JupyterNotebook/tutorial/02_flux_framework.ipynb b/2024-RADIUSS-AWS/JupyterNotebook/tutorial/02_flux_framework.ipynb
index 6d6305f..e4030be 100644
--- a/2024-RADIUSS-AWS/JupyterNotebook/tutorial/02_flux_framework.ipynb
+++ b/2024-RADIUSS-AWS/JupyterNotebook/tutorial/02_flux_framework.ipynb
@@ -8,12 +8,15 @@
"![](\"Flux-logo.svg\"){kind=logo}
\n",
"![](\"img/flux-instance-w-tbon.png\")
\n",
@@ -48,25 +51,46 @@
"Image created by Vanessa Sochat for Flux Framework Components documentation\n",
"