From 5dd90af286f91b2cf1b082d3f54107e73ef5f999 Mon Sep 17 00:00:00 2001
From: vsoch <vsoch@users.noreply.github.com>
Date: Wed, 13 Dec 2023 15:04:05 -0700
Subject: [PATCH 1/5] rename: scheduler-plugin directory to src

Problem: the scheduler-plugin directory has source code for fluence, but
is confusing alongside sig-scheduler-plugin
Solution: rename scheduler-plugin to src so it is clearly source code.

Signed-off-by: vsoch <vsoch@users.noreply.github.com>
---
 README.md                                                   | 6 +++++-
 {scheduler-plugin => src}/LICENSE                           | 0
 {scheduler-plugin => src}/Makefile                          | 0
 {scheduler-plugin => src}/build/scheduler/Dockerfile        | 0
 {scheduler-plugin => src}/fluence/cmd/main.go               | 0
 {scheduler-plugin => src}/fluence/cmd/main.go.bk            | 0
 .../fluence/fluxcli-grpc/fluxcli.pb.go                      | 0
 .../fluence/fluxcli-grpc/fluxcli.proto                      | 0
 .../fluence/fluxcli-grpc/fluxcli_grpc.pb.go                 | 0
 {scheduler-plugin => src}/fluence/fluxion/fluxion.go        | 0
 {scheduler-plugin => src}/fluence/go.mod                    | 0
 {scheduler-plugin => src}/fluence/go.sum                    | 0
 {scheduler-plugin => src}/fluence/jgf/jgf.go                | 0
 {scheduler-plugin => src}/fluence/jobspec/jobspec.go        | 0
 {scheduler-plugin => src}/fluence/jobspec/types.go          | 0
 {scheduler-plugin => src}/fluence/utils/types.go            | 0
 {scheduler-plugin => src}/fluence/utils/utils.go            | 0
 17 files changed, 5 insertions(+), 1 deletion(-)
 rename {scheduler-plugin => src}/LICENSE (100%)
 rename {scheduler-plugin => src}/Makefile (100%)
 rename {scheduler-plugin => src}/build/scheduler/Dockerfile (100%)
 rename {scheduler-plugin => src}/fluence/cmd/main.go (100%)
 rename {scheduler-plugin => src}/fluence/cmd/main.go.bk (100%)
 rename {scheduler-plugin => src}/fluence/fluxcli-grpc/fluxcli.pb.go (100%)
 rename {scheduler-plugin => src}/fluence/fluxcli-grpc/fluxcli.proto (100%)
 rename {scheduler-plugin => src}/fluence/fluxcli-grpc/fluxcli_grpc.pb.go (100%)
 rename {scheduler-plugin => src}/fluence/fluxion/fluxion.go (100%)
 rename {scheduler-plugin => src}/fluence/go.mod (100%)
 rename {scheduler-plugin => src}/fluence/go.sum (100%)
 rename {scheduler-plugin => src}/fluence/jgf/jgf.go (100%)
 rename {scheduler-plugin => src}/fluence/jobspec/jobspec.go (100%)
 rename {scheduler-plugin => src}/fluence/jobspec/types.go (100%)
 rename {scheduler-plugin => src}/fluence/utils/types.go (100%)
 rename {scheduler-plugin => src}/fluence/utils/utils.go (100%)

diff --git a/README.md b/README.md
index ce6b6f3..006eb57 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,11 @@ that you can easily use to deploy Fluence right away! You'll simply need to clon
 Here are the quick install steps:
 
 ```bash
-$ git clone https://github.com/openshift-psap/scheduler-plugins.git -b fluence
+# This clones the upstream scheduler plugins code, we will add fluence to it!
+$ git clone --depth 1  https://github.com/kubernetes-sigs/scheduler-plugins.git upstream
+
+# Add fluence assets
+cp -R 
 $ cd scheduler-plugins/manifests/install/charts
 $ helm install \
   --set scheduler.image=ghcr.io/flux-framework/fluence:latest \
diff --git a/scheduler-plugin/LICENSE b/src/LICENSE
similarity index 100%
rename from scheduler-plugin/LICENSE
rename to src/LICENSE
diff --git a/scheduler-plugin/Makefile b/src/Makefile
similarity index 100%
rename from scheduler-plugin/Makefile
rename to src/Makefile
diff --git a/scheduler-plugin/build/scheduler/Dockerfile b/src/build/scheduler/Dockerfile
similarity index 100%
rename from scheduler-plugin/build/scheduler/Dockerfile
rename to src/build/scheduler/Dockerfile
diff --git a/scheduler-plugin/fluence/cmd/main.go b/src/fluence/cmd/main.go
similarity index 100%
rename from scheduler-plugin/fluence/cmd/main.go
rename to src/fluence/cmd/main.go
diff --git a/scheduler-plugin/fluence/cmd/main.go.bk b/src/fluence/cmd/main.go.bk
similarity index 100%
rename from scheduler-plugin/fluence/cmd/main.go.bk
rename to src/fluence/cmd/main.go.bk
diff --git a/scheduler-plugin/fluence/fluxcli-grpc/fluxcli.pb.go b/src/fluence/fluxcli-grpc/fluxcli.pb.go
similarity index 100%
rename from scheduler-plugin/fluence/fluxcli-grpc/fluxcli.pb.go
rename to src/fluence/fluxcli-grpc/fluxcli.pb.go
diff --git a/scheduler-plugin/fluence/fluxcli-grpc/fluxcli.proto b/src/fluence/fluxcli-grpc/fluxcli.proto
similarity index 100%
rename from scheduler-plugin/fluence/fluxcli-grpc/fluxcli.proto
rename to src/fluence/fluxcli-grpc/fluxcli.proto
diff --git a/scheduler-plugin/fluence/fluxcli-grpc/fluxcli_grpc.pb.go b/src/fluence/fluxcli-grpc/fluxcli_grpc.pb.go
similarity index 100%
rename from scheduler-plugin/fluence/fluxcli-grpc/fluxcli_grpc.pb.go
rename to src/fluence/fluxcli-grpc/fluxcli_grpc.pb.go
diff --git a/scheduler-plugin/fluence/fluxion/fluxion.go b/src/fluence/fluxion/fluxion.go
similarity index 100%
rename from scheduler-plugin/fluence/fluxion/fluxion.go
rename to src/fluence/fluxion/fluxion.go
diff --git a/scheduler-plugin/fluence/go.mod b/src/fluence/go.mod
similarity index 100%
rename from scheduler-plugin/fluence/go.mod
rename to src/fluence/go.mod
diff --git a/scheduler-plugin/fluence/go.sum b/src/fluence/go.sum
similarity index 100%
rename from scheduler-plugin/fluence/go.sum
rename to src/fluence/go.sum
diff --git a/scheduler-plugin/fluence/jgf/jgf.go b/src/fluence/jgf/jgf.go
similarity index 100%
rename from scheduler-plugin/fluence/jgf/jgf.go
rename to src/fluence/jgf/jgf.go
diff --git a/scheduler-plugin/fluence/jobspec/jobspec.go b/src/fluence/jobspec/jobspec.go
similarity index 100%
rename from scheduler-plugin/fluence/jobspec/jobspec.go
rename to src/fluence/jobspec/jobspec.go
diff --git a/scheduler-plugin/fluence/jobspec/types.go b/src/fluence/jobspec/types.go
similarity index 100%
rename from scheduler-plugin/fluence/jobspec/types.go
rename to src/fluence/jobspec/types.go
diff --git a/scheduler-plugin/fluence/utils/types.go b/src/fluence/utils/types.go
similarity index 100%
rename from scheduler-plugin/fluence/utils/types.go
rename to src/fluence/utils/types.go
diff --git a/scheduler-plugin/fluence/utils/utils.go b/src/fluence/utils/utils.go
similarity index 100%
rename from scheduler-plugin/fluence/utils/utils.go
rename to src/fluence/utils/utils.go

From 2c29a650f42ecaa3f260f785ff5247e9df239607 Mon Sep 17 00:00:00 2001
From: vsoch <vsoch@users.noreply.github.com>
Date: Thu, 14 Dec 2023 00:32:40 -0700
Subject: [PATCH 2/5] update scheduler Dockerfile to use flux-sched

Problem: the current src/build/scheduler/Dockerfile is using an older
flux-sched branch, and flux-sched has since been refactored to build
with cmake.
Solution: update the Dockerfile to use the latest flux-sched,
and find libraries based on cmake install. Note that several of
the needed libs are currently built as STATIC and need to be
SHARED for this to work. We are currently patching with sed
and will improve upon this if/when a solution is added upstream.

Signed-off-by: vsoch <vsoch@users.noreply.github.com>
---
 .github/workflows/build-deploy.yaml |  31 +++----
 .gitignore                          |   3 +-
 README.md                           | 132 +++++++++++++++++++++------
 src/Makefile                        |   8 +-
 src/build/scheduler/Dockerfile      | 133 +++++++++++++++-------------
 5 files changed, 197 insertions(+), 110 deletions(-)

diff --git a/.github/workflows/build-deploy.yaml b/.github/workflows/build-deploy.yaml
index 6784013..1a7e8e0 100644
--- a/.github/workflows/build-deploy.yaml
+++ b/.github/workflows/build-deploy.yaml
@@ -17,24 +17,20 @@ jobs:
     runs-on: ubuntu-latest
     name: build fluence
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-go@v3
       with:
-        go-version: ^1.18.1
+        go-version: ^1.19
 
-    - name: Build Container
-      run: |
-        git clone -b fluence https://github.com/openshift-psap/scheduler-plugins ./plugins
-        cd ./plugins
-        make local-image LOCAL_REGISTRY=ghcr.io/flux-framework LOCAL_IMAGE=fluence
+    - name: Build Containers
+      run: make build REGISTRY=ghcr.io/flux-framework SCHEDULER_IMAGE=fluence
 
-    - name: Tag and Push Release Image
+    - name: Tag Release Image
       if: (github.event_name == 'release')
       run: |
         tag=${GITHUB_REF#refs/tags/}
-        echo "Tagging and releasing ${{ env.container }}:${tag}"
+        echo "Tagging and releasing ${{ env.container}}:${tag}"        
         docker tag ${{ env.container }}:latest ${{ env.container }}:${tag}
-        docker push ${{ env.container }}:${tag}
 
     - name: GHCR Login
       if: (github.event_name != 'pull_request')
@@ -46,7 +42,7 @@ jobs:
 
     - name: Deploy Container
       if: (github.event_name != 'pull_request')
-      run: docker push ${{ env.container }}:latest
+      run: docker push ${{ env.container }} --all-tags
       
   build-sidecar:
     permissions:
@@ -56,23 +52,20 @@ jobs:
     runs-on: ubuntu-latest
     name: build sidecar
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-go@v3
       with:
-        go-version: ^1.18.1
+        go-version: ^1.19
 
     - name: Build Container
-      run: |
-        cd scheduler-plugin
-        make LOCAL_REGISTRY=ghcr.io/flux-framework
+      run: make build-sidecar REGISTRY=ghcr.io/flux-framework SIDECAR_IMAGE=fluence-sidecar
 
-    - name: Tag and Push Release Image
+    - name: Tag Release Image
       if: (github.event_name == 'release')
       run: |
         tag=${GITHUB_REF#refs/tags/}
         echo "Tagging and releasing ${{ env.container }}:${tag}"
         docker tag ${{ env.container }}:latest ${{ env.container }}:${tag}
-        docker push ${{ env.container }}:${tag}
 
     - name: GHCR Login
       if: (github.event_name != 'pull_request')
@@ -84,4 +77,4 @@ jobs:
 
     - name: Deploy Container
       if: (github.event_name != 'pull_request')
-      run: docker push ${{ env.container }}:latest
+      run: docker push ${{ env.container }} --all-tags
diff --git a/.gitignore b/.gitignore
index 9369c58..fa1845c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 plugins
-scheduler-plugins
\ No newline at end of file
+upstream
+scheduler-plugins
diff --git a/README.md b/README.md
index 006eb57..94833c6 100644
--- a/README.md
+++ b/README.md
@@ -18,15 +18,14 @@ For background on the Flux framework and the Fluxion scheduler, you can take a l
 
 We provide a set of pre-build containers [alongside the repository](https://github.com/orgs/flux-framework/packages?repo_name=flux-k8s)
 that you can easily use to deploy Fluence right away! You'll simply need to clone the proper helm charts, and then install to your cluster.
-Here are the quick install steps:
+We provide helper commands to do that.
 
 ```bash
 # This clones the upstream scheduler plugins code, we will add fluence to it!
-$ git clone --depth 1  https://github.com/kubernetes-sigs/scheduler-plugins.git upstream
+$ make prepare
 
 # Add fluence assets
-cp -R 
-$ cd scheduler-plugins/manifests/install/charts
+$ cd upstream/manifests/install/charts
 $ helm install \
   --set scheduler.image=ghcr.io/flux-framework/fluence:latest \
   --set scheduler.sidecarimage=ghcr.io/flux-framework/fluence-sidecar \
@@ -51,24 +50,63 @@ There are two images we will be building:
  - the scheduler sidecar: built from the repository here
  - the scheduler: built from [this branch of scheduler-plugins](https://github.com/openshift-psap/scheduler-plugins/blob/fluence/build/scheduler/Dockerfile)
 
-We will be adding more notes about how these containers work together.
+#### All at once (Sidecar + Scheduler)
 
-#### Build Sidecar
+**recommended**
+
+This will run the full builds for all containers in one step, which includes:
 
-To build the plugin containers, cd into [scheduler-plugins](./scheduler-plugins) and run `make`:
+1. Building the fluence sidecar from source code in [src](src)
+2. Cloning the upstream kubernetes-sigs/plugin-schedulers respository to ./upstream
+3. Building the scheduler container
+
+From the root here:
 
 ```bash
-cd scheduler-plugin
 make
 ```
 
+or customize the naming of your registry or local images:
+
+```bash
+make REGISTRY=vanessa SCHEDULER_IMAGE=fluence SIDECAR_IMAGE=fluence-sidecar
+```
+
+As an alternative, you can do each of the steps separately or manually (detailed below).
+
+<details>
+
+<summary> Manual Build Instructions </summary>
+
+#### Build Sidecar
+
+To build the plugin containers, we will basically be running `make` from the [src](src) directory. We have wrapped that for you
+in the Makefile:
+
+```bash
+make build-sidecar
+```
+
 To build for a custom registry (e.g., "vanessa' on Docker Hub):
 
 ```bash
-make LOCAL_REGISTRY=vanessa
+make build-sidecar REGISTRY=vanessa
+```
+
+And specify the sidecar image name too:
+
+```bash
+make build-sidecar REGISTRY=vanessa SIDECAR_IMAGE=another-sidecar
 ```
 
-This will create the scheduler plugin main container, which can be tagged and pushed to the preferred registry. As an example,
+The equivalent manual command is:
+
+```bash
+cd src
+make
+```
+
+Using either of the approaches above, this will create the scheduler plugin main container, which can be tagged and pushed to the preferred registry. As an example,
 here we push to the result of the build above:
 
 ```bash
@@ -77,10 +115,49 @@ docker push docker.io/vanessa/fluence-sidecar:latest
 
 #### Build Scheduler
 
-First, clone the repository:
+Note that you can run this entire process like:
 
 ```bash
-$ git clone -b fluence https://github.com/openshift-psap/scheduler-plugins ./plugins
+make prepare
+make build
+```
+
+Or customize the name of the scheduler image:
+
+```bash
+make prepare
+make build REGISTRY=vanessa
+```
+
+For a custom scheduler or controller image (we just need the scheduler):
+
+```bash
+make build REGISTRY=vanessa CONTROLLER_IMAGE=fluence-controller SCHEDULER_IMAGE=fluence
+```
+
+To walk through it manually, first, clone the upstream scheduler-plugins repository:
+
+```bash
+$ git clone https://github.com/kubernetes-sigs/scheduler-plugins ./upstream
+```
+
+We need to add our fluence package to the scheduler plugins to build. You can do that manully as follows:
+
+```bash
+# These are entirely new directory structures
+cp -R sig-scheduler-plugins/pkg/fluence ./upstream/pkg/fluence
+cp -R sig-scheduler-plugins/manifests/fluence ./upstream/manifests/fluence
+
+# These are files with subtle changes to add fluence
+cp sig-scheduler-plugins/cmd/scheduler/main.go ./upstream/cmd/scheduler/main.go
+cp sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/templates/deployment.yaml ./upstream/manifests/install/charts/as-a-second-scheduler/templates/deployment.yaml
+cp sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/templates/values.yaml ./upstream/manifests/install/charts/as-a-second-scheduler/templates/values.yaml
+```
+
+Then change directory to the scheduler plugins repository. 
+
+```bash
+cd ./upstream
 ```
 
 And build! You'll most likely want to set a custom registry and image name again:
@@ -90,12 +167,14 @@ And build! You'll most likely want to set a custom registry and image name again
 $ make local-image
 
 # this will build to docker.io/vanessa/fluence
-$ make local-image LOCAL_REGISTRY=vanessa LOCAL_IMAGE=fluence
+$ make local-image REGISTRY=vanessa CONTROLLER_IMAGE=fluence
 ```
 
+</details>
+
 **Important** the make command above produces _two images_ and you want to use the first that is mentioned in the output (not the second, which is a controller).
 
-And push to your registry for later!
+Whatever build approach you use, you'll want to push to your registry for later discovery!
 
 ```bash
 $ docker push docker.io/vanessa/fluence
@@ -115,23 +194,18 @@ $ kind create cluster
 ### Install Fluence
 
 For some background, the [Scheduling Framework](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) provided by
-Kubernetes means that our container is going to provide specific endpoints to allow for custom scheduling.
-To make it possible to use our own plugin, we need to install our fork of [kubernetes-sigs/scheduler-plugins](https://github.com/kubernetes-sigs/scheduler-plugins) at [openshift-psap/scheduler-plugins](https://github.com/openshift-psap/scheduler-plugins.git) which will be done through Helm charts. We will be customizing the values.yaml file to point to our image. First, clone the repository:
-
-```bash
-$ git clone https://github.com/openshift-psap/scheduler-plugins.git -b fluence
-```
-
-Install the charts, using the images you just built:
+Kubernetes means that our container is going to provide specific endpoints to allow for custom scheduling. At this point you can follow the instructions
+under [deploy](#deploy) to ensure you have cloned the upstream kubernetes-sigs/scheduler-plugins and installed fluence. This section will provide
+more details to inspect attributes available to you. Let's say that you ran
 
 ```bash
-$ cd scheduler-plugins/manifests/install/charts
+$ make prepare
 ```
 
-Here is how to see values allowed. Note the name of the directory - "as-a-second-scheduler" - this is what we are going to be doing - installing
-Fluxion as a second scheduler!
+You could then inspect values with helm:
 
 ```bash
+$ cd upstream/manifests/install/charts
 $ helm show values as-a-second-scheduler/
 ```
 
@@ -171,22 +245,26 @@ plugins:
 </details>
 
 Note that this plugin is going to allow us to create a Deployment with our plugin to be used as a scheduler!
-So we are good installing the defaults.
+The `helm install` shown under [deploy](#deploy) is how you can install to your cluster, and then proceed to testing below.
+Here would be an example using custom images:
 
 ```bash
+$ cd upstream/manifests/install/charts
 $ helm install \
   --set scheduler.image=vanessa/fluence:latest \
   --set scheduler.sidecarimage=vanessa/fluence-sidecar \
     schedscheduler-plugins as-a-second-scheduler/
 ```
 
+Next you can move down to testing the install.
+
 ### Testing Install
 
 The installation process will run one scheduler and one controller pod for the Scheduler Plugin Framework in the default namespace.
 You can double check that everything is running as follows:
 
 ```bash
-$ kubectl get pods -n scheduler-plugins
+$ kubectl get pods
 ```
 ```console
 NAME                                            READY   STATUS    RESTARTS   AGE
diff --git a/src/Makefile b/src/Makefile
index 14c4045..a32efce 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -1,5 +1,11 @@
+
+FLUX_SCHED_ROOT ?= /home/flux-sched
+INSTALL_PREFIX ?= /usr
 COMMONENVVAR=GOOS=$(shell uname -s | tr A-Z a-z)
-BUILDENVVAR=CGO_CFLAGS="-I/home/flux-sched/resource/reapi/bindings/c -I/home/flux-install/include" CGO_LDFLAGS="-L/home/flux-install/lib -lreapi_cli  -L/home/flux-install/lib -lresource -L/home/flux-install/lib -lflux-idset -lstdc++ -lczmq -ljansson -lhwloc -lboost_system -L/home/flux-install/lib -lflux-hostlist -lboost_graph -lyaml-cpp"
+
+# This is what worked
+# GOOS=linux CGO_CFLAGS="-I/home/flux-sched/resource/reapi/bindings/c" CGO_LDFLAGS="-L/usr/lib -L/home/flux-sched/resource -lresource -L/home/flux-sched/resource/libjobspec -ljobspec_conv -L/home/flux-sched/resource/reapi/bindings -lreapi_cli -lflux-idset -lstdc++ -lczmq -ljansson -lhwloc -lboost_system -lflux-hostlist -lboost_graph -lyaml-cpp" go build -ldflags '-w' -o bin/server cmd/main.go
+BUILDENVVAR=CGO_CFLAGS="-I${FLUX_SCHED_ROOT}/resource/reapi/bindings/c" CGO_LDFLAGS="-L${INSTALL_PREFIX}/lib -L${FLUX_SCHED_ROOT}/resource -lresource -L${FLUX_SCHED_ROOT}/resource/libjobspec -ljobspec_conv -L/${FLUX_SCHED_ROOT}/resource/reapi/bindings -lreapi_cli -lflux-idset -lstdc++ -lczmq -ljansson -lhwloc -lboost_system -lflux-hostlist -lboost_graph -lyaml-cpp"
 
 LOCAL_REGISTRY=localhost:5000
 LOCAL_IMAGE=fluence-sidecar:latest
diff --git a/src/build/scheduler/Dockerfile b/src/build/scheduler/Dockerfile
index 4592482..2f8d4d2 100644
--- a/src/build/scheduler/Dockerfile
+++ b/src/build/scheduler/Dockerfile
@@ -1,24 +1,27 @@
-FROM ubuntu:latest as basegoflux
+FROM ubuntu:latest as base
 
 RUN apt -y update && apt -y upgrade && apt -y clean && apt -y autoremove
+ENV DEBIAN_FRONTEND=noninteractive
+ENV GO_VERSION=1.19.10
+ENV INSTALL_PREFIX=/usr
 
-RUN DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends tzdata && apt -y --no-install-recommends install \
-    libc6-dev \
-    wget \
-    systemd \
-    git \
+RUN apt install -y --no-install-recommends tzdata && \
+    apt -y --no-install-recommends install \
+    aspell \
+    aspell-en \
     autoconf \
     automake \
-    libtool \
-    libelf-dev \ 
+    curl \
+    git \    
+    libc6-dev \
+    libczmq-dev \
+    libmpich-dev \
     libncurses5-dev \
+    libelf-dev \ 
     libssl-dev \
-    openssh-client \
-    make \
+    libtool \
     libsodium-dev \
     libzmq3-dev \
-    libczmq-dev \
-    uuid-dev \
     libjansson-dev \
     liblz4-dev \
     libhwloc-dev \
@@ -26,6 +29,8 @@ RUN DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends tzdata
     lua5.1 \
     liblua5.1-dev \
     lua-posix \
+    make \
+    openssh-client \
     python3-dev \
     python3-cffi \
     python3-six \
@@ -34,22 +39,26 @@ RUN DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends tzdata
     python3-sphinx \
     python3-pip \
     python3-setuptools \
-    aspell \
-    aspell-en \
-    libmpich-dev && apt -y clean  && apt -y autoremove
+    systemd \
+    wget \
+    uuid-dev && \
+    apt -y clean  && apt -y autoremove
+
+RUN echo 'alias python="/usr/bin/python3.8"' >> /root/.bashrc && \
+    echo 'alias pip="/usr/bin/pip3"' >> /root/.bashrc && \
+    . /root/.bashrc
 
-RUN \
-   echo 'alias python="/usr/bin/python3.8"' >> /root/.bashrc && \
-   echo 'alias pip="/usr/bin/pip3"' >> /root/.bashrc && \
-   . /root/.bashrc
+RUN echo 'set number' >> /root/.vimrc 
 
-RUN \
-   echo 'set number' >> /root/.vimrc 
+# Install cmake for new build system
+RUN curl -s -L https://github.com/Kitware/CMake/releases/download/v3.26.4/cmake-3.26.4-linux-$(uname -m).sh > cmake.sh ;\
+    bash cmake.sh --prefix=/usr/local --skip-license ;\
+    rm cmake.sh
 
-# Remove python2
+# Remove Python 2
 RUN apt purge -y python2.7-minimal
-# You already have Python3 but
-# don't care about the version
+
+# Python 3 should be linked to python
 RUN ln -s /usr/bin/python3 /usr/bin/python
 RUN apt install -y python3-pip \
     &&  apt -y --no-install-recommends install \
@@ -65,38 +74,41 @@ RUN apt install -y python3-pip \
     libedit-dev \
     libarchive-dev \
     pkg-config && apt -y clean  && apt -y autoremove
-RUN python --version
-RUN cd /home/ && mkdir flux-install
-WORKDIR /home/
-RUN git clone https://github.com/flux-framework/flux-core.git 
 
-RUN cd /home/flux-core/ && ./autogen.sh && PYTHON_VERSION=3 ./configure --prefix=/home/flux-install \ 
-    && make && make install && cd /home && rm -rf /home/flux-core
+RUN git clone https://github.com/flux-framework/flux-core.git /home/flux-core && \
+    cd /home/flux-core/ && \
+    ./autogen.sh && \
+    PYTHON_VERSION=3 ./configure --prefix=${INSTALL_PREFIX} && \ 
+    make && make install && \
+    cd ../ && \
+    rm -rf flux-core
 
-# Install go 19
-WORKDIR /home
-RUN wget https://go.dev/dl/go1.19.10.linux-amd64.tar.gz  && tar -xvf go1.19.10.linux-amd64.tar.gz && \
-         mv go /usr/local && rm go1.19.10.linux-amd64.tar.gz
+# Install go 19.10
+RUN wget https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz  && tar -xvf go${GO_VERSION}.linux-amd64.tar.gz && \
+    mv go /usr/local && rm go${GO_VERSION}.linux-amd64.tar.gz
 
-ENV GOROOT "/usr/local/go"
-ENV GOPATH "/go"
-ENV PATH "$GOROOT/bin:$PATH"
+ENV GOROOT=/usr/local/go
+ENV GOPATH=/go
+ENV PATH="$GOROOT/bin:$PATH"
 RUN mkdir -p /go/src
-
-ENV PATH "/home/flux-install/bin:$PATH"
-ENV LD_LIBRARY_PATH "/home/flux-install/lib/flux:/home/flux-install/lib"
 RUN flux keygen
 
-RUN git clone https://github.com/cmisale/flux-sched.git --branch golang-pr --single-branch && \ 
-    cd /home/flux-sched/ \
-	&& ./autogen.sh && PYTHON_VERSION=3.8 ./configure --prefix=/home/flux-install && make && make install
-RUN	cd /home/flux-sched && \
-    cp -r resource/reapi/bindings/c/.libs/* resource/.libs/* /home/flux-install/lib/ \
-	&& cp -r resource/reapi/bindings/go/src/fluxcli /go/src/ \
-	&& mv  resource/reapi/bindings /tmp \
-	&& cd /home && mkdir -p flux-sched/resource/reapi && mv /tmp/bindings flux-sched/resource/reapi
+ENV WITH_GO=yes
+RUN git clone https://github.com/flux-framework/flux-sched.git /home/flux-sched && \ 
+    cd /home/flux-sched/ && \
+    # Ensure we pin to variant that has STATIC - will update when fix is in
+    git fetch && git checkout v0.31.0 && \
+    # These need to be shared libraries 
+    # https://github.com/flux-framework/flux-sched/pull/1094
+    sed -i 's/add_library(resource STATIC/add_library(resource SHARED/g' resource/CMakeLists.txt && \
+    sed -i 's/add_library ( reapi_module STATIC/add_library ( reapi_module SHARED/g' resource/reapi/bindings/CMakeLists.txt && \
+    sed -i 's/add_library ( reapi_cli STATIC/add_library ( reapi_cli SHARED/g' resource/reapi/bindings/CMakeLists.txt && \
+    sed -i 's/add_library ( jobspec_conv STATIC/add_library ( jobspec_conv SHARED/g' resource/libjobspec/CMakeLists.txt && \
+    PYTHON_VERSION=3 ./configure --prefix=${INSTALL_PREFIX} && \
+    make && make install
 
-RUN apt purge -y git  python3-dev \
+RUN apt purge -y \
+    python3-dev \
     python3-cffi \
     python3-six \
     python3-yaml \
@@ -106,20 +118,17 @@ RUN apt purge -y git  python3-dev \
     python3-setuptools \
     && apt -y clean && apt -y autoremove
 
-# BUILD SHARED LIBRARY
-# RUN cp -r /go/src/fluxcli /usr/local/go/src/
-# WORKDIR /usr/local/go/src/fluxcli
-
-# RUN CGO_CFLAGS="-I/root/flux-sched/resource/hlapi/bindings/c -I/root/flux-install/include" CGO_LDFLAGS="-L/root/flux-install/lib/ -lreapi_cli  -L/root/flux-install/lib/ -lresource -lstdc++ -lczmq -ljansson -lhwloc -lboost_system -L/root/flux-install/lib -lflux-hostlist -lboost_graph -lyaml-cpp" go install -buildmode=shared -linkshared fluxcli
 ENV PATH=/usr/local/go/bin:$PATH
 RUN apt -y update && apt -y upgrade && apt install --no-install-recommends -y protobuf-compiler curl && \
-   go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 && \ 
-   go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1
+    go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26 && \ 
+    go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1
 
-COPY fluence /go/src/fluence/
-COPY Makefile /go/src/fluence/
+# These need to be on the LD_LIBRARY_PATH for the server to find at runtime
+# This mimcs what we use to build server
+ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib:/home/flux-sched/resource:/home/flux-sched/resource/libjobspec:/home/flux-sched/resource/reapi/bindings"
+COPY fluence Makefile /go/src/fluence/
 WORKDIR /go/src/fluence/
-RUN go mod tidy
-RUN make server
-RUN mkdir -p /home/data/jobspecs && mkdir -p /home/data/jgf
-RUN chmod -R ugo+rwx /home/data
\ No newline at end of file
+RUN go mod tidy && \
+    make server FLUX_SCHED_ROOT=/home/flux-sched INSTALL_PREFIX=${INSTALL_PREFIX} && \
+    mkdir -p /home/data/jobspecs /home/data/jgf && \
+    chmod -R ugo+rwx /home/data

From 298011537c2a101f087aeaefe02ba70d31908b0b Mon Sep 17 00:00:00 2001
From: vsoch <vsoch@users.noreply.github.com>
Date: Thu, 14 Dec 2023 12:59:34 -0700
Subject: [PATCH 3/5] add remainder of files (main.go and charts) to build
 fluence

Problem: we have a small subset of files that fluence uses that are not included.
Solution: we can add them to the repository and they can be copied over
into the sig-scheduler-plugins branch. The changes in these files are minimal

Signed-off-by: vsoch <vsoch@users.noreply.github.com>
---
 README.md                                     | 19 +++--
 sig-scheduler-plugins/cmd/scheduler/main.go   | 68 ++++++++++++++++
 .../templates/deployment.yaml                 | 81 +++++++++++++++++++
 .../charts/as-a-second-scheduler/values.yaml  | 36 +++++++++
 4 files changed, 196 insertions(+), 8 deletions(-)
 create mode 100644 sig-scheduler-plugins/cmd/scheduler/main.go
 create mode 100644 sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/templates/deployment.yaml
 create mode 100644 sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/values.yaml

diff --git a/README.md b/README.md
index 94833c6..546c928 100644
--- a/README.md
+++ b/README.md
@@ -277,7 +277,7 @@ Let's now check logs for containers to check that everything is OK.
 First, let's look at logs for the sidecar container:
 
 ```bash
-$ kubectl logs -n scheduler-plugins fluence-6bbcbc6bbf-xjfx6 
+$ kubectl logs fluence-6bbcbc6bbf-xjfx6 
 ```
 ```console
 Defaulted container "sidecar" out of: sidecar, scheduler-plugins-scheduler
@@ -296,7 +296,7 @@ Match policy:  {"matcher_policy": "lonode"}
 And for the fluence container:
 
 ```bash
-$ kubectl logs -n scheduler-plugins fluence-6bbcbc6bbf-xjfx6 -c scheduler-plugins-scheduler
+$ kubectl logs fluence-6bbcbc6bbf-xjfx6 -c scheduler-plugins-scheduler
 ```
 
 If you haven't done anything, you'll likely just see health checks.
@@ -306,7 +306,8 @@ If you haven't done anything, you'll likely just see health checks.
 Let's now run a simple example! Change directory into this directory:
 
 ```bash
-cd ../../../../examples/simple_example
+# This is from the root of flux-k8s
+cd examples/simple_example
 ```
 
 And then we want to deploy two pods, one assigned to the `default-scheduler` and the other
@@ -320,14 +321,14 @@ spec:
 Here is how to create the pods:
 
 ```bash
-$ kubectl apply -f default-scheduler-pod.yaml
-$ kubectl apply -f fluence-scheduler-pod.yaml
+kubectl apply -f default-scheduler-pod.yaml
+kubectl apply -f fluence-scheduler-pod.yaml
 ```
 
 Once it was created, aside from checking that it ran OK, I could verify by looking at the scheduler logs again:
 
 ```bash
-$ kubectl logs -n scheduler-plugins fluence-6bbcbc6bbf-xjfx6
+kubectl logs fluence-6bbcbc6bbf-xjfx6
 ```
 ```bash
 Defaulted container "sidecar" out of: sidecar, scheduler-plugins-scheduler
@@ -380,10 +381,12 @@ FINAL NODE RESULT:
 I was trying to look for a way to see the assignment, and maybe we can see it here (this is the best I could come up with!)
 
 ```bash
-$ kubectl get events -o wide
+kubectl get events -o wide
+```
+```bash
+kubectl get events -o wide |  awk {'print $4" " $5" " $6'} | column -t
 ```
 ```console
-$ kubectl get events -o wide |  awk {'print $4" " $5" " $6'} | column -t
 REASON                     OBJECT                                        SUBOBJECT
 pod/default-scheduler-pod  default-scheduler                             Successfully
 pod/default-scheduler-pod  spec.containers{default-scheduler-container}  kubelet,
diff --git a/sig-scheduler-plugins/cmd/scheduler/main.go b/sig-scheduler-plugins/cmd/scheduler/main.go
new file mode 100644
index 0000000..d9a580a
--- /dev/null
+++ b/sig-scheduler-plugins/cmd/scheduler/main.go
@@ -0,0 +1,68 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"os"
+
+	"k8s.io/component-base/cli"
+	_ "k8s.io/component-base/metrics/prometheus/clientgo" // for rest client metric registration
+	_ "k8s.io/component-base/metrics/prometheus/version"  // for version metric registration
+	"k8s.io/kubernetes/cmd/kube-scheduler/app"
+
+	"sigs.k8s.io/scheduler-plugins/pkg/capacityscheduling"
+	"sigs.k8s.io/scheduler-plugins/pkg/coscheduling"
+	"sigs.k8s.io/scheduler-plugins/pkg/networkaware/networkoverhead"
+	"sigs.k8s.io/scheduler-plugins/pkg/networkaware/topologicalsort"
+	"sigs.k8s.io/scheduler-plugins/pkg/noderesources"
+	"sigs.k8s.io/scheduler-plugins/pkg/noderesourcetopology"
+	"sigs.k8s.io/scheduler-plugins/pkg/podstate"
+	"sigs.k8s.io/scheduler-plugins/pkg/preemptiontoleration"
+	"sigs.k8s.io/scheduler-plugins/pkg/qos"
+	"sigs.k8s.io/scheduler-plugins/pkg/trimaran/loadvariationriskbalancing"
+	"sigs.k8s.io/scheduler-plugins/pkg/trimaran/lowriskovercommitment"
+	"sigs.k8s.io/scheduler-plugins/pkg/trimaran/targetloadpacking"
+	"sigs.k8s.io/scheduler-plugins/pkg/fluence"
+	// Ensure scheme package is initialized.
+	_ "sigs.k8s.io/scheduler-plugins/apis/config/scheme"
+)
+
+func main() {
+	// Register custom plugins to the scheduler framework.
+	// Later they can consist of scheduler profile(s) and hence
+	// used by various kinds of workloads.
+	command := app.NewSchedulerCommand(
+		app.WithPlugin(capacityscheduling.Name, capacityscheduling.New),
+		app.WithPlugin(coscheduling.Name, coscheduling.New),
+		app.WithPlugin(loadvariationriskbalancing.Name, loadvariationriskbalancing.New),
+		app.WithPlugin(networkoverhead.Name, networkoverhead.New),
+		app.WithPlugin(topologicalsort.Name, topologicalsort.New),
+		app.WithPlugin(noderesources.AllocatableName, noderesources.NewAllocatable),
+		app.WithPlugin(noderesourcetopology.Name, noderesourcetopology.New),
+		app.WithPlugin(preemptiontoleration.Name, preemptiontoleration.New),
+		app.WithPlugin(targetloadpacking.Name, targetloadpacking.New),
+		app.WithPlugin(lowriskovercommitment.Name, lowriskovercommitment.New),
+		// Sample plugins below.
+		// app.WithPlugin(crossnodepreemption.Name, crossnodepreemption.New),
+		app.WithPlugin(podstate.Name, podstate.New),
+		app.WithPlugin(qos.Name, qos.New),
+		app.WithPlugin(fluence.Name, fluence.New),
+	)
+
+	code := cli.Run(command)
+	os.Exit(code)
+}
diff --git a/sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/templates/deployment.yaml b/sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/templates/deployment.yaml
new file mode 100644
index 0000000..161b356
--- /dev/null
+++ b/sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/templates/deployment.yaml
@@ -0,0 +1,81 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: {{ .Values.controller.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: scheduler-plugins-controller
+spec:
+  replicas: {{ .Values.controller.replicaCount }}
+  selector:
+    matchLabels:
+      app: scheduler-plugins-controller
+  template:
+    metadata:
+      labels:
+        app: scheduler-plugins-controller
+    spec:
+      serviceAccountName: {{ .Values.controller.name }}
+      containers:
+      - name: scheduler-plugins-controller
+        image: {{ .Values.controller.image }}
+        imagePullPolicy: IfNotPresent
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    component: scheduler
+  name: {{ .Values.scheduler.name }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  selector:
+    matchLabels:
+      component: scheduler
+  replicas: {{ .Values.scheduler.replicaCount }}
+  template:
+    metadata:
+      labels:
+        component: scheduler
+    spec:
+      serviceAccountName: {{ .Values.scheduler.name }}
+      containers:
+      - image: {{ .Values.scheduler.sidecarimage }}
+        imagePullPolicy: Always
+        command:
+        - /go/src/fluence/bin/server
+        - --policy={{ .Values.scheduler.policy }}
+        name: sidecar
+      - command:
+        - /bin/kube-scheduler
+        - --config=/etc/kubernetes/scheduler-config.yaml
+        - -v=9
+        image: {{ .Values.scheduler.image }}
+        imagePullPolicy: Always
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 10259
+            scheme: HTTPS
+          initialDelaySeconds: 15
+        name: scheduler-plugins-scheduler
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 10259
+            scheme: HTTPS
+        resources:
+          requests:
+            cpu: '0.1'
+        securityContext:
+          privileged: false
+        volumeMounts:
+        - name: scheduler-config
+          mountPath: /etc/kubernetes
+          readOnly: true
+      hostNetwork: false
+      hostPID: false
+      volumes:
+      - name: scheduler-config
+        configMap:
+          name: scheduler-config
diff --git a/sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/values.yaml b/sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/values.yaml
new file mode 100644
index 0000000..e467e45
--- /dev/null
+++ b/sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/values.yaml
@@ -0,0 +1,36 @@
+# Default values for scheduler-plugins-as-a-second-scheduler.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+scheduler:
+  name: fluence
+  image: registry.k8s.io/scheduler-plugins/kube-scheduler:v0.27.8
+  replicaCount: 1
+  leaderElect: false
+  sidecarimage: quay.io/cmisale1/fluence-sidecar:latest
+  policy: lonode
+
+controller:
+  name: scheduler-plugins-controller
+  image: registry.k8s.io/scheduler-plugins/controller:v0.27.8
+  replicaCount: 1
+
+# LoadVariationRiskBalancing and TargetLoadPacking are not enabled by default
+# as they need extra RBAC privileges on metrics.k8s.io.
+
+plugins:
+  enabled: ["Fluence"]
+  disabled: ["CapacityScheduling","NodeResourceTopologyMatch","NodeResourcesAllocatable","PrioritySort","Coscheduling"] # only in-tree plugins need to be defined here
+
+# Customize the enabled plugins' config.
+# Refer to the "pluginConfig" section of manifests/<plugin>/scheduler-config.yaml.
+# For example, for Coscheduling plugin, you want to customize the permit waiting timeout to 10 seconds:
+pluginConfig:
+- name: Coscheduling
+  args:
+    permitWaitingTimeSeconds: 10 # default is 60
+# Or, customize the other plugins
+# - name: NodeResourceTopologyMatch
+#   args:
+#     scoringStrategy:
+#       type: MostAllocated # default is LeastAllocated

From 22b3bfe5f3d16f32dca1220abdb7520e0ed72687 Mon Sep 17 00:00:00 2001
From: vsoch <vsoch@users.noreply.github.com>
Date: Thu, 14 Dec 2023 13:01:15 -0700
Subject: [PATCH 4/5] add Makefile to simplify container builds

Problem: the container builds (for fluence scheduler and sidecar) can be confusing
Solution: create a Makefile that helps with the complexity.
Signed-off-by: vsoch <vsoch@users.noreply.github.com>
---
 Makefile | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 Makefile

diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..dc87d50
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,39 @@
+CLONE_UPSTREAM ?= ./upstream
+UPSTREAM ?= https://github.com/kubernetes-sigs/scheduler-plugins
+BASH ?= /bin/bash
+DOCKER ?= docker
+TAG ?= latest
+
+# These are passed to build the sidecar
+REGISTRY ?= ghcr.io/flux-framework
+SIDECAR_IMAGE ?= fluence-sidecar:latest
+CONTROLLER_IMAGE ?= fluence-controller
+SCHEDULER_IMAGE ?= fluence
+
+.PHONY: all build build-sidecar prepare push push-sidecar push-controller
+
+all: build-sidecar prepare build
+
+build-sidecar: 
+	make -C ./src LOCAL_REGISTRY=${REGISTRY} LOCAL_IMAGE=${SIDECAR_IMAGE}
+
+prepare: 
+	if [ -d "$(CLONE_UPSTREAM)" ]; then echo "Upstream is cloned"; else git clone $(UPSTREAM) ./$(CLONE_UPSTREAM); fi
+	# These are entirely new directory structures
+	cp -R sig-scheduler-plugins/pkg/fluence $(CLONE_UPSTREAM)/pkg/fluence
+	cp -R sig-scheduler-plugins/manifests/fluence $(CLONE_UPSTREAM)/manifests/fluence
+	# These are files with subtle changes to add fluence
+	cp sig-scheduler-plugins/cmd/scheduler/main.go ./upstream/cmd/scheduler/main.go
+	cp sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/templates/deployment.yaml $(CLONE_UPSTREAM)/manifests/install/charts/as-a-second-scheduler/templates/deployment.yaml
+	cp sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/values.yaml $(CLONE_UPSTREAM)/manifests/install/charts/as-a-second-scheduler/values.yaml
+
+build:
+	REGISTRY=${REGISTRY} IMAGE=${SCHEDULER_IMAGE} CONTROLLER_IMAGE=${CONTROLLER_IMAGE} $(BASH) $(CLONE_UPSTREAM)/hack/build-images.sh
+
+push-sidecar:
+	$(DOCKER) push $(REGISTRY)/$(SIDECAR_IMAGE):$(TAG) --all-tags
+
+push-controller:
+	$(DOCKER) push $(REGISTRY)/$(CONTROLLER_IMAGE):$(TAG) --all-tags
+
+push: push-sidecar push-controller

From 105562e4662e86a1b8ce1e19762678a6c0dd6309 Mon Sep 17 00:00:00 2001
From: vsoch <vsoch@users.noreply.github.com>
Date: Thu, 14 Dec 2023 13:51:52 -0700
Subject: [PATCH 5/5] update fluence scheduler build to use newer API

Problem: we made changes to the flux-sched reapi go module, and did not update here.
Solution: update the client interface here, ensuring that we also install exactly
the version of flux-sched that we are cloning. I am also cleaning up the README to
not use $ before commands (easier to copy paste), the go.mod/sum, and the utils
function to print an error code and not an integer (part of the refactor)

Signed-off-by: vsoch <vsoch@users.noreply.github.com>
---
 .github/workflows/build-deploy.yaml |  8 ++-
 README.md                           | 44 +++++++--------
 src/build/scheduler/Dockerfile      |  7 ++-
 src/fluence/fluxion/fluxion.go      | 60 ++++++++++----------
 src/fluence/go.mod                  |  2 +-
 src/fluence/go.sum                  |  4 +-
 src/fluence/utils/utils.go          | 87 ++++++++++++++---------------
 7 files changed, 106 insertions(+), 106 deletions(-)

diff --git a/.github/workflows/build-deploy.yaml b/.github/workflows/build-deploy.yaml
index 1a7e8e0..c993aa9 100644
--- a/.github/workflows/build-deploy.yaml
+++ b/.github/workflows/build-deploy.yaml
@@ -23,7 +23,9 @@ jobs:
         go-version: ^1.19
 
     - name: Build Containers
-      run: make build REGISTRY=ghcr.io/flux-framework SCHEDULER_IMAGE=fluence
+      run: |
+        make prepare
+        make build REGISTRY=ghcr.io/flux-framework SCHEDULER_IMAGE=fluence
 
     - name: Tag Release Image
       if: (github.event_name == 'release')
@@ -58,7 +60,9 @@ jobs:
         go-version: ^1.19
 
     - name: Build Container
-      run: make build-sidecar REGISTRY=ghcr.io/flux-framework SIDECAR_IMAGE=fluence-sidecar
+      run: |
+        make prepare
+        make build-sidecar REGISTRY=ghcr.io/flux-framework SIDECAR_IMAGE=fluence-sidecar
 
     - name: Tag Release Image
       if: (github.event_name == 'release')
diff --git a/README.md b/README.md
index 546c928..2addaa7 100644
--- a/README.md
+++ b/README.md
@@ -22,11 +22,11 @@ We provide helper commands to do that.
 
 ```bash
 # This clones the upstream scheduler plugins code, we will add fluence to it!
-$ make prepare
+make prepare
 
 # Add fluence assets
-$ cd upstream/manifests/install/charts
-$ helm install \
+cd upstream/manifests/install/charts
+helm install \
   --set scheduler.image=ghcr.io/flux-framework/fluence:latest \
   --set scheduler.sidecarimage=ghcr.io/flux-framework/fluence-sidecar \
     schedscheduler-plugins as-a-second-scheduler/
@@ -138,7 +138,7 @@ make build REGISTRY=vanessa CONTROLLER_IMAGE=fluence-controller SCHEDULER_IMAGE=
 To walk through it manually, first, clone the upstream scheduler-plugins repository:
 
 ```bash
-$ git clone https://github.com/kubernetes-sigs/scheduler-plugins ./upstream
+git clone https://github.com/kubernetes-sigs/scheduler-plugins ./upstream
 ```
 
 We need to add our fluence package to the scheduler plugins to build. You can do that manully as follows:
@@ -151,7 +151,7 @@ cp -R sig-scheduler-plugins/manifests/fluence ./upstream/manifests/fluence
 # These are files with subtle changes to add fluence
 cp sig-scheduler-plugins/cmd/scheduler/main.go ./upstream/cmd/scheduler/main.go
 cp sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/templates/deployment.yaml ./upstream/manifests/install/charts/as-a-second-scheduler/templates/deployment.yaml
-cp sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/templates/values.yaml ./upstream/manifests/install/charts/as-a-second-scheduler/templates/values.yaml
+cp sig-scheduler-plugins/manifests/install/charts/as-a-second-scheduler/values.yaml ./upstream/manifests/install/charts/as-a-second-scheduler/values.yaml
 ```
 
 Then change directory to the scheduler plugins repository. 
@@ -164,10 +164,10 @@ And build! You'll most likely want to set a custom registry and image name again
 
 ```bash
 # This will build to localhost
-$ make local-image
+make local-image
 
 # this will build to docker.io/vanessa/fluence
-$ make local-image REGISTRY=vanessa CONTROLLER_IMAGE=fluence
+make local-image REGISTRY=vanessa CONTROLLER_IMAGE=fluence
 ```
 
 </details>
@@ -177,7 +177,7 @@ $ make local-image REGISTRY=vanessa CONTROLLER_IMAGE=fluence
 Whatever build approach you use, you'll want to push to your registry for later discovery!
 
 ```bash
-$ docker push docker.io/vanessa/fluence
+docker push docker.io/vanessa/fluence
 ```
 
 ### Prepare Cluster
@@ -188,7 +188,7 @@ These steps will require a Kubernetes cluster to install to, and having pushed t
 create a local one with `kind`:
 
 ```bash
-$ kind create cluster
+kind create cluster
 ```
 
 ### Install Fluence
@@ -199,14 +199,14 @@ under [deploy](#deploy) to ensure you have cloned the upstream kubernetes-sigs/s
 more details to inspect attributes available to you. Let's say that you ran
 
 ```bash
-$ make prepare
+make prepare
 ```
 
 You could then inspect values with helm:
 
 ```bash
-$ cd upstream/manifests/install/charts
-$ helm show values as-a-second-scheduler/
+cd upstream/manifests/install/charts
+helm show values as-a-second-scheduler/
 ```
 
 <details>
@@ -249,8 +249,8 @@ The `helm install` shown under [deploy](#deploy) is how you can install to your
 Here would be an example using custom images:
 
 ```bash
-$ cd upstream/manifests/install/charts
-$ helm install \
+cd upstream/manifests/install/charts
+helm install \
   --set scheduler.image=vanessa/fluence:latest \
   --set scheduler.sidecarimage=vanessa/fluence-sidecar \
     schedscheduler-plugins as-a-second-scheduler/
@@ -264,7 +264,7 @@ The installation process will run one scheduler and one controller pod for the S
 You can double check that everything is running as follows:
 
 ```bash
-$ kubectl get pods
+kubectl get pods
 ```
 ```console
 NAME                                            READY   STATUS    RESTARTS   AGE
@@ -277,18 +277,18 @@ Let's now check logs for containers to check that everything is OK.
 First, let's look at logs for the sidecar container:
 
 ```bash
-$ kubectl logs fluence-6bbcbc6bbf-xjfx6 
+kubectl logs fluence-6bbcbc6bbf-xjfx6 
 ```
 ```console
 Defaulted container "sidecar" out of: sidecar, scheduler-plugins-scheduler
 This is the fluxion grpc server
-Created cli context  &{}
-&{}
+Created flux resource client  &{0x3bd33d0}
+&{ctx:0x3bd33d0}
 Number nodes  1
 node in flux group  kind-control-plane
-Node  kind-control-plane  flux cpu  6
-Node  kind-control-plane  total mem  16132255744
-Can request at most  6  exclusive cpu
+Node  kind-control-plane  flux cpu  10
+Node  kind-control-plane  total mem  32992821248
+Can request at most  10  exclusive cpu
 Match policy:  {"matcher_policy": "lonode"}
 [GRPCServer] gRPC Listening on [::]:4242
 ```
@@ -296,7 +296,7 @@ Match policy:  {"matcher_policy": "lonode"}
 And for the fluence container:
 
 ```bash
-$ kubectl logs fluence-6bbcbc6bbf-xjfx6 -c scheduler-plugins-scheduler
+kubectl logs fluence-6bbcbc6bbf-xjfx6 -c scheduler-plugins-scheduler
 ```
 
 If you haven't done anything, you'll likely just see health checks.
diff --git a/src/build/scheduler/Dockerfile b/src/build/scheduler/Dockerfile
index 2f8d4d2..18c4bd7 100644
--- a/src/build/scheduler/Dockerfile
+++ b/src/build/scheduler/Dockerfile
@@ -128,7 +128,10 @@ RUN apt -y update && apt -y upgrade && apt install --no-install-recommends -y pr
 ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib:/home/flux-sched/resource:/home/flux-sched/resource/libjobspec:/home/flux-sched/resource/reapi/bindings"
 COPY fluence Makefile /go/src/fluence/
 WORKDIR /go/src/fluence/
-RUN go mod tidy && \
+
+# This is the 0.31.0 tag of flux-sched (same as we install above)
+RUN go get -u github.com/flux-framework/flux-sched/resource/reapi/bindings/go/src/fluxcli@250eac78a6753253fc8353a3504d7e843d1b6b24 && \
+    go mod tidy && \
     make server FLUX_SCHED_ROOT=/home/flux-sched INSTALL_PREFIX=${INSTALL_PREFIX} && \
     mkdir -p /home/data/jobspecs /home/data/jgf && \
-    chmod -R ugo+rwx /home/data
+    chmod -R ugo+rwx /home/data
\ No newline at end of file
diff --git a/src/fluence/fluxion/fluxion.go b/src/fluence/fluxion/fluxion.go
index 643fa9b..7ef532a 100644
--- a/src/fluence/fluxion/fluxion.go
+++ b/src/fluence/fluxion/fluxion.go
@@ -1,68 +1,64 @@
 package fluxion
 
 import (
-	
-	"github.com/cmisale/flux-sched/resource/hlapi/bindings/go/src/fluxcli"
-	"github.com/flux-framework/flux-k8s/flux-plugin/fluence/utils" 
-	"github.com/flux-framework/flux-k8s/flux-plugin/fluence/jobspec"
+	"os"
+
 	pb "github.com/flux-framework/flux-k8s/flux-plugin/fluence/fluxcli-grpc"
+	"github.com/flux-framework/flux-k8s/flux-plugin/fluence/jobspec"
+	"github.com/flux-framework/flux-k8s/flux-plugin/fluence/utils"
+	"github.com/flux-framework/flux-sched/resource/reapi/bindings/go/src/fluxcli"
 
 	"context"
 	"errors"
 	"fmt"
-	"io/ioutil"
 )
 
 type Fluxion struct {
-	fctx 		*fluxcli.ReapiCtx
+	cli *fluxcli.ReapiClient
 	pb.UnimplementedFluxcliServiceServer
 }
 
-func (f *Fluxion) Context() *fluxcli.ReapiCtx {
-	return f.fctx
-}
-
 func (f *Fluxion) InitFluxion(policy *string, label *string) {
-	f.fctx = fluxcli.NewReapiCli()
+	f.cli = fluxcli.NewReapiClient()
 
-	fmt.Println("Created cli context ", f.fctx)
-	fmt.Printf("%+v\n", f.fctx)
+	fmt.Println("Created flux resource client ", f.cli)
+	fmt.Printf("%+v\n", f.cli)
 	filename := "/home/data/jgf/kubecluster.json"
 	err := utils.CreateJGF(filename, label)
 	if err != nil {
 		return
 	}
-	
-	jgf, err := ioutil.ReadFile(filename)
+
+	jgf, err := os.ReadFile(filename)
 	if err != nil {
 		fmt.Println("Error reading JGF")
 		return
 	}
-	
+
 	p := "{}"
 	if *policy != "" {
 		p = string("{\"matcher_policy\": \"" + *policy + "\"}")
 		fmt.Println("Match policy: ", p)
-	} 
-	
-	fluxcli.ReapiCliInit(f.fctx, string(jgf), p)
+	}
 
+	f.cli.InitContext(string(jgf), p)
 }
 
 func (s *Fluxion) Cancel(ctx context.Context, in *pb.CancelRequest) (*pb.CancelResponse, error) {
 	fmt.Printf("[GRPCServer] Received Cancel request %v\n", in)
-	err := fluxcli.ReapiCliCancel(s.fctx, int64(in.JobID), true)
-	if err < 0 {
+	err := s.cli.Cancel(int64(in.JobID), true)
+	if err != nil {
 		return nil, errors.New("Error in Cancel")
 	}
 
-	dr := &pb.CancelResponse{JobID: in.JobID, Error: int32(err)}
+	// Why would we have an error code here if we check above?
+	// This (I think) should be an error code for the specific job
+	dr := &pb.CancelResponse{JobID: in.JobID}
 	fmt.Printf("[GRPCServer] Sending Cancel response %v\n", dr)
 
-	fmt.Printf("[CancelRPC] Errors so far: %s\n", fluxcli.ReapiCliGetErrMsg(s.fctx))
-
-	reserved, at, overhead, mode, fluxerr := fluxcli.ReapiCliInfo(s.fctx, int64(in.JobID))
+	fmt.Printf("[CancelRPC] Errors so far: %s\n", s.cli.GetErrMsg())
 
+	reserved, at, overhead, mode, fluxerr := s.cli.Info(int64(in.JobID))
 	fmt.Println("\n\t----Job Info output---")
 	fmt.Printf("jobid: %d\nreserved: %t\nat: %d\noverhead: %f\nmode: %s\nerror: %d\n", in.JobID, reserved, at, overhead, mode, fluxerr)
 
@@ -74,17 +70,17 @@ func (s *Fluxion) Match(ctx context.Context, in *pb.MatchRequest) (*pb.MatchResp
 	filename := "/home/data/jobspecs/jobspec.yaml"
 	jobspec.CreateJobSpecYaml(in.Ps, in.Count, filename)
 
-	spec, err := ioutil.ReadFile(filename)
+	spec, err := os.ReadFile(filename)
 	if err != nil {
 		return nil, errors.New("Error reading jobspec")
 	}
 
 	fmt.Printf("[GRPCServer] Received Match request %v\n", in)
-	reserved, allocated, at, overhead, jobid, fluxerr := fluxcli.ReapiCliMatchAllocate(s.fctx, false, string(spec))
+	reserved, allocated, at, overhead, jobid, fluxerr := s.cli.MatchAllocate(false, string(spec))
 	utils.PrintOutput(reserved, allocated, at, overhead, jobid, fluxerr)
 
-	fmt.Printf("[MatchRPC] Errors so far: %s\n", fluxcli.ReapiCliGetErrMsg(s.fctx))
-	if fluxerr != 0 {
+	fmt.Printf("[MatchRPC] Errors so far: %s\n", s.cli.GetErrMsg())
+	if fluxerr != nil {
 		return nil, errors.New("Error in ReapiCliMatchAllocate")
 	}
 
@@ -93,12 +89,12 @@ func (s *Fluxion) Match(ctx context.Context, in *pb.MatchRequest) (*pb.MatchResp
 	}
 
 	nodetasks := utils.ParseAllocResult(allocated)
-	
+
 	nodetaskslist := make([]*pb.NodeAlloc, len(nodetasks))
 	for i, result := range nodetasks {
-		nodetaskslist[i] = &pb.NodeAlloc {
+		nodetaskslist[i] = &pb.NodeAlloc{
 			NodeID: result.Basename,
-			Tasks: int32(result.CoreCount)/in.Ps.Cpu,
+			Tasks:  int32(result.CoreCount) / in.Ps.Cpu,
 		}
 	}
 	mr := &pb.MatchResponse{PodID: in.Ps.Id, Nodelist: nodetaskslist, JobID: int64(jobid)}
diff --git a/src/fluence/go.mod b/src/fluence/go.mod
index 88808d4..5a14548 100644
--- a/src/fluence/go.mod
+++ b/src/fluence/go.mod
@@ -3,7 +3,7 @@ module github.com/flux-framework/flux-k8s/flux-plugin/fluence
 go 1.16
 
 require (
-	github.com/cmisale/flux-sched/resource/hlapi/bindings/go/src/fluxcli v0.0.0-20220921153849-5b631bfccecf
+	github.com/flux-framework/flux-sched/resource/reapi/bindings/go v0.0.0-20231213021445-250eac78a675
 	google.golang.org/grpc v1.38.0
 	google.golang.org/protobuf v1.26.0
 	gopkg.in/yaml.v2 v2.4.0
diff --git a/src/fluence/go.sum b/src/fluence/go.sum
index 22d523f..19e571c 100644
--- a/src/fluence/go.sum
+++ b/src/fluence/go.sum
@@ -64,8 +64,6 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/cmisale/flux-sched/resource/hlapi/bindings/go/src/fluxcli v0.0.0-20220915175752-563ad82d6cb3 h1:iyAYjW/7RZeGXViInC6xKXalgwXdfat/iwcXlGpO42c=
-github.com/cmisale/flux-sched/resource/hlapi/bindings/go/src/fluxcli v0.0.0-20220915175752-563ad82d6cb3/go.mod h1:YECCCsYTFtX4YCub9nBnvEmNj319mPNS8ERpvpKH1nU=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
@@ -100,6 +98,8 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZM
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/flux-framework/flux-sched/resource/reapi/bindings/go v0.0.0-20231213021445-250eac78a675 h1:FgEA3pnL/kDoLaVOUDa401yainApQJaow9jeBPg4dek=
+github.com/flux-framework/flux-sched/resource/reapi/bindings/go v0.0.0-20231213021445-250eac78a675/go.mod h1:yhmzNyn45YhoxEohh1Sl3h3izLMqL7qpcvmYTRpv7eY=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
diff --git a/src/fluence/utils/utils.go b/src/fluence/utils/utils.go
index 452c025..2d6d932 100644
--- a/src/fluence/utils/utils.go
+++ b/src/fluence/utils/utils.go
@@ -3,19 +3,20 @@ package utils
 import (
 	"context"
 	"fmt"
+
 	// "strings"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	corev1 "k8s.io/api/core/v1"
-	"github.com/flux-framework/flux-k8s/flux-plugin/fluence/jgf"
 	"encoding/json"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/kubernetes"
+
+	"github.com/flux-framework/flux-k8s/flux-plugin/fluence/jgf"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	resourcehelper "k8s.io/kubectl/pkg/util/resource"
-	"k8s.io/apimachinery/pkg/api/resource"
 )
 
-
 func CreateJGF(filename string, label *string) error {
 	ctx := context.Background()
 	config, err := rest.InClusterConfig()
@@ -30,7 +31,7 @@ func CreateJGF(filename string, label *string) error {
 		return err
 	}
 	nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
-	
+
 	var fluxgraph jgf.Fluxjgf
 	fluxgraph = jgf.InitJGF()
 	// subnets := make(map[string]string)
@@ -53,11 +54,11 @@ func CreateJGF(filename string, label *string) error {
 		if *label != "" {
 			_, fluxnode := node.Labels[*label]
 			if !fluxnode {
-				fmt.Println("Skipping node ",  node.GetName())
+				fmt.Println("Skipping node ", node.GetName())
 				continue
 			}
-		} 
-			
+		}
+
 		fmt.Println("node in flux group ", node.GetName())
 		if !node.Spec.Unschedulable {
 			fieldselector, err := fields.ParseSelector("spec.nodeName=" + node.GetName() + ",status.phase!=" + string(corev1.PodSucceeded) + ",status.phase!=" + string(corev1.PodFailed))
@@ -75,24 +76,22 @@ func CreateJGF(filename string, label *string) error {
 			// Here we build subnets according to topology.kubernetes.io/zone label
 			subnetName := node.Labels["topology.kubernetes.io/zone"]
 			subnet := fluxgraph.MakeSubnet(sdnCount, subnetName)
-			sdnCount = sdnCount+1
+			sdnCount = sdnCount + 1
 			fluxgraph.MakeEdge(cluster, subnet, "contains")
 			fluxgraph.MakeEdge(subnet, cluster, "in")
-	
-		
+
 			reqs := computeTotalRequests(pods)
 			cpuReqs := reqs[corev1.ResourceCPU]
 			memReqs := reqs[corev1.ResourceMemory]
-			
+
 			avail := node.Status.Allocatable.Cpu().MilliValue()
-			totalcpu := int64((avail-cpuReqs.MilliValue())/1000) //- 1
+			totalcpu := int64((avail - cpuReqs.MilliValue()) / 1000) //- 1
 			fmt.Println("Node ", node.GetName(), " flux cpu ", totalcpu)
-			totalAllocCpu = totalAllocCpu+totalcpu
+			totalAllocCpu = totalAllocCpu + totalcpu
 			totalmem = node.Status.Allocatable.Memory().Value() - memReqs.Value()
 			fmt.Println("Node ", node.GetName(), " total mem ", totalmem)
 			gpuAllocatable, hasGpuAllocatable := node.Status.Allocatable["nvidia.com/gpu"]
 
-			
 			// reslist := node.Status.Allocatable
 			// resources := make([]corev1.ResourceName, 0, len(reslist))
 			// for resource := range reslist {
@@ -101,28 +100,27 @@ func CreateJGF(filename string, label *string) error {
 			// }
 			// for _, resource := range resources {
 			// 	value := reslist[resource]
-				
+
 			// 	fmt.Printf(" %s:\t%s\n", resource, value.String())
 			// }
 
-
 			workernode := fluxgraph.MakeNode(node_index, false, node.Name)
 			fluxgraph.MakeEdge(subnet, workernode, "contains") // this is rack otherwise
-			fluxgraph.MakeEdge(workernode, subnet, "in") // this is rack otherwise
+			fluxgraph.MakeEdge(workernode, subnet, "in")       // this is rack otherwise
 
 			// socket := fluxgraph.MakeSocket(0, "socket")
 			// fluxgraph.MakeEdge(workernode, socket, "contains")
 			// fluxgraph.MakeEdge(socket, workernode, "in")
-			
+
 			if hasGpuAllocatable {
 				fmt.Println("GPU Resource quantity ", gpuAllocatable.Value())
 				//MakeGPU(index int, name string, size int) string {
-				for index :=0; index < int(gpuAllocatable.Value()); index++ {
+				for index := 0; index < int(gpuAllocatable.Value()); index++ {
 					gpu := fluxgraph.MakeGPU(index, "nvidiagpu", 1)
 					fluxgraph.MakeEdge(workernode, gpu, "contains") // workernode was socket
 					fluxgraph.MakeEdge(gpu, workernode, "in")
 				}
-				
+
 			}
 
 			for index := 0; index < int(totalcpu); index++ {
@@ -145,7 +143,7 @@ func CreateJGF(filename string, label *string) error {
 			fractionmem := totalmem >> 30
 			// fractionmem := (totalmem/totalcpu) >> 20
 			// fmt.Println("Creating ", fractionmem, " vertices with ", 1<<10, " MB of mem")
-			for i:=0; i < /*int(totalcpu)*/int(fractionmem); i++ {
+			for i := 0; i < /*int(totalcpu)*/ int(fractionmem); i++ {
 				mem := fluxgraph.MakeMemory(i, "memory", "MB", int(1<<10))
 				fluxgraph.MakeEdge(workernode, mem, "contains")
 				fluxgraph.MakeEdge(mem, workernode, "in")
@@ -185,32 +183,31 @@ func computeTotalRequests(podList *corev1.PodList) (total map[corev1.ResourceNam
 	return
 }
 
-
 type allocation struct {
-	Type 		string
-	Name 		string
-	Basename 	string
-	CoreCount	int
+	Type      string
+	Name      string
+	Basename  string
+	CoreCount int
 }
 
-func ParseAllocResult(allocated string) []allocation{
+func ParseAllocResult(allocated string) []allocation {
 	var dat map[string]interface{}
 	result := make([]allocation, 0)
 	corecount := 0
 	if err := json.Unmarshal([]byte(allocated), &dat); err != nil {
-        panic(err)
-    }
+		panic(err)
+	}
 	// fmt.Println("PRINTING DATA:\n", dat)
 	// graph := dat["graph"]
 	// fmt.Println("GET GRAPH:\n ", graph)
 	nodes := dat["graph"].(interface{})
-	str1 := nodes.(map[string]interface {})
-    // fmt.Println("GET NODES:\n", str1["nodes"])
-	str2 := str1["nodes"].([]interface {})
+	str1 := nodes.(map[string]interface{})
+	// fmt.Println("GET NODES:\n", str1["nodes"])
+	str2 := str1["nodes"].([]interface{})
 	// fmt.Println("NODES:\n", len(str2))
 	for _, item := range str2 {
 		// fmt.Println("ITEM: ", item)
-		str1 = item.(map[string]interface {})
+		str1 = item.(map[string]interface{})
 		metadata := str1["metadata"].(map[string]interface{})
 		// fmt.Println("TYPE: ", metadata["type"])
 		if metadata["type"].(string) == "core" {
@@ -219,10 +216,10 @@ func ParseAllocResult(allocated string) []allocation{
 		// fmt.Println("BASENAME: ", metadata["basename"])
 		if metadata["type"].(string) == "node" {
 			result = append(result, allocation{
-				Type : metadata["type"].(string),
-				Name : metadata["name"].(string),
-				Basename : metadata["basename"].(string),
-				CoreCount : corecount,
+				Type:      metadata["type"].(string),
+				Name:      metadata["name"].(string),
+				Basename:  metadata["basename"].(string),
+				CoreCount: corecount,
 			})
 			corecount = 0
 			// result.Type = metadata["type"].(string)
@@ -235,8 +232,8 @@ func ParseAllocResult(allocated string) []allocation{
 	return result
 }
 
-////// Utility functions
-func PrintOutput(reserved bool, allocated string, at int64, overhead float64, jobid uint64, fluxerr int) {
+// //// Utility functions
+func PrintOutput(reserved bool, allocated string, at int64, overhead float64, jobid uint64, fluxerr error) {
 	fmt.Println("\n\t----Match Allocate output---")
-	fmt.Printf("jobid: %d\nreserved: %t\nallocated: %s\nat: %d\noverhead: %f\nerror: %d\n", jobid, reserved, allocated, at, overhead, fluxerr)
-}
\ No newline at end of file
+	fmt.Printf("jobid: %d\nreserved: %t\nallocated: %s\nat: %d\noverhead: %f\nerror: %w\n", jobid, reserved, allocated, at, overhead, fluxerr)
+}