Merge pull request #40 from CanDIG/daisieh/vault-opa

DIG-1169, DIG-1402: Opa uses Vault for secret storage

Dockerfile

@@ -1,32 +1,33 @@
 ARG venv_python
-ARG alpine_version
-FROM python:${venv_python}-alpine${alpine_version}
+FROM python:${venv_python}
 
 LABEL Maintainer="CanDIG Project"
 LABEL "candigv2"="opa"
 
 USER root
 
-RUN addgroup -S candig && adduser -S candig -G candig
+RUN groupadd -r candig && useradd -rm candig -g candig
 
-RUN apk update
-
-RUN apk add --no-cache \
+RUN apt-get update && apt-get -y install \
 	bash \
 	expect \
 	jq \
-	curl
+	curl \
+	vim \
+	git
 
-COPY ./ /app/
+COPY requirements.txt /app/requirements.txt
 
 RUN pip install --no-cache-dir -r /app/requirements.txt
 
-WORKDIR /app/
+COPY ./ /app/
 
 RUN chown -R candig:candig /app
 
 USER candig
 
+WORKDIR /app/
+
 RUN touch /app/initial_setup
 
 ENTRYPOINT ["bash", "/app/entrypoint.sh"]

entrypoint.sh

@@ -2,21 +2,38 @@
 
 set -Euo pipefail
 
+OPA_ROOT_TOKEN=$(cat /run/secrets/opa-root-token)
+
 if [[ -f "/app/initial_setup" ]]; then
     sed -i s/CLIENT_ID/$KEYCLOAK_CLIENT_ID/ /app/permissions_engine/idp.rego && sed -i s/CLIENT_ID/$KEYCLOAK_CLIENT_ID/ /app/permissions_engine/authz.rego
     sed -i s/OPA_SITE_ADMIN_KEY/$OPA_SITE_ADMIN_KEY/ /app/permissions_engine/idp.rego && sed -i s/OPA_SITE_ADMIN_KEY/$OPA_SITE_ADMIN_KEY/ /app/permissions_engine/authz.rego
 
     OPA_SERVICE_TOKEN=$(cat /run/secrets/opa-service-token)
     sed -i s/OPA_SERVICE_TOKEN/$OPA_SERVICE_TOKEN/ /app/permissions_engine/authz.rego
 
-    OPA_ROOT_TOKEN=$(cat /run/secrets/opa-root-token)
     sed -i s/OPA_ROOT_TOKEN/$OPA_ROOT_TOKEN/ /app/permissions_engine/authz.rego
-    echo "initializing idp"
-    python3 /app/permissions_engine/initialize_idp.py
-    rm /app/initial_setup
+
+    echo "initializing stores"
+    python3 /app/initialize_vault_store.py
+    if [[ $? -eq 0 ]]; then
+        rm /app/initial_setup
+        rm /app/bearer.txt
+        echo "setup complete"
+    else
+        echo "!!!!!! INITIALIZATION FAILED, TRY AGAIN !!!!!!"
+    fi
 fi
 
+
 while [ 0 -eq 0 ]
 do
-  sleep 60
+  echo "storing vault token"
+  python3 get_vault_store_token.py
+  if [[ $? -eq 0 ]]; then
+      echo "vault token stored"
+      sleep 300
+  else
+      echo "vault token not stored"
+      sleep 30
+  fi
 done

get_vault_store_token.py

@@ -0,0 +1,22 @@
+import json
+import os
+from authx.auth import get_vault_token_for_service
+import sys
+import requests
+
+
+# get the token for the opa store
+try:
+    with open("/run/secrets/opa-root-token") as f:
+        OPA_ROOT_TOKEN = f.read().strip()
+        opa_token = get_vault_token_for_service("opa")
+        headers = {
+            "X-Opa": OPA_ROOT_TOKEN,
+            "Content-Type": "application/json; charset=utf-8"
+        }
+        payload = f"{{\"token\": \"{opa_token}\"}}"
+        response = requests.put(url=f"{os.getenv('OPA_URL')}/v1/data/store_token", headers=headers, data=payload)
+        print(response.text)
+except Exception as e:
+    print(str(e))
+    sys.exit(1)

initialize_vault_store.py

@@ -0,0 +1,39 @@
+import json
+import os
+from authx.auth import set_service_store_secret, add_provider_to_opa
+import sys
+
+## Initializes Vault's opa service store with the information for our IDP and the data in access.json and paths.json
+
+results = []
+
+try:
+    with open('/app/bearer.txt') as f:
+        try:
+            token = f.read().strip()
+            response, status_code = set_service_store_secret("opa", key="data", value=json.dumps({"keys":[]}))
+            response = add_provider_to_opa(token, os.getenv("KEYCLOAK_REALM_URL"))
+            results.append(response)
+        except Exception as e:
+            print(str(e))
+            sys.exit(1)
+
+    with open('/app/permissions_engine/access.json') as f:
+        data = f.read()
+        response, status_code = set_service_store_secret("opa", key="access", value=data)
+        if status_code != 200:
+            sys.exit(2)
+        results.append(response)
+
+    with open('/app/permissions_engine/paths.json') as f:
+        data = f.read()
+        response, status_code = set_service_store_secret("opa", key="paths", value=data)
+        if status_code != 200:
+            sys.exit(3)
+        results.append(response)
+except Exception as e:
+    print(str(e))
+    sys.exit(4)
+
+# print(json.dumps(results, indent=4))
+sys.exit(0)

permissions_engine/authz.rego

@@ -54,10 +54,13 @@ allow {
     decode_verify_token_output[_][2].realm_access.roles[_] == "OPA_SITE_ADMIN_KEY"
 }
 
+import data.store_token.token as token
+keys = http.send({"method": "get", "url": "http://vault:8200/v1/opa/data", "headers": {"X-Vault-Token": token}}).body.data.keys
+
 decode_verify_token_output[issuer] := output {
     some i
-    issuer := data.keys[i].iss
-    cert := data.keys[i].cert
+    issuer := keys[i].iss
+    cert := keys[i].cert
     output := io.jwt.decode_verify(     # Decode and verify in one-step
         input.identity,
         {                         # With the supplied constraints:

permissions_engine/idp.rego

@@ -4,10 +4,14 @@ package idp
 #
 # Store decode and verified token
 #
+
+import data.store_token.token as token
+keys = http.send({"method": "get", "url": "http://vault:8200/v1/opa/data", "headers": {"X-Vault-Token": token}}).body.data.keys
+
 decode_verify_token_output[issuer] := output {
     some i
-    issuer := data.keys[i].iss
-    cert := data.keys[i].cert
+    issuer := keys[i].iss
+    cert := keys[i].cert
     output := io.jwt.decode_verify(     # Decode and verify in one-step
         input.token,
         {                         # With the supplied constraints:

permissions_engine/permissions.rego

@@ -1,17 +1,22 @@
 package permissions
 #
 # This is the set of policy definitions for the permissions engine.
-# 
+#
 
 default datasets = []
 
-get_input_paths = data.paths.get
-post_input_paths = data.paths.post
+import data.store_token.token as token
+access = http.send({"method": "get", "url": "http://vault:8200/v1/opa/access", "headers": {"X-Vault-Token": token}}).body.data.access
+
+paths = http.send({"method": "get", "url": "http://vault:8200/v1/opa/paths", "headers": {"X-Vault-Token": token}}).body.data.paths
+
+get_input_paths = paths.get
+post_input_paths = paths.post
 
 #
-# Provided: 
+# Provided:
 # input = {
-#     'token': user token 
+#     'token': user token
 #     'method': method requested at data service
 #     'path': path to request at data service
 # }
@@ -27,7 +32,7 @@ import data.idp.email
 
 default registered_allowed = []
 
-registered_allowed = data.access.registered_datasets {
+registered_allowed = access.registered_datasets {
     valid_token         # extant, valid token
     trusted_researcher  # has claim we're using for registered access
 }
@@ -38,7 +43,7 @@ registered_allowed = data.access.registered_datasets {
 
 default controlled_allowed = []
 
-controlled_allowed = data.access.controlled_access_list[email]{
+controlled_allowed = access.controlled_access_list[email]{
     valid_token                  # extant, valid token
 }
 
@@ -47,20 +52,20 @@ controlled_allowed = data.access.controlled_access_list[email]{
 #
 
 # allowed datasets
-datasets = array.concat(array.concat(data.access.open_datasets, registered_allowed), controlled_allowed)
+datasets = array.concat(array.concat(access.open_datasets, registered_allowed), controlled_allowed)
 {
     input.body.method = "GET"
     regex.match(get_input_paths[_], input.body.path) == true
 }
 
-datasets = array.concat(array.concat(data.access.open_datasets, registered_allowed), controlled_allowed)
+datasets = array.concat(array.concat(access.open_datasets, registered_allowed), controlled_allowed)
 {
     input.body.method = "POST"
     regex.match(post_input_paths[_], input.body.path) == true
 }
 
 # allowed datasets for counting
-datasets = array.concat(data.access.open_datasets, data.access.opt_in_datasets) {
+datasets = array.concat(access.open_datasets, access.opt_in_datasets) {
     valid_token == true
     input.method = "GET"
     regex.match(get_input_paths[_], input.body.path) == true

requirements.txt

@@ -1 +1,3 @@
-requests
+requests
+jq
+candigv2-authx@git+https://github.com/CanDIG/[email protected]