Feature / Simplify working with secrets (#229)

* New tasks in auth tool to add and remove basic secrets

* Change the component description for the auth tool

* Rename secret tool in code

* Rename secret tool project

* Use secret-tool in integration workflow

* Set main class in secret tool project

* Use secrets for storage and metadb sensitive config in CI

* Bump netty version for compliance

* Allow console input when running as a Gradle task in CI

* Handle secrets for metadata DAL

* Fix setup of secrets in integration workflow

* Handle secrets for metadb deploy tool

* Handle secrets for DAL-level integration tests

* Remove properties prefix in JDBC unit tests

* Remove properties prefix for JDBC setup

* Make JDBC setup accept plugin config as well as processed properties

* Make JDBC setup accept plugin config as well as processed properties

* Make metadata DAL a plugin, so it can use the regular plugin loader

* Put start and stop methods into JDBC metadata DAL

* Print a warning in plugin manager if a plugin fails to load

* Use the new DAL plugin to load the DAL in the metadata service

* Use the new JDBC setup in the DB deploy tool (respects plugin secrets)

* Use new DAL setup where required in test code

* Always process resources in meta svc build (to make the DAL plugin available)

* Move design principles into the dev folder (doc is for published docs)

* Fix JDBC integration tests after updates

* Update references to secret-tool in the sandbox guide

* Add a documentation page on using secrets

* Update authentication docs to better reference the secret-tool

* Update storage docs to better reference the secret-tool

* Update metadata store docs to better reference the secret-tool

* Add a note about securing the H2 password in the sandbox quick start guide

* Set test file path for HTTP/1 proxy test (was using old doc file that was moved)

.github/config/int-metadb-mariadb.yaml

@@ -36,9 +36,10 @@ metadata:
       dialect: MARIADB
       jdbcUrl: //metadb:3306/trac
       mariadb.user: trac_admin
-      mariadb.password: trac_admin
       pool.size: 2
       pool.overflow: 5
+    secrets:
+      mariadb.password: metadb_secret
 
 instances:
   meta:

.github/config/int-metadb-mysql.yaml

@@ -36,9 +36,10 @@ metadata:
       dialect: MYSQL
       jdbcUrl: //metadb:3306/trac
       mysql.user: trac_admin
-      mysql.password: trac_admin
       pool.size: 2
       pool.overflow: 5
+    secrets:
+      mysql.password: metadb_secret
 
 instances:
   meta:

.github/config/int-metadb-postgresql.yaml

@@ -36,9 +36,10 @@ metadata:
       dialect: POSTGRESQL
       jdbcUrl: //metadb:5432/trac
       postgresql.user: trac_admin
-      postgresql.password: trac_admin
       pool.size: 2
       pool.overflow: 5
+    secrets:
+      postgresql.password: metadb_secret
 
 instances:
   meta:

.github/config/int-metadb-sqlserver.yaml

@@ -37,9 +37,10 @@ metadata:
       jdbcUrl: //metadb:1433
       sqlserver.databaseName: master
       sqlserver.user: sa
-      sqlserver.password: tR4c_aDm!n
       pool.size: 2
       pool.overflow: 5
+    secrets:
+      sqlserver.password: metadb_secret
 
 instances:
   meta:

.github/config/int-storage-s3.yaml

@@ -58,7 +58,8 @@ storage:
         bucket: ${TRAC_AWS_BUCKET}
         path: int-storage-s3
         accessKeyId: ${TRAC_AWS_ACCESS_KEY_ID}
-        secretAccessKey: ${TRAC_AWS_SECRET_ACCESS_KEY}
+      secrets:
+        secretAccessKey: storage_secret
 
 
 repositories:

.github/workflows/integration.yml

@@ -79,7 +79,8 @@ jobs:
           sed -i "s#/path/to#${deploy_dir}#g" etc/trac-platform.yaml
           sed -i '/jwtExpiry: 86400/a \ \ disableAuth: true' etc/trac-platform.yaml
           
-          bin/auth-tool run --task create_root_auth_key EC 256
+          bin/secret-tool run --task init_secrets
+          bin/secret-tool run --task create_root_auth_key EC 256
           bin/deploy-metadb run --task deploy_schema
           bin/deploy-metadb run --task add_tenant ACME_CORP "ACME Rockets always get the Coyote!"
 
@@ -241,6 +242,7 @@ jobs:
               MYSQL_DATABASE: trac,
               MYSQL_USER: trac_admin,
               MYSQL_PASSWORD: trac_admin,
+              METADB_SECRET: trac_admin,
               MYSQL_ALLOW_EMPTY_PASSWORD: yes }
 
           - { DB_NAME: MariaDB,
@@ -253,6 +255,7 @@ jobs:
               MYSQL_DATABASE: trac,
               MYSQL_USER: trac_admin,
               MYSQL_PASSWORD: trac_admin,
+              METADB_SECRET: trac_admin,
               MYSQL_ALLOW_EMPTY_PASSWORD: yes }
 
           - { DB_NAME: PostgreSQL,
@@ -264,6 +267,7 @@ jobs:
               TRAC_SECRET_KEY: hjXks83bX=wxMr,
               POSTGRES_DB: trac,
               POSTGRES_USER: trac_admin,
+              METADB_SECRET: trac_admin,
               POSTGRES_PASSWORD: trac_admin }
 
           - { DB_NAME: SQLServer,
@@ -275,7 +279,8 @@ jobs:
               TRAC_SECRET_KEY: unHkj>weN2jSl,
               MSSQL_PID: Developer,
               ACCEPT_EULA: Y,
-              SA_PASSWORD: "tR4c_aDm!n" }
+              SA_PASSWORD: "tR4c_aDm!n",
+              METADB_SECRET: "tR4c_aDm!n" }
 
     env: ${{ matrix.database }}
 
@@ -308,24 +313,24 @@ jobs:
         run: ./gradlew trac-svc-meta:testClasses
 
       # Auth tool will also create the secrets file if it doesn't exist
-      - name: Prepare Auth Keys
+      - name: Prepare secrets
         run: |
-          ./gradlew auth-tool:run --args="\
-          --config ${{ env.TRAC_CONFIG_FILE }} \
-          --secret-key ${{ env.TRAC_SECRET_KEY }} \
-          --task create_root_auth_key EC 256"
+          ./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task init_secrets"
+          ./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task create_root_auth_key EC 256"
+          
+          echo "${METADB_SECRET}" | ./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task add_secret metadb_secret"
 
       # The name and description of the test tenant are verified in one of the test cases so they need to match
       # MetadataReapApiTest listTenants()
-      - name: Prepare Database
+      - name: Prepare database
         run: |
           ./gradlew deploy-metadb:run --args="\
           --config ${{ env.TRAC_CONFIG_FILE }} \
           --secret-key ${{ env.TRAC_SECRET_KEY }} \
           --task deploy_schema \
           --task add_tenant ACME_CORP 'Test tenant [ACME_CORP]'"
 
-      - name: Integration Tests
+      - name: Integration tests
         run: ./gradlew trac-svc-meta:integration -DintegrationTags="int-metadb"
 
       # If the tests fail, make the output available for download
@@ -360,7 +365,8 @@ jobs:
               BUILD_aws_storage: true,
               TRAC_CONFIG_FILE: '.github/config/int-storage-s3.yaml',
               TRAC_SECRET_KEY: storage_s3_secrets,
-              S3_BUCKET: not-configured }
+              S3_BUCKET: not-configured,
+              STORGE_SECRET: "not_configured" }
 
 
     env: ${{ matrix.storage }}
@@ -384,16 +390,16 @@ jobs:
         run: ./gradlew testClasses
 
       # Auth tool will also create the secrets file if it doesn't exist
-      - name: Prepare Auth Keys
+      - name: Prepare secrets
         run: |
-          ./gradlew auth-tool:run --args="\
-          --config ${{ env.TRAC_CONFIG_FILE }} \
-          --secret-key ${{ env.TRAC_SECRET_KEY }} \
-          --task create_root_auth_key EC 256"
+          ./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task init_secrets"
+          ./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task create_root_auth_key EC 256"
+          
+          echo "${STORAGE_SECRET}" | ./gradlew secret-tool:run --args="--config ${{ env.TRAC_CONFIG_FILE }} --task add_secret storage_secret"
 
       # No need to prepare DB, it is done by the platform test setup
 
-      - name: Integration Tests
+      - name: Integration tests
         run: ./gradlew integration -DintegrationTags="int-storage"
 
       # If the tests fail, make the output available for download

doc/design_principals.md → dev/design_principals.md

@@ -15,6 +15,8 @@ choices. Suggestions / discussion are very welcome!
 
 3. Simplicity (should always be on the list)!
 
+   * As simple as possible, but no simpler
+
 4. Prefer open standards
 
 5. Minimise dependencies

doc/deployment/authentication.rst

@@ -29,7 +29,8 @@ the platform and gateway config files.
 You will also need to add an authentication block in both config files, specifying the issuer
 and expiry times for JWT tokens. If you know the DNS address that TRAC will be served from you
 could use this as the JWT issuer, other options could be the user ID of a service account you
-have set up to run TRAC, or a TRAC reserved identifier such as "trac_system".
+have set up to run TRAC, or a TRAC reserved identifier such as "trac_system". The root authentication
+key is stored in the secret store, so make sure you have a secret store configured too.
 
 .. code-block:: yaml
 
@@ -41,13 +42,12 @@ have set up to run TRAC, or a TRAC reserved identifier such as "trac_system".
       jwtIssuer: http://localhost:8080/
       jwtExpiry: 7200
 
-The auth-tool utility can be used to generate the root signing key, it will be written into the
-secret store. The available key types are elliptic curve (EC) or RSA. Elliptic curve keys are
-considered to give better security with better performance at lower key sizes. For this reason
-we recommended EC 256 keys.
+The secret-tool utility can be used to generate the root signing key. The available key types are
+elliptic curve (EC) or RSA. Elliptic curve keys are considered to give better security with better
+performance at lower key sizes. For this reason we recommended EC 256 keys.
 
-Make sure you have set the *TRAC_SECRET_KEY* environment variable before running *auth_tool*. For
-sandbox deployments, this can be set in *etc/env.sh* (or *etc\\env.bat* on Windows).
+Make sure you have initialized the secret store and set the *TRAC_SECRET_KEY* environment variable
+before running *secret-tool*. For more details on the *secret-tool*, see :doc:`secrets`.
 
 .. tab-set::
 
@@ -57,15 +57,15 @@ sandbox deployments, this can be set in *etc/env.sh* (or *etc\\env.bat* on Windo
         .. code-block:: shell
 
             cd /opt/trac/current
-            bin/auth-tool run --task create_root_auth_key EC 256
+            bin/secret-tool run --task create_root_auth_key EC 256
 
     .. tab-item:: Windows
         :sync: platform_windows
 
         .. code-block:: batch
 
             cd /d C:\trac\tracdap-sandbox-<version>
-            bin\auth-tool.bat run --task create_root_auth_key EC 256
+            bin\secret-tool.bat run --task create_root_auth_key EC 256
 
 .. note::
     Running the *create_root_auth_key* command a second time will replace the root authentication key,
@@ -109,7 +109,7 @@ these settings into the *config* section of the gateway config file.
       users.url: local_users.p12
       users.key: local_users_key
 
-You will need to initialize the user database and add at least one user. The *auth-tool* utility will let
+You will need to initialize the user database and add at least one user. The *secret-tool* utility will let
 you do this. The add_user command is interactive and will ask for details to create a user. You can remove
 users later using the *delete_user* command.
 
@@ -121,21 +121,21 @@ users later using the *delete_user* command.
         .. code-block:: shell
 
             cd /opt/trac/current
-            bin/auth-tool run --task init_trac_users
-            bin/auth-tool run --task add_user
+            bin/secret-tool run --task init_trac_users
+            bin/secret-tool run --task add_user
 
-            bin/auth-tool run --task delete_user <user_id>
+            bin/secret-tool run --task delete_user <user_id>
 
     .. tab-item:: Windows
         :sync: platform_windows
 
         .. code-block:: batch
 
             cd /d C:\trac\tracdap-sandbox-<version>
-            bin\auth-tool.bat run --task init_trac_users
-            bin\auth-tool.bat run --task add_user
+            bin\secret-tool.bat run --task init_trac_users
+            bin\secret-tool.bat run --task add_user
 
-            bin\auth-tool.bat run --task delete_user <user_id>
+            bin\secret-tool.bat run --task delete_user <user_id>
 
 Once the user database is created you can enable the basic authentication provider. To do this,
 replace the provider section in the authentication block of the gateway config file and set the

doc/deployment/index.rst

@@ -14,6 +14,7 @@ Deployment
 
     sandbox
     platform
+    secrets
     metadata_store
     authentication
     storage

doc/deployment/metadata_store.rst

@@ -5,10 +5,20 @@ Metadata Store
 To use a different SQL backend, get the JDBC driver JAR for your chosen database and copy it into the
 *plugins* folder of the metadata service installation. Then edit the main configuration file
 (usually trac-platform.yaml) with the correct settings from below.
-Pool size and overflow options are always available, the values below are examples only.
 
-Oracle support is available but not actively tested in CI due to licensing issues. If you would like support for
-a different SQL dialect, please `get in touch <https://github.com/finos/tracdap/issues>`_.
+Properties for the JDBC datasource are passed directly to the JDBC driver. For example, if you are using
+the SQLSERVER dialect, then *sqlserver.databaseName* will be passed to the driver as the *databaseName*,
+*sqlserver.user* will be passed as *user* and so on. The properties supported by each driver are described
+in their respective documentation pages. Pool size and overflow options are available for all dialects.
+
+For database accounts secured with passwords, you may want to use the TRAC secrets configuration mechanism
+which allows any property to be stored as a secret in the secret store. For details on setting up secrets
+see :doc:`secrets`. In a cloud deployments it may be possible to use service roles, which removes the need
+to keep credentials in the configuration altogether. Other authentication mechanisms such as Kerberos are
+also possible and can be achieved by setting the correct JDBC properties, environment variables,
+Java properties etc. If you have a particular requirement that does not work through the standard mechanisms,
+please `get in touch <https://github.com/finos/tracdap/issues>`_.
+
 
 **H2 configuration**
 
@@ -27,6 +37,10 @@ a different SQL dialect, please `get in touch <https://github.com/finos/tracdap/
           pool.size: 10
           pool.overflow: 5
 
+.. note::
+    H2 is mostly used in development scenarios where the password is not sensitive.
+    If you want to use a secret for the H2 password, you can!
+
 **MySQL configuration**
 
 .. code-block:: yaml
@@ -39,9 +53,10 @@ a different SQL dialect, please `get in touch <https://github.com/finos/tracdap/
           dialect: MYSQL
           jdbcUrl: //<host>:<port>/<database>
           mysql.user: <username>
-          mysql.password: <password>
           pool.size: 10
           pool.overflow: 5
+        secrets:
+          mysql.password: metadb_password
 
 **MariaDB configuration**
 
@@ -55,9 +70,10 @@ a different SQL dialect, please `get in touch <https://github.com/finos/tracdap/
           dialect: MARIADB
           jdbcUrl: //<host>:<port>/<database>
           mariadb.user: <username>
-          mariadb.password: <password>
           pool.size: 10
           pool.overflow: 5
+        secrets:
+          mariadb.password: metadb_password
 
 **PostgreSQL configuration**
 
@@ -71,9 +87,10 @@ a different SQL dialect, please `get in touch <https://github.com/finos/tracdap/
           dialect: POSTGRESQL
           jdbcUrl: //<host>:<port>/<database>
           postgresql.user: <username>
-          postgresql.password: <password>
           pool.size: 10
           pool.overflow: 5
+        secrets:
+          postgresql.password: metadb_password
 
 **SQL Server configuration**
 
@@ -88,7 +105,11 @@ a different SQL dialect, please `get in touch <https://github.com/finos/tracdap/
           jdbcUrl: //<host>:<port>
           sqlserver.databaseName: <database>
           sqlserver.user: <username>
-          sqlserver.password: <password>
           pool.size: 10
           pool.overflow: 5
+        secrets:
+          sqlserver.password: metadb_password
 
+.. note::
+    Oracle support is available but not actively tested in CI due to licensing issues. If you would like support for
+    a different SQL dialect, please `get in touch <https://github.com/finos/tracdap/issues>`_.