Fix / Executor start-up sequence (#294)

* Update plugin manager API

* Use update load API for storage plugins

* Update platform plugin API to distinguish between regular and config plugins, regular plugins get config manager on load

* Update config manager for changes to plugin API

* Update all plugins to the new loading API

* Update tests for config plugins

* Pass config manager into data formats (format plugins require config manager)

* Change default service not found message in base TracPlugin

* Minor refactor in plugin manager

* Put start/top methods in executor plugin API

* Start / stop methods for local executor

* Call executor start/stop in orchestrator service

* Do not try to set event loop group for ssh executor (we need to know what it's doing)

* Reduce executor polling interval to limit race conditions

* Decrease polling interval for local executor

* Do not audit dev-only dependencies with npm audit

tracdap-api/packages/web/package.json

@@ -40,7 +40,7 @@
     "buildApi": "node api_builder.js",
     "compliance-owasp": "owasp-dependency-check --project \"tracdap-web-api\" --suppression ../../../dev/compliance/owasp-false-positives.xml --failOnCVSS 4 --data ../../../build/compliance-cache/nvd_web_api --out ../../../build/compliance/web-api-owasp/",
     "compliance-licenses": "license-checker-rseidelsohn --onlyAllow \"MIT; Apache-2.0; BSD; BSD-3-Clause; ISC; BSD-2-Clause; Unlicense; WTFPL; Python-2.0\" --excludePackages \"spdx-exceptions;spdx-license-ids\" --json --out ../../../build/compliance/web-api-licenses/license-report.json",
-    "compliance-audit": "npm audit > ../../../build/compliance/web-api-npm-audit/npm-audit.txt",
+    "compliance-audit": "npm audit --omit dev > ../../../build/compliance/web-api-npm-audit/npm-audit.txt",
     "pbtsTask": "pbts -o tracdap.d.ts tracdap.js"
   }
 }

tracdap-libs/tracdap-lib-auth/src/main/java/org/finos/tracdap/common/auth/external/common/CommonAuthPlugin.java

@@ -17,7 +17,8 @@
 package org.finos.tracdap.common.auth.external.common;
 
 import org.finos.tracdap.common.auth.external.IAuthProvider;
-import org.finos.tracdap.common.exception.EUnexpected;
+import org.finos.tracdap.common.config.ConfigManager;
+import org.finos.tracdap.common.exception.EPluginNotAvailable;
 import org.finos.tracdap.common.plugin.PluginServiceInfo;
 import org.finos.tracdap.common.plugin.TracPlugin;
 
@@ -46,14 +47,15 @@ public List<PluginServiceInfo> serviceInfo() {
     }
 
     @Override @SuppressWarnings("unchecked")
-    protected <T> T createService(String serviceName, Properties properties) {
+    protected <T> T createService(String serviceName, Properties properties, ConfigManager configManager) {
 
         if (serviceName.equals(GUEST_PROVIDER))
             return (T) new GuestAuthProvider(properties);
 
         if (serviceName.equals(BASIC_PROVIDER))
             return (T) new BasicAuthProvider(properties);
 
-        throw new EUnexpected();
+        var message = String.format("Plugin [%s] does not support the service [%s]", pluginName(), serviceName);
+        throw new EPluginNotAvailable(message);
     }
 }

tracdap-libs/tracdap-lib-common/src/main/java/org/finos/tracdap/common/config/ConfigManager.java

@@ -173,7 +173,7 @@ public URI configRoot() {
      * Allow client code to resolve config files explicitly
      *
      * <p>This is not normally needed by the TRAC services, which read config though this class.
-     * But some of the utility programs can use this method to find config files for updating.
+     * But the TRAC utility programs can use this method to find config files for updating.
      *
      * @param relativePath Relative URL of the config file to resolve
      * @return The resolved absolute URL of the config file
@@ -504,7 +504,7 @@ private IConfigLoader configLoaderForProtocol(String protocol) {
             throw new EConfigLoad(message);
         }
 
-        return plugins.createService(IConfigLoader.class, protocol);
+        return plugins.createConfigService(IConfigLoader.class, protocol, new Properties());
     }
 
     private ISecretLoader secretLoaderForProtocol(String protocol, Map<String, String> configMap) {
@@ -518,7 +518,7 @@ private ISecretLoader secretLoaderForProtocol(String protocol, Map<String, Strin
         }
 
         var secretProps = buildSecretProps(configMap);
-        var secretLoader = plugins.createService(ISecretLoader.class, protocol, secretProps);
+        var secretLoader = plugins.createConfigService(ISecretLoader.class, protocol, secretProps);
         secretLoader.init(this);
 
         return secretLoader;

tracdap-libs/tracdap-lib-common/src/main/java/org/finos/tracdap/common/config/local/LocalConfigPlugin.java

@@ -18,7 +18,7 @@
 
 import org.finos.tracdap.common.config.IConfigLoader;
 import org.finos.tracdap.common.config.ISecretLoader;
-import org.finos.tracdap.common.exception.EUnexpected;
+import org.finos.tracdap.common.exception.EPluginNotAvailable;
 import org.finos.tracdap.common.plugin.PluginServiceInfo;
 import org.finos.tracdap.common.plugin.TracPlugin;
 
@@ -48,14 +48,15 @@ public List<PluginServiceInfo> serviceInfo() {
     }
 
     @Override @SuppressWarnings("unchecked")
-    protected <T> T createService(String serviceName, Properties properties) {
+    protected <T> T createConfigService(String serviceName, Properties properties) {
 
         if (serviceName.equals(FILE_LOADER))
             return (T) new LocalConfigLoader();
 
         if (serviceName.equals(JKS_SECRET_LOADER))
             return (T) new JksSecretLoader(properties);
 
-        throw new EUnexpected();
+        var message = String.format("Plugin [%s] does not support the service [%s]", pluginName(), serviceName);
+        throw new EPluginNotAvailable(message);
     }
 }

tracdap-libs/tracdap-lib-common/src/main/java/org/finos/tracdap/common/plugin/IPluginManager.java

@@ -28,9 +28,11 @@ public interface IPluginManager {
 
     boolean isServiceAvailable(Class<?> serviceClass, String protocol);
 
-    <T> T createService(Class<T> serviceClass, ConfigManager configManager, PluginConfig pluginConfig);
+    <T> T createService(Class<T> serviceClass, PluginConfig pluginConfig, ConfigManager configManager);
 
-    <T> T createService(Class<T> serviceClass, String protocol);
+    <T> T createService(Class<T> serviceClass, String protocol, ConfigManager configManager);
 
-    <T> T createService(Class<T> serviceClass, String protocol, Properties properties);
+    <T> T createConfigService(Class<T> serviceClass, PluginConfig pluginConfig);
+
+    <T> T createConfigService(Class<T> serviceClass, String protocol, Properties properties);
 }

tracdap-libs/tracdap-lib-common/src/main/java/org/finos/tracdap/common/plugin/ITracPlugin.java

@@ -16,8 +16,11 @@
 
 package org.finos.tracdap.common.plugin;
 
+import org.finos.tracdap.common.config.ConfigManager;
+import org.finos.tracdap.config.PluginConfig;
+
 import java.util.List;
-import java.util.Properties;
+
 
 public interface ITracPlugin {
 
@@ -27,5 +30,7 @@ public interface ITracPlugin {
 
     List<String> protocols(Class<?> service);
 
-    <T> T createService(Class<T> serviceClass, String protocol, Properties properties);
+    <T> T createService(Class<T> serviceClass, PluginConfig config, ConfigManager configManager);
+
+    <T> T createConfigService(Class<T> serviceClass, PluginConfig config);
 }

tracdap-libs/tracdap-lib-common/src/main/java/org/finos/tracdap/common/plugin/PluginManager.java

@@ -147,29 +147,49 @@ public boolean isServiceAvailable(Class<?> serviceClass, String protocol) {
     }
 
     @Override
-    public <T> T createService(Class<T> serviceClass, ConfigManager configManager, PluginConfig pluginConfig) {
+    public <T> T createService(Class<T> serviceClass, PluginConfig pluginConfig, ConfigManager configManager) {
 
-        var protocol = pluginConfig.getProtocol();
-        var properties = new Properties();
-        properties.putAll(pluginConfig.getPropertiesMap());
+        var plugin = lookupPlugin(serviceClass, pluginConfig.getProtocol());
 
-        for (var secret : pluginConfig.getSecretsMap().entrySet()) {
-            var secretKey = secret.getKey();
-            var secretValue = configManager.loadPassword(secret.getValue());
-            properties.put(secretKey, secretValue);
-        }
+        return plugin.createService(serviceClass, pluginConfig, configManager);
+    }
+
+    @Override
+    public <T> T createService(Class<T> serviceClass, String protocol, ConfigManager configManager) {
 
-        return createService(serviceClass, protocol, properties);
+        var pluginConfig = PluginConfig.newBuilder()
+                .setProtocol(protocol)
+                .build();
+
+        return createService(serviceClass, pluginConfig, configManager);
     }
 
     @Override
-    public <T> T createService(Class<T> serviceClass, String protocol) {
+    public <T> T createConfigService(Class<T> serviceClass, PluginConfig pluginConfig) {
+
+        var plugin = lookupPlugin(serviceClass, pluginConfig.getProtocol());
 
-        return createService(serviceClass, protocol, new Properties());
+        return plugin.createConfigService(serviceClass, pluginConfig);
     }
 
     @Override
-    public <T> T createService(Class<T> serviceClass, String protocol, Properties properties) {
+    public <T> T createConfigService(Class<T> serviceClass, String protocol, Properties properties) {
+
+        var pluginConfig = PluginConfig.newBuilder()
+                .setProtocol(protocol);
+
+        for (var property: properties.entrySet()) {
+            var key = property.getKey().toString();
+            var value = property.getValue().toString();
+            pluginConfig.putProperties(key, value);
+        }
+
+        var plugin = lookupPlugin(serviceClass, protocol);
+
+        return plugin.createConfigService(serviceClass, pluginConfig.build());
+    }
+
+    private <T> ITracPlugin lookupPlugin(Class<T> serviceClass, String protocol) {
 
         var pluginKey = new PluginKey(serviceClass, protocol);
 
@@ -196,9 +216,7 @@ public <T> T createService(Class<T> serviceClass, String protocol, Properties pr
             throw new EPluginNotAvailable(message);
         }
 
-        var plugin = plugins.get(pluginKey);
-
-        return plugin.createService(serviceClass, protocol, properties);
+        return plugins.get(pluginKey);
     }
 
     private String prettyTypeName(String rawTypeName, boolean caps) {

tracdap-libs/tracdap-lib-common/src/main/java/org/finos/tracdap/common/plugin/TracPlugin.java

@@ -16,14 +16,28 @@
 
 package org.finos.tracdap.common.plugin;
 
+import org.finos.tracdap.common.config.ConfigManager;
+import org.finos.tracdap.common.exception.EPluginNotAvailable;
+import org.finos.tracdap.config.PluginConfig;
+
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
 import java.util.stream.Collectors;
 
 public abstract class TracPlugin implements ITracPlugin {
 
-    protected abstract <T> T createService(String serviceName, Properties properties);
+    protected <T> T createService(String serviceName, Properties properties, ConfigManager configManager) {
+
+        var message = String.format("Plugin [%s] does not support the service [%s]", pluginName(), serviceName);
+        throw new EPluginNotAvailable(message);
+    }
+
+    protected <T> T createConfigService(String serviceName, Properties properties) {
+
+        var message = String.format("Plugin [%s] does not support the service [%s]", pluginName(), serviceName);
+        throw new EPluginNotAvailable(message);
+    }
 
     @Override
     public final List<String> protocols(Class<?> service) {
@@ -43,7 +57,34 @@ public final List<String> protocols(Class<?> service) {
     }
 
     @Override
-    public final <T> T createService(Class<T> serviceClass, String protocol, Properties properties) {
+    public final <T> T createService(Class<T> serviceClass, PluginConfig config, ConfigManager configManager) {
+
+        var serviceInfo = lookupService(serviceClass, config.getProtocol());
+
+        var properties = new Properties();
+        properties.putAll(config.getPropertiesMap());
+
+        for (var secret : config.getSecretsMap().entrySet()) {
+            var secretKey = secret.getKey();
+            var secretValue = configManager.loadPassword(secret.getValue());
+            properties.put(secretKey, secretValue);
+        }
+
+        return createService(serviceInfo.serviceName(), properties, configManager);
+    }
+
+    @Override
+    public final <T> T createConfigService(Class<T> serviceClass, PluginConfig config) {
+
+        var serviceInfo = lookupService(serviceClass, config.getProtocol());
+
+        var properties = new Properties();
+        properties.putAll(config.getPropertiesMap());
+
+        return createConfigService(serviceInfo.serviceName(), properties);
+    }
+
+    private <T> PluginServiceInfo lookupService(Class<T> serviceClass, String protocol) {
 
         var psi = serviceInfo();
 
@@ -54,10 +95,10 @@ public final <T> T createService(Class<T> serviceClass, String protocol, Propert
         if (matchingPsi.isEmpty())
             throw new IllegalArgumentException();
 
-        return createService(matchingPsi.get().serviceName(), properties);
+        return matchingPsi.get();
     }
 
-    protected static boolean psiMatch(PluginServiceInfo psi, Class<?> service, String protocol) {
+    private static boolean psiMatch(PluginServiceInfo psi, Class<?> service, String protocol) {
 
         if (psi.serviceClass() != service)
             return false;

tracdap-libs/tracdap-lib-common/src/test/java/org/finos/tracdap/common/config/test/TestConfigPlugin.java

@@ -17,7 +17,7 @@
 package org.finos.tracdap.common.config.test;
 
 import org.finos.tracdap.common.config.IConfigLoader;
-import org.finos.tracdap.common.exception.EUnexpected;
+import org.finos.tracdap.common.exception.EPluginNotAvailable;
 import org.finos.tracdap.common.plugin.PluginServiceInfo;
 import org.finos.tracdap.common.plugin.TracPlugin;
 
@@ -46,15 +46,16 @@ public List<PluginServiceInfo> serviceInfo() {
     }
 
     @Override @SuppressWarnings("unchecked")
-    protected <T> T createService(String serviceName, Properties properties) {
+    protected <T> T createConfigService(String serviceName, Properties properties) {
 
         if (tempDir == null || !Files.exists(tempDir))
             throw new RuntimeException("Temp dir must be set for TestConfigPlugin");
 
         if (serviceName.equals(SERVICE_NAME))
             return (T) new TestConfigLoader(tempDir);
 
-        throw new EUnexpected();
+        var message = String.format("Plugin [%s] does not support the service [%s]", pluginName(), serviceName);
+        throw new EPluginNotAvailable(message);
     }
 
     public static void setCurrentTempDir(Path tempDir) {

tracdap-libs/tracdap-lib-data/src/main/java/org/finos/tracdap/common/codec/CodecManager.java

@@ -16,6 +16,7 @@
 
 package org.finos.tracdap.common.codec;
 
+import org.finos.tracdap.common.config.ConfigManager;
 import org.finos.tracdap.common.exception.EPluginNotAvailable;
 import org.finos.tracdap.common.plugin.IPluginManager;
 import org.slf4j.Logger;
@@ -27,16 +28,18 @@ public class CodecManager implements ICodecManager {
     private final Logger log = LoggerFactory.getLogger(getClass());
 
     private final IPluginManager plugins;
+    private final ConfigManager configManager;
 
-    public CodecManager(IPluginManager plugins) {
+    public CodecManager(IPluginManager plugins, ConfigManager configManager) {
         this.plugins = plugins;
+        this.configManager = configManager;
     }
 
     @Override
     public ICodec getCodec(String format) {
 
         if (plugins.isServiceAvailable(ICodec.class, format))
-            return plugins.createService(ICodec.class, format);
+            return plugins.createService(ICodec.class, format, configManager);
 
         else {
 

tracdap-libs/tracdap-lib-data/src/main/java/org/finos/tracdap/common/data/CoreDataPlugin.java

@@ -21,7 +21,8 @@
 import org.finos.tracdap.common.codec.arrow.ArrowStreamCodec;
 import org.finos.tracdap.common.codec.csv.CsvCodec;
 import org.finos.tracdap.common.codec.json.JsonCodec;
-import org.finos.tracdap.common.exception.EUnexpected;
+import org.finos.tracdap.common.config.ConfigManager;
+import org.finos.tracdap.common.exception.EPluginNotAvailable;
 import org.finos.tracdap.common.plugin.PluginServiceInfo;
 import org.finos.tracdap.common.plugin.TracPlugin;
 import org.finos.tracdap.common.storage.IFileStorage;
@@ -60,7 +61,7 @@ public List<PluginServiceInfo> serviceInfo() {
     }
 
     @Override @SuppressWarnings("unchecked")
-    public <T> T createService(String serviceName, Properties properties) {
+    protected  <T> T createService(String serviceName, Properties properties, ConfigManager configManager) {
 
         switch (serviceName) {
 
@@ -69,9 +70,9 @@ public <T> T createService(String serviceName, Properties properties) {
             case ARROW_FILE_CODEC_NAME: return (T) new ArrowFileCodec();
             case CSV_CODEC_NAME: return (T) new CsvCodec();
             case JSON_CODEC_NAME: return (T) new JsonCodec();
-
-            default:
-                throw new EUnexpected();
         }
+
+        var message = String.format("Plugin [%s] does not support the service [%s]", pluginName(), serviceName);
+        throw new EPluginNotAvailable(message);
     }
 }