chore: branch fixing update-do-swarm-workers (#45)

* chore: recreated branch fixing update-do-swarm-workers

* chore: changed readme

* fix(do/swarm-workers): fix minor issues

* fix(do/swarm-workers): fix config

---------

Co-authored-by: Anthony Phothirath <[email protected]>
Co-authored-by: Olivier Pichon <[email protected]>

generators/do-swarm-workers/templates/do-swarm-workers/Pulumi.stack.yaml

@@ -1,12 +1,19 @@
 config:
-  aws:region: ap-southeast-1
-  <%= projectName %>:doResourcesProject: <%= prefix %>-do-resources
+#  <%= projectName %>:count: "1"
   <%= projectName %>:image: docker-20-04
   <%= projectName %>:name: <%= prefix %>-<%= environment %>
+  <%= projectName %>:pathToSshKeysFolder: "../../ssh-keys"
+#  <%= projectName %>:projectId: DO PROJECT ID
+#  <%= projectName %>:projectStack: <%= prefix %>-do-resources
   <%= projectName %>:protect: "false"
   <%= projectName %>:publicKeyNames:
     - KEY_NAME
   <%= projectName %>:region: sgp1
   <%= projectName %>:retainOnDelete: "false"
   <%= projectName %>:size: s-1vcpu-1gb
-  <%= projectName %>:username: USERNAME
+  <%= projectName %>:sshKeyNames: 
+    - ...
+  <%= projectName %>:userGroups: group1,group2
+  <%= projectName %>:username: USERNAME
+#  <%= projectName %>:vpcUuid: VPC UUID
+#  <%= projectName %>:vpcStack: <%= prefix %>-do-resources

generators/do-swarm-workers/templates/do-swarm-workers/Pulumi.yaml

@@ -3,4 +3,4 @@ runtime:
   name: nodejs
   options:
     typescript: true
-description: DigitalOcean resources for Docker swarm workers
+description: DigitalOcean droplets to be used as Docker swarm workers

generators/do-swarm-workers/templates/do-swarm-workers/README.md

@@ -1,6 +1,8 @@
-# DigitalOcean Docker swarm workers
+# DigitalOcean swarm workers
 
-This project provisions worker nodes in DigitalOcean for a Docker Swarm. The worker nodes are assigned to a manager node using the ansible-do generator by modifying the inventory file (hosts) and running the command make setup.swarm
+Provisions one or more DigitalOcean droplets to be used as Docker swarm worker nodes.
+
+The droplets are not joined to a Docker swarm. This is expected to be handled by an Ansible playbook.
 
 ## Requirements
 
@@ -12,7 +14,7 @@ This project provisions worker nodes in DigitalOcean for a Docker Swarm. The wor
 
 ## Usage
 
-* Cd into the `do-swarm-workers` folder.
+All commands must be run from inside the Pulumi project folder.
 
 * Install dependencies 
 
@@ -22,7 +24,7 @@ npm install
 
 * Copy `.env.example` to `.env` and update your DigitalOcean Personal Access Token.
 
-* Set the default organization 
+* If using the Pulumi cloud as backend, set the default organization 
 
 ```bash
 pulumi org set-default {your organization}
@@ -31,11 +33,11 @@ pulumi org set-default {your organization}
 * Initialize and select the appropriate stack
 
 ```bash
-pulumi stack init [staging|production]
-pulumi stack select [staging|production]
+pulumi stack init {stack}
+pulumi stack select {stack}
 ```
 
-* Update the stack config `Pulimi.[staging|production].yaml` with the appropriate values for your project.
+* Update the stack config `Pulimi.{stack}.yaml` with the appropriate values for your project.
 
 * Run `pulumi up`
 
@@ -47,25 +49,89 @@ pulumi destroy
 
 ## Resources provisioned
 
-### TODO
+### DigitalOcean droplets
 
-## Resource names
+One or multiple DigitalOcean droplets.
 
-Resources are given a unique physical name by adding a suffix common to all names. This ensures that physical names are unique but also that they are related. It becomes easy to understand which resources werer created as part of the same batch. Because the suffix is used in the volume name, it must be lowercase and alphanumeric. We recommend using a datestamp in the form of `YYYYMMDD`. 
+### DigitalOcean project resource
 
-## Configuration settings
+If `projectId` or `projectStack` are set in the stack config, the corresponding project will be associated with the droplet.
+
+If `projectId` is defined, then it will be assumed to be the id of the project.
+
+If `projectId` is not defined, then `projectStack` is examined.
+
+`projectStack` is expected to be in the form of `[project][:id_output`]`, where:
+
+* `project` is the name of the Pulumi project where the DigitalOcean project was provisioned. The default value is `<%= prefix %>-do-resources`.
+* `id_output` is the name of the output that represents the DigitalOcean Project's id. The default value is `projectId`.
+
+The organization and stack of the project are assumed to be identical to the droplet's organization and stack.
+
+
+### VPC
+
+If `vpcUuid` or `vpcStack` are set in the stack config, the droplet will be attached to the VPC.
+
+If `vpcId` is defined, then it will be assumed to be the id of the VPC.
+
+If `vpcId` is not defined, then `vpcStack` is examined. 
+
+`vpcStack` is expected to be in the form of `[project][:id_output`]`, where:
+
+* `project` is the name of the Pulumi project where the VPC was provisioned. The default value is `<%= prefix %>-do-resources`.
+* `id_output` is the name of the output that represents the VPC's id. The default value is `vpcId`.
+
+The organization and stack of the VPC are assumed to be identical to the droplet's organization and stack.
 
-# DigitalOcean Project Settings
+### Local command
+
+The droplet's SSH key will be added to the user's `~/.ssh/knwn_hosts` file when created.
+
+The entry wil be deleted from `~/.ssh/known_hosts` when the droplet is destroyed.
+
+Note: this only works for the user running the `pulumi up` or `pulumi destroy` command. 
+
+### User 
+
+If `username` is defined in the stack config, a user account of that name will be created on the droplet.
+
+The user will be added to the `sudo` group as well as to any other groups defined by the `userGroups` stack config variable. This variable's format is a comma-concatenated set of group names, eg `docker,wheel`.
+
+The mechanism to add SSH keys to the user does not use the SSH keys registered in DigitalOcean. Rather, the public keys are expected to be stored in a folder (specified via the `pathToSshKeysFolder` stack config variable). The keys to add to the user are defined by the `publicKeyNames` stack config variable. This variable's value is expected to be an array of filenames with the `.pub` extension omitted. 
+
+For example, if the SSHkeys folder contains the public key files `alice.pub` and `bob.pub`, then the `publicSShKeys` variable is an array containing any of the following values: `alice`, `bob`.
+
+### Root user
+
+If `username` is defined, then root's SSH access to the droplet will be disabled.
+
+DigitalOcean SSH keys will be added to the root account. The names of the SSH keys to add are defined by the `sshKeyNames` stack config variable.
+
+If none are defined, then root access will be password-based.
+
+## Configuration settings
 
-| Setting         | Type    | Default                          | Description                                     |
-|------------------|---------|----------------------------------|-------------------------------------------------|
-| `doResourcesProject`          | string  | `<%=prefix%>-do-resources`                   | do-resources project name |
-| `image`          | string  | `docker-20.04`                   | Image used |
-| `name`          | string  | `stack-name ` | DO project name                                |                     |
-| `protect`       | string  | `"false"`                       | Protect resources from accidental deletion     |
-| `publicKeyNames`| array   | `- KEY_NAME`                    | Public SSH key names                           |
-| `region`        | string  | `sgp1`                          | DO region                                      |
-| `retainOnDelete`| string  | `"false"`                       | Retain resources when destroyed               |
-| `size`          | string  | `s-1vcpu-1gb`                   | Size of block volume to create                 |   
-| `username`      | string  | `USERNAME`                      | User for resource access                       |
 
+| Setting | Type | Default | Description |
+|---------|------|---------|-------------|
+| count | string | `1` | Number of workers |
+| image | string | `ubuntu-24-10-x64` | DO dropletimage |
+| name | string | `{stack}` | DO droplet name | 
+| packages | string[] | | Packages to install on the droplet |
+| pathToSshKeysFolder | string | `../../ssh-keys` | Path to folder containing public key files |
+| projectId | string |  | Id of the DigitalOcean project to which the droplet is associated |
+| projectStack | string |  | Name of the Pulumi project where the DigitalOcean project was provisioned |
+| protect | boolean | false | Protect resources from accidental deletion |
+| publicKeyNames | string[] |  | Names of public SSH keys to attach to the droplet's user |
+| region | string | | DO region |
+| retainOnDelete | boolean | false | Retain resources when destroyed |
+| size | number | `s-1vcpu-1gb` | Size of the droplet |
+| sshKeyNames | string[] | | Names of DigitalOcean SSH keys associated with the root user | 
+| swapFile | string |  | Path to the swap file |
+| swapSize | number |  | Size of the swap file |
+| userDataTemplate | string | `./cloud-config.njx` | Path to user data template |
+| userGroups | comma-separated strings |  | Groups to which the user belongs |
+| username | string |  | Name of the user to create in the droplet |
+| vpcId | string |  | Id of the DigitalOcean VPC to which the droplet is associated |
+| vpcStack | string |  | Name of the Pulumi project where the DigitalOcean VPC was provisioned |

generators/do-swarm-workers/templates/do-swarm-workers/cloud-config.njx

@@ -2,7 +2,7 @@
 users:
 {% for user in users %}
   - name: {{ user.username }}
-    groups: {{ user.groups }}
+    groups: docker{% if user.groups %},{{ user.groups }}{% endif %}
     shell: /bin/bash
     sudo: ['ALL=(ALL) NOPASSWD:ALL']
     ssh-authorized-keys: 
@@ -47,4 +47,4 @@ runcmd:
 {% for volume
  in volumes %}
   - chown -R {{ volume.user }}:{{ volume.group }} {{ volume.path }}
-{% endfor %}
+{% endfor %}

generators/do-swarm-workers/templates/do-swarm-workers/config.ts

@@ -9,47 +9,73 @@ import {
 import getPublicKeys from "./public-keys";
 
 export const getConfig = async () => {
-  const organization = getOrganization();
   const stack = getStack();
   const stackConfig = new Config();
 
-  const doResourcesProject = stackConfig.get("doResourcesProject") || "do-resources";
+  const packages = stackConfig.getObject<string[]>("packages") || [];
 
-  const resourcesStack = new StackReference(`${organization}/${doResourcesProject}/${stack}`);
+  const pathToSshKeysFolder = stackConfig.get("pathToSshKeysFolder") || "../../ssh-keys";
 
-  const projectIdOutput = await resourcesStack.getOutputDetails("projectId");
-  const projectId = getValue<string>(projectIdOutput);
+  const publicKeyNames = stackConfig.requireObject("publicKeyNames") as string[];
 
-  const reservedIpIdOutput = await resourcesStack.getOutputDetails("reservedIpId");
-  const reservedIpId = getValue<string>(reservedIpIdOutput);
+  let projectId = stackConfig.get("projectId");
+
+  if (!projectId) {
+    const outputs = await getOutputs(
+      "projectStack",
+      "projectId"
+    );
+
+    projectId = outputs ? outputs[0] : undefined;
+  }
+
+  const sshKeyNames = stackConfig.requireObject("sshKeyNames") as string[];
+
+  const userDataTemplate = stackConfig.get("userDataTemplate") || "./cloud-config.njx";
 
   const username = stackConfig.require("username");
 
-  const vpcUuidOutput = await resourcesStack.getOutputDetails("vpcId");
-  const vpcUuid = getValue<string>(vpcUuidOutput);
+  const userGroups = stackConfig.get("userGroups");
+  const groups = userGroups ? `sudo,${userGroups}` : "sudo";
 
-  const publicKeyNames = stackConfig.requireObject("publicKeyNames") as string[];
-  const pathToSshKeysFolder = stackConfig.get("pathToSshKeysFolder") || "../../ssh-keys";
+  let vpcId = stackConfig.get("vpcId");
+  let vpcIpRange = undefined as unknown as string;
+
+  if (!vpcId) {
+    const outputs = await getOutputs(
+      "vpcStack",
+      "vpcId,vpcIpRange"
+    );
+
+    if (outputs) {
+      vpcId = outputs[0] as string;
+      vpcIpRange = outputs[1] as string;
+    }
+  }
 
   return {
+    count: stackConfig.getNumber("count") || 1,
     image: stackConfig.require("image"),
     name: stackConfig.get("name") || stack,
+    packages,
+    pathToSshKeysFolder,
     projectId,
     protect: stackConfig.getBoolean("protect"),
     region: stackConfig.require("region"),
-    reservedIpId,
     retainOnDelete: stackConfig.getBoolean("retainOnDelete"),
     size: stackConfig.require("size"),
-    userDataTemplate: "./cloud-config.njx",
+    sshKeyNames,
+    userDataTemplate,
     users: [
       {
         username,
-        groups: "sudo, docker",
+        groups,
         publicKeys: getPublicKeys(publicKeyNames, pathToSshKeysFolder)
 
       },
     ],
-    vpcUuid,
+    vpcId,
+    vpcIpRange,
   };
 };
 
@@ -67,4 +93,55 @@ function getValue<T>(input: StackReferenceOutputDetails, defaultValue?: T): T {
   }
 
   return defaultValue;
-}
+}
+
+const stacks: { [key: string]: StackReference } = {};
+
+async function getOutputs(
+  stackConfigVar: string,
+  defaultOutputs: string
+): Promise<undefined | string[]> {
+  const organization = getOrganization();
+  const stack = getStack();
+  const stackConfig = new Config();
+
+  const config = stackConfig.get(stackConfigVar);
+
+  if (!config) {
+    return undefined;
+  }
+
+  let [project, outputNamesString] = config.split(":");
+
+  if (!project) {
+    return undefined;
+  }
+
+  if (!outputNamesString) {
+    outputNamesString = defaultOutputs;
+  }
+
+  if (!outputNamesString) {
+    return undefined;
+  }
+
+  const outputNames = outputNamesString.split(","); 
+
+  const stackName = `${organization}/${project}/${stack}`;
+  let otherStack = stacks[stackName];
+
+  if (!otherStack) {
+    otherStack = new StackReference(stackName);
+    stacks[stackName] = otherStack;
+  }
+
+  let outputs = [];
+
+  for (var i = 0, name = null; name = outputNames[i]; i++) {
+    const output = await otherStack.getOutputDetails(name);
+
+    outputs.push(getValue<string>(output) as string)
+  }
+
+  return outputs;
+}