Improves installation scripts

.gitignore

@@ -166,6 +166,9 @@ runner.yml
 # command scripts for runner
 servers/execution/runner/lifecycle*
 
+# Python virtual-env
+dtaas-venv/
+
 # certs
 *.pem
 

docs/developer/index.md

@@ -30,6 +30,12 @@ There is a devcontainer configuration (`.devcontainer/devcontainer.json`)
 for the project. Please use it to get a dockerized development environment.
 DevContainer is the easiest way to get started.
 
+### DevContainers
+
+There is a devcontainer configuration (`.devcontainer/devcontainer.json`)
+for the project. Please use it to get a dockerized development environment.
+DevContainer is the easiest way to get started.
+
 ### Ubuntu/Linux
 
 The code base has been developed for most part on
@@ -92,15 +98,52 @@ Then, open powershell with **administrative** priviledges and run the
 following commands in the given order:
 
 ```bash
-powershell -F script/base.ps1
-powershell -F script/env.ps1
-powershell -F script/docker.ps1
+powershell -executionpolicy bypass -F script/base.ps1
+powershell -executionpolicy bypass -F script/env.ps1
+powershell -executionpolicy bypass -F script/docker.ps1
 ```
 
 :warning: The docker images are large and are likely to consume
 about 5GB of bandwidth and 15GB of space.
 You will have to download the docker images on a really good network.
 
+<!--
+TODO insert when githooks works
+
+### git hooks
+
+The git-hooks will ensure that your commits are formatted
+correctly and that the tests pass before you
+push the commits to remote repositories.
+
+The project uses pre-commit for managing git hooks. Install git hooks using
+
+```bash
+pre-commit install
+```
+
+The git hooks run during pre-commit (`git commit`) and
+pre-push (`git push`) stages. No special flags and options are
+needed to run these git hooks.
+
+You can also run the git-hooks manually before committing or pushing
+by using the run commands below. The autoupdate command will set the
+revisions of the git repos used in the .pre-commit-config.yaml up to date.
+
+```bash
+pre-commit run --hook-stage pre-commit # runs format and syntax checks
+pre-commit run --hook-stage pre-push   # runs test
+pre-commit autoupdate                  # update hooks to latests versions
+```
+
+Be aware that the some tests may take a long time to run.
+If you want to skip the tests or formatting,
+you can use the `--no-verify` flag
+on `git commit` or `git push`. Please use this
+option with care.
+
+-->
+
 ## :building_construction: Development Workflow
 
 To manage collaboration by multiple developers on the software,
@@ -113,10 +156,11 @@ a development workflow is in place. Each developer should follow these steps:
    [Codecov](https://docs.codecov.com/docs/quick-start)
    for your fork. The codecov does not require secret token
    for public repositories.
-1. Install git-hooks for the project.
+1. nvm use 22 (if nvm or node isn't installed)
 1. Use
    [Fork, Branch, PR](https://gun.io/news/2017/01/how-to-github-fork-branch-and-pull-request/)
    workflow.
+1. Active the Python Virtual Enviornment by calling (`./dtaas-venv/bin/activate`)
 1. Work in your fork and open a PR from your working
    branch to your `feature/distributed-demo` branch.
    The PR will run all the github actions, code climate and codecov checks.

script/base.ps1

@@ -28,6 +28,10 @@ if (-not (Get-Command docker-desktop -ErrorAction SilentlyContinue)) {
     # https://docs.docker.com/engine/install/ubuntu/
     choco install -y docker-desktop
 
+    $dockerDesktopPath = Join-Path $env:ProgramFiles "Docker\Docker\resources\cli-plugins"
+    # Ensure docker-desktop is in the system path
+    [Environment]::SetEnvironmentVariable("Path", $env:Path + ";$dockerDesktopPath", [EnvironmentVariableTarget]::Machine)
+
     # Check if docker group exists, if not, create it
     $dockerGroup = 'docker'
     if (-not (Get-LocalGroup -Name $dockerGroup -ErrorAction SilentlyContinue)) {
@@ -61,21 +65,22 @@ if (-not (Get-Command docker-desktop -ErrorAction SilentlyContinue)) {
 
     # Enable Docker service
     Set-Service -Name "com.docker.service" -StartupType Automatic
-
-    # Install docker-compose from https://docs.docker.com/compose/install/other/
-    $url = "https://github.com/docker/compose/releases/download/v2.23.3/docker-compose-windows-x86_64.exe"
-    $outputPath = Join-Path $env:ProgramFiles "Docker\docker-compose.exe"
-    Invoke-WebRequest -Uri $url -OutFile $outputPath
-    # Ensure docker-compose is in the system path
-    [Environment]::SetEnvironmentVariable("Path", $env:Path + ";$outputPath", [EnvironmentVariableTarget]::Machine)
 }
 
 # Check if Node.js is already installed
 if (-not (Get-Command node -ErrorAction SilentlyContinue)) {
     # Install Node.js environment if not already installed
-    choco install -y nodejs-lts
+    choco install -y nvm
+    $nvmInstallPath = Join-Path "C:\ProgramData" "nvm"
+    [Environment]::SetEnvironmentVariable("Path", $env:Path + ";$nvmInstallPath", [EnvironmentVariableTarget]::Machine)
+    Write-Host "installing nvm"
 }
 
+# Install node
+nvm install 22
+nvm use 22
+
+
 # Install Yarn
 choco install -y yarn
 

script/base.sh

@@ -14,7 +14,8 @@ sudo apt-get install -y \
     apache2-utils \
     net-tools \
     python3-dev \
-    python3-pip
+    python3-pip \
+    python3-venv
 
 sudo mkdir -p /etc/apt/keyrings
 if [ ! -f /etc/apt/keyrings/docker.gpg ]
@@ -26,7 +27,7 @@ then
 fi
 
 sudo apt-get update -y
-sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
+sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
 sudo groupadd docker || true
 sudo usermod -aG docker "$USER" || true
 
@@ -52,42 +53,3 @@ docker run hello-world
 sudo systemctl enable docker.service
 sudo systemctl enable containerd.service
 
-
-#install docker-compose from https://docs.docker.com/compose/install/other/
-sudo curl -SL "https://github.com/docker/compose/releases/download/v2.23.3/docker-compose-linux-x86_64" \
-  -o /usr/local/bin/docker-compose
-sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
-sudo chmod 755 /usr/local/bin/docker-compose /usr/bin/docker-compose
-
-
-# Install nodejs environment
-sudo apt-get update
-sudo apt-get install -y ca-certificates curl gnupg
-sudo mkdir -p /etc/apt/keyrings
-if [ ! -f /etc/apt/keyrings/nodesource.gpg ]
-then
-  curl -fsSL "https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key" | \
-    sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
-fi
-NODE_MAJOR=20
-printf "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] \
-  https://deb.nodesource.com/node_%s.x nodistro main" "$NODE_MAJOR" | \
-  sudo tee /etc/apt/sources.list.d/nodesource.list
-
-sudo apt-get update
-sudo apt-get install -y nodejs
-sudo npm install -g [email protected]
-
-
-if [ ! -f /usr/share/keyrings/yarnkey.gpg ]
-then
-  curl -sL "https://dl.yarnpkg.com/debian/pubkey.gpg" | gpg --dearmor | \
-    sudo tee /usr/share/keyrings/yarnkey.gpg >/dev/null
-  printf "deb [signed-by=/usr/share/keyrings/yarnkey.gpg] https://dl.yarnpkg.com/debian stable main \n" | \
-    sudo tee /etc/apt/sources.list.d/yarn.list
-fi
-
-sudo apt-get update -y
-sudo apt-get install -y yarn
-sudo npm install -g serve
-sudo npm install -g pm2

script/env.ps1

@@ -21,9 +21,6 @@ if (-not ([Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarg
     [Environment]::SetEnvironmentVariable("Path", "$env:Path;$(Get-Command python3 | Select-Object -ExpandProperty Directory)", [EnvironmentVariableTarget]::Machine)
 }
 
-# Install mkdocs
-pip install mkdocs
-
 # Install Ruby: https://github.com/oneclick/rubyinstaller2/releases/download/RubyInstaller-3.1.2-1/rubyinstaller-devkit-3.1.2-1-x64.exe
 choco install -y ruby
 
@@ -36,9 +33,18 @@ if (-not ([Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarg
 # Install markdownlint
 gem install mdl
 
+python -m venv ./dtaas-venv
+./dtaas-venv/Scripts/Activate
+
+# Install mkdocs
+pip install mkdocs
+
 # Install mkdocs plugins
 pip install mkdocs-material python-markdown-math mkdocs-open-in-new-tab mkdocs-with-pdf qrcode
 
+# Deativate venv for python
+deactivate
+
 # Ensure Python scripts directory is added to PATH (for pip)
 if (-not ([Environment]::GetEnvironmentVariable("Path", [EnvironmentVariableTarget]::Machine) -like "*Python3*")) {
     Write-Host "Adding Python scripts to PATH (for pip)..."

script/env.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 source script/base.sh
+source script/nvm.sh
 
 # Install openssl for certificate generation
 sudo apt-get install -y wget openssl
@@ -10,13 +11,19 @@ npx --yes playwright install-deps
 
 #-------------
 printf "\n\n Installing required python packages...."
-sudo apt install -y python3-pip
-sudo -H pip3 install mkdocs
-sudo -H pip3 install mkdocs-material
-sudo -H pip3 install python-markdown-math
-sudo -H pip3 install mkdocs-open-in-new-tab
-sudo -H pip3 install mkdocs-with-pdf
-sudo -H pip3 install qrcode
+python3 -m venv ./dtaas-venv
+
+chmod +x ./dtaas-venv/bin/activate
+./dtaas-venv/bin/activate
+
+./dtaas-venv/bin/pip3 install mkdocs
+./dtaas-venv/bin/pip3 install mkdocs-material
+./dtaas-venv/bin/pip3 install python-markdown-math
+./dtaas-venv/bin/pip3 install mkdocs-open-in-new-tab
+./dtaas-venv/bin/pip3 install mkdocs-with-pdf
+./dtaas-venv/bin/pip3 install qrcode
+
+deactivate
 
 # Install markdownlint
 sudo apt-get install -y rubygems
@@ -27,5 +34,3 @@ sudo apt-get install -y shellcheck
 
 # Install madge for generating dependency graphs of typescript projects
 sudo apt-get install -y graphviz
-sudo npm install -g madge
-

script/nvm.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# Install nodejs environment
+sudo apt-get install -y ca-certificates curl gnupg
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash
+
+export NVM_DIR="$HOME/.nvm"
+[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"  # This loads nvm
+[ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"  # This loads nvm bash_completion
+
+nvm install 22
+nvm use 22
+nvm alias default 22
+
+# Install yarn
+if [ ! -f /usr/share/keyrings/yarnkey.gpg ]
+then
+  curl -sL "https://dl.yarnpkg.com/debian/pubkey.gpg" | gpg --dearmor | \
+    sudo tee /usr/share/keyrings/yarnkey.gpg >/dev/null
+  printf "deb [signed-by=/usr/share/keyrings/yarnkey.gpg] https://dl.yarnpkg.com/debian stable main \n" | \
+    sudo tee /etc/apt/sources.list.d/yarn.list
+fi
+sudo apt-get update
+sudo apt-get install -y yarn
+
+npm install -g serve pm2 madge