From bbd435fd4e6d7db4c05a1223bcc30a00cd877142 Mon Sep 17 00:00:00 2001 From: Jacopo Andrea Giola Date: Wed, 22 Nov 2023 15:00:44 +0100 Subject: [PATCH] feat: add GitLab build pipelines (#12) Co-authored-by: simoscarano <121295755+simoscarano@users.noreply.github.com> --- README.md | 23 ++- bin/gitlab/config.json | 6 + bin/gitlab/install | 151 ++++++++++++++++ containers/README.md | 8 + docs/10-starting-guide.md | 28 +++ docs/20-gitlab-installation.md | 80 +++++++++ docs/gitlab-ci/10-templates.md | 65 +++++++ docs/gitlab-ci/20-npm.md | 161 ++++++++++++++++++ docs/gitlab-ci/30-yarn.md | 160 +++++++++++++++++ docs/gitlab-ci/40-golang.md | 106 ++++++++++++ docs/gitlab-ci/50-docker.md | 106 ++++++++++++ docs/gitlab-ci/60-licenses.md | 36 ++++ docs/gitlab-ci/70-sysdig.md | 59 +++++++ gitlab-ci/base/Application.gitlab-ci.yml | 15 ++ gitlab-ci/base/CustomOverrides.gitlab-ci.yml | 1 + gitlab-ci/base/Governance.gitlab-ci.yml | 30 ++++ .../base/GovernanceOverrides.gitlab-ci.yml | 23 +++ gitlab-ci/base/jobs/docker.gitlab-ci.yml | 94 ++++++++++ gitlab-ci/base/jobs/golang.gitlab-ci.yml | 61 +++++++ gitlab-ci/base/jobs/licenses.gitlab-ci.yml | 38 +++++ gitlab-ci/base/jobs/npm.gitlab-ci.yml | 81 +++++++++ gitlab-ci/base/jobs/sysdig.gitlab-ci.yml | 107 ++++++++++++ gitlab-ci/base/jobs/yarn.gitlab-ci.yml | 88 ++++++++++ .../marketplace/golang-template.gitlab-ci.yml | 41 +++++ .../marketplace/npm-template.gitlab-ci.yml | 41 +++++ .../marketplace/yarn-template.gitlab-ci.yml | 41 +++++ 26 files changed, 1649 insertions(+), 1 deletion(-) create mode 100644 bin/gitlab/config.json create mode 100755 bin/gitlab/install create mode 100644 containers/README.md create mode 100644 docs/10-starting-guide.md create mode 100644 docs/20-gitlab-installation.md create mode 100644 docs/gitlab-ci/10-templates.md create mode 100644 docs/gitlab-ci/20-npm.md create mode 100644 docs/gitlab-ci/30-yarn.md create mode 100644 docs/gitlab-ci/40-golang.md create mode 100644 docs/gitlab-ci/50-docker.md create mode 100644 docs/gitlab-ci/60-licenses.md create mode 100644 docs/gitlab-ci/70-sysdig.md create mode 100644 gitlab-ci/base/Application.gitlab-ci.yml create mode 100644 gitlab-ci/base/CustomOverrides.gitlab-ci.yml create mode 100644 gitlab-ci/base/Governance.gitlab-ci.yml create mode 100644 gitlab-ci/base/GovernanceOverrides.gitlab-ci.yml create mode 100644 gitlab-ci/base/jobs/docker.gitlab-ci.yml create mode 100644 gitlab-ci/base/jobs/golang.gitlab-ci.yml create mode 100644 gitlab-ci/base/jobs/licenses.gitlab-ci.yml create mode 100644 gitlab-ci/base/jobs/npm.gitlab-ci.yml create mode 100644 gitlab-ci/base/jobs/sysdig.gitlab-ci.yml create mode 100644 gitlab-ci/base/jobs/yarn.gitlab-ci.yml create mode 100644 gitlab-ci/marketplace/golang-template.gitlab-ci.yml create mode 100644 gitlab-ci/marketplace/npm-template.gitlab-ci.yml create mode 100644 gitlab-ci/marketplace/yarn-template.gitlab-ci.yml diff --git a/README.md b/README.md index f1e4152..c540278 100644 --- a/README.md +++ b/README.md @@ -1 +1,22 @@ -# Public Pipelines +# Mia-Platform Public Pipelines + +![GitLab CI/CD Supported] +![Node.js with NPM Supported] ![Node.js with Yarn Supported] ![Go Supported] + +The goals of this repository is to provide to user reusable pipelines for different languages for building libraries +and Docker images to use in their projects and follow the best practicies for setting up comprehensive pipelines +for building a project, assessing its security, and keep track of its dependencies. You can see this as a starting +step for your Platform Engeneering team for setting up a comprehensive suite of tools for their pipelines that can be +easy to use, and meet regulatory compliance for your company. + +## How to Contribute + +You can contribute to this repository in adding support for new languages or improve the default steps for the +exsisting ones, improve the documentation, improve the Docker images and so on. We welcome everybody! + +You can find more reading the [documentation](./docs/10_starting-guide.md). + +[Gitlab CI/CD Supported]: https://img.shields.io/badge/GitLab-CI%2FCD-orange?logo=gitlab&style=for-the-badge +[Node.js with NPM Supported]: https://img.shields.io/badge/18%2C20-black?style=for-the-badge&logo=npm&label=NPM%20Node.js&color=3C873A +[Node.js with Yarn Supported]: https://img.shields.io/badge/18%2C20-black?style=for-the-badge&logo=yarn&label=Yarn%20Node.js&color=3C873A +[Go Supported]: https://img.shields.io/badge/1.20%2C1.21-black?style=for-the-badge&logo=go&label=Go&color=007E9A diff --git a/bin/gitlab/config.json b/bin/gitlab/config.json new file mode 100644 index 0000000..9ca445c --- /dev/null +++ b/bin/gitlab/config.json @@ -0,0 +1,6 @@ +{ + "instance_url": "https://gitlab.com", + "group_id": "", + "pipeline_images_base_name": "ghrc.io/mia-platform", + "default_visibility": "internal" +} diff --git a/bin/gitlab/install b/bin/gitlab/install new file mode 100755 index 0000000..401b7b0 --- /dev/null +++ b/bin/gitlab/install @@ -0,0 +1,151 @@ +#! /usr/bin/env bash + +set -o pipefail +set -o errexit + +## This script will setup a repository in your chosen GitLab group or in a predefined root groups. +## Then all the public pipelines yaml files will be commited in the newly created project and a pipeline +## will be set to automatically open merge requests with eventual updates. + +## The needed values from your self-maanged instance or GitLab SaaS are the following: +## - a valid token that has the 'api' and 'write_repository' scopes on the target GitLab instance +## - the username of the user that has created the token +## - a different path where to find the configuration file for the script + +# local configuration +readonly temp_project_folder_path=/tmp/tmp-pipelines +basedir=$(dirname "$0") + +readonly basedir + +# parameters read from the user +TOKEN="" +USERNAME="" +CONFIG_PATH="${basedir}/config.json" + +# original values that the final user can change +readonly ORIGINAL_CONTAINER_PATH="ghcr.io/mia-platform" +readonly ORIGINAL_INCLUDE_PATH="mia-platform/pipeline-templates" +readonly DEFAULT_GROUP_NAME="" + +function print_usage() { + cat < `gitlab > config.json`](../bin/gitlab/config.json) and below you can find the default content: + +```json +{ + "instance_url": "https://gitlab.com", + "group_id": "", + "pipeline_images_base_name": "ghrc.io/mia-platform", + "default_visibility": "internal" +} +``` + +For an explanation of the various keys and possible values reference the following table: + +| key | value | +| --- | --- | +| **instance_url** | The url of the GitLab instance, default to the GitLab SaaS installation | +| **group_id** | The id of the parent group where to create the pipeline template project, if left empty a `mia-platform` group will be created on the root of the instance, for GitLab SaaS this values cannot be empty | +| **pipeline_images_base_name** | If you cannot access directly the public repositories where the images for the jobs are hosted you can change this to the mirror where they will be available | +| **default_visibility** | The visibility set for the project and optional group that will be created, by default they will be set as internal, but you can set `public` or `private` if you need to | + +## Install the Templates + +1. clone the repository on your local machine + + ```sh + git clone https://github.com/mia-platform/public-pipelines.git + ``` + +1. edit the default values of the confiugration file with your favorite editor + + ```sh + ${EDITOR} ./public-pipelines/bin/gitlab/config.json + ``` + +1. set the GitLab username and token as env variables + + ```sh + export GITLAB_USERNAME="" + export GITLAB_TOKEN="" + ``` + +1. run the script + + ```sh + ./public-pipelines/bin/gitlab/install -u "${GITLAB_USERNAME}" \ + -t "${GITLAB_TOKEN}" -c \ + "./public-pipelines/bin/gitlab/config.json" + ``` + +1. after the run is complete you can delete the cloned repository if you want + + ```sh + rm -fr ./public-pipelines + ``` + +## Setting Pipeline Templates + +If you are a Premium subscriber you can use the newly created project as an instance or group template repository. +All pipelines used for the public marketplace can also be used as templates for your projects that do not use the console. + +[git]: https://git-scm.com (Git is a free and open source distributed version control system ) +[jq]: https://jqlang.github.io/jq/ (jq is a lightweight and flexible command-line JSON processor) diff --git a/docs/gitlab-ci/10-templates.md b/docs/gitlab-ci/10-templates.md new file mode 100644 index 0000000..01cf072 --- /dev/null +++ b/docs/gitlab-ci/10-templates.md @@ -0,0 +1,65 @@ +# GitLab CI/CD Templates + +The GitLab CI/CD templates are composed by multiple files and here you can find a brief overview of what +they contains, and how, the various jobs that they will add, works and can be customized. + +## Main Files + +### Application.gitlab-ci.yml + +The `Application.gitlab-ci.yml` file is the file that will import +every other files. It will not declare any jobs, but will contain imports of all the other files and the GitLab +templates that will form the base for your pipelines. This will allow you to only import this file in your +`.gitlab-ci.yaml` file inside your repository. + +### Governance.gitlab-ci.yml + +In the `Governance.gitlab-ci.yml` file there will be added all the jobs +and templates that add security and compliance steps to the pipelines. +This is a separeted file so if you are an Ultimate subscriber you can set it as a [Compliance pipelines] in your +GitLab environment groups. + +The pipeline is conditionally included inside the `Application.gitlab-ci.yml` file if we detect that the Compliance +pipelines feature is turned off. + +Once set to be enforced at the Group and Project level. This file will be the file GitLab CI processes and runs first. +Everything set and defined in this file will be immutable and unable to be changed or overwritten by the development +teams. This file will invoke the `.gitlab-ci.yml`` file in a team's repository; there the team can define any number +of jobs or variables. However, what's defined in this file will take precedent over anything defined in the +individual projects. + +In this file will also explicited the standard stages for the pipelines: + +- `build` +- `sast` +- `test` +- `container-build` +- `compliance` +- `deploy` + +### GovernanceOverrides.gitlab-ci.yml + +In the `GovernanceOverrides.gitlab-ci.yml`file there will be added overrides for the security steps included inside +the Governance file. + +### CustomOverrides.gitlab-ci.yml + +In the `CustomOverrides.gitlab-ci.yml` file the maintainers of the templates for your organization can add all the +overrides needed by them to conform the pipeline templates to their GitLab environment. +We reccomend to use only this file to ad your overrides to simplify eventual upgrades of the templates after the +first installation. + +## Jobs Folder + +The main part of the pipeline templates are implemented inside the `jobs` folder, inside that folder you can find +various files that create the jobs you can use to compose your pipelines. + +- [npm](./20-npm.md) +- [yarn](./30-yarn.md) +- [golang](./40-golang.md) +- [docker](./50-docker.md) +- [licenses](./60-licenses.md) +- [sysdig](./70-sysdig.md) + +[Compliance pipelines]: https://docs.gitlab.com/ee/user/group/compliance_frameworks.html#compliance-pipelines + (GitLab compliance pipelines documentation site) diff --git a/docs/gitlab-ci/20-npm.md b/docs/gitlab-ci/20-npm.md new file mode 100644 index 0000000..2a14794 --- /dev/null +++ b/docs/gitlab-ci/20-npm.md @@ -0,0 +1,161 @@ +# npm + +The `npm` file will provide the basic blocks for managing a **Node.js** project managed by `npm`. +The file will add a series of hidden jobs that can be extended inside the `.gitlab-ci.yml` file in the relative +project on GitLab. + +The templates will also try to setup useful default for the project caches and reports that can be visualized inside +GitLab. The cache is composed by two different blocks; the first one that is used only on the `.npm-build` job is aimed +to speed up the `npm ci` command and is the npm global cache for already downloaded packages. The second one is then +used inside the other jobs and by default contains only the `node_modules` folders available inside the project. +If you have to change the paths of the second cache you can override the `cache.paths` property of the `.npm` job +and it will be set for every other job. + +This file will set the following env variables in the global space. + +| Key | Default Value | Description | +| --- | --- | --- | +| NPM_CONFIG_USERCONFIG | "" | custom path to the npm user configuration, if left empty is $HOME/.npm | +| NPM_CONFIG_CACHE | $CI_PROJECT_DIR/.npm | custom path for the npm cache, if you change it you must override the `.npm-build` job cache configurations to point to the new path, we higly discourage that | +| NODE_IMAGE_TAG | "20" | the node major version to use in the pipeline, our image will support the tls version that you can find at this [link] | + +## .npm-build + +This job is used for setting up the repository for the subsequent stages of the pipeline, is responsible to download +the project dependencies, refresh the ones that are dependent on the operating system or Node.js version and to +build/transpile the source files if needed. +To do this the job will run the `ci`, `rebuild` and `run build --if-present` command of `npm`. The last one is run +conditionally because not every project need to be built, if your project must be built, we reccomend to add the +relevant folders created by the process to the cache for being available to other npm related jobs and/or to an +artifact if they must be availabe to other jobs in the pipeline. + +This job will set `NPM_CONFIG_PREFER_OFFLINE` env variable to true for avoiding if possible the redownload of already +cached dependencies to improve install time. +We also use the `NPM_INSTALL_CLI_OPTS` env variable to set specific flags for the `ci` command and the `NPM_CLI_OPTS` +one for the `rebuild` and `build` ones. + +The `before_script` is left empty to allow you to set env variables, or setup access to private repositories, even if +if possible we reccomend to configure npm via its env variables that use the `NPM_CONFIG_` prefix or the two previously +mentioned variables for setting up additional flags. + +### Usage Example + +```yaml +install-app: + stage: build + extends: .npm-build + variables: + NPM_CLI_OPTS: --ignore-scripts + NPM_CONFIG_REGISTRY: https://npm-cache.internal.local/ +``` + +### Jobs variables + +| Key | Default Value | Description | +| --- | --- | --- | +| NPM_INSTALL_CLI_OPTS | "" | the `ci` command of npm support different options than the other comands you can set them here | +| NPM_CONFIG_PREFER_OFFLINE | true | control if the installation for the module will always attempt to download them from the repository or use the version in the cache if available | +| NPM_CLI_OPTS | "" | the `rebuild` and `run build` commands will use this variable for additional options | + +### Image + +The job will use the `${CONTAINER_PATH}/node-pipeline:${NODE_IMAGE_TAG}` image to run its scripts. + +## .npm-lint + +The `.npm-lint` hidden job can be used to add a lint check of the code pushed on the repository for enforcing +company mandated style, running linter to catch common programming mistakes that render the code problematic, etc. + +To do so we try to run a script called `lint`, in this script you can run your preferred tool like `ESLint`, +`JSHint`, `Prettier` and/or any other program you want to use. + +As this can be seen as a SAST job we are configuring it as allowed to fail to avoid to block the pipeline if the +command is missing, missconfigured or if found issues that can be resolved in subsequent commits. For this reason we +also set the default rule of the job to run only on branches and on merge request pipelines and avoid to run it on +tags. + +The `before_script` is left empty to allow you to set env variables, or setup access to private repositories, even if +if possible we reccomend to configure npm via its env variables that use the `NPM_CONFIG_` prefix or `NPM_CLI_OPTS` for +setting up additional flags. + +### Usage + +```yaml +lint-app: + stage: sast + extends: .npm-lint +``` + +### Job variables + +| Key | Default Value | Description | +| --- | --- | --- | +| NPM_CLI_OPTS | "" | the `run lint` command will use this variable for additional options | + +### Image + +The job will use the `${CONTAINER_PATH}/node-pipeline:${NODE_IMAGE_TAG}` image to run its scripts. + +## .npm-test + +In `.npm-test` we try to run the `coverage` script of your `package.json`. In this script we expect that you will run +your test framework with a coverage package enabled for creating a report in the cobertura standard and if possibile +a junit report. This files will be picked up via the relative reporters for GitLab to process and we will try to find +possible coverage file names `cobertura-coverage.xml` and junit report called `junit.xml` in every subfolders of the +project. + +The `before_script` is left empty to allow you to set env variables, or setup access to private repositories, even if +if possible we reccomend to configure npm via its env variables that use the `NPM_CONFIG_` prefix or `NPM_CLI_OPTS` for +setting up additional flags. + +### Usage + +```yaml +test-app: + stage: test + extends: .npm-test +``` + +### Job variables + +| Key | Default Value | Description | +| --- | --- | --- | +| NPM_CLI_OPTS | "" | the `run coverage` command will use this variable for additional options | + +### Image + +The job will use the `${CONTAINER_PATH}/node-pipeline:${NODE_IMAGE_TAG}` image to run its scripts. + +## .npm-publish + +The last job we provide is `.npm-publish` and as its name suggest, is a job you can use for publishing your +package to a remote index, like npm.js or an internal one used by your organization like JFrog Artifactory. + +The command that is invoked is `npm publish` and you can personalized it using the `NPM_CLI_OPTS` or via configuring +npm via the `NPM_CONFIG_` variables. + +Remember to setup a rules that is conformant with your organization policies for uploading the package, like running +the job on tags and/or via manual approval. + +### Usage + +```yaml +publish-package: + stage: deploy + extends: .npm-publish + + rules: + - if: $CI_COMMIT_TAG +``` + +### Job variables + +| Key | Default Value | Description | +| --- | --- | --- | +| NPM_CLI_OPTS | "" | the `publish` command will use this variable for additional options | + +### Image + +The job will use the `${CONTAINER_PATH}/node-pipeline:${NODE_IMAGE_TAG}` image to run its scripts. + +[link]: https://github.com/nodejs/release#release-schedule (Node.js LTS release schedule) diff --git a/docs/gitlab-ci/30-yarn.md b/docs/gitlab-ci/30-yarn.md new file mode 100644 index 0000000..002b1b3 --- /dev/null +++ b/docs/gitlab-ci/30-yarn.md @@ -0,0 +1,160 @@ +# Yarn + +The `yarn` file will provide the basic blocks for managing a **Node.js** project managed by `Yarn`. +The file will add a series of hidden jobs that can be extended inside the `.gitlab-ci.yml` file in the relative +project on GitLab. + +Inside the provided image, the latest version of Yarn is installed but we reccomend to use the preferred workflow of +Yarn and set the version you want to use with `yarn set version `. This command will set the appropriate +configuration inside the `package.json` and `.yarnrc.yml` files. + +The templates will also try to setup useful default for the project caches and reports that can be visualized inside +GitLab. The cache is composed by two different blocks; the first one that is used only on the `.yarn-build` job is aimed +to speed up the `yarn install` command and is the yarn cache for already downloaded packages. The second one is then +used inside the other jobs and by default contains only the `node_modules` folders available inside the project. +If you have to add more paths, or prefer to use the pnp linker, you cna override the `cache.paths` property of the +`.yarn` job and it will be set for every other job. + +This file will set the following env variables in the global space. + +| Key | Default Value | Description | +| --- | --- | --- | +| NPM_CONFIG_USERCONFIG | "" | custom path to the npm user configuration, if left empty is $HOME/.npm, it can be used with yarn classic compatibility mode | +| YARN_CACHE_FOLDER | $CI_PROJECT_DIR/.yarn/cache | custom path for the yarn cache, if you change it you must override the `.yarn-build` job cache configurations to point to the new path, we higly discourage that | +| NODE_IMAGE_TAG | "20" | the node major version to use in the pipeline, our image will support the tls version that you can find at this [link] | + +## .yarn-build + +This job is used for setting up the repository for the subsequent stages of the pipeline, is responsible to download +the project dependencies, refresh the ones that are dependent on the operating system or Node.js version and to +build/transpile the source files if needed. +To do this the job will run the `install` and `run build` command of `yarn`. The last one is run only if found inside +the `package.json` file because not every project need to be built, if your project must be built, we reccomend to +add the relevant folders created by the process to the cache for being available to other yarn related jobs and/or to an +artifact if they must be availabe to other jobs in the pipeline. + +We use the `YARNPKG_INSTALL_CLI_OPTS` env variable to set specific flags for the `install` command and the +`YARNPKG_CLI_OPTS` one for `run build`. + +The `before_script` is left empty to allow you to set env variables, or setup access to private repositories, even if +if possible we reccomend to configure yarn via its env variables that use the `YARN_` prefix or the two previously +mentioned variables for setting up additional flags. + +### Usage Example + +```yaml +install-app: + stage: build + extends: .yarn-build +``` + +### Jobs variables + +| Key | Default Value | Description | +| --- | --- | --- | +| YARNPKG_INSTALL_CLI_OPTS | "" | the `install` command of yarn support different options than the other comands you can set them here | +| YARNPKG_CLI_OPTS | "" | the `run build` command will use this variable for additional options | + +### Image + +The job will use the `${CONTAINER_PATH}/node-pipeline:${NODE_IMAGE_TAG}` image to run its scripts. + +## .yarn-lint + +The `.yarn-lint` hidden job can be used to add a lint check of the code pushed on the repository for enforcing +company mandated style, running linter to catch common programming mistakes that render the code problematic, etc. + +To do so we try to run a script called `lint`, in this script you can run your preferred tool like `ESLint`, +`JSHint`, `Prettier` and/or any other program you want to use. + +As this can be seen as a SAST job we are configuring it as allowed to fail to avoid to block the pipeline if the +command is missing, missconfigured or if found issues that can be resolved in subsequent commits. For this reason we +also set the default rule of the job to run only on branches and on merge request pipelines and avoid to run it on +tags. + +The `before_script` is left empty to allow you to set env variables, or setup access to private repositories, even if +if possible we reccomend to configure yarn via its env variables that use the `YARN_` prefix or `YARNPKG_CLI_OPTS` for +setting up additional flags. + +### Usage + +```yaml +lint-app: + stage: sast + extends: .yarn-lint +``` + +### Job variables + +| Key | Default Value | Description | +| --- | --- | --- | +| YARNPKG_CLI_OPTS | "" | the `run lint` command will use this variable for additional options | + +### Image + +The job will use the `${CONTAINER_PATH}/node-pipeline:${NODE_IMAGE_TAG}` image to run its scripts. + +## .yarn-test + +In `.yarn-test` we try to run the `coverage` script of your `package.json`. In this script we expect that you will run +your test framework with a coverage package enabled for creating a report in the cobertura standard and if possibile +a junit report. This files will be picked up via the relative reporters for GitLab to process and we will try to find +possible coverage file names `cobertura-coverage.xml` and junit report called `junit.xml` in every subfolders of the +project. + +The `before_script` is left empty to allow you to set env variables, or setup access to private repositories, even if +if possible we reccomend to configure yarn via its env variables that use the `YARN_` prefix or `YARNPKG_CLI_OPTS` for +setting up additional flags. + +### Usage + +```yaml +test-app: + stage: test + extends: .yarn-test +``` + +### Job variables + +| Key | Default Value | Description | +| --- | --- | --- | +| YARNPKG_CLI_OPTS | "" | the `run coverage` command will use this variable for additional options | + +### Image + +The job will use the `${CONTAINER_PATH}/node-pipeline:${NODE_IMAGE_TAG}` image to run its scripts. + +## .yarn-publish + +The last job we provide is `.yarn-publish` and its variant if you still use yarn v1 `.yarn-classi-publish` and as +its name suggest, is a job you can use for publishing your package to a remote index, like npm.js or an internal +one used by your organization like JFrog Artifactory. + +The command that is invoked is `yarn npm publish`, or `yarn publish` for the classic variant, and you can personalized +it using the `YARNPKG_CLI_OPTS` or via configuring yarn via the `YARN_` variables. + +Remember to setup a rules that is conformant with your organization policies for uploading the package, like running +the job on tags and/or via manual approval. + +### Usage + +```yaml +publish-package: + stage: deploy + extends: .yarn-publish + + rules: + - if: $CI_COMMIT_TAG +``` + +### Job variables + +| Key | Default Value | Description | +| --- | --- | --- | +| YARNPKG_CLI_OPTS | "" | the `npm publish` command will use this variable for additional options | + +### Image + +The job will use the `${CONTAINER_PATH}/node-pipeline:${NODE_IMAGE_TAG}` image to run its scripts. + +[link]: https://github.com/nodejs/release#release-schedule (Node.js LTS release schedule) diff --git a/docs/gitlab-ci/40-golang.md b/docs/gitlab-ci/40-golang.md new file mode 100644 index 0000000..3fd54b0 --- /dev/null +++ b/docs/gitlab-ci/40-golang.md @@ -0,0 +1,106 @@ +# Golang + +The `golang` file will provide the basic blocks for managing a **Go** project. +The file will add a series of hidden jobs that can be extended inside the `.gitlab-ci.yml` file in the relative +project on GitLab. + +The templates will also try to setup useful default for the project caches and reports that can be visualized inside +GitLab. The cache is configured via the `cahce.paths` property of the `.golang` job, it is configured to be equals to +the `GOPATH` property, but is not referenced directly, if you change the value of the variable you have to override +the cache configuration. The cache for go are crated inside the `.go-build` job using `go get` and then is downloaded +in all other job for later reuse. + +This file will import the following env variables in the global space. + +| Key | Default Value | Description | +| --- | --- | --- | +| GO_MAIN_MODULE_PATH | $CI_PROJECT_DIR | the path where the main module is found | +| GOPATH | $CI_PROJECT_DIR/.cache | the golang packages cache directory, if you change this path you must override the job cache configuration | +| GOLANG_IMAGE_TAG | 1.20 | the golang version of the image where to run the scripts, we support the latest two minor version that you can find on [go.dev site] | + +## .go-build + +This job is used for downloading all the dependencies of the project and try to do a full build to check if there are +no errors in the program. The download is done via calling `go get -t ./...`, with this command all the dependencies, +included the ones needed for running tests will be downloaded and refreshed in the cache. +After the download a `go build` command is called using the `GO_MAIN_MODULE_PATH` as target for the project. + +The `before_script` is left empty to allow you to set env variables, or setup access to private repositories. + +### Usage + +```yaml +build-app: + stage: build + extends: .go-build +``` + +### Jobs variables + +| Key | Default Value | Description | +| --- | --- | --- | +| GO_CLI_OPTS | "" | optional env for passign custom configuration to the `get` and `build` command | + +### Image + +The job will use the `${CONTAINER_PATH}/go-pipeline:${GOLANG_IMAGE_TAG}` image to run its scripts. + +## .go-lint + +The `.go-lint` hidden job can be used to add a lint check of the code pushed in the repository for enforcing +company mandated style, running linter to catch common programming mistakes that render the code problematic, etc. + +To do so we use the [`golangci-lint`] cli, and will we search for a configuration file present at +`GOLANG_LINT_CI_CONFIG`, if you don't have this file in your repository set the env to the **empty string** for +running the tool with the default configurations. The job is allowed to fail to avoid blocking merges for +formatting error, every developer is responsible to check if this job will report errors and to fix them appropriately. + +The `before_script` is left empty to allow you to set env variables, or setup access to private repositories. + +### Usage + +```yaml +lint-app: + stage: sast + extends: .go-lint +``` + +### Jobs variables + +| Key | Default Value | Description | +| --- | --- | --- | +| GOLANG_LINT_CI_CONFIG | "$CI_PROJECT_DIR/.golangci.yaml" | path of the `golangci-lint` configuration file | + +### Image + +The job will use the `${CONTAINER_PATH}/go-pipeline:${GOLANG_IMAGE_TAG}` image to run its scripts. + +## .go-test + +In `.go-test` you can find commands for running tests and generating a coverage file. We run the test with che +`covermode` set to `atomic` and the `race` flag enabled; you can also customize it via the `GO_CLI_OPTS` variables +for adding more flags and change the `GO_TEST_PATH` for indicating custom location for tests files. + +The `before_script` is left empty to allow you to set env variables, or setup access to private repositories. + +### Usage + +```yaml +test: + stage: test + extends: .go-test +``` + +### Jobs variables + +| Key | Default Value | Description | +| --- | --- | --- | +| GO_CLI_OPTS | "" | optional env for passign custom configuration to the test command | +| GO_TEST_PATH | "$GO_MAIN_MODULE_PATH" | you can use this variable to set a different path where to run tests | + +### Image + +The job will use the `${CONTAINER_PATH}/go-pipeline:${GOLANG_IMAGE_TAG}` image to run its scripts. + +[go.dev site]: https://go.dev/dl/ (Golang supported versions) +[`golangci-lint`]: https://golangci-lint.run (golangci-lint is a Go linters aggregator) diff --git a/docs/gitlab-ci/50-docker.md b/docs/gitlab-ci/50-docker.md new file mode 100644 index 0000000..ce4e4f7 --- /dev/null +++ b/docs/gitlab-ci/50-docker.md @@ -0,0 +1,106 @@ +# Docker + +The `docker` file contains the building blocks needed for building, generating the SBOM and sign a Docker image. +The file will add a series of hidden jobs that can be extended inside the `.gitlab-ci.yml` file in the relative +project on GitLab. + +The templates will ensure to build a multi-architecture image using the `IMAGE_PLATFORMS` variable for letting the +user to add or remove platforms as they see fit. + +The jobs will generate a set of env variables if not already present for permitting a full configuration of the build +process, these variables will start with the `CI_APPLICATION` prefix and are the following: + +- `CI_APPLICATION_REPOSITORY`: this variables will contain the full image name, the value is dependent on if the + GitLab Repository feature is turned on and if the `REGISTRY` environment is set +- `CI_APPLICATION_REPOSITORY_REGISTRY`: will contains the `REGISTRY` content and will be used to perform a login to + the remote registry +- `CI_APPLICATION_REPOSITORY_USER`: is the username used during login, will contain the `REGISTRY_USER` content +- `CI_APPLICATION_REPOSITORY_PASSWORD`: is the username used during login, will contain the `REGISTRY_PASSWORD` content +- `CI_APPLICATION_TAG`: contain the tag to apply to the image, and contains by default, `latest` if the current branch + is the `CI_DEFAULT_BRANCH`, the `CI_COMMIT_TAG` content if exists or the `CI_COMMIT_REF_SLUG` in all other cases + +This file will import the following env variables in the global space. + +| Key | Default Value | Description | +| --- | --- | --- | +| REGISTRY | $CI_REGISTRY | the remote registry where to evenutally upload the image | +| IMAGE_NAME | "" | the image name to use if the remote registry is not the GitLab one | +| REGISTRY_USER | $CI_REGISTRY_USER | username of the user that will upload the image to the registry | +| REGISTRY_PASSWORD | $CI_REGISTRY_PASSWORD | password of the user that will upload the image to the registry | +| DOCKER_IMAGE_TAG | "1" | the golang version of the image where to run the scripts, we will always use the latest docker version available | +| IMAGE_PLATFORMS | linux/amd64,linux/arm64 | defualt platforms to build the image | +| ENABLE_SEMVER_TAG_WITHOUT_VERSION_PREFIX | "" | setting this variable to "1" or "true" will remove the `v` prefix from the docker tag if it is a valid semver | + +## .docker-build + +This job will log in to the remote registry and will run the `docker buildx build` command for building and pushing a +docker container to it. It will build the image for the platforms indicated inside the `IMAGE_PLATFORMS` env variable. +The job will run the commands using the values inside `CI_APPLICATION_REPOSITORY_USER`, +`CI_APPLICATION_REPOSITORY_PASSWORD`, `CI_APPLICATION_REPOSITORY_REGISTRY`, `CI_APPLICATION_REPOSITORY` and +`CI_APPLICATION_TAG` variables. Additionally you can set the `DOCKERBUILD_ADDITIONAL_FLAGS` variable for add +additional flags to the command. + +The `CI_COMMIT_SHA` and `CI_APPLICATION_TAG` variables will be passed respectively as the `COMMIT_SHA` and `VERSION` +build variables automatically. + +The image produced by this job will use a tag that is the `CI_COMMIT_TAG` content or `CI_COMMIT_SHORT_SHA`, this +is for aiding a process of promoting images after the run of the container scanning stage. + +### Usage + +```yaml +docker:image: + stage: container-build + extends: .docker-build + + rules: + - if: $CI_COMMIT_BRANCH + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + variables: + DOCKERBUILD_OUTPUT: "type=image,push=false" + - if: $CI_OPEN_MERGE_REQUESTS + when: never +``` + +#### Jobs variables + +| Key | Default Value | Description | +| --- | --- | --- | +| DOCKERBUILD_DIR | $CI_PROJECT_DIR | the path to pass as a context to the build command | +| DOCKERFILE_PATH | $CI_PROJECT_DIR/Dockerfile | the path of the dockerfile to use | +| DOCKERBUILD_OUTPUT | type=image,push=true | the output to set for `buildx build` command, you can use this variable to change it | + +#### Image + +The job will use the `${CONTAINER_PATH}/docker-pipeline:${DOCKER_IMAGE_TAG}` image to run its scripts. + +## .docker-deploy + +This job can be used to retag images from the `CI_COMMIT_SHORT_SHA` tag to `CI_APPLICATION_TAG` if no `CI_COMMIT_TAG` +is detected or it will generate the SBOM for the current `CI_COMMIT_TAG` tag and will try to perfrom a cryptografic +signing of the image if a `COSIGN_PRIVATE_KEY_PATH` or a `SIGSTORE_ID_TOKEN` variable is found. + +### Usage + +```yaml +docker:deploy: + extends: .docker-deploy + stage: deploy + + rules: + - if: $CI_COMMIT_BRANCH + - if: $CI_COMMIT_TAG +``` + +#### Job variables + +| Key | Default Value | Description | +| --- | --- | --- | +| COSIGN_PRIVATE_KEY_PATH | "" | path to a private key for usign with cosing | +| SIGSTORE_ID_TOKEN | "" | if you are on GitLab SaaS environment, you can follow the [official guide] to setup keyless signing | + +#### Image + +The job will use the `${CONTAINER_PATH}/docker-pipeline:${DOCKER_IMAGE_TAG}` image to run its scripts. + +[official guide]: https://docs.gitlab.com/ee/ci/yaml/signing_examples.html (GitLab documentation on keyless signing) diff --git a/docs/gitlab-ci/60-licenses.md b/docs/gitlab-ci/60-licenses.md new file mode 100644 index 0000000..7a33ebb --- /dev/null +++ b/docs/gitlab-ci/60-licenses.md @@ -0,0 +1,36 @@ +# Licenses + +The `licenses` file contains an additional SAST job that introduce a dependency scanner based on syft that can be +used as an alternative to the one implementend in the Ultimate offering of GitLab. +This job will not integrate with the GitLab flow for managing and reviewing dependencies inside the GitLab UI, but +you can use the `LICENSES_BLOCKLIST` env variable for explicity block licences that your project cannot import. + +We reccomend to turn off this scanner if you are paying for the Ultimate subscription, for its better integration +with the GitLab UI and flow. + +## syft-dependency_scanning + +The job will run syft for generating a SBOM report in **SPDX** and **CycloneDX** format. The cdx format is passed as +an artifact report for the job. + +After the report generation it will use `jq` for parsing the SPDX one and try to find any licence listed inside +the `LICENSES_BLOCKLIST` variable. If the variable is empty no control will be performed. + +The job will pickup any change made in the `.ds-analyzer` hidden job, and will follow the same rule for +`DEPENDENCY_SCANNING_DISABLED` and `DS_EXCLUDED_ANALYZERS` variables that the official GitLab guide is reccomending. + +### Usage + +You don't have to do anything for setting up the job in your pipeline, if you want to disable it you can set +`DEPENDENCY_SCANNING_DISABLED` to `true` or add `syft` to the list in `DS_EXCLUDED_ANALYZERS`. + +### Jobs variables + +| Key | Default Value | Description | +| --- | --- | --- | +| LICENSES_BLOCKLIST | "" | a list of valid SPDX idendifiers separated by `,` do not add any blank space in the list | +| SYFT_IMAGE_TAG | "1" | the tag of the image where to run the scripts | + +### Image + +The job will use the `${CONTAINER_PATH}/base-pipeline:${SYFT_IMAGE_TAG}` image to run its scripts. diff --git a/docs/gitlab-ci/70-sysdig.md b/docs/gitlab-ci/70-sysdig.md new file mode 100644 index 0000000..0a83c1f --- /dev/null +++ b/docs/gitlab-ci/70-sysdig.md @@ -0,0 +1,59 @@ +# Sysdig + +The `sysdig` file will add the capability to use the secure service of [Sysdig] for scan the docker images built for +vulnerabilities. The file is supporting both scanning engine, the legacy one and the new one via two different +jobs, by default the new engine is used via the new `sysdig-cli-scanner` cli but you can opt in to use the legacy one +setting the `SYSDIG_LEGACY_SCAN` to **1** or **true**. + +As with the default GitLab container scanner the jobs can be turned off setting the variable `CONTAINER_SCANNING_DISABLED` +to **1** or **true**. + +This file will import the following env variables in the global space. + +| Key | Default Value | Description | +| --- | --- | --- | +| SYSDIG_IMAGE_TAG | "1" | the tag for the sysdig image where the scripts will run | + +## sysdig-container_scanning + +This job will use the new container scanning cli from sysdig for creating a report of the docker image that you are +building. You will need a valid subscription with sysdig for using the container scanning functionality. + +### Usage + +The job is automatically added to your pipline if `CONTAINER_SCANNING_DISABLED` is not set to `1` or `true` +and you have set the `SYSDIG_SECURE_TOKEN` variable. + +### Job variables + +| Key | Default Value | Description | +| --- | --- | --- | +|SYSDIG_SECURE_TOKEN | "" | the secure token from sysdig for calling their APIs | +|SYSDIG_SECURE_BACKEND_ENDPOINT | "" | the secure backend endpoint of sysdig for your tenancy | + +### Image + +The job will use the `${CONTAINER_PATH}/sysdig-pipeline:${SYSDIG_IMAGE_TAG}` image to run its scripts. + +## sysdig-legacy-container_scanning + +This job will use the legacy inline scanner of sysdig, but the rest configurations are the same of the previous jobs. +This job may be selected instead of the previous one by setting the `SYSDIG_LEGACY_SCAN` variable to `true`. + +### Usage + +The job is automatically added to your pipline if `CONTAINER_SCANNING_DISABLED` is not set to `1` or `true`, +you have set the `SYSDIG_SECURE_TOKEN` variable and `SYSDIG_LEGACY_SCAN` is set to `true` + +### Job variables + +| Key | Default Value | Description | +| --- | --- | --- | +|SYSDIG_SECURE_TOKEN | "" | the secure token from sysdig for calling their APIs | +|SYSDIG_SECURE_BACKEND_ENDPOINT | "" | the secure backend endpoint of sysdig for your tenancy | + +### Image + +The job will use the `${CONTAINER_PATH}/sysdig-pipeline:${SYSDIG_IMAGE_TAG}` image to run its scripts. + +[Sysdig]: https://sysdig.com (Security Tools for Containers, Kubernetes, and Cloud) diff --git a/gitlab-ci/base/Application.gitlab-ci.yml b/gitlab-ci/base/Application.gitlab-ci.yml new file mode 100644 index 0000000..0e1448a --- /dev/null +++ b/gitlab-ci/base/Application.gitlab-ci.yml @@ -0,0 +1,15 @@ +# This file is used as meta include, and will only contains includes and user overridable variables +include: +- local: jobs/npm.gitlab-ci.yml +- local: jobs/yarn.gitlab-ci.yml +- local: jobs/golang.gitlab-ci.yml +- local: jobs/docker.gitlab-ci.yml +- local: Governance.gitlab-ci.yml + rules: + - if: $GITLAB_FEATURES !~ /\bcompliance_pipeline_configuration\b/ + +variables: + CONTAINER_PATH: ghcr.io/mia-platform # Default container path for purpose build containers. + +default: + image: ${CONTAINER_PATH}/base-pipeline:1 # Sets default image if one is not specified diff --git a/gitlab-ci/base/CustomOverrides.gitlab-ci.yml b/gitlab-ci/base/CustomOverrides.gitlab-ci.yml new file mode 100644 index 0000000..9b53325 --- /dev/null +++ b/gitlab-ci/base/CustomOverrides.gitlab-ci.yml @@ -0,0 +1 @@ +# This file is for your global overrides for tailoring the pipeline templates to your GitLab environment. diff --git a/gitlab-ci/base/Governance.gitlab-ci.yml b/gitlab-ci/base/Governance.gitlab-ci.yml new file mode 100644 index 0000000..7bc84f3 --- /dev/null +++ b/gitlab-ci/base/Governance.gitlab-ci.yml @@ -0,0 +1,30 @@ +# This file includes compliance steps for the pipelines and can be set as a Compliance Framework pipelines file +include: +# NOTE: These links point to the latest templates for development in GitLab canonical project, +# therefore the actual templates that were included could be different from the contents in the links. +# To view the actual templates, please replace `master` to the specific GitLab version when +# the pipeline started running e.g. `v15.9.0-ee`. +- template: Jobs/SAST.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml +- template: Jobs/Dependency-Scanning.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml +- template: Jobs/Container-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml +- template: Jobs/Secret-Detection.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml + +- local: jobs/licenses.gitlab-ci.yml +- local: jobs/sysdig.gitlab-ci.yml +- local: jobs/gitlab-cs.gitlab-ci.yml +- local: GovernanceOverrides.gitlab-ci.yml +- local: CustomOverrides.gitlab-ci.yml + +- project: $CI_PROJECT_PATH + file: $CI_CONFIG_PATH + ref: $CI_COMMIT_REF_NAME + rules: + - if: $GITLAB_FEATURES =~ /\bcompliance_pipeline_configuration\b/ + +stages: +- build +- sast +- test +- container-build +- compliance +- deploy diff --git a/gitlab-ci/base/GovernanceOverrides.gitlab-ci.yml b/gitlab-ci/base/GovernanceOverrides.gitlab-ci.yml new file mode 100644 index 0000000..82b4844 --- /dev/null +++ b/gitlab-ci/base/GovernanceOverrides.gitlab-ci.yml @@ -0,0 +1,23 @@ +# This files is used to overrides configurations included inside security templates and pipelines + +.sast-analyzer: + stage: sast + +.secret-analyzer: + stage: sast + +.ds-analyzer: + stage: sast + +container_scanning: + stage: compliance + + script: + - !reference [.container-base, script] + - export CS_REGISTRY_USER=${CS_REGISTRY_USER:-$CI_APPLICATION_REPOSITORY_USER} + - export CS_REGISTRY_PASSWORD=${CS_REGISTRY_PASSWORD:-$CI_APPLICATION_REPOSITORY_PASSWORD} + - export CS_IMAGE=${CS_IMAGE:-$CI_APPLICATION_REPOSITORY:$push_tag} + - gtcs scan + +.sysdig-scan: + stage: compliance diff --git a/gitlab-ci/base/jobs/docker.gitlab-ci.yml b/gitlab-ci/base/jobs/docker.gitlab-ci.yml new file mode 100644 index 0000000..9826381 --- /dev/null +++ b/gitlab-ci/base/jobs/docker.gitlab-ci.yml @@ -0,0 +1,94 @@ +variables: + REGISTRY: $CI_REGISTRY + IMAGE_NAME: "" + REGISTRY_USER: $CI_REGISTRY_USER + REGISTRY_PASSWORD: $CI_REGISTRY_PASSWORD + DOCKER_IMAGE_TAG: 1 + IMAGE_PLATFORMS: linux/amd64,linux/arm64 + +.container-base: + script: + - | + # override CI_APPLICATION_REPOSITORY if needed based on REGISTRY value + if [[ "${REGISTRY}" != "${CI_REGISTRY}" ]]; then + export CI_APPLICATION_REPOSITORY=${CI_APPLICATION_REPOSITORY:-$REGISTRY/$IMAGE_NAME} + else + export CI_APPLICATION_REPOSITORY=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE} + fi + - export CI_APPLICATION_REPOSITORY_REGISTRY=${CI_APPLICATION_REPOSITORY_REGISTRY:-$REGISTRY} + - export CI_APPLICATION_REPOSITORY_USER=${CI_APPLICATION_REPOSITORY_USER:-$REGISTRY_USER} + - export CI_APPLICATION_REPOSITORY_PASSWORD=${CI_APPLICATION_REPOSITORY_PASSWORD:-$REGISTRY_PASSWORD} + - | + # override CI_APPLICATION_TAG if needed based on CI_COMMIT_TAG and CI_COMMIT_REF_NAME value + if [[ "${CI_COMMIT_TAG}" ]]; then + export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG} + elif [[ "${CI_COMMIT_REF_NAME}" == "${CI_DEFAULT_BRANCH}" ]]; then + export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-latest} + else + export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_REF_SLUG} + fi + - push_tag=${CI_COMMIT_TAG:-$CI_COMMIT_SHORT_SHA} + - | + if [[ "${ENABLE_SEMVER_TAG_WITHOUT_VERSION_PREFIX}" == "true" || "${ENABLE_SEMVER_TAG_WITHOUT_VERSION_PREFIX}" == "1" ]]; then + CI_APPLICATION_TAG="$(echo "${CI_APPLICATION_TAG}" | perl -pe 's/^v(?P(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?:[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)$/$+{semver}/')" + push_tag="$(echo "${push_tag}" | perl -pe 's/^v(?P(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)(?:-(?:(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?:[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)$/$+{semver}/')" + fi + - "echo Running on ref ${CI_COMMIT_REF_NAME}: tag = ${CI_APPLICATION_TAG}" + +.docker-base: + image: + name: ${CONTAINER_PATH}/docker-pipeline:${DOCKER_IMAGE_TAG} + + variables: + # These are usually specified by the entrypoint, however the + # Kubernetes executor doesn't run entrypoints + # https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4125 + DOCKER_HOST: tcp://docker:2376 + DOCKER_TLS_CERTDIR: "/certs" + DOCKER_TLS_VERIFY: 1 + DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" + DOCKER_DRIVER: overlay2 + + services: + - name: docker:24-dind + + dependencies: [] + +.docker-build: + extends: .docker-base + + variables: + DOCKERBUILD_DIR: $CI_PROJECT_DIR + DOCKERFILE_PATH: $CI_PROJECT_DIR/Dockerfile + DOCKERBUILD_OUTPUT: "type=image,push=true" + + script: + - !reference [.container-base, script] + - docker info + - docker buildx version + - oras version + - source /usr/local/lib/docker_helpers.sh + - docker_login "${CI_APPLICATION_REPOSITORY_USER}" "${CI_APPLICATION_REPOSITORY_PASSWORD}" "${CI_APPLICATION_REPOSITORY_REGISTRY}" + - setup_docker_context "${CI_PROJECT_ID}" + - docker_build "${DOCKERBUILD_DIR}" "${DOCKERFILE_PATH}" "${DOCKERBUILD_OUTPUT}" "${IMAGE_PLATFORMS}" "${CI_APPLICATION_REPOSITORY}:${push_tag}" "${CI_COMMIT_SHA}" "${CI_APPLICATION_TAG}" "${DOCKERBUILD_ADDITIONAL_FLAGS}" + - cleanup_docker_context "${CI_PROJECT_ID}" + +.docker-deploy: + extends: .docker-base + + script: + - !reference [.container-base, script] + - source /usr/local/lib/docker_helpers.sh + - docker_login "${CI_APPLICATION_REPOSITORY_USER}" "${CI_APPLICATION_REPOSITORY_PASSWORD}" "${CI_APPLICATION_REPOSITORY_REGISTRY}" + - | + if [[ -z "${CI_COMMIT_TAG}" ]]; then + docker_retag_image "${CI_APPLICATION_REPOSITORY}:${push_tag}" "${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG}" + else + syft version + cosign version + docker_create_sbom_and_sign_image "${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG}" + fi + + artifacts: + paths: + - "**.spdx.json" diff --git a/gitlab-ci/base/jobs/golang.gitlab-ci.yml b/gitlab-ci/base/jobs/golang.gitlab-ci.yml new file mode 100644 index 0000000..cdd67b4 --- /dev/null +++ b/gitlab-ci/base/jobs/golang.gitlab-ci.yml @@ -0,0 +1,61 @@ +variables: + GO_MAIN_MODULE_PATH: $CI_PROJECT_DIR + GOPATH: $CI_PROJECT_DIR/.cache + GOLANG_IMAGE_TAG: "1.20" + +.golang: + image: ${CONTAINER_PATH}/golang-pipeline:${GOLANG_IMAGE_TAG} + + cache: + key: + files: + - go.sum + prefix: ${CI_COMMIT_REF_SLUG} + paths: + - .cache + policy: pull + unprotect: true + +.go-build: + extends: .golang + + script: + - go version + - go get ${GO_CLI_OPTS} -t ./... + - go build ${GO_CLI_OPTS} "${GO_MAIN_MODULE_PATH}" + + cache: + policy: pull-push + +.go-lint: + extends: .golang + + variables: + GOLANG_LINT_CI_CONFIG: $CI_PROJECT_DIR/.golangci.yaml + + script: + - golangci-lint --version + - golangci-lint run --config="${GOLANG_LINT_CI_CONFIG}" + + rules: + - if: $CI_MERGE_REQUEST_IID + - if: $CI_OPEN_MERGE_REQUESTS + when: never + - if: $CI_COMMIT_BRANCH + + allow_failure: true + +.go-test: + extends: .golang + + variables: + GO_TEST_PATH: $GO_MAIN_MODULE_PATH + + script: + - go test -race -coverprofile=coverage.out -covermode=atomic ${GO_CLI_OPTS} "${GO_TEST_PATH}" + - go tool cover -func=coverage.out + + artifacts: + name: $CI_JOB_NAME_SLUG + paths: + - "coverage.out" diff --git a/gitlab-ci/base/jobs/licenses.gitlab-ci.yml b/gitlab-ci/base/jobs/licenses.gitlab-ci.yml new file mode 100644 index 0000000..a55f6ba --- /dev/null +++ b/gitlab-ci/base/jobs/licenses.gitlab-ci.yml @@ -0,0 +1,38 @@ +variables: + SYFT_IMAGE_TAG: 1 + +syft-dependency_scanning: + extends: .ds-analyzer + image: ${CONTAINER_PATH}/base-pipeline:${SYFT_IMAGE_TAG} + + variables: + LICENSES_BLOCKLIST: "" + + script: + - syft version + - syft "${CI_PROJECT_DIR}" -o spdx-json=gl-sbom-report.spdx.json -o cyclonedx-json=gl-sbom-report.cdx.json + - if [ -z "${LICENSES_BLOCKLIST}" ]; then exit 0; fi; + - 'echo -e "Blocked Licenses: ${LICENSES_BLOCKLIST}"' + - licenses_json=$(echo "${LICENSES_BLOCKLIST}" | sed 's/,/","/g' | awk '{print "[\"" $0 "\"]"}') + - blocked_licenses=$(jq -r --argjson licenses "${licenses_json}" '.packages[] | select(.licenseDeclared as $ld | any($licenses[]; . == $ld)) | [.name, .licenseDeclared] | join(" ")' gl-sbom-report.spdx.json | column --table) + - if [ -z "${blocked_licenses}" ]; then exit 0; fi; + - echo "${blocked_licenses}" + - exit 1 + + rules: + - if: $DEPENDENCY_SCANNING_DISABLED == 'true' || $DEPENDENCY_SCANNING_DISABLED == '1' + when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /syft/ + when: never + - if: $CI_MERGE_REQUEST_IID + - if: $CI_OPEN_MERGE_REQUESTS + when: never + - if: $CI_COMMIT_BRANCH + + artifacts: + paths: + - "**/gl-sbom-*.spdx.json" + - "**/gl-sbom-*.cdx.json" + reports: + cyclonedx: "**/gl-sbom-*.cdx.json" + dependency_scanning: null diff --git a/gitlab-ci/base/jobs/npm.gitlab-ci.yml b/gitlab-ci/base/jobs/npm.gitlab-ci.yml new file mode 100644 index 0000000..aa716ec --- /dev/null +++ b/gitlab-ci/base/jobs/npm.gitlab-ci.yml @@ -0,0 +1,81 @@ +variables: + NPM_CONFIG_USERCONFIG: "" + NPM_CONFIG_CACHE: $CI_PROJECT_DIR/.npm + NODE_IMAGE_TAG: "20" + +.npm: + image: ${CONTAINER_PATH}/node-pipeline:${NODE_IMAGE_TAG} + + cache: + key: + files: + - package-lock.json + prefix: ${CI_COMMIT_REF_SLUG} + paths: + - "**/node_modules" + policy: pull + unprotect: true + +.npm-build: + extends: .npm + variables: + NPM_CONFIG_PREFER_OFFLINE: "true" + + script: + - npm version + - npm ci ${NPM_INSTALL_CLI_OPTS} + - npm rebuild ${NPM_CLI_OPTS} + - npm run build --if-present ${NPM_CLI_OPTS} + + cache: + - key: $CI_COMMIT_REF_SLUG + paths: + - .npm + policy: pull-push + unprotect: true + - key: + files: + - package-lock.json + prefix: ${CI_COMMIT_REF_SLUG} + paths: !reference [.npm, cache, paths] + policy: pull-push + unprotect: true + +.npm-lint: + extends: .npm + dependencies: [] + allow_failure: true + + script: + - npm run lint ${NPM_CLI_OPTS} + + rules: + - if: $CI_MERGE_REQUEST_IID + - if: $CI_OPEN_MERGE_REQUESTS + when: never + - if: $CI_COMMIT_BRANCH + +.npm-test: + extends: .npm + + script: + - npm run coverage ${NPM_CLI_OPTS} + + artifacts: + name: $CI_JOB_NAME_SLUG + paths: + - "**/cobertura-coverage.xml" + - "**/junit.xml" + reports: + coverage_report: + coverage_format: cobertura + path: "**/cobertura-coverage.xml" + junit: + - "**/junit.xml" + +.npm-publish: + extends: .npm + dependencies: [] + + script: + - npm publish ${NPM_CLI_OPTS} diff --git a/gitlab-ci/base/jobs/sysdig.gitlab-ci.yml b/gitlab-ci/base/jobs/sysdig.gitlab-ci.yml new file mode 100644 index 0000000..2e4fd69 --- /dev/null +++ b/gitlab-ci/base/jobs/sysdig.gitlab-ci.yml @@ -0,0 +1,107 @@ +variables: + SYSDIG_IMAGE_TAG: 1 + +sysdig: + stage: test + image: + name: ${CONTAINER_PATH}/sysdig-pipeline:${SYSDIG_IMAGE_TAG} + + dependencies: [] + + variables: + SYSDIG_SECURE_TOKEN: "" # the sysdig secure token of your instance + SYSDIG_SECURE_BACKEND_ENDPOINT: "" # the sysdig secure backend endpoint for your instance + GIT_STRATEGY: none # if you need files in the repository chagne the strategy to fetch + + script: + - echo "${CI_JOB_NAME} is used for configuration only, and its script should not be executed" + - exit 1 + + rules: + - when: never + + artifacts: + paths: + - container-scanning-report.json + when: always + +.sysdig-scan: + extends: sysdig + + script: + - !reference [.container-base, script] + - if [ -z "${SYSDIG_SECURE_BACKEND_ENDPOINT}" ]; then echo "SYSDIG_SECURE_BACKEND_ENDPOINT is not set."; exit 1; fi; + - echo "${CI_APPLICATION_REPOSITORY}:${push_tag}" > registry_image.txt + + after_script: + - if [ ! -f "registry_image.txt" ]; then exit 0; fi; + - registry_image=$(cat registry_image.txt) + - echo "${CI_APPLICATION_REPOSITORY_PASSWORD}" | oras login --username "${CI_APPLICATION_REPOSITORY_USER}" --password-stdin "${CI_APPLICATION_REPOSITORY_REGISTRY}" + - | + oras manifest fetch ${registry_image} | jq -n -r -c '.manifests[].digest' | \ + while read digest; do + full_image_name="${registry_image}@${digest}" + oras manifest delete --force "${full_image_name}" + done + - oras manifest delete --force "${registry_image}" + +sysdig-container_scanning: + extends: .sysdig-scan + + variables: + SECURE_API_TOKEN: "${SYSDIG_SECURE_TOKEN}" + + script: + - !reference [.sysdig-scan, script] + - sysdig-cli-scanner --version + - REGISTRY_USER="${CI_APPLICATION_REPOSITORY_USER}" REGISTRY_PASSWORD="${CI_APPLICATION_REPOSITORY_PASSWORD}" sysdig-cli-scanner --console-log --apiurl "${SYSDIG_SECURE_BACKEND_ENDPOINT}" "${CI_APPLICATION_REPOSITORY}:${push_tag}" --json-scan-result="${CI_PROJECT_DIR}/container-scanning-report.json" + - rm registry_image.txt + + rules: + - if: $CONTAINER_SCANNING_DISABLED == 'true' || $CONTAINER_SCANNING_DISABLED == '1' + when: never + - if: $SYSDIG_LEGACY_SCAN == 'true' || $SYSDIG_LEGACY_SCAN == '1' + when: never + - if: $CI_COMMIT_BRANCH && + $SYSDIG_SECURE_TOKEN + +sysdig-legacy-container_scanning: + extends: .sysdig-scan + + variables: + SYSDIG_INLINE_IMAGE: "quay.io/sysdig/secure-inline-scan:2" + # These are usually specified by the entrypoint, however the + # Kubernetes executor doesn't run entrypoints + # https://gitlab.com/gitlab-org/gitlab-runner/-/issues/4125 + DOCKER_HOST: tcp://docker:2376 + DOCKER_TLS_CERTDIR: "/certs" + DOCKER_TLS_VERIFY: 1 + DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" + DOCKER_DRIVER: overlay2 + + services: + - name: docker:24-dind + + script: + - !reference [.sysdig-scan, script] + - docker pull ${SYSDIG_INLINE_IMAGE} + - docker inspect ${SYSDIG_INLINE_IMAGE} + - | + docker run --rm -v ${CI_PROJECT_DIR}:/scan \ + ${SYSDIG_INLINE_IMAGE} \ + "${CI_APPLICATION_REPOSITORY}:${push_tag}" \ + --sysdig-url "${SYSDIG_SECURE_BACKEND_ENDPOINT}" \ + --sysdig-token "${SYSDIG_SECURE_TOKEN}" \ + --registry-auth-basic "${CI_APPLICATION_REPOSITORY_USER}:${CI_APPLICATION_REPOSITORY_PASSWORD}" \ + --write-json "/scan/container-scanning-report.json" + - rm registry_image.txt + + rules: + - if: $CONTAINER_SCANNING_DISABLED == 'true' || $CONTAINER_SCANNING_DISABLED == '1' + when: never + - if: $SYSDIG_LEGACY_SCAN == 'false' || $SYSDIG_LEGACY_SCAN == '0' + when: never + # Enable job if we want the legacy scanner, the token is set and the pipeline is for a branch or tag + - if: $CI_COMMIT_BRANCH && + $SYSDIG_LEGACY_SCAN && + $SYSDIG_SECURE_TOKEN diff --git a/gitlab-ci/base/jobs/yarn.gitlab-ci.yml b/gitlab-ci/base/jobs/yarn.gitlab-ci.yml new file mode 100644 index 0000000..1958025 --- /dev/null +++ b/gitlab-ci/base/jobs/yarn.gitlab-ci.yml @@ -0,0 +1,88 @@ +variables: + NPM_CONFIG_USERCONFIG: "" + YARN_CACHE_FOLDER: $CI_PROJECT_DIR/.yarn/cache + NODE_IMAGE_TAG: "20" + +.yarn: + image: ${CONTAINER_PATH}/node-pipeline:${NODE_IMAGE_TAG} + + cache: + key: + files: + - yarn.lock + prefix: ${CI_COMMIT_REF_SLUG} + paths: + - "**/node_modules" + policy: pull + unprotect: true + +.yarn-build: + extends: .yarn + + script: + - yarn --version + - yarn install --immutable ${YARNPKG_INSTALL_CLI_OPTS} + - | + if [[ "$(jq '.scripts | has("build")' "${CI_PROJECT_DIR}/package.json")" = "true" ]]; then + yarn run build ${YARNPKG_CLI_OPTS} + fi + + cache: + - key: "${CI_COMMIT_REF_SLUG}" + paths: + - .yarn/cache + policy: pull-push + unprotect: true + - key: + files: + - yarn.lock + prefix: ${CI_COMMIT_REF_SLUG} + paths: !reference [.yarn, cache, paths] + policy: pull-push + unprotect: true + +.yarn-lint: + extends: .yarn + + script: + - yarn run lint ${YARNPKG_CLI_OPTS} + + rules: + - if: $CI_MERGE_REQUEST_IID + - if: $CI_OPEN_MERGE_REQUESTS + when: never + - if: $CI_COMMIT_BRANCH + + allow_failure: true + +.yarn-test: + extends: .yarn + + script: + - yarn run coverage ${YARNPKG_CLI_OPTS} + + artifacts: + name: $CI_JOB_NAME_SLUG + paths: + - "**/cobertura-coverage.xml" + - "**/junit.xml" + reports: + coverage_report: + coverage_format: cobertura + path: "**/cobertura-coverage.xml" + junit: + - "**/junit.xml" + +.yarn-publish: + extends: .yarn + dependencies: [] + + script: + - yarn npm publish ${YARNPKG_CLI_OPTS} + +.yarn-classic-publish: + extends: .yarn + dependencies: [] + + script: + - yarn publish ${YARNPKG_CLI_OPTS} diff --git a/gitlab-ci/marketplace/golang-template.gitlab-ci.yml b/gitlab-ci/marketplace/golang-template.gitlab-ci.yml new file mode 100644 index 0000000..b2fd69d --- /dev/null +++ b/gitlab-ci/marketplace/golang-template.gitlab-ci.yml @@ -0,0 +1,41 @@ +include: +- project: mia-platform/pipeline-templates + file: Application.gitlab-ci.yml + ref: HEAD +- template: Workflows/MergeRequest-Pipelines.gitlab-ci.yml + +variables: + GOLANG_IMAGE_TAG: "1.21" + IMAGE_NAME: mia_template_image_name_placeholder + +build: + extends: .go-build + stage: build + +lint: + extends: .go-lint + stage: sast + +test: + extends: .go-test + stage: test + +docker:build: + extends: .docker-build + stage: container-build + + rules: + - if: $CI_COMMIT_BRANCH + - if: $CI_MERGE_REQUEST_IID + variables: + DOCKERBUILD_OUTPUT: "type=image,push=false" + - if: $CI_OPEN_MERGE_REQUESTS + when: never + +docker:deploy: + extends: .docker-deploy + stage: deploy + + rules: + - if: $CI_COMMIT_BRANCH + - if: $CI_COMMIT_TAG diff --git a/gitlab-ci/marketplace/npm-template.gitlab-ci.yml b/gitlab-ci/marketplace/npm-template.gitlab-ci.yml new file mode 100644 index 0000000..e85a458 --- /dev/null +++ b/gitlab-ci/marketplace/npm-template.gitlab-ci.yml @@ -0,0 +1,41 @@ +include: +- project: mia-platform/pipeline-templates + file: Application.gitlab-ci.yml + ref: HEAD +- template: Workflows/MergeRequest-Pipelines.gitlab-ci.yml + +variables: + NODE_IMAGE_TAG: "20" + IMAGE_NAME: mia_template_image_name_placeholder + +build: + extends: .npm-build + stage: build + +lint: + extends: .npm-lint + stage: sast + +test: + extends: .npm-test + stage: test + +docker:build: + extends: .docker-build + stage: container-build + + rules: + - if: $CI_COMMIT_BRANCH + - if: $CI_MERGE_REQUEST_IID + variables: + DOCKERBUILD_OUTPUT: "type=image,push=false" + - if: $CI_OPEN_MERGE_REQUESTS + when: never + +docker:deploy: + extends: .docker-deploy + stage: deploy + + rules: + - if: $CI_COMMIT_BRANCH + - if: $CI_COMMIT_TAG diff --git a/gitlab-ci/marketplace/yarn-template.gitlab-ci.yml b/gitlab-ci/marketplace/yarn-template.gitlab-ci.yml new file mode 100644 index 0000000..3454d07 --- /dev/null +++ b/gitlab-ci/marketplace/yarn-template.gitlab-ci.yml @@ -0,0 +1,41 @@ +include: +- project: mia-platform/pipeline-templates + file: Application.gitlab-ci.yml + ref: HEAD +- template: Workflows/MergeRequest-Pipelines.gitlab-ci.yml + +variables: + NODE_IMAGE_TAG: "20" + IMAGE_NAME: mia_template_image_name_placeholder + +build: + extends: .yarn-build + stage: build + +lint: + extends: .yarn-lint + stage: sast + +test: + extends: .yarn-test + stage: test + +docker:build: + extends: .docker-build + stage: container-build + + rules: + - if: $CI_COMMIT_BRANCH + - if: $CI_MERGE_REQUEST_IID + variables: + DOCKERBUILD_OUTPUT: "type=image,push=false" + - if: $CI_OPEN_MERGE_REQUESTS + when: never + +docker:deploy: + extends: .docker-deploy + stage: deploy + + rules: + - if: $CI_COMMIT_BRANCH + - if: $CI_COMMIT_TAG