Skip to content

Commit

Permalink
Remove apk go requirement when possible (#2318)
Browse files Browse the repository at this point in the history
* Install revive from ghcr docker binary

* Install actionlint from docker image's binary to remove apk go

* Install checkmake from docker image binary

* Use latest instead of tags for modified docker images

* Use a builder image for revive as the released binaries return broken version number

* Edit comment in actionlint Dockerfile instructions

* Build dustilock in a stage and copy go binary

* Specify GOBIN folder for revive install in builder

* Remove unneeded GOPATH and GOROOT variables

Unneeded since we are not building go packages from source anymore

* Include fixes from linters

* Add `COPY --link --from` by using dockerfile frontend syntax

* Revert: Remove unneeded GOPATH and GOROOT variables

* Run build.sh

* Run ./build.sh after rebase

* Include apk go requirement for golangci-lint

* Add changelog entry

* Handle duplicated FROM and COPY instructions

* cSpell fixes
  • Loading branch information
echoix authored Feb 19, 2023
1 parent bed68e1 commit fb924c7
Show file tree
Hide file tree
Showing 155 changed files with 800 additions and 517 deletions.
22 changes: 20 additions & 2 deletions .automation/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -212,8 +212,13 @@ def generate_flavor(flavor, flavor_info):
os.makedirs(os.path.dirname(dockerfile), exist_ok=True)
copyfile(f"{REPO_HOME}/Dockerfile", dockerfile)
flavor_label = flavor_info["label"]
comment = f"# MEGA-LINTER FLAVOR [{flavor}]: {flavor_label}"
comment = f"# MEGALINTER FLAVOR [{flavor}]: {flavor_label}"
with open(dockerfile, "r+", encoding="utf-8") as f:
first_line = f.readline().rstrip()
if first_line.startswith("# syntax="):
comment = f"{first_line}\n{comment}"
else:
f.seek(0)
content = f.read()
f.seek(0)
f.truncate()
Expand Down Expand Up @@ -300,14 +305,27 @@ def build_dockerfile(
for dockerfile_item in item["install"]["dockerfile"]:
# FROM
if dockerfile_item.startswith("FROM"):
if dockerfile_item in all_dockerfile_items:
dockerfile_item = (
"# Next FROM line commented because already managed by another linter\n"
"# " + "\n# ".join(dockerfile_item.splitlines())
)
docker_from += [dockerfile_item]
# ARG
elif dockerfile_item.startswith("ARG"):
docker_arg += [dockerfile_item]
# COPY
elif dockerfile_item.startswith("COPY"):
if dockerfile_item in all_dockerfile_items:
dockerfile_item = (
"# Next COPY line commented because already managed by another linter\n"
"# " + "\n# ".join(dockerfile_item.splitlines())
)
docker_copy += [dockerfile_item]
docker_other += ["# Managed with " + dockerfile_item]
docker_other += [
"# Managed with "
+ "\n# ".join(dockerfile_item.splitlines())
]
# Already used item
elif (
dockerfile_item in all_dockerfile_items
Expand Down
4 changes: 4 additions & 0 deletions .github/linters/.cspell.json
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@
"Fmegalinter",
"GHSA",
"GITLEAKS",
"GOBIN",
"GOLANGCI",
"GOODCHECK",
"GOPATH",
Expand Down Expand Up @@ -539,6 +540,7 @@
"descriptionless",
"devcontainer",
"devcontainers",
"devel",
"devskim",
"dimen",
"dimendef",
Expand Down Expand Up @@ -864,6 +866,7 @@
"mlinters",
"moarvm",
"mprettier",
"mrtazz",
"msgon",
"mstruebing",
"msvs",
Expand Down Expand Up @@ -977,6 +980,7 @@
"packagename",
"pagebreak",
"pageref",
"pandoc",
"parallelization",
"paren",
"parens",
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ Note: Can be used with `oxsecurity/megalinter@beta` in your GitHub Action mega-l
- CI
- Use docker/build-push-action to build docker images and akhilerm/tag-push-action to release by retagging and pushing beta images instead of rebuilding them
- Authenticate to GitHub API during docker build to avoid reaching limits
- Remove apk go package install in images where possible to decrease image sizes, by @echoix in <https://github.com/oxsecurity/megalinter/pull/2318>

- Fixes
- Replace deprecated spectral package, by @bdovaz in by @bdovaz in <https://github.com/oxsecurity/megalinter/pull/2340>
Expand Down
106 changes: 62 additions & 44 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
# syntax=docker/dockerfile:1
###########################################
###########################################
## Dockerfile to run MegaLinter ##
Expand All @@ -10,15 +11,33 @@
## @generated by .automation/build.py using descriptor files, please do not update manually ##
#############################################################################################
#FROM__START
FROM rhysd/actionlint:latest as actionlint
# shellcheck is a dependency for actionlint

FROM koalaman/shellcheck:stable as shellcheck
# Next FROM line commented because already managed by another linter
# FROM koalaman/shellcheck:stable as shellcheck
FROM mvdan/shfmt:latest-alpine as shfmt
FROM hadolint/hadolint:v2.12.0-alpine as hadolint
FROM mstruebing/editorconfig-checker:2.7.0 as editorconfig-checker
FROM golang:1.19-alpine as revive
## The golang image used as a builder is a temporary workaround
## for the released revive binaries not returning version numbers (devel).
## The install command should then be what is commented in the go.megalinter-descriptor.yml
RUN GOBIN=/usr/bin go install github.com/mgechev/revive@latest

FROM ghcr.io/assignuser/chktex-alpine:latest as chktex
FROM mrtazz/checkmake:latest as checkmake
FROM yoheimuta/protolint:latest as protolint
FROM golang:alpine as dustilock
RUN GOBIN=/usr/bin go install github.com/checkmarx/[email protected]

FROM zricethezav/gitleaks:v8.15.3 as gitleaks
FROM ghcr.io/terraform-linters/tflint:v0.45.0 as tflint
FROM tenable/terrascan:1.18.0 as terrascan
FROM alpine/terragrunt:latest as terragrunt
# Next FROM line commented because already managed by another linter
# FROM alpine/terragrunt:latest as terragrunt
FROM checkmarx/kics:alpine as kics
#FROM__END

Expand Down Expand Up @@ -69,7 +88,6 @@ RUN apk add --update --no-cache \
openssh \
docker \
openrc \
go \
icu-libs \
libcurl \
libintl \
Expand All @@ -93,9 +111,11 @@ RUN apk add --update --no-cache \
php81-simplexml \
composer \
dpkg \
py3-pyflakes \
nodejs \
npm \
yarn \
go \
openssl \
readline-dev \
g++ \
Expand Down Expand Up @@ -280,17 +300,26 @@ ENV PATH="/root/.cargo/bin:${PATH}"
#############################################################################################

#COPY__START
COPY --from=shfmt /bin/shfmt /usr/bin/
COPY --from=hadolint /bin/hadolint /usr/bin/hadolint
COPY --from=editorconfig-checker /usr/bin/ec /usr/bin/editorconfig-checker
COPY --from=chktex /usr/bin/chktex /usr/bin/
COPY --from=protolint /usr/local/bin/protolint /usr/bin/
COPY --from=gitleaks /usr/bin/gitleaks /usr/bin/
COPY --from=tflint /usr/local/bin/tflint /usr/bin/
COPY --from=terrascan /go/bin/terrascan /usr/bin/
COPY --from=terragrunt /usr/local/bin/terragrunt /usr/bin/
COPY --from=terragrunt /bin/terraform /usr/bin/
COPY --from=kics /app/bin/kics /usr/bin/
COPY --link --from=actionlint /usr/local/bin/actionlint /usr/bin/actionlint
# shellcheck is a dependency for actionlint

COPY --link --from=shellcheck /bin/shellcheck /usr/bin/shellcheck
# Next COPY line commented because already managed by another linter
# COPY --link --from=shellcheck /bin/shellcheck /usr/bin/shellcheck
COPY --link --from=shfmt /bin/shfmt /usr/bin/
COPY --link --from=hadolint /bin/hadolint /usr/bin/hadolint
COPY --link --from=editorconfig-checker /usr/bin/ec /usr/bin/editorconfig-checker
COPY --link --from=revive /usr/bin/revive /usr/bin/revive
COPY --link --from=chktex /usr/bin/chktex /usr/bin/
COPY --link --from=checkmake /checkmake /usr/bin/checkmake
COPY --link --from=protolint /usr/local/bin/protolint /usr/bin/
COPY --link --from=dustilock /usr/bin/dustilock /usr/bin/dustilock
COPY --link --from=gitleaks /usr/bin/gitleaks /usr/bin/
COPY --link --from=tflint /usr/local/bin/tflint /usr/bin/
COPY --link --from=terrascan /go/bin/terrascan /usr/bin/
COPY --link --from=terragrunt /usr/local/bin/terragrunt /usr/bin/
COPY --link --from=terragrunt /bin/terraform /usr/bin/
COPY --link --from=kics /app/bin/kics /usr/bin/
COPY --from=kics /app/bin/assets /opt/kics/assets/
#COPY__END

Expand Down Expand Up @@ -380,8 +409,9 @@ RUN echo y|sfdx plugins:install sfdx-hardis \
# ENV PATH="${PATH}:/root/.dotnet/tools:/usr/share/dotnet"

# actionlint installation
ENV GO111MODULE=on
RUN go install github.com/rhysd/actionlint/cmd/actionlint@latest && go clean --cache
# Managed with COPY --link --from=actionlint /usr/local/bin/actionlint /usr/bin/actionlint
# # shellcheck is a dependency for actionlint
# Managed with COPY --link --from=shellcheck /bin/shellcheck /usr/bin/shellcheck

# arm-ttk installation
ENV ARM_TTK_PSD1="${ARM_TTK_DIRECTORY}/arm-ttk-master/arm-ttk/arm-ttk.psd1"
Expand All @@ -396,14 +426,11 @@ RUN curl --retry 5 --retry-delay 5 -sLO "${ARM_TTK_URI}" \
&& chmod +x /usr/bin/bash-exec \

# shellcheck installation
&& ML_THIRD_PARTY_DIR="/third-party/shellcheck" \
&& mkdir -p ${ML_THIRD_PARTY_DIR} \
&& wget -qO- "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz" | tar -xJv --directory ${ML_THIRD_PARTY_DIR} \
&& mv "${ML_THIRD_PARTY_DIR}/shellcheck-stable/shellcheck" /usr/bin/ \
&& find ${ML_THIRD_PARTY_DIR} -type f -not -name 'LICENSE*' -delete -o -type d -empty -delete \
# Managed with # Next COPY line commented because already managed by another linter
# # COPY --link --from=shellcheck /bin/shellcheck /usr/bin/shellcheck

# shfmt installation
# Managed with COPY --from=shfmt /bin/shfmt /usr/bin/
# Managed with COPY --link --from=shfmt /bin/shfmt /usr/bin/

# bicep_linter installation
&& curl --retry 5 --retry-delay 5 -sLo ${BICEP_EXE} "${BICEP_URI}" \
Expand All @@ -428,20 +455,21 @@ RUN curl --retry 5 --retry-delay 5 -sLO "${ARM_TTK_URI}" \
&& rm -r dart-sdk/ \

# hadolint installation
# Managed with COPY --from=hadolint /bin/hadolint /usr/bin/hadolint
# Managed with COPY --link --from=hadolint /bin/hadolint /usr/bin/hadolint

# editorconfig-checker installation
# Managed with COPY --from=editorconfig-checker /usr/bin/ec /usr/bin/editorconfig-checker
# Managed with COPY --link --from=editorconfig-checker /usr/bin/ec /usr/bin/editorconfig-checker

# dotenv-linter installation
&& wget -q -O - https://raw.githubusercontent.com/dotenv-linter/dotenv-linter/master/install.sh | sh -s \

# golangci-lint installation
&& wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh \
&& golangci-lint --version \
&& golangci-lint --version


# revive installation
&& go install github.com/mgechev/revive@latest && go clean --cache
# Managed with COPY --link --from=revive /usr/bin/revive /usr/bin/revive

# checkstyle installation
RUN --mount=type=secret,id=GITHUB_TOKEN CHECKSTYLE_LATEST=$(curl -s \
Expand Down Expand Up @@ -487,7 +515,7 @@ RUN wget --quiet https://github.com/pmd/pmd/releases/download/pmd_releases%2F${P
&& find ${ML_THIRD_PARTY_DIR} -type f -not -name 'LICENSE*' -delete -o -type d -empty -delete \

# chktex installation
# Managed with COPY --from=chktex /usr/bin/chktex /usr/bin/
# Managed with COPY --link --from=chktex /usr/bin/chktex /usr/bin/
&& cd ~ && touch .chktexrc && cd / \

# luacheck installation
Expand All @@ -506,9 +534,7 @@ RUN wget --quiet https://github.com/pmd/pmd/releases/download/pmd_releases%2F${P
&& cd / \

# checkmake installation
&& ( [ -d /usr/local/bin ] || mkdir -p /usr/local/bin ) \
&& wget -q "https://github.com/mrtazz/checkmake/releases/download/0.2.1/checkmake-0.2.1.linux.amd64" -O /usr/local/bin/checkmake \
&& chmod 755 /usr/local/bin/checkmake \
# Managed with COPY --link --from=checkmake /checkmake /usr/bin/checkmake

# perlcritic installation
&& curl --retry 5 --retry-delay 5 -sL https://cpanmin.us/ | perl - -nq --no-wget Perl::Critic
Expand Down Expand Up @@ -537,7 +563,7 @@ RUN composer global require --ignore-platform-reqs overtrue/phplint ^5.3 \
# RUN pwsh -c 'Install-Module -Name PSScriptAnalyzer -RequiredVersion ${PSSA_VERSION} -Scope AllUsers -Force'

# protolint installation
# Managed with COPY --from=protolint /usr/local/bin/protolint /usr/bin/
# Managed with COPY --link --from=protolint /usr/local/bin/protolint /usr/bin/

# lintr installation
&& mkdir -p /home/r-library \
Expand Down Expand Up @@ -565,18 +591,10 @@ ENV PATH="~/.raku/bin:/opt/rakudo-pkg/bin:/opt/rakudo-pkg/share/perl6/site/bin:$
RUN dotnet tool install --global Microsoft.CST.DevSkim.CLI \

# dustilock installation
&& ML_THIRD_PARTY_DIR=/download/dustilock && \
mkdir -p ${ML_THIRD_PARTY_DIR} && \
git clone https://github.com/Checkmarx/dustilock.git ${ML_THIRD_PARTY_DIR} && \
cd ${ML_THIRD_PARTY_DIR} && \
go build && go clean --cache && \
chmod +x dustilock && \
mv "${ML_THIRD_PARTY_DIR}/dustilock" /usr/bin/ && \
find ${ML_THIRD_PARTY_DIR} -type f -not -name 'LICENSE*' -delete -o -type d -empty -delete && \
cd / \
# Managed with COPY --link --from=dustilock /usr/bin/dustilock /usr/bin/dustilock

# gitleaks installation
# Managed with COPY --from=gitleaks /usr/bin/gitleaks /usr/bin/
# Managed with COPY --link --from=gitleaks /usr/bin/gitleaks /usr/bin/

# syft installation
&& curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin \
Expand Down Expand Up @@ -622,19 +640,19 @@ RUN dotnet tool install --global Microsoft.CST.DevSkim.CLI \
&& dotnet tool install --global TSQLLint \

# tflint installation
# Managed with COPY --from=tflint /usr/local/bin/tflint /usr/bin/
# Managed with COPY --link --from=tflint /usr/local/bin/tflint /usr/bin/

# terrascan installation
# Managed with COPY --from=terrascan /go/bin/terrascan /usr/bin/
# Managed with COPY --link --from=terrascan /go/bin/terrascan /usr/bin/

# terragrunt installation
# Managed with COPY --from=terragrunt /usr/local/bin/terragrunt /usr/bin/
# Managed with COPY --link --from=terragrunt /usr/local/bin/terragrunt /usr/bin/

# terraform-fmt installation
# Managed with COPY --from=terragrunt /bin/terraform /usr/bin/
# Managed with COPY --link --from=terragrunt /bin/terraform /usr/bin/

# kics installation
# Managed with COPY --from=kics /app/bin/kics /usr/bin/
# Managed with COPY --link --from=kics /app/bin/kics /usr/bin/
&& mkdir -p /opt/kics/assets
ENV KICS_QUERIES_PATH=/opt/kics/assets/queries KICS_LIBRARIES_PATH=/opt/kics/assets/libraries
# Managed with COPY --from=kics /app/bin/assets /opt/kics/assets/
Expand Down
31 changes: 15 additions & 16 deletions flavors/ci_light/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# MEGA-LINTER FLAVOR [ci_light]: Optimized for CI items (Dockerfile, Jenkinsfile, JSON/YAML schemas,XML
# syntax=docker/dockerfile:1
# MEGALINTER FLAVOR [ci_light]: Optimized for CI items (Dockerfile, Jenkinsfile, JSON/YAML schemas,XML
###########################################
###########################################
## Dockerfile to run MegaLinter ##
Expand All @@ -11,8 +12,10 @@
## @generated by .automation/build.py using descriptor files, please do not update manually ##
#############################################################################################
#FROM__START
FROM koalaman/shellcheck:stable as shellcheck
FROM mvdan/shfmt:latest-alpine as shfmt
FROM hadolint/hadolint:v2.12.0-alpine as hadolint
FROM mrtazz/checkmake:latest as checkmake
FROM zricethezav/gitleaks:v8.15.3 as gitleaks
#FROM__END

Expand Down Expand Up @@ -165,38 +168,34 @@ ENV PATH="/root/.cargo/bin:${PATH}"
#############################################################################################

#COPY__START
COPY --from=shfmt /bin/shfmt /usr/bin/
COPY --from=hadolint /bin/hadolint /usr/bin/hadolint
COPY --from=gitleaks /usr/bin/gitleaks /usr/bin/
COPY --link --from=shellcheck /bin/shellcheck /usr/bin/shellcheck
COPY --link --from=shfmt /bin/shfmt /usr/bin/
COPY --link --from=hadolint /bin/hadolint /usr/bin/hadolint
COPY --link --from=checkmake /checkmake /usr/bin/checkmake
COPY --link --from=gitleaks /usr/bin/gitleaks /usr/bin/
#COPY__END

#############################################################################################
## @generated by .automation/build.py using descriptor files, please do not update manually ##
#############################################################################################
#OTHER__START
# shellcheck installation
RUN ML_THIRD_PARTY_DIR="/third-party/shellcheck" \
&& mkdir -p ${ML_THIRD_PARTY_DIR} \
&& wget -qO- "https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz" | tar -xJv --directory ${ML_THIRD_PARTY_DIR} \
&& mv "${ML_THIRD_PARTY_DIR}/shellcheck-stable/shellcheck" /usr/bin/ \
&& find ${ML_THIRD_PARTY_DIR} -type f -not -name 'LICENSE*' -delete -o -type d -empty -delete \
# Managed with COPY --link --from=shellcheck /bin/shellcheck /usr/bin/shellcheck

# shfmt installation
# Managed with COPY --from=shfmt /bin/shfmt /usr/bin/
# Managed with COPY --link --from=shfmt /bin/shfmt /usr/bin/

# hadolint installation
# Managed with COPY --from=hadolint /bin/hadolint /usr/bin/hadolint
# Managed with COPY --link --from=hadolint /bin/hadolint /usr/bin/hadolint

# dotenv-linter installation
&& wget -q -O - https://raw.githubusercontent.com/dotenv-linter/dotenv-linter/master/install.sh | sh -s \
RUN wget -q -O - https://raw.githubusercontent.com/dotenv-linter/dotenv-linter/master/install.sh | sh -s \

# checkmake installation
&& ( [ -d /usr/local/bin ] || mkdir -p /usr/local/bin ) \
&& wget -q "https://github.com/mrtazz/checkmake/releases/download/0.2.1/checkmake-0.2.1.linux.amd64" -O /usr/local/bin/checkmake \
&& chmod 755 /usr/local/bin/checkmake \
# Managed with COPY --link --from=checkmake /checkmake /usr/bin/checkmake

# gitleaks installation
# Managed with COPY --from=gitleaks /usr/bin/gitleaks /usr/bin/
# Managed with COPY --link --from=gitleaks /usr/bin/gitleaks /usr/bin/

# trivy installation
&& wget --tries=5 -q -O - https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
Expand Down
Loading

0 comments on commit fb924c7

Please sign in to comment.