Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: fix multi arch packages deploy #742

Merged
merged 29 commits into from
Jan 20, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/actions/build-cpp/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
FROM debian:bookworm-slim
FROM ubuntu:noble

ENV BUILD_DEPS "g++ cmake make git pkgconf jq python3 python3-pip python3-setuptools ca-certificates libasan6 zip curl wget"
ENV BUILD_DEPS "g++ cmake make git pkgconf jq python3 python3-pip python3-setuptools ca-certificates zip curl wget gcc-aarch64-linux-gnu g++-aarch64-linux-gnu"

COPY ./entrypoint.sh /entrypoint.sh

Expand Down
5 changes: 5 additions & 0 deletions .github/actions/build-cpp/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,11 @@ inputs:
required: true
default: "ON"

arch:
description: "arch"
required: true
default: "amd64"

file:
description: "Dockerfile used to build the image"
required: true
Expand Down
34 changes: 29 additions & 5 deletions .github/actions/build-cpp/entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,40 @@ function build() {
cp -rf /github/workspace/3rd/ /pktvisor-src/3rd/
cp -rf /github/workspace/libs/ /pktvisor-src/libs/
cp -rf /github/workspace/docker/ /pktvisor-src/docker/
cp -rf /github/workspace/golang/ /pktvisor-src/golang/
cp -rf /github/workspace/build/ /pktvisor-src/build/
cp -rf /github/workspace/golang/ /pktvisor-src/golang/
cp -rf /github/workspace/integration_tests/ /pktvisor-src/integration_tests/
cp -rf /github/workspace/cmake/ /pktvisor-src/cmake/
cp -rf /github/workspace/CMakeLists.txt /pktvisor-src/
cp -rf /github/workspace/conanfile.py /pktvisor-src/
cd /pktvisor-src/build/
cp -rf /github/workspace/.conanrc /pktvisor-src/
cd /pktvisor-src/
conan profile detect -f
PKG_CONFIG_PATH=/local/lib/pkgconfig cmake -DCMAKE_BUILD_TYPE=$INPUT_BUILD_TYPE -DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=./cmake/conan_provider.cmake -DASAN=$INPUT_ASAN ..
make all -j 4
cd /pktvisor-src/build/
if [ "$INPUT_ARCH" == "amd64" ]; then
PKG_CONFIG_PATH=/local/lib/pkgconfig cmake .. -DCMAKE_BUILD_TYPE=$INPUT_BUILD_TYPE \
-DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=./cmake/conan_provider.cmake -DASAN=$INPUT_ASAN
elif [ "$INPUT_ARCH" == "arm64" ]; then
echo "[settings]
os=Linux
arch=armv8
compiler=gcc
compiler.version=13
compiler.cppstd=17
compiler.libcxx=libstdc++11
build_type=$INPUT_BUILD_TYPE
[buildenv]
CC=/usr/bin/aarch64-linux-gnu-gcc
CXX=/usr/bin/aarch64-linux-gnu-g++
" | tee "$(conan config home)/profiles/host"
conan install .. --profile host --build missing
source $INPUT_BUILD_TYPE/generators/conanbuild.sh
PKG_CONFIG_PATH=/local/lib/pkgconfig cmake .. -DCMAKE_BUILD_TYPE=$INPUT_BUILD_TYPE \
-DCMAKE_PROJECT_TOP_LEVEL_INCLUDES=./cmake/conan_provider.cmake -DASAN=$INPUT_ASAN \
-DCONAN_HOST_PROFILE="host" -DCORRADE_RC_PROGRAM=$(command -v corrade-rc) \
-DCONAN_INSTALL_ARGS=--build=never
fi
cmake --build . --config $INPUT_BUILD_TYPE -- -j 4
}

function move() {
Expand All @@ -49,7 +73,7 @@ function publishToBugsplat() {
./dump_syms /github/workspace/pktvisord > pktvisor.sym
PKTVISOR_VERSION=$(cat VERSION)
ls -lha
./symupload -k $INPUT_BUGSPLAT_KEY pktvisor.sym $INPUT_BUGSPLAT_SYMBOL_URL$PKTVISOR_VERSION 2>/dev/null
./symupload -k $INPUT_BUGSPLAT_KEY pktvisor.sym $INPUT_BUGSPLAT_SYMBOL_URL$INPUT_ARCH$PKTVISOR_VERSION 2>/dev/null
fi
}

Expand Down
147 changes: 106 additions & 41 deletions .github/workflows/build-develop.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ env:
CTEST_OUTPUT_ON_FAILURE: 1
CONAN_NON_INTERACTIVE: 1
CONAN_REVISIONS_ENABLED: 1
IMAGE_NAME: netboxlabs/pktvisor

jobs:
unit-tests-mac:
Expand Down Expand Up @@ -61,7 +62,7 @@ jobs:
- name: Test
working-directory: ${{github.workspace}}/build
shell: bash
run: ctest -C $BUILD_TYPE
run: ctest -C $BUILD_TYPE --output-on-failure

unit-tests-linux:
runs-on: ubuntu-latest
Expand All @@ -82,8 +83,8 @@ jobs:
uses: actions/cache@v4
with:
path: ${{github.workspace}}/build/p/
key: conan-${{ runner.os }}-${{ hashFiles('conanfile.py', '*/conanfile.py') }}
restore-keys: conan-${{ runner.os }}-
key: conan-${{ runner.os }}-amd64-${{ hashFiles('conanfile.py', '*/conanfile.py') }}
restore-keys: conan-${{ runner.os }}-amd64-

- name: Configure CMake
shell: bash
Expand All @@ -104,7 +105,7 @@ jobs:
- name: Test
working-directory: ${{github.workspace}}/build
shell: bash
run: sudo ctest -C $BUILD_TYPE
run: sudo ctest --output-on-failure

build-win64:
runs-on: windows-2019
Expand Down Expand Up @@ -231,19 +232,20 @@ jobs:
uses: actions/cache@v4
with:
path: ${{github.workspace}}/build/p/
key: conan-${{ runner.os }}-${{ hashFiles('conanfile.py', '*/conanfile.py') }}
restore-keys: conan-${{ runner.os }}-
key: conan-${{ runner.os }}-amd64-${{ hashFiles('conanfile.py', '*/conanfile.py') }}
restore-keys: conan-${{ runner.os }}-amd64-

- name: Build pktvisord + push symbol to bugsplat.com
- name: Build pktvisord + push symbol to backtrace.io
uses: ./.github/actions/build-cpp
with:
context: "."
build_type: "Release"
asan: "OFF"
bugsplat_key: ${{secrets.BUGSPLAT_KEY}}
bugsplat_symbol_url: ${{secrets.BUGSPLAT_SYMBOL_URL}}
bugsplat: "true"
bugsplat: "false"
file: "./Dockerfile"
arch: "amd64"

- name: Build pktvisor-cli
uses: ./.github/actions/build-go
Expand All @@ -255,15 +257,14 @@ jobs:
run: ls -lha .

- name: Get VERSION
id: build
run: |
echo "VERSION=${{needs.unit-tests-linux.outputs.version_number}}" >> $GITHUB_ENV

- name: Generate ref tag (develop)
run: echo "REF_TAG=latest-develop" >> $GITHUB_ENV

- name: Debug ref tag
run: echo ${{ env.REF_TAG }}
- name: Docker meta
id: meta
uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 #v5.6.1
with:
images: ${{ env.IMAGE_NAME }}

- name: Login to Docker Hub
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 #v3.3.0
Expand All @@ -289,24 +290,33 @@ jobs:
uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 #v3.8.0

- name: Build + push - pktvisor (multi-arch)
env:
IMAGE_NAME: netboxlabs/pktvisor
uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 #v6.10.0
id: docker_build
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
file: ./docker/Dockerfile.crashhandler
platforms: linux/amd64
push: true
tags: |
${{ env.IMAGE_NAME }}:${{ needs.unit-tests-linux.outputs.version_number }}
${{ env.IMAGE_NAME }}:${{ env.REF_TAG }}
outputs: type=docker,dest=/tmp/amd64.tar
labels: ${{ steps.meta.outputs.labels }}
outputs: type=image,"name=${{ env.IMAGE_NAME }}",push-by-digest=true,name-canonical=true,push=true

- name: Export digest
run: |
mkdir -p /tmp/digests
digest="${{ steps.docker_build.outputs.digest }}"
touch "/tmp/digests/${digest#sha256:}"

- name: Upload digest
uses: actions/upload-artifact@v4
with:
name: digests-linux-amd64
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1

build-app-image-x64:
needs: [ package-amd64 ]
needs: [ merge-packages ]
runs-on: ubuntu-latest
#if: github.event_name != 'pull_request'
steps:
- uses: actions/checkout@v4

Expand All @@ -317,6 +327,13 @@ jobs:
id: conan
uses: turtlebrowser/get-conan@c171f295f3f507360ee018736a6608731aa2109d #v1.2

- name: Setup Conan Cache
uses: actions/cache@v4
with:
path: ${{github.workspace}}/build/p/
key: conan-${{ runner.os }}-amd64-${{ hashFiles('conanfile.py', '*/conanfile.py') }}
restore-keys: conan-${{ runner.os }}-amd64-

- name: Configure CMake to generate VERSION
shell: bash
working-directory: ${{github.workspace}}/build
Expand Down Expand Up @@ -365,15 +382,13 @@ jobs:

- name: Build + push - pktvisor-prom-write
env:
IMAGE_NAME: netboxlabs/pktvisor-prom-write
PROM_IMAGE_NAME: netboxlabs/pktvisor-prom-write
working-directory: ${{github.workspace}}/centralized_collection/prometheus/docker-grafana-agent
run: |
docker build . --file Dockerfile --build-arg PKTVISOR_TAG=${{ env.REF_TAG }} --tag ${{ env.IMAGE_NAME }}:${{ env.VERSION }} --tag ${{ env.IMAGE_NAME }}:${{ env.REF_TAG }}
docker push -a ${{ env.IMAGE_NAME }}
docker build . --file Dockerfile --build-arg PKTVISOR_TAG=${{ env.REF_TAG }} --tag ${{ env.PROM_IMAGE_NAME }}:${{ env.VERSION }} --tag ${{ env.PROM_IMAGE_NAME }}:${{ env.REF_TAG }}
docker push -a ${{ env.PROM_IMAGE_NAME }}

- name: Generate AppImage
env:
IMAGE_NAME: netboxlabs/pktvisor
working-directory: ${{github.workspace}}/appimage
run: |
DEV_IMAGE="${{ env.IMAGE_NAME }}:${{ env.VERSION }}" DEV_MODE=t make pktvisor-x86_64.AppImage
Expand Down Expand Up @@ -406,8 +421,8 @@ jobs:
path: ${{github.workspace}}/build/p/
key: conan-${{ runner.os }}-arm64-${{ hashFiles('conanfile.py', '*/conanfile.py') }}
restore-keys: conan-${{ runner.os }}-arm64-

- name: Build pktvisord + push symbol to bugsplat.com
- name: Build pktvisord + push symbol to backtrace.io
uses: ./.github/actions/build-cpp
with:
context: "."
Expand All @@ -417,6 +432,7 @@ jobs:
bugsplat_symbol_url: ${{secrets.BUGSPLAT_SYMBOL_URL}}
bugsplat: "false"
file: "./Dockerfile"
arch: "arm64"

- name: Build pktvisor-cli
uses: ./.github/actions/build-go
Expand All @@ -428,11 +444,11 @@ jobs:
- name: Debug artifacts
run: ls -lha .

- name: Generate ref tag (develop)
run: echo "REF_TAG=latest-develop" >> $GITHUB_ENV

- name: Debug ref tag
run: echo ${{ env.REF_TAG }}
- name: Docker meta
id: meta
uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 #v5.6.1
with:
images: ${{ env.IMAGE_NAME }}

- name: Login to Docker Hub
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 #v3.3.0
Expand All @@ -459,16 +475,65 @@ jobs:

- name: Build + push - pktvisor (multi-arch)
id: docker_build
env:
IMAGE_NAME: netboxlabs/pktvisor
uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 #v6.10.0
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
file: ./docker/Dockerfile.crashhandler
platforms: linux/arm64
push: true
labels: ${{ steps.meta.outputs.labels }}
outputs: type=image,"name=${{ env.IMAGE_NAME }}",push-by-digest=true,name-canonical=true,push=true

- name: Export digest
run: |
mkdir -p /tmp/digests
digest="${{ steps.docker_build.outputs.digest }}"
touch "/tmp/digests/${digest#sha256:}"

- name: Upload digest
uses: actions/upload-artifact@v4
with:
name: digests-linux-arm64
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1

merge-packages:
runs-on: ubuntu-latest
needs: [unit-tests-linux, package-amd64, package-arm64]
steps:
- name: Download digests
uses: actions/download-artifact@v4
with:
path: /tmp/digests
pattern: digests-*
merge-multiple: true

- name: Login to Docker Hub
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 #v3.3.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}

- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 #v3.8.0

- name: Docker meta
id: meta
uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 #v5.6.1
with:
images: ${{ env.IMAGE_NAME }}
tags: |
${{ env.IMAGE_NAME }}:${{ needs.unit-tests-linux.outputs.version_number }}
${{ env.IMAGE_NAME }}:${{ env.REF_TAG }}
outputs: type=docker,dest=/tmp/arm64.tar
type=raw,value=latest-develop
type=raw,value=${{ needs.unit-tests-linux.outputs.version_number }}

- name: Create manifest list and push
working-directory: /tmp/digests
run: |
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
$(printf '${{ env.IMAGE_NAME }}@sha256:%s ' *)

- name: Inspect image
run: |
docker buildx imagetools inspect ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
2 changes: 1 addition & 1 deletion .github/workflows/build-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ jobs:
- name: Test
working-directory: ${{github.workspace}}/build
shell: bash
run: sudo ctest -C $BUILD_TYPE
run: sudo ctest --output-on-failure

prebuild-package:
needs: [ unit-tests ]
Expand Down
7 changes: 7 additions & 0 deletions .github/workflows/build_cross.yml
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,13 @@ jobs:
LDFLAGS=${{matrix.ldflags}}
EOF

- name: Setup Conan Cache
uses: actions/cache@v4
with:
path: ${{github.workspace}}/src/build/p/
key: conan-${{ runner.os }}-${{matrix.arch}}-${{ hashFiles('**/conanfile.py') }}
restore-keys: conan-${{ runner.os }}-${{matrix.arch}}-

- name: Install dependencies
working-directory: ${{github.workspace}}/src
run: |
Expand Down
Loading
Loading