Skip to content

Commit

Permalink
Merge branch 'develop' into test-ci
Browse files Browse the repository at this point in the history
  • Loading branch information
stefankiesz authored Sep 27, 2024
2 parents c785b7b + eabf283 commit 55a8b64
Show file tree
Hide file tree
Showing 30 changed files with 1,301 additions and 320 deletions.
75 changes: 53 additions & 22 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ jobs:
- name: Run tests
run: |
cd build
source ../scripts/get-iot-certs.sh ci
./tst/producer_test
mac-os-build-clang:
Expand Down Expand Up @@ -81,6 +82,37 @@ jobs:
- name: Run tests
run: |
cd build
source ../scripts/get-iot-certs.sh ci
./tst/producer_test
mac-os-m1-build-clang:
runs-on: macos-13-xlarge
env:
AWS_KVS_LOG_LEVEL: 2
permissions:
id-token: write
contents: read
steps:
- name: Clone repository
uses: actions/checkout@v3
- name: Build repository
run: |
brew install pkgconfig
brew unlink openssl # it seems the libcurl is trying to access this openssl despite explicitly setting it to our build
mkdir build && cd build
sh -c 'cmake .. -DBUILD_TEST=TRUE -DCOMPILER_WARNINGS=TRUE -DCMAKE_C_COMPILER=$(brew --prefix llvm@15)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm@15)/bin/clang++;cmake .. -DBUILD_TEST=TRUE -DCOMPILER_WARNINGS=TRUE -DCMAKE_C_COMPILER=$(brew --prefix llvm@15)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm@15)/bin/clang++'
make
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1-node16
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
role-session-name: ${{ secrets.AWS_ROLE_SESSION_NAME }}
aws-region: ${{ secrets.AWS_REGION }}
role-duration-seconds: 10800
- name: Run tests
run: |
cd build
source ../scripts/get-iot-certs.sh ci
./tst/producer_test
mac-os-m1-build-clang:
Expand Down Expand Up @@ -178,6 +210,7 @@ jobs:
- name: Run tests
run: |
cd build
source ../scripts/get-iot-certs.sh ci
./tst/producer_test
mac-os-build-clang-local-openssl:
Expand Down Expand Up @@ -209,40 +242,30 @@ jobs:
- name: Run tests
run: |
cd build
source ../scripts/get-iot-certs.sh ci
./tst/producer_test
linux-gcc-code-coverage:
ubuntu-os-gcc-build-lws-mbedtls:
runs-on: ubuntu-20.04
env:
AWS_KVS_LOG_LEVEL: 2
CC: gcc
CXX: g++
permissions:
id-token: write
contents: read
steps:
- name: Clone repository
uses: actions/checkout@v3
- name: Install dependencies
run: |
sudo apt clean && sudo apt update
sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'
- name: Build repository
run: |
mkdir build && cd build
cmake .. -DCODE_COVERAGE=TRUE -DBUILD_TEST=TRUE -DBUILD_COMMON_LWS=TRUE
cmake .. -DBUILD_COMMON_LWS=ON -DBUILD_COMMON_CURL=OFF -DUSE_OPENSSL=OFF -DUSE_MBEDTLS=ON
make
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1-node16
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
role-session-name: ${{ secrets.AWS_ROLE_SESSION_NAME }}
aws-region: ${{ secrets.AWS_REGION }}
role-duration-seconds: 10800
- name: Run tests
run: |
cd build
ulimit -c unlimited -S
timeout --signal=SIGABRT 150m ./tst/producer_test --gtest_break_on_failure
- name: Code coverage
run: |
cd build
for test_file in $(find cproducer.dir kvsCommonCurl.dir kvsCommonLws.dir -name '*.gcno'); do gcov $test_file; done
bash <(curl -s https://codecov.io/bash)
address-sanitizer:
runs-on: ubuntu-20.04
Expand All @@ -253,6 +276,10 @@ jobs:
CC: clang
CXX: clang++
AWS_KVS_LOG_LEVEL: 2
# The unit tests will have both LWS and CURL enabled, both linking to globalMemCalloc. Our PIC is
# has quite a few global definitions that get linked twice. TODO: see if we can list common dependency
# to remove this option
ASAN_OPTIONS: detect_odr_violation=0
steps:
- name: Clone repository
uses: actions/checkout@v3
Expand All @@ -272,6 +299,7 @@ jobs:
run: |
cd build
ulimit -c unlimited -S
source ../scripts/get-iot-certs.sh ci
timeout --signal=SIGABRT 150m ./tst/producer_test --gtest_break_on_failure
undefined-behavior-sanitizer:
Expand Down Expand Up @@ -302,8 +330,10 @@ jobs:
run: |
cd build
ulimit -c unlimited -S
source ../scripts/get-iot-certs.sh ci
timeout --signal=SIGABRT 150m ./tst/producer_test --gtest_break_on_failure
# memory-sanitizer:
# runs-on: ubuntu-20.04
# permissions:
Expand Down Expand Up @@ -378,7 +408,7 @@ jobs:
- name: Build repository
run: |
mkdir build && cd build
cmake .. -DBUILD_TEST=TRUE
cmake .. -DBUILD_TEST=TRUE -DBUILD_COMMON_LWS=ON
make
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1-node16
Expand All @@ -391,6 +421,7 @@ jobs:
run: |
cd build
ulimit -c unlimited -S
source ../scripts/get-iot-certs.sh ci
timeout --signal=SIGABRT 150m ./tst/producer_test --gtest_break_on_failure
windows-msvc:
Expand All @@ -405,7 +436,7 @@ jobs:
uses: actions/checkout@v3
- name: Install dependencies
run: |
choco install nasm strawberryperl
choco install nasm strawberryperl pkgconfiglite
- name: Build repository
run: |
$env:Path += ';C:\Strawberry\perl\site\bin;C:\Strawberry\perl\bin;C:\Strawberry\c\bin;C:\Program Files\NASM;D:\a\amazon-kinesis-video-streams-producer-c\amazon-kinesis-video-streams-producer-c\open-source\lib;D:\a\amazon-kinesis-video-streams-producer-c\amazon-kinesis-video-streams-producer-c\open-source\bin'
Expand All @@ -421,7 +452,7 @@ jobs:
- name: Run tests
run: |
$env:Path += ';C:\Strawberry\perl\site\bin;C:\Strawberry\perl\bin;C:\Strawberry\c\bin;C:\Program Files\NASM;D:\a\amazon-kinesis-video-streams-producer-c\amazon-kinesis-video-streams-producer-c\open-source\lib;D:\a\amazon-kinesis-video-streams-producer-c\amazon-kinesis-video-streams-producer-c\open-source\bin'
& "D:\a\amazon-kinesis-video-streams-producer-c\amazon-kinesis-video-streams-producer-c\build\tst\producer_test.exe" --gtest_filter="-ProducerFunctionalityTest.pressure_on_buffer_duration_fail_new_connection_at_token_rotation"
& "D:\a\amazon-kinesis-video-streams-producer-c\amazon-kinesis-video-streams-producer-c\build\tst\producer_test.exe" --gtest_filter="-IoTCredentialTest.*:ProducerFunctionalityTest.pressure_on_buffer_duration_fail_new_connection_at_token_rotation"
arm64-cross-compilation:
runs-on: ubuntu-20.04
Expand Down
51 changes: 51 additions & 0 deletions .github/workflows/codecov.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
name: Producer C SDK Codecov

on:
push:
branches:
- develop
- master
pull_request:
branches:
- develop
- master

jobs:
linux-gcc-code-coverage:
runs-on: ubuntu-20.04
env:
AWS_KVS_LOG_LEVEL: 2
permissions:
id-token: write
contents: read
steps:
- name: Clone repository
uses: actions/checkout@v3
- name: Build repository
run: |
sudo apt install lcov
mkdir build && cd build
cmake .. -DCODE_COVERAGE=TRUE -DBUILD_TEST=TRUE -DBUILD_COMMON_LWS=TRUE
make
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1-node16
with:
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
role-session-name: ${{ secrets.AWS_ROLE_SESSION_NAME }}
aws-region: ${{ secrets.AWS_REGION }}
role-duration-seconds: 10800
- name: Run tests
run: |
cd build
ulimit -c unlimited -S
source ../scripts/get-iot-certs.sh ci
timeout --signal=SIGABRT 150m ./tst/producer_test --gtest_break_on_failure
- name: Generate and process coverage data with lcov
run: |
cd build
lcov --capture --directory CMakeFiles/cproducer.dir --directory CMakeFiles/kvsCommonCurl.dir --directory CMakeFiles/kvsCommonLws.dir --output-file coverage.info
lcov --list coverage.info
lcov --remove coverage.info '*jsmn*' -o coverage_filtered.info
lcov --list coverage_filtered.info
rm coverage.info
bash <(curl -s https://codecov.io/bash) -f ./coverage_filtered.info -t ${{ secrets.CODECOV_TOKEN }} -Z
42 changes: 42 additions & 0 deletions .github/workflows/version-check.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
name: Check Version Mismatch between PR branch and master.

on:
pull_request:
branches:
- master

jobs:
check-version:
runs-on: ubuntu-latest
steps:
- name: Checkout PR branch
uses: actions/checkout@v4

- name: Get version from PR
id: pr_version
run: |
PR_VERSION=$(grep -Po 'KinesisVideoProducerC VERSION \K[0-9]+\.[0-9]+\.[0-9]+' CMakeLists.txt)
echo "PR_VERSION=$PR_VERSION" >> "$GITHUB_ENV"
echo "PR Version: $PR_VERSION"
- name: Checkout master branch
uses: actions/checkout@v4
with:
ref: master

- name: Get version from master
id: master_version
run: |
MASTER_VERSION=$(grep -Po 'KinesisVideoProducerC VERSION \K[0-9]+\.[0-9]+\.[0-9]+' CMakeLists.txt)
echo "MASTER_VERSION=$MASTER_VERSION" >> "$GITHUB_ENV"
echo "Master version: $MASTER_VERSION"
- name: Compare versions
run: |
echo "Comparing PR Version: $PR_VERSION with Master Version: $MASTER_VERSION"
if [ "$MASTER_VERSION" == "$PR_VERSION" ]; then
echo "Please update the version in CMakeLists.txt file (project(KinesisVideoProducerC VERSION <ver-string> LANGUAGES C). Any PR getting merged to master requires a version update"
exit 1
else
echo "Version update detected."
fi
2 changes: 2 additions & 0 deletions CMake/Dependencies/libkvspic-CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,12 @@ include(ExternalProject)
ExternalProject_Add(libkvspic-download
GIT_REPOSITORY https://github.com/awslabs/amazon-kinesis-video-streams-pic.git
GIT_TAG v1.1.0

SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/kvspic-src"
BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/kvspic-build"
CMAKE_ARGS
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DKVS_DEFAULT_STACK_SIZE=${KVS_DEFAULT_STACK_SIZE}
CONFIGURE_COMMAND ""
BUILD_COMMAND ""
INSTALL_COMMAND ""
Expand Down
2 changes: 1 addition & 1 deletion CMake/Dependencies/libwebsockets-CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ endif()

ExternalProject_Add(project_libwebsockets
GIT_REPOSITORY https://github.com/warmcat/libwebsockets.git
GIT_TAG v3.2.3
GIT_TAG v4.3.3
PREFIX ${CMAKE_CURRENT_BINARY_DIR}/build
CMAKE_ARGS
-DCMAKE_INSTALL_PREFIX=${OPEN_SRC_INSTALL_PREFIX}
Expand Down
8 changes: 6 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ set(KINESIS_VIDEO_PRODUCER_C_MINOR_VERSION 5)
set(KINESIS_VIDEO_PRODUCER_C_PATCH_VERSION 2)
set(KINESIS_VIDEO_PRODUCER_C_VERSION ${KINESIS_VIDEO_PRODUCER_C_MAJOR_VERSION}.${KINESIS_VIDEO_PRODUCER_C_MINOR_VERSION}.${KINESIS_VIDEO_PRODUCER_C_PATCH_VERSION})

set(KINESIS_VIDEO_PRODUCER_C_MAJOR_VERSION 1)
set(KINESIS_VIDEO_PRODUCER_C_MINOR_VERSION 5)
set(KINESIS_VIDEO_PRODUCER_C_PATCH_VERSION 1)
set(KINESIS_VIDEO_PRODUCER_C_VERSION ${KINESIS_VIDEO_PRODUCER_C_MAJOR_VERSION}.${KINESIS_VIDEO_PRODUCER_C_MINOR_VERSION}.${KINESIS_VIDEO_PRODUCER_C_PATCH_VERSION})

include(GNUInstallDirs)


Expand Down Expand Up @@ -289,8 +294,7 @@ if(BUILD_COMMON_LWS)
set_target_properties(kvsCommonLws PROPERTIES VERSION ${KINESIS_VIDEO_PRODUCER_C_VERSION} SOVERSION ${KINESIS_VIDEO_PRODUCER_C_MAJOR_VERSION})
endif()
target_link_libraries(kvsCommonLws
${OPENSSL_CRYPTO_LIBRARY}
${OPENSSL_SSL_LIBRARY}
${PRODUCER_CRYPTO_LIBRARY}
${LIBWEBSOCKETS_LIBRARIES}
kvspicUtils)

Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ To run the samples with IoT credential provider:
3. Build the changes: `make`
4. Run the sample using the instructions in previous section.


### Fragment metadata

`./kvsVideoOnlyRealtimeStreamingSample` is the only sample that has the [fragment metadata](https://docs.aws.amazon.com/kinesisvideostreams/latest/dg/how-meta.html) implemented out of the box.
Expand Down
2 changes: 1 addition & 1 deletion scripts/generate-iot-credential.sh
Original file line number Diff line number Diff line change
Expand Up @@ -112,4 +112,4 @@ export AWS_IOT_CORE_CREDENTIAL_ENDPOINT=$(cat iot-credential-provider.txt)
export AWS_IOT_CORE_CERT=$(pwd)"/"$iotCert
export AWS_IOT_CORE_PRIVATE_KEY=$(pwd)"/"$iotPrivateKey
export AWS_IOT_CORE_ROLE_ALIAS=$iotRoleAlias
export AWS_IOT_CORE_THING_NAME=$thingName
export AWS_IOT_CORE_THING_NAME=$thingName
28 changes: 28 additions & 0 deletions scripts/get-iot-certs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#!/bin/bash

prefix=$1
thingName="${prefix}_thing"
thingTypeName="${prefix}_thing_type"
iotPolicyName="${prefix}_policy"
iotRoleName="${prefix}_role"
iotRoleAlias="${prefix}_role_alias"
iotCert="${prefix}_certificate.pem"
iotPublicKey="${prefix}_public.key"
iotPrivateKey="${prefix}_private.key"

# Create the certificate to which you must attach the policy for IoT that you created above.
aws iot create-keys-and-certificate --set-as-active --certificate-pem-outfile $iotCert --public-key-outfile $iotPublicKey --private-key-outfile $iotPrivateKey > certificate
# Attach the policy for IoT (KvsCameraIoTPolicy created above) to this certificate.
aws iot attach-policy --policy-name $iotPolicyName --target $(jq --raw-output '.certificateArn' certificate)
# Attach your IoT thing (kvs_example_camera_stream) to the certificate you just created:
aws iot attach-thing-principal --thing-name $thingName --principal $(jq --raw-output '.certificateArn' certificate)
# In order to authorize requests through the IoT credentials provider, you need the IoT credentials endpoint which is unique to your AWS account ID. You can use the following command to get the IoT credentials endpoint.
aws iot describe-endpoint --endpoint-type iot:CredentialProvider --output text > iot-credential-provider.txt
# In addition to the X.509 cerficiate created above, you must also have a CA certificate to establish trust with the back-end service through TLS. You can get the CA certificate using the following command:
curl 'https://www.amazontrust.com/repository/SFSRootCAG2.pem' --output cacert.pem

export AWS_IOT_CORE_CREDENTIAL_ENDPOINT=$(cat iot-credential-provider.txt)
export AWS_IOT_CORE_CERT=$(pwd)"/"$iotCert
export AWS_IOT_CORE_PRIVATE_KEY=$(pwd)"/"$iotPrivateKey
export AWS_IOT_CORE_ROLE_ALIAS=$iotRoleAlias
export AWS_IOT_CORE_THING_NAME=$thingName
2 changes: 1 addition & 1 deletion src/source/CallbacksProvider.c
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ STATUS createDefaultCallbacksProviderWithFileAuth(PCHAR credentialsFilePath, PCH
}

if (pAuthCallbacks != NULL) {
freeIotAuthCallbacks(&pAuthCallbacks);
freeFileAuthCallbacks(&pAuthCallbacks);
}

if (pStreamCallbacks != NULL) {
Expand Down
4 changes: 2 additions & 2 deletions src/source/Common/AwsV4Signer.c
Original file line number Diff line number Diff line change
Expand Up @@ -916,14 +916,14 @@ STATUS uriDecodeString(PCHAR pSrc, UINT32 srcLen, PCHAR pDst, PUINT32 pDstLen)
PCHAR pCurPtr = pSrc, pDec = pDst;
CHAR ch;

CHK(pSrc != NULL && pDstLen != NULL, STATUS_NULL_ARG);

// Set the source length to max if not specified
strLen = (srcLen == 0) ? MAX_UINT32 : srcLen;

// Set the remaining length
remaining = (pDst == NULL) ? MAX_UINT32 : *pDstLen;

CHK(pSrc != NULL && pDstLen != NULL, STATUS_NULL_ARG);

while (((UINT32) (pCurPtr - pSrc) < strLen) && ((ch = *pCurPtr) != '\0')) {
if (ch == '%') {
CHK((UINT32) (pCurPtr - pSrc) + decLen <= strLen && *(pCurPtr + 1) != '\0' && *(pCurPtr + 2) != '\0', STATUS_INVALID_ARG);
Expand Down
2 changes: 1 addition & 1 deletion src/source/Common/IotCredentialProvider.c
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ STATUS createIotCredentialProviderWithTime(PCHAR iotGetCredentialEndpoint, PCHAR
pIotCredentialProvider->getCurrentTimeFn = (getCurrentTimeFn == NULL) ? commonDefaultGetCurrentTimeFunc : getCurrentTimeFn;
pIotCredentialProvider->customData = customData;

CHK(STRNLEN(iotGetCredentialEndpoint, MAX_URI_CHAR_LEN + 1) <= MAX_URI_CHAR_LEN, MAX_URI_CHAR_LEN);
CHK(STRNLEN(iotGetCredentialEndpoint, MAX_URI_CHAR_LEN + 1) <= MAX_URI_CHAR_LEN, STATUS_INVALID_ARG);
STRNCPY(pIotCredentialProvider->iotGetCredentialEndpoint, iotGetCredentialEndpoint, MAX_URI_CHAR_LEN);

CHK(STRNLEN(roleAlias, MAX_ROLE_ALIAS_LEN + 1) <= MAX_ROLE_ALIAS_LEN, STATUS_MAX_ROLE_ALIAS_LEN_EXCEEDED);
Expand Down
Loading

0 comments on commit 55a8b64

Please sign in to comment.