Skip to content

Commit

Permalink
Merge branch 'release-1.1' into java/bump-protobuf-valkey-417
Browse files Browse the repository at this point in the history
  • Loading branch information
@Yury-Fridlyand
Yury-Fridlyand authored Jan 7, 2025
2 parents b4add58 + f19609f commit b10cc05
Show file tree
Hide file tree
Showing 7 changed files with 1,988 additions and 4,816 deletions.
57 changes: 57 additions & 0 deletions .github/workflows/ort-sweeper.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
name: ORT - Trigger periodic checks for relevant branches

on:
schedule:
- cron: "0 0 * * *" # Runs daily at 00:00 UTC

jobs:
trigger-ort-check:
runs-on: ubuntu-latest

steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Fetch relevant branches
id: get-branches
run: |
# Get all branches matching 'release-*' and include 'main'
branches=$(git ls-remote --heads origin | awk -F'/' '/refs\/heads\/release-/ {print $NF}')
branches="main $branches"
echo "::set-output name=branches::$branches"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- name: Trigger ORT Check workflows
uses: actions/github-script@v6
with:
script: |
const branches = "${{ steps.get-branches.outputs.branches }}".split(" ");
const workflowFile = "ort.yml";
const triggerWorkflow = async (branch) => {
try {
console.log(`Triggering workflow for branch: ${branch}`);
await github.rest.actions.createWorkflowDispatch({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: workflowFile,
ref: branch, // The branch where workflow_dispatch is triggered
inputs: {
branch_name: branch
}
});
console.log(`Successfully triggered workflow for branch: ${branch}`);
} catch (error) {
core.setFailed(error.message);
}
};
// Fire all workflow dispatch requests concurrently
const promises = branches
.filter(branch => branch) // Skip empty branches
.map(branch => triggerWorkflow(branch));
await Promise.allSettled(promises);
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
238 changes: 143 additions & 95 deletions .github/workflows/ort.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,57 +2,57 @@
name: The OSS Review Toolkit (ORT)

on:
schedule:
- cron: "0 0 * * *"
pull_request:
paths:
- .github/workflows/ort.yml
- .github/workflows/run-ort-tools/action.yml
- utils/get_licenses_from_ort.py

workflow_dispatch:
inputs:
branch:
description: 'The branch to run against the ORT tool'
required: true
version:
description: 'The release version of GLIDE'
required: true
inputs:
branch_name:
description: "The branch to run against the ORT tool"
required: true
default: "main"

jobs:
run-ort:
if: github.repository_owner == 'valkey-io'
name: Create attribution files
runs-on: ubuntu-latest

# 1. For workflow_dispatch, always allow
# 2. For pull_request, run if branch is not autogenerated ort-diff-for- branches
if: >
github.event_name == 'workflow_dispatch' ||
(github.event_name == 'pull_request' && !startsWith(github.head_ref, 'ort-diff-for-'))
strategy:
fail-fast: false
env:
PYTHON_ATTRIBUTIONS: "python/THIRD_PARTY_LICENSES_PYTHON"
NODE_ATTRIBUTIONS: "node/THIRD_PARTY_LICENSES_NODE"
RUST_ATTRIBUTIONS: "glide-core/THIRD_PARTY_LICENSES_RUST"
JAVA_ATTRIBUTIONS: "java/THIRD_PARTY_LICENSES_JAVA"
fail-fast: false
env:
PYTHON_ATTRIBUTIONS: "python/THIRD_PARTY_LICENSES_PYTHON"
NODE_ATTRIBUTIONS: "node/THIRD_PARTY_LICENSES_NODE"
RUST_ATTRIBUTIONS: "glide-core/THIRD_PARTY_LICENSES_RUST"
JAVA_ATTRIBUTIONS: "java/THIRD_PARTY_LICENSES_JAVA"
INPUT_TARGET_BRANCH: ${{ github.event.inputs.branch_name }}
EVENT_NAME: ${{ github.event_name }}
HEAD_REF: ${{ github.head_ref }}

steps:
- name: Set the release version
shell: bash
run: |
export version=`if [ "$EVENT_NAME" == 'schedule' ] || [ "$EVENT_NAME" == 'pull_request' ]; then echo '255.255.255'; else echo "$INPUT_VERSION"; fi`
echo "RELEASE_VERSION=${version}" >> $GITHUB_ENV
env:
EVENT_NAME: ${{ github.event_name }}
INPUT_VERSION: ${{ github.event.inputs.version }}

- name: Set the base branch
- name: Setup target branch and commit
run: |
export BASE_BRANCH=`if [ "$EVENT_NAME" == 'schedule' ]; then echo 'main'; elif [ "$EVENT_NAME" == 'workflow_dispatch' ]; then echo "$INPUT_BRANCH"; else echo ""; fi`
echo "Base branch is: ${BASE_BRANCH}"
echo "BASE_BRANCH=${BASE_BRANCH}" >> $GITHUB_ENV
env:
EVENT_NAME: ${{ github.event_name }}
INPUT_BRANCH: ${{ github.event.inputs.branch }}
if [ "$EVENT_NAME" == 'workflow_dispatch' ]; then
echo "TARGET_BRANCH=$INPUT_TARGET_BRANCH" >> $GITHUB_ENV
elif [ "$EVENT_NAME" == 'pull_request' ]; then
echo "TARGET_BRANCH=$HEAD_REF" >> $GITHUB_ENV
fi
- name: Checkout
- name: Checkout target branch
uses: actions/checkout@v4
with:
submodules: "true"
ref: ${{ env.BASE_BRANCH }}
ref: ${{ env.TARGET_BRANCH }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
token: ${{ secrets.GITHUB_TOKEN }}
fetch-depth: 0 # Fetch all history for all branches and tags

- name: Setup target commit
run: |
echo "TARGET_COMMIT=`git rev-parse HEAD`" >> $GITHUB_ENV
- name: Set up JDK 11 for the ORT package
uses: actions/setup-java@v4
Expand Down Expand Up @@ -82,7 +82,7 @@ jobs:
- name: Install Rust toolchain
uses: dtolnay/[email protected]

- name: Install ORT
- name: Build and install ORT
if: steps.cache-ort.outputs.cache-hit != 'true'
working-directory: ./ort/
run: |
Expand All @@ -100,8 +100,7 @@ jobs:
EOF
cat ~/.ort/config/config.yml
### NodeJS ###

### NodeJS ###
- name: Set up Node.js 16.x
uses: actions/setup-node@v4
with:
Expand All @@ -110,28 +109,27 @@ jobs:
- name: Create package.json file for the Node wrapper
uses: ./.github/workflows/node-create-package-file
with:
release_version: ${{ env.RELEASE_VERSION }}
os: "ubuntu-latest"
release_version: ${{ env.TARGET_COMMIT }}
os: "ubuntu-latest"

- name: Fix Node base NPM package.json file for ORT
working-directory: ./node/npm/glide
run: |
# Remove the glide-rs dependency to avoid duplication
sed -i '/ "glide-rs":/d' ../../package.json
export pkg_name=valkey-glide-base
export package_version="${{ env.RELEASE_VERSION }}"
export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi`
mv package.json package.json.tmpl
envsubst < package.json.tmpl > "package.json"
cat package.json
# Remove the glide-rs dependency to avoid duplication
sed -i '/ "glide-rs":/d' ../../package.json
export pkg_name=valkey-glide-base
export package_version="${{ env.TARGET_COMMIT }}"
export scope=`if [ "$NPM_SCOPE" != '' ]; then echo "$NPM_SCOPE/"; fi`
mv package.json package.json.tmpl
envsubst < package.json.tmpl > "package.json"
cat package.json
- name: Run ORT tools for Node
uses: ./.github/workflows/run-ort-tools
with:
folder_path: "${{ github.workspace }}/node"

### Python ###
folder_path: "${{ github.workspace }}/node"

### Python ###
- name: Set up Python 3.10
uses: actions/setup-python@v5
with:
Expand All @@ -148,15 +146,13 @@ jobs:
with:
folder_path: "${{ github.workspace }}/python"

### Rust ###

- name: Run ORT tools for Rust
### Rust glide-core ###
- name: Run ORT tools for glide-core
uses: ./.github/workflows/run-ort-tools
with:
folder_path: "${{ github.workspace }}/glide-core"

### Java ###

### Java ###
- name: Set up JDK 11
uses: actions/setup-java@v4
with:
Expand All @@ -168,21 +164,7 @@ jobs:
with:
folder_path: "${{ github.workspace }}/java"

### Process results ###

- name: Check for diff
run: |
cp python/ort_results/NOTICE_DEFAULT $PYTHON_ATTRIBUTIONS
cp node/ort_results/NOTICE_DEFAULT $NODE_ATTRIBUTIONS
cp glide-core/ort_results/NOTICE_DEFAULT $RUST_ATTRIBUTIONS
cp java/ort_results/NOTICE_DEFAULT $JAVA_ATTRIBUTIONS
GIT_DIFF=`git diff $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS`
if [ -n "$GIT_DIFF" ]; then
echo "FOUND_DIFF=true" >> $GITHUB_ENV
else
echo "FOUND_DIFF=false" >> $GITHUB_ENV
fi
### Get licenses ###
- name: Retrieve licenses list
working-directory: ./utils
run: |
Expand All @@ -192,26 +174,7 @@ jobs:
echo EOF
} >> "$GITHUB_ENV"
### Create PR ###

- name: Create pull request
if: ${{ env.FOUND_DIFF == 'true' && github.event_name != 'pull_request' }}
run: |
export BRANCH_NAME=`if [ "$EVENT_NAME" == 'schedule' ] || [ "$EVENT_NAME" == 'pull_request' ]; then echo 'scheduled-ort'; else echo "ort-v$INPUT_VERSION"; fi`
echo "Creating pull request from branch ${BRANCH_NAME} to branch ${{ env.BASE_BRANCH }}"
git config --global user.email "[email protected]"
git config --global user.name "ort-bot"
git checkout -b ${BRANCH_NAME}
git add $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS
git commit -m "Updated attribution files" -s
git push --set-upstream origin ${BRANCH_NAME} -f
title="Updated attribution files for ${BRANCH_NAME}"
gh pr create -B ${{ env.BASE_BRANCH }} -H ${BRANCH_NAME} --title "${title}" --body 'Created by Github action.\n${{ env.LICENSES_LIST }}'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
EVENT_NAME: ${{ github.event_name }}
INPUT_VERSION: ${{ github.event.inputs.version }}

### Upload licenses ###
- name: Get current date
id: date
run: |
Expand All @@ -235,3 +198,88 @@ jobs:
path: |
utils/skipped_package_list.txt
retention-days: 30

- name: Upload the unknown/unapproved package list
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: unapproved-package-list-${{ steps.date.outputs.date }}
path: |
utils/unapproved_package_list.txt
retention-days: 30

### TODO: Fail if there are unapproved packages ###

### Check for attributions diff ###
- name: Check for diff
run: |
cp python/ort_results/NOTICE_DEFAULT $PYTHON_ATTRIBUTIONS
cp node/ort_results/NOTICE_DEFAULT $NODE_ATTRIBUTIONS
cp glide-core/ort_results/NOTICE_DEFAULT $RUST_ATTRIBUTIONS
cp java/ort_results/NOTICE_DEFAULT $JAVA_ATTRIBUTIONS
GIT_DIFF=`git diff $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS`
if [ -n "$GIT_DIFF" ]; then
echo "FOUND_DIFF=true" >> $GITHUB_ENV
else
echo "FOUND_DIFF=false" >> $GITHUB_ENV
fi
### Create PR, Note a potential race on the source branch ###
- name: Create pull request
if: ${{ env.FOUND_DIFF == 'true' && github.event_name != 'pull_request' }}
run: |
export ORT_DIFF_BRANCH_NAME="ort-diff-for-$TARGET_BRANCH"
echo "Creating pull request from branch $ORT_DIFF_BRANCH_NAME to branch $TARGET_BRANCH"
git config --global user.email "[email protected]"
git config --global user.name "ort-bot"
git checkout -b ${ORT_DIFF_BRANCH_NAME}
git add $PYTHON_ATTRIBUTIONS $NODE_ATTRIBUTIONS $RUST_ATTRIBUTIONS $JAVA_ATTRIBUTIONS
git commit -m "Updated attribution files" -s
git push --set-upstream origin ${ORT_DIFF_BRANCH_NAME} -f
# Check if PR already exists
existing_pr=$(gh pr list --base ${TARGET_BRANCH} --head ${ORT_DIFF_BRANCH_NAME} --json number --jq '.[0].number')
if [ -z "$existing_pr" ]; then
# Create a new PR if none exists
title="Updated attribution files for commit ${TARGET_COMMIT}"
gh pr create -B ${TARGET_BRANCH} -H ${ORT_DIFF_BRANCH_NAME} --title "${title}" --body "Created by Github action. ${{ env.LICENSES_LIST }}"
echo "Pull request created successfully."
else
# Update the existing PR
echo "Pull request #$existing_pr already exists. Updating branch."
gh pr edit $existing_pr --title "Updated attribution files for commit ${TARGET_COMMIT}" --body "Created by Github action. ${{ env.LICENSES_LIST }}"
echo "Pull request updated successfully."
fi
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
INPUT_VERSION: ${{ github.event.inputs.version }}

### Warn of outdated attributions for PR ###
- name: Warn of outdated attributions due to the PR
if: ${{ env.FOUND_DIFF == 'true' && github.event_name == 'pull_request' }}
run: |
ATTRIBUTION_FILES=(
"${{ env.PYTHON_ATTRIBUTIONS }}"
"${{ env.NODE_ATTRIBUTIONS }}"
"${{ env.RUST_ATTRIBUTIONS }}"
"${{ env.JAVA_ATTRIBUTIONS }}"
)
MESSAGE="WARNING! The attribution files differ in this PR. Please ensure an updating PR is issued using a scheduled or manual run of this workflow!"
# Echo the message to the console
echo "$MESSAGE"
# Emit a general warning in the action log
echo "::warning::$MESSAGE"
# Loop through the attribution files
for FILE in "${ATTRIBUTION_FILES[@]}"; do
if git diff --quiet "$FILE"; then
continue
else
# Emit a warning associated with the changed file
echo "::warning file=$FILE::WARNING! The attribution file '$FILE' differs in this PR."
fi
done
Loading

0 comments on commit b10cc05

Please sign in to comment.