Merge remote-tracking branch 'upstream/unstable' into engine-api-1261

rjd15372 · Dec 13, 2024 · a0eefee · a0eefee
2 parents 02b2395 + b60097b
commit a0eefee
Show file tree

Hide file tree

Showing 81 changed files with 6,017 additions and 1,877 deletions.
diff --git a/.github/workflows/build-release-packages.yml b/.github/workflows/build-release-packages.yml
@@ -3,14 +3,20 @@ name: Build Release Packages
 on:
   release:
     types: [published]
-
+  push:
+    paths:
+      - '.github/workflows/build-release-packages.yml'
+      - '.github/workflows/call-build-linux-arm-packages.yml'
+      - '.github/workflows/call-build-linux-x86_64-packages.yml'
+      - 'utils/releasetools/build-config.json'
   workflow_dispatch:
     inputs:
       version:
         description: Version of Valkey to build
         required: true
 
 permissions:
+  id-token: write
   contents: read
 
 jobs:
@@ -20,8 +26,8 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       version: ${{ steps.get_version.outputs.VERSION }}
+      is_test: ${{ steps.check-if-testing.outputs.IS_TEST }}
     steps:
-
       - run: |
           echo "Version: ${{ inputs.version || github.ref_name }}"
         shell: bash
@@ -32,8 +38,13 @@ jobs:
       - name: Get the version
         id: get_version
         run: |
-          VERSION="${INPUT_VERSION}"
+          if [[ "${{ github.event_name }}" == "push" ]]; then
+            VERSION=${{ github.ref_name }}
+          else
+            VERSION="${INPUT_VERSION}"
+          fi
           if [ -z "${VERSION}" ]; then
+            echo "Error: No version specified"
             exit 1
           fi
           echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
@@ -43,6 +54,16 @@ jobs:
           # only ever be a tag
           INPUT_VERSION: ${{ inputs.version || github.ref_name }}
 
+      - name: Check if we are testing
+        id: check-if-testing
+        run: |
+          if [[ "${{ github.event_name }}" == "push" ]]; then
+            echo "IS_TEST=true" >> $GITHUB_OUTPUT
+          else
+            echo "IS_TEST=false" >> $GITHUB_OUTPUT
+          fi
+        shell: bash
+
   generate-build-matrix:
     name: Generating build matrix
     runs-on: ubuntu-latest
@@ -56,7 +77,7 @@ jobs:
       - uses: ./.github/actions/generate-package-build-matrix
         id: set-matrix
         with:
-          ref: ${{ inputs.version || github.ref_name }}
+          ref: ${{ needs.release-build-get-meta.outputs.version }}
 
   release-build-linux-x86-packages:
     needs:
@@ -67,11 +88,10 @@ jobs:
       version: ${{ needs.release-build-get-meta.outputs.version }}
       ref: ${{ inputs.version || github.ref_name }}
       build_matrix: ${{ needs.generate-build-matrix.outputs.x86_64-build-matrix }}
+      region: us-west-2
     secrets:
-      token: ${{ secrets.GITHUB_TOKEN }}
-      bucket: ${{ secrets.AWS_S3_BUCKET }}
-      access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
-      secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }}
+      bucket_name: ${{ needs.release-build-get-meta.outputs.is_test == 'true' && secrets.AWS_TEST_BUCKET || secrets.AWS_S3_BUCKET }}
+      role_to_assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
 
   release-build-linux-arm-packages:
     needs:
@@ -82,8 +102,7 @@ jobs:
       version: ${{ needs.release-build-get-meta.outputs.version }}
       ref: ${{ inputs.version || github.ref_name }}
       build_matrix: ${{ needs.generate-build-matrix.outputs.arm64-build-matrix }}
+      region: us-west-2
     secrets:
-      token: ${{ secrets.GITHUB_TOKEN }}
-      bucket: ${{ secrets.AWS_S3_BUCKET }}
-      access_key_id: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
-      secret_access_key: ${{ secrets.AWS_S3_ACCESS_KEY }}
+      bucket_name: ${{ needs.release-build-get-meta.outputs.is_test == 'true' && secrets.AWS_TEST_BUCKET || secrets.AWS_S3_BUCKET }}
+      role_to_assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
diff --git a/.github/workflows/call-build-linux-arm-packages.yml b/.github/workflows/call-build-linux-arm-packages.yml
@@ -15,21 +15,20 @@ on:
         description: The build targets to produce as a JSON matrix.
         type: string
         required: true
+      region:
+        description: The AWS region to push packages into.
+        type: string
+        required: true
     secrets:
-      token:
-        description: The Github token or similar to authenticate with.
+      bucket_name:
+        description: The S3 bucket to push packages into.
+        required: true
+      role_to_assume:
+        description: The role to assume for the S3 bucket.
         required: true
-      bucket:
-        description: The name of the S3 bucket to push packages into.
-        required: false
-      access_key_id:
-        description: The S3 access key id for the bucket.
-        required: false
-      secret_access_key:
-        description: The S3 secret access key for the bucket.
-        required: false
 
 permissions:
+  id-token: write
   contents: read
 
 jobs:
@@ -46,6 +45,12 @@ jobs:
         with:
           ref: ${{ inputs.version }}
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ inputs.region }}
+          role-to-assume: ${{ secrets.role_to_assume }}
+
       - name: Make Valkey
         uses: uraimo/run-on-arch-action@v2
         with:
@@ -65,15 +70,5 @@ jobs:
           mkdir -p packages-files
           cp -rfv $TAR_FILE_NAME.tar* packages-files/
 
-      - name: Install AWS cli.
-        run: |
-          sudo apt-get install -y awscli
-
-      - name: Configure AWS credentials
-        run: |
-          aws configure set region us-west-2
-          aws configure set aws_access_key_id ${{ secrets.access_key_id }}
-          aws configure set aws_secret_access_key ${{ secrets.secret_access_key }}
-
       - name: Sync to S3
-        run: aws s3 sync packages-files s3://${{secrets.bucket}}/releases/
+        run: aws s3 sync packages-files s3://${{ secrets.bucket_name }}/releases/
diff --git a/.github/workflows/call-build-linux-x86-packages.yml b/.github/workflows/call-build-linux-x86-packages.yml
@@ -15,28 +15,27 @@ on:
         description: The build targets to produce as a JSON matrix.
         type: string
         required: true
+      region:
+        description: The AWS region to upload the packages to.
+        type: string
+        required: true
     secrets:
-      token:
-        description: The Github token or similar to authenticate with.
+      bucket_name:
+        description: The name of the S3 bucket to upload the packages to.
+        required: true
+      role_to_assume:
+        description: The role to assume for the S3 bucket.
         required: true
-      bucket:
-        description: The name of the S3 bucket to push packages into.
-        required: false
-      access_key_id:
-        description: The S3 access key id for the bucket.
-        required: false
-      secret_access_key:
-        description: The S3 secret access key for the bucket.
-        required: false
 
 permissions:
+  id-token: write
   contents: read
 
 jobs:
   build-valkey:
     # Capture source tarball and generate checksum for it
     name: Build package ${{ matrix.distro.target }} ${{ matrix.distro.arch }}
-    runs-on: "ubuntu-latest"
+    runs-on: ${{matrix.distro.target}}
     strategy:
       fail-fast: false
       matrix: ${{ fromJSON(inputs.build_matrix) }}
@@ -46,6 +45,12 @@ jobs:
         with:
           ref: ${{ inputs.version }}
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ inputs.region }}
+          role-to-assume: ${{ secrets.role_to_assume }}
+
       - name: Install dependencies
         run: sudo apt-get update && sudo apt-get install -y build-essential libssl-dev libsystemd-dev
 
@@ -63,15 +68,5 @@ jobs:
           mkdir -p packages-files
           cp -rfv $TAR_FILE_NAME.tar* packages-files/
 
-      - name: Install AWS cli.
-        run: |
-          sudo apt-get install -y awscli
-
-      - name: Configure AWS credentials
-        run: |
-          aws configure set region us-west-2
-          aws configure set aws_access_key_id ${{ secrets.access_key_id }}
-          aws configure set aws_secret_access_key ${{ secrets.secret_access_key }}
-
       - name: Sync to S3
-        run: aws s3 sync packages-files s3://${{secrets.bucket}}/releases/
+        run: aws s3 sync packages-files s3://${{ secrets.bucket_name }}/releases/
diff --git a/GOVERNANCE.md b/GOVERNANCE.md
@@ -2,7 +2,9 @@
 
 The Valkey project is managed by a Technical Steering Committee (TSC) composed of the maintainers of the Valkey repository.
 The Valkey project includes all of the current and future repositories under the Valkey-io organization.
-Maintainers are defined as individuals with full commit access to a repository, which shall be in sync with the MAINTAINERS.md file in a given projects repository.
+Committers are defined as individuals with write access to the code within a repository.
+Maintainers are defined as individuals with full access to a repository and own its governance.
+Both maintainers and committers should be clearly listed in the MAINTAINERS.md file in a given projects repository.
 Maintainers of other repositories within the Valkey project are not members of the TSC unless explicitly added.
 
 ## Technical Steering Committee

diff --git a/MAINTAINERS.md b/MAINTAINERS.md
@@ -16,8 +16,16 @@ Maintainers listed in alphabetical order by their github ID.
 | Zhao Zhao           | [soloestoy](https://github.com/soloestoy)       | Alibaba     |
 | Viktor Söderqvist   | [zuiderkwast](https://github.com/zuiderkwast)   | Ericsson    |
 
+## Current Committers
 
-### Former Maintainers
+Committers listed in alphabetical order by their github ID.
+
+| Committer           | GitHub ID                                       | Affiliation |
+| ------------------- | ----------------------------------------------- | ----------- |
+| Harkrishn Patro     | [hpatro](https://github.com/hpatro)             | Amazon      |
+| Ran Shidlansik      | [ranshid](https://github.com/ranshid)           | Amazon      |
+
+### Former Maintainers and Committers 
 
 | Maintainer          | GitHub ID                                       | Affiliation |
 | ------------------- | ----------------------------------------------- | ----------- |
diff --git a/cmake/Modules/SourceFiles.cmake b/cmake/Modules/SourceFiles.cmake
@@ -10,6 +10,7 @@ set(VALKEY_SERVER_SRCS
     ${CMAKE_SOURCE_DIR}/src/ae.c
     ${CMAKE_SOURCE_DIR}/src/anet.c
     ${CMAKE_SOURCE_DIR}/src/dict.c
+    ${CMAKE_SOURCE_DIR}/src/hashtable.c
     ${CMAKE_SOURCE_DIR}/src/kvstore.c
     ${CMAKE_SOURCE_DIR}/src/sds.c
     ${CMAKE_SOURCE_DIR}/src/zmalloc.c

diff --git a/src/Makefile b/src/Makefile
@@ -411,7 +411,7 @@ endif
 ENGINE_NAME=valkey
 SERVER_NAME=$(ENGINE_NAME)-server$(PROG_SUFFIX)
 ENGINE_SENTINEL_NAME=$(ENGINE_NAME)-sentinel$(PROG_SUFFIX)
-ENGINE_SERVER_OBJ=threads_mngr.o adlist.o quicklist.o ae.o anet.o dict.o kvstore.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o memory_prefetch.o io_threads.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o cluster_legacy.o cluster_slot_stats.o crc16.o endianconv.o slowlog.o eval.o bio.o rio.o rand.o memtest.o syscheck.o crcspeed.o crccombine.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o valkey-check-rdb.o valkey-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o allocator_defrag.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o socket.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script_lua.o script.o functions.o function_lua.o commands.o strl.o connection.o unix.o logreqres.o rdma.o
+ENGINE_SERVER_OBJ=threads_mngr.o adlist.o quicklist.o ae.o anet.o dict.o hashtable.o kvstore.o server.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o memory_prefetch.o io_threads.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o cluster_legacy.o cluster_slot_stats.o crc16.o endianconv.o slowlog.o eval.o bio.o rio.o rand.o memtest.o syscheck.o crcspeed.o crccombine.o crc64.o bitops.o sentinel.o notify.o setproctitle.o blocked.o hyperloglog.o latency.o sparkline.o valkey-check-rdb.o valkey-check-aof.o geo.o lazyfree.o module.o evict.o expire.o geohash.o geohash_helper.o childinfo.o allocator_defrag.o defrag.o siphash.o rax.o t_stream.o listpack.o localtime.o lolwut.o lolwut5.o lolwut6.o acl.o tracking.o socket.o tls.o sha256.o timeout.o setcpuaffinity.o monotonic.o mt19937-64.o resp_parser.o call_reply.o script_lua.o script.o functions.o function_lua.o commands.o strl.o connection.o unix.o logreqres.o rdma.o
 ENGINE_CLI_NAME=$(ENGINE_NAME)-cli$(PROG_SUFFIX)
 ENGINE_CLI_OBJ=anet.o adlist.o dict.o valkey-cli.o zmalloc.o release.o ae.o serverassert.o crcspeed.o crccombine.o crc64.o siphash.o crc16.o monotonic.o cli_common.o mt19937-64.o strl.o cli_commands.o
 ENGINE_BENCHMARK_NAME=$(ENGINE_NAME)-benchmark$(PROG_SUFFIX)