Merge branch 'main' into fix_remain_remove_concurrency

.github/scripts/check_juicefs_log.sh

@@ -0,0 +1,9 @@
+#!/bin/bash -e
+for log_file in /var/log/juicefs.log $HOME/.juicefs/juicefs.log; do
+    if [ -f $log_file ]; then
+        break
+    fi
+done
+echo "tail -1000 $log_file"
+tail -1000 $log_file
+grep -i "<FATAL>\|panic" $log_file && exit 1 || true

.github/scripts/command/info.sh

@@ -12,7 +12,7 @@ test_info_big_file(){
     prepare_test
     ./juicefs format $META_URL myjfs
     ./juicefs mount -d $META_URL /jfs
-    dd if=/dev/urandom of=/jfs/bigfile bs=16M count=1024
+    dd if=/dev/zero of=/jfs/bigfile bs=1M count=4096
     ./juicefs info /jfs/bigfile
     ./juicefs rmr /jfs/bigfile
     df -h /jfs

.github/scripts/hypo/fs_sdk_test.py

@@ -132,14 +132,14 @@ def test_issue_1424(self):
         state.readline(file=v2, mode='r', offset=1708,  user='root', whence=0)
         state.teardown()
 
-    def test_issue_1425(self):
-        # SEE: https://github.com/juicedata/jfs/issues/1425
-        state = JuicefsMachine()
-        v1 = state.init_folders()
-        v2 = state.create_file(content=b'a', file_name='a', parent=v1, umask=18,  user='root')
-        v3 = state.mkdir(mode=0, parent=v1, subdir='b', umask=18,  user='root')
-        state.rename_dir(entry=v3, new_entry_name=v2, parent=v1, umask=18,  user='root')
-        state.teardown()
+    # def test_issue_1425(self):
+    #     # SEE: https://github.com/juicedata/jfs/issues/1425
+    #     state = JuicefsMachine()
+    #     v1 = state.init_folders()
+    #     v2 = state.create_file(content=b'a', file_name='a', parent=v1, umask=18,  user='root')
+    #     v3 = state.mkdir(mode=0, parent=v1, subdir='b', umask=18,  user='root')
+    #     state.rename_dir(entry=v3, new_entry_name=v2, parent=v1, umask=18,  user='root')
+    #     state.teardown()
 
     def test_issue_1442(self):
         # SEE: https://github.com/juicedata/jfs/issues/1442
@@ -149,14 +149,14 @@ def test_issue_1442(self):
         state.set_xattr(file=v2, flag=0, name='user.0', user='root', value=b'\x01\x01\x00\x01')
         state.teardown()
 
-    def test_issue_1443(self):
-        # SEE: https://github.com/juicedata/jfs/issues/1443
-        state = JuicefsMachine()
-        v1 = state.init_folders()
-        v2 = state.create_file(content=b'bcb', file_name='bcba', parent=v1, umask=18, user='root')
-        v3 = state.hardlink(src_file=v2, link_file_name='a', parent=v1, umask=18, user='root')
-        state.rename_file(entry=v2, new_entry_name=v3, parent=v1, umask=18, user='root')
-        state.teardown()
+    # def test_issue_1443(self):
+    #     # SEE: https://github.com/juicedata/jfs/issues/1443
+    #     state = JuicefsMachine()
+    #     v1 = state.init_folders()
+    #     v2 = state.create_file(content=b'bcb', file_name='bcba', parent=v1, umask=18, user='root')
+    #     v3 = state.hardlink(src_file=v2, link_file_name='a', parent=v1, umask=18, user='root')
+    #     state.rename_file(entry=v2, new_entry_name=v3, parent=v1, umask=18, user='root')
+    #     state.teardown()
 
     def test_issue_1449(self):
         # SEE: https://github.com/juicedata/jfs/issues/1449

.github/scripts/hypo/sync.py

@@ -69,13 +69,11 @@ def equal(self, result1, result2):
     @rule(target=Files, 
           parent = Folders.filter(lambda x: x != multiple()), 
           file_name = st_entry_name, 
-          mode = st_open_mode, 
-          content = st_content, 
           umask = st_umask, 
             )
-    def create_file(self, parent, file_name, content, mode='x', user='root', umask=0o022):
-        result1 = self.fsop1.do_create_file(parent, file_name, mode, content, user, umask)
-        result2 = self.fsop2.do_create_file(parent, file_name, mode, content, user, umask)
+    def create_file(self, parent, file_name, content='s', mode='x', user='root', umask=0o022):
+        result1 = self.fsop1.do_create_file(parent=parent, file_name=file_name, mode=mode, content=content, user=user, umask=umask)
+        result2 = self.fsop2.do_create_file(parent=parent, file_name=file_name, mode=mode, content=content, user=user, umask=umask)
         assert self.equal(result1, result2), f'\033[31mcreate_file:\nresult1 is {result1}\nresult2 is {result2}\033[0m'
         if isinstance(result1, Exception):
             return multiple()

.github/scripts/pysdk/pysdk_test.py

@@ -156,6 +156,7 @@ def _test_link(self, file1, file2):
     def test_link(self):
         self._test_link(self.file1, self.file2)
 
+@unittest.skip("Skipping SummaryTests")
 class SummaryTests(unittest.TestCase):
     # /test/dir1/file
     #      /dir2

.github/workflows/cache.yml

@@ -0,0 +1,76 @@
+name: "cache"
+
+on:
+  push:
+    branches:
+      - 'main'
+      - 'release-**'
+    paths:
+      - '**/cache.yml'
+      - '**/cache.sh'
+  pull_request:
+    branches:
+      - 'main'
+      - 'release-**'
+    paths:
+      - '**/cache.yml'
+      - '**/cache.sh'
+  schedule:
+    - cron:  '30 20 * * *'
+  workflow_dispatch:
+
+jobs:
+  cache:
+    timeout-minutes: 60
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+
+      - name: mount coverage dir
+        timeout-minutes: 5
+        uses: ./.github/actions/mount-coverage-dir
+        with:
+          mount_point: cover
+          access_key: ${{ secrets.CI_COVERAGE_AWS_AK }}
+          secret_key: ${{ secrets.CI_COVERAGE_AWS_SK }}
+          token: ${{ secrets.CI_COVERAGE_AWS_TOKEN }}
+
+      - name: Build
+        timeout-minutes: 10
+        uses: ./.github/actions/build
+
+      - name: Test
+        run: |
+          sudo GOCOVERDIR=$(pwd)/cover .github/scripts/cache.sh
+
+      - name: Check juicefs log
+        if: always()
+        run: |
+          sudo .github/scripts/check_juicefs_log.sh
+      
+      - name: Check /tmp/juicefs.log
+        if: always()
+        run: |
+          sudo tail -n 1000 /tmp/juicefs.log
+
+      - name: upload coverage report
+        timeout-minutes: 5
+        uses: ./.github/actions/upload-coverage
+        with:
+          UPLOAD_TOKEN: ${{ secrets.CI_COVERAGE_FILE_UPLOAD_AUTH_TOKEN }}
+
+      - name: Send Slack Notification
+        if: failure()
+        uses: juicedata/slack-notify-action@main
+        with:
+          channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}"
+          slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"  
+
+      - name: Setup upterm session
+        if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1)
+        # if: failure()
+        timeout-minutes: 60
+        uses: lhotari/action-upterm@v1

.github/workflows/command.yml

@@ -77,44 +77,48 @@ jobs:
         uses: ./.github/actions/build
 
       - name: Test Gc
-        timeout-minutes: 30
+        timeout-minutes: 10
         run: |
           sudo META=${{matrix.meta}} .github/scripts/command/gc.sh      
 
       - name: Test Config
+        timeout-minutes: 10
         run: |
           sudo META=${{matrix.meta}} .github/scripts/command/config.sh
     
       - name: Test acl
-        timeout-minutes: 30
+        timeout-minutes: 10
         run: |
           sudo META=${{matrix.meta}} .github/scripts/command/acl.sh     
 
       - name: Test Clone
-        timeout-minutes: 30
+        timeout-minutes: 10
         run: |
           sudo META=${{matrix.meta}} .github/scripts/command/clone.sh
 
       - name: Test fsck
-        timeout-minutes: 30
+        timeout-minutes: 10
         run: |
           sudo META=${{matrix.meta}} .github/scripts/command/fsck.sh 
       
       - name: Test Gateway
-        timeout-minutes: 30
+        timeout-minutes: 10
         run: |
           sudo META=${{matrix.meta}} .github/scripts/command/gateway.sh
       
       - name: Test Debug
-        timeout-minutes: 30
+        timeout-minutes: 10
         run: |
           sudo META=${{matrix.meta}} .github/scripts/command/debug.sh
       
       - name: Test Info
+        timeout-minutes: 10
         run: |
           sudo META=${{matrix.meta}} .github/scripts/command/info.sh
 
       - name: Test Format
+        if: false
+        timeout-minutes: 10
         run: |
           sudo META=${{matrix.meta}} .github/scripts/command/format.sh
   
@@ -239,7 +243,7 @@ jobs:
         timeout-minutes: 30
         run: |
           if [[ "${{matrix.meta}}" == "mysql" || "${{matrix.meta}}" == "tikv" ]]; then
-            sudo META=${{matrix.meta}} .github/scripts/command/cache.sh
+            sudo META=${{matrix.meta}} .github/scripts/cache.sh
           fi
 
       - name: Test Graceful upgrade

.github/workflows/fsrand.yml

@@ -127,7 +127,7 @@ jobs:
       - name: check fsrand.log
         if: always()
         run: | 
-          tail -500 fsrand.log
+          [[ -f fsrand.log ]] && tail -500 fsrand.log || true
 
       - name: check juicefs.log
         if: always()