From af64b71afd4ba793a38971f304a3a182fb95b1bd Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Mon, 2 Sep 2024 17:57:04 +0800 Subject: [PATCH 01/10] meta: in compaction don't skip the first slice if it's duplicated --- pkg/meta/base_test.go | 15 +++++++++++++++ pkg/meta/slice.go | 6 ++++++ 2 files changed, 21 insertions(+) diff --git a/pkg/meta/base_test.go b/pkg/meta/base_test.go index 9c58c1d9aa89..b5c25d0b81f2 100644 --- a/pkg/meta/base_test.go +++ b/pkg/meta/base_test.go @@ -1571,6 +1571,21 @@ func testCompaction(t *testing.T, m Meta, trash bool) { t.Fatalf("inode %d should be compacted, but have %d slices, size %d,%d,%d", inode, len(slices), slices[0].Len, slices[1].Len, slices[2].Len) } + + m.NewSlice(ctx, &sliceId) + _ = m.Write(ctx, inode, 3, 0, Slice{Id: sliceId, Size: 2338508, Len: 2338508}, time.Now()) + _ = m.CopyFileRange(ctx, inode, 3*ChunkSize, inode, 4*ChunkSize, 2338508, 0, nil, nil) + _ = m.Fallocate(ctx, inode, fallocZeroRange, 4*ChunkSize, ChunkSize, nil) + _ = m.CopyFileRange(ctx, inode, 3*ChunkSize, inode, 4*ChunkSize, 2338508, 0, nil, nil) + if c, ok := m.(compactor); ok { + c.compactChunk(inode, 4, false, true) + } + if st := m.Read(ctx, inode, 4, &slices); st != 0 { + t.Fatalf("read inode %d chunk 4: %s", inode, st) + } + if len(slices) != 1 || slices[0].Len != 2338508 { + t.Fatalf("inode %d should be compacted, but have %d slices, size %d", inode, len(slices), slices[0].Len) + } } func testConcurrentWrite(t *testing.T, m Meta) { diff --git a/pkg/meta/slice.go b/pkg/meta/slice.go index 081276cf6e8c..59d4b37f698c 100644 --- a/pkg/meta/slice.go +++ b/pkg/meta/slice.go @@ -183,6 +183,7 @@ func compactChunk(ss []*slice) (uint32, uint32, []Slice) { func skipSome(chunk []*slice) int { var skipped int var total = len(chunk) +OUT: for skipped < total { ss := chunk[skipped:] pos, size, c := compactChunk(ss) @@ -198,6 +199,11 @@ func skipSome(chunk []*slice) int { // it's not the first slice, compact it break } + for _, s := range ss[1:] { + if *s == *first { + break OUT + } + } skipped++ } return skipped From f5de55db56c9c907f859ec9a18ed3e480baff6fb Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Tue, 3 Sep 2024 10:47:06 +0800 Subject: [PATCH 02/10] cleanup skipSome --- pkg/meta/base_test.go | 9 ++++----- pkg/meta/slice.go | 23 +++++------------------ 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/pkg/meta/base_test.go b/pkg/meta/base_test.go index b5c25d0b81f2..68a35bfeb08b 100644 --- a/pkg/meta/base_test.go +++ b/pkg/meta/base_test.go @@ -1548,8 +1548,8 @@ func testCompaction(t *testing.T, m Meta, trash bool) { if st := m.Read(ctx, inode, 0, &slices); st != 0 { t.Fatalf("read 0: %s", st) } - if len(slices) != 1 || slices[0].Len != 3<<20 { - t.Fatalf("inode %d should be compacted, but have %d slices, size %d", inode, len(slices), slices[0].Len) + if len(slices) != 2 { + t.Fatalf("inode %d should be compacted, but have %d slices, size %d: %+v", inode, len(slices), slices[0].Len, slices) } m.NewSlice(ctx, &sliceId) @@ -1567,9 +1567,8 @@ func testCompaction(t *testing.T, m Meta, trash bool) { t.Fatalf("read 1: %s", st) } // compact twice: 4515328+2607724-2338508 = 4784544; 8829056+1074933-2338508-4784544=2780937 - if len(slices) != 3 || slices[0].Len != 2338508 || slices[1].Len != 4784544 || slices[2].Len != 2780937 { - t.Fatalf("inode %d should be compacted, but have %d slices, size %d,%d,%d", - inode, len(slices), slices[0].Len, slices[1].Len, slices[2].Len) + if len(slices) != 1 { + t.Fatalf("inode %d should be compacted, but have %d slices: %+v", inode, len(slices), slices) } m.NewSlice(ctx, &sliceId) diff --git a/pkg/meta/slice.go b/pkg/meta/slice.go index 59d4b37f698c..d10b64742b0e 100644 --- a/pkg/meta/slice.go +++ b/pkg/meta/slice.go @@ -183,27 +183,14 @@ func compactChunk(ss []*slice) (uint32, uint32, []Slice) { func skipSome(chunk []*slice) int { var skipped int var total = len(chunk) -OUT: + _, size, _ := compactChunk(chunk) for skipped < total { - ss := chunk[skipped:] - pos, size, c := compactChunk(ss) - first := ss[0] - if first.len < (1<<20) || first.len*5 < size || size == 0 { - // it's too small + _, size1, _ := compactChunk(chunk[skipped+1:]) + reduced := size - size1 + if size1 == 0 || reduced < chunk[skipped].len || reduced*5 < size || reduced < 2<<20 { break } - isFirst := func(pos uint32, s Slice) bool { - return pos == first.pos && s.Id == first.id && s.Off == first.off && s.Len == first.len - } - if !isFirst(pos, c[0]) { - // it's not the first slice, compact it - break - } - for _, s := range ss[1:] { - if *s == *first { - break OUT - } - } + size = size1 skipped++ } return skipped From 93b84b678aef84d1a30034b754d143caf0489e45 Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Fri, 6 Sep 2024 14:50:41 +0800 Subject: [PATCH 03/10] try skip head and tail --- pkg/meta/base.go | 12 ++---------- pkg/meta/base_test.go | 4 ++-- pkg/meta/slice.go | 19 ++++++++++++++++--- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/pkg/meta/base.go b/pkg/meta/base.go index e8231e30754a..97a41b3c02c4 100644 --- a/pkg/meta/base.go +++ b/pkg/meta/base.go @@ -2049,21 +2049,13 @@ func (m *baseMeta) compactChunk(inode Ino, indx uint32, once, force bool) { if len(ss) > maxCompactSlices { ss = ss[:maxCompactSlices] } - skipped := skipSome(ss) + skipped, tail := skipSome(ss) + ss = ss[:tail] compacted := ss[skipped:] pos, size, slices := compactChunk(compacted) if len(compacted) < 2 || size == 0 { return } - for _, s := range ss[:skipped] { - if pos+size > s.pos && s.pos+s.len > pos { - var sstring string - for _, s := range ss { - sstring += fmt.Sprintf("\n%+v", *s) - } - panic(fmt.Sprintf("invalid compaction skipped %d, pos %d, size %d; slices: %s", skipped, pos, size, sstring)) - } - } var id uint64 if st = m.NewSlice(Background, &id); st != 0 { diff --git a/pkg/meta/base_test.go b/pkg/meta/base_test.go index 68a35bfeb08b..27f7e95f5309 100644 --- a/pkg/meta/base_test.go +++ b/pkg/meta/base_test.go @@ -1566,8 +1566,8 @@ func testCompaction(t *testing.T, m Meta, trash bool) { if st := m.Read(ctx, inode, 2, &slices); st != 0 { t.Fatalf("read 1: %s", st) } - // compact twice: 4515328+2607724-2338508 = 4784544; 8829056+1074933-2338508-4784544=2780937 - if len(slices) != 1 { + // 8829056 - 2338508 = 6490548 + if len(slices) != 3 || slices[0].Len != 2338508 || slices[1].Len != 6490548 || slices[2].Len != 1074933 { t.Fatalf("inode %d should be compacted, but have %d slices: %+v", inode, len(slices), slices) } diff --git a/pkg/meta/slice.go b/pkg/meta/slice.go index d10b64742b0e..aeb6af6f5b4b 100644 --- a/pkg/meta/slice.go +++ b/pkg/meta/slice.go @@ -180,11 +180,11 @@ func compactChunk(ss []*slice) (uint32, uint32, []Slice) { return pos, size, chunk } -func skipSome(chunk []*slice) int { +func skipSome(chunk []*slice) (int, int) { var skipped int var total = len(chunk) _, size, _ := compactChunk(chunk) - for skipped < total { + for skipped+1 < total { _, size1, _ := compactChunk(chunk[skipped+1:]) reduced := size - size1 if size1 == 0 || reduced < chunk[skipped].len || reduced*5 < size || reduced < 2<<20 { @@ -193,5 +193,18 @@ func skipSome(chunk []*slice) int { size = size1 skipped++ } - return skipped + tail := total + for skipped+1 < tail { + if chunk[tail-1].id == 0 { + break + } + _, size1, _ := compactChunk(chunk[skipped : tail-1]) + reduced := size - size1 + if size1 == 0 || reduced < chunk[tail-1].len || reduced*5 < size || reduced < 2<<20 { + break + } + size = size1 + tail-- + } + return skipped, tail } From 8f3b1e3b356afbd5ec6fc711c750bcba169f9f3e Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Fri, 6 Sep 2024 16:31:43 +0800 Subject: [PATCH 04/10] fix unit test --- pkg/meta/base_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/meta/base_test.go b/pkg/meta/base_test.go index 27f7e95f5309..7cf4cd3b97fa 100644 --- a/pkg/meta/base_test.go +++ b/pkg/meta/base_test.go @@ -1548,8 +1548,8 @@ func testCompaction(t *testing.T, m Meta, trash bool) { if st := m.Read(ctx, inode, 0, &slices); st != 0 { t.Fatalf("read 0: %s", st) } - if len(slices) != 2 { - t.Fatalf("inode %d should be compacted, but have %d slices, size %d: %+v", inode, len(slices), slices[0].Len, slices) + if len(slices) != 3 || slices[0].Len != 589824 || slices[1].Len != 458752 || slices[2].Len != 2097152 { + t.Fatalf("inode %d should be compacted, but have %d slices: %+v", inode, len(slices), slices) } m.NewSlice(ctx, &sliceId) From e0349937a97e409cf21c21c97368d162b3531421 Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Fri, 6 Sep 2024 17:30:01 +0800 Subject: [PATCH 05/10] fix unit test --- pkg/meta/base_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/meta/base_test.go b/pkg/meta/base_test.go index 7cf4cd3b97fa..49390f56488e 100644 --- a/pkg/meta/base_test.go +++ b/pkg/meta/base_test.go @@ -1541,14 +1541,14 @@ func testCompaction(t *testing.T, m Meta, trash bool) { _ = m.Write(ctx, inode, 0, uint32(0), Slice{Id: sliceId, Size: 1 << 20, Len: 64 << 10}, time.Now()) m.NewSlice(ctx, &sliceId) _ = m.Write(ctx, inode, 0, uint32(128<<10), Slice{Id: sliceId, Size: 2 << 20, Len: 128 << 10}, time.Now()) - _ = m.Write(ctx, inode, 0, uint32(0), Slice{Id: 0, Size: 1 << 20, Len: 1 << 20}, time.Now()) + _ = m.Fallocate(ctx, inode, fallocZeroRange, 0, 1<<20, nil) if c, ok := m.(compactor); ok { c.compactChunk(inode, 0, false, true) } if st := m.Read(ctx, inode, 0, &slices); st != 0 { t.Fatalf("read 0: %s", st) } - if len(slices) != 3 || slices[0].Len != 589824 || slices[1].Len != 458752 || slices[2].Len != 2097152 { + if len(slices) != 2 || slices[0].Len != 1048576 || slices[1].Len != 2097152 { t.Fatalf("inode %d should be compacted, but have %d slices: %+v", inode, len(slices), slices) } From f0bb8fab54b6a9bb6a289fffe55f72993fb9b475 Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Fri, 6 Sep 2024 20:55:19 +0800 Subject: [PATCH 06/10] don't skip head if it's covered by zero slice --- pkg/meta/slice.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/meta/slice.go b/pkg/meta/slice.go index aeb6af6f5b4b..726447253553 100644 --- a/pkg/meta/slice.go +++ b/pkg/meta/slice.go @@ -184,12 +184,19 @@ func skipSome(chunk []*slice) (int, int) { var skipped int var total = len(chunk) _, size, _ := compactChunk(chunk) +OUT: for skipped+1 < total { _, size1, _ := compactChunk(chunk[skipped+1:]) reduced := size - size1 - if size1 == 0 || reduced < chunk[skipped].len || reduced*5 < size || reduced < 2<<20 { + pos, length := chunk[skipped].pos, chunk[skipped].len + if size1 == 0 || reduced < length || reduced*5 < size || reduced < 2<<20 { break } + for _, s := range chunk[skipped+1:] { + if s.id == 0 && pos+length > s.pos && s.pos+s.len > pos { + break OUT + } + } size = size1 skipped++ } From d6fe2a6e090d8ab417a76cb08dbe61824b864560 Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Tue, 10 Sep 2024 21:16:21 +0800 Subject: [PATCH 07/10] adjust skipSome --- pkg/meta/base.go | 2 +- pkg/meta/slice.go | 111 ++++++++++++++++++++++++++++------------------ 2 files changed, 70 insertions(+), 43 deletions(-) diff --git a/pkg/meta/base.go b/pkg/meta/base.go index 97a41b3c02c4..b1721e2841f9 100644 --- a/pkg/meta/base.go +++ b/pkg/meta/base.go @@ -2052,7 +2052,7 @@ func (m *baseMeta) compactChunk(inode Ino, indx uint32, once, force bool) { skipped, tail := skipSome(ss) ss = ss[:tail] compacted := ss[skipped:] - pos, size, slices := compactChunk(compacted) + size, _, pos, slices := compactChunk(ss, skipped) if len(compacted) < 2 || size == 0 { return } diff --git a/pkg/meta/slice.go b/pkg/meta/slice.go index 726447253553..3cab2af95e49 100644 --- a/pkg/meta/slice.go +++ b/pkg/meta/slice.go @@ -154,64 +154,91 @@ func buildSlice(ss []*slice) []Slice { return chunk } -func compactChunk(ss []*slice) (uint32, uint32, []Slice) { - var chunk = buildSlice(ss) +func interact(pos uint32, a Slice, b *slice) bool { + return pos+a.Len > b.pos && b.pos+b.len > pos +} + +func compactChunk(slices []*slice, skip int) (uint32, uint32, uint32, []Slice) { var pos uint32 - n := len(chunk) - for n > 1 { - if chunk[0].Id == 0 { - pos += chunk[0].Len - chunk = chunk[1:] - n-- - } else if chunk[n-1].Id == 0 { - chunk = chunk[:n-1] - n-- - } else { + ss := buildSlice(slices[skip:]) + if len(ss) > 0 && ss[0].Id == 0 && ss[0].Size > 0 { + pos += ss[0].Len + ss = ss[1:] + } + var head, tail int +HEAD: + for ; head < len(ss)-1; head++ { + if ss[head].Id > 0 { break } + for _, s := range slices[:skip] { + if interact(pos, ss[head], s) { // FIXME: shrink zero slice + break HEAD + } + } + pos += ss[head].Len } - if n == 1 && chunk[0].Id == 0 { - chunk[0].Len = 1 + ss = ss[head:] +TAIL: + for tail = len(ss); tail > 1; tail-- { + if ss[tail-1].Id > 0 { + break + } + for _, s := range slices[:skip] { + if interact(pos, ss[tail-1], s) { + break TAIL + } + } + } + ss = ss[:tail] + var write, delete uint32 + for _, c := range ss { + write += c.Len } - var size uint32 - for _, c := range chunk { - size += c.Len + for _, s := range slices[skip:] { + if s.id > 0 { + delete += s.size + } } - return pos, size, chunk + return write, delete, pos, ss } -func skipSome(chunk []*slice) (int, int) { - var skipped int - var total = len(chunk) - _, size, _ := compactChunk(chunk) +func skipSome(slices []*slice) (int, int) { + var head, tail int + write, delete, _, _ := compactChunk(slices, 0) OUT: - for skipped+1 < total { - _, size1, _ := compactChunk(chunk[skipped+1:]) - reduced := size - size1 - pos, length := chunk[skipped].pos, chunk[skipped].len - if size1 == 0 || reduced < length || reduced*5 < size || reduced < 2<<20 { - break + for ; head < len(slices); head++ { + var p uint32 + ss := buildSlice(slices[head:]) + if len(ss) > 0 && ss[0].Id == 0 && ss[0].Size > 0 { // padding + p += ss[0].Len + ss = ss[1:] } - for _, s := range chunk[skipped+1:] { - if s.id == 0 && pos+length > s.pos && s.pos+s.len > pos { + for _, c := range ss { + if c.Id == 0 && c.Size > 0 && interact(p, c, slices[head]) { break OUT } + p += c.Len } - size = size1 - skipped++ - } - tail := total - for skipped+1 < tail { - if chunk[tail-1].id == 0 { + + write1, delete1, _, _ := compactChunk(slices, head+1) + reduced := write - write1 + // saved := delete - delete1 + if write < delete && (write1 == 0 || reduced < slices[head].len || reduced*5 < write || reduced < 2<<20) { break } - _, size1, _ := compactChunk(chunk[skipped : tail-1]) - reduced := size - size1 - if size1 == 0 || reduced < chunk[tail-1].len || reduced*5 < size || reduced < 2<<20 { + write = write1 + delete = delete1 + } + for tail = len(slices); tail > head; tail-- { + write1, delete1, _, _ := compactChunk(slices[:tail-1], head) + reduced := write - write1 + // saved := delete - delete1 + if write < delete && (write1 == 0 || reduced < slices[tail-1].len || reduced*5 < write || reduced < 2<<20) { break } - size = size1 - tail-- + write = write1 + delete = delete1 } - return skipped, tail + return head, tail } From a21fba46c874ad21f4c92739efe13a132f79cd61 Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Wed, 11 Sep 2024 15:42:13 +0800 Subject: [PATCH 08/10] adjust compactChunk --- pkg/meta/base.go | 15 +++++--- pkg/meta/redis.go | 21 ++++++++--- pkg/meta/slice.go | 92 +++++++++++++++++++++++++---------------------- pkg/meta/sql.go | 18 +++++++--- pkg/meta/tkv.go | 18 +++++++--- 5 files changed, 103 insertions(+), 61 deletions(-) diff --git a/pkg/meta/base.go b/pkg/meta/base.go index b1721e2841f9..c96c701cfc40 100644 --- a/pkg/meta/base.go +++ b/pkg/meta/base.go @@ -104,7 +104,7 @@ type engine interface { doWrite(ctx Context, inode Ino, indx uint32, off uint32, slice Slice, mtime time.Time, numSlices *int, delta *dirStat, attr *Attr) syscall.Errno doTruncate(ctx Context, inode Ino, flags uint8, length uint64, delta *dirStat, attr *Attr, skipPermCheck bool) syscall.Errno doFallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64, delta *dirStat, attr *Attr) syscall.Errno - doCompactChunk(inode Ino, indx uint32, origin []byte, ss []*slice, skipped int, pos uint32, id uint64, size uint32, delayed []byte) syscall.Errno + doCompactChunk(inode Ino, indx uint32, origin []byte, dslices, wslices []*slice, skipped int, delayed []byte) syscall.Errno doGetParents(ctx Context, inode Ino) map[Ino]int doUpdateDirStat(ctx Context, batch map[Ino]dirStat) error @@ -2052,7 +2052,7 @@ func (m *baseMeta) compactChunk(inode Ino, indx uint32, once, force bool) { skipped, tail := skipSome(ss) ss = ss[:tail] compacted := ss[skipped:] - size, _, pos, slices := compactChunk(ss, skipped) + size, deleted, wslices, rslices := compactChunk(ss) if len(compacted) < 2 || size == 0 { return } @@ -2061,8 +2061,13 @@ func (m *baseMeta) compactChunk(inode Ino, indx uint32, once, force bool) { if st = m.NewSlice(Background, &id); st != 0 { return } - logger.Debugf("compact %d:%d: skipped %d slices (%d bytes) %d slices (%d bytes)", inode, indx, skipped, pos, len(compacted), size) - err := m.newMsg(CompactChunk, slices, id) + for _, s := range wslices { + if s.size > 0 { + s.id = id + } + } + logger.Debugf("compact %d:%d: skipped %d slices, will compact %d slices (%d bytes), write %d bytes", inode, indx, skipped, len(compacted), deleted, size) + err := m.newMsg(CompactChunk, rslices, id) if err != nil { if !strings.Contains(err.Error(), "not exist") && !strings.Contains(err.Error(), "not found") { logger.Warnf("compact %d %d with %d slices: %s", inode, indx, len(compacted), err) @@ -2084,7 +2089,7 @@ func (m *baseMeta) compactChunk(inode Ino, indx uint32, once, force bool) { for _, s := range ss { origin = append(origin, marshalSlice(s.pos, s.id, s.size, s.off, s.len)...) } - st = m.en.doCompactChunk(inode, indx, origin, compacted, skipped, pos, id, size, dsbuf) + st = m.en.doCompactChunk(inode, indx, origin, compacted, wslices, skipped, dsbuf) if st == syscall.EINVAL { logger.Infof("compaction for %d:%d is wasted, delete slice %d (%d bytes)", inode, indx, id, size) m.deleteSlice(id, size) diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go index c875eb97ef69..21c3448caba2 100644 --- a/pkg/meta/redis.go +++ b/pkg/meta/redis.go @@ -2837,10 +2837,18 @@ func (r *redisMeta) doCleanupDelayedSlices(edge int64) (int, error) { return count, err } -func (m *redisMeta) doCompactChunk(inode Ino, indx uint32, origin []byte, ss []*slice, skipped int, pos uint32, id uint64, size uint32, delayed []byte) syscall.Errno { +func (m *redisMeta) doCompactChunk(inode Ino, indx uint32, origin []byte, dslices, wslices []*slice, skipped int, delayed []byte) syscall.Errno { + var id uint64 + var size uint32 + for _, s := range wslices { + if s.id > 0 { + id, size = s.id, s.size + break + } + } var rs []*redis.IntCmd // trash disabled: check reference of slices if delayed == nil { - rs = make([]*redis.IntCmd, len(ss)) + rs = make([]*redis.IntCmd, len(dslices)) } key := m.chunkKey(inode, indx) ctx := Background @@ -2861,7 +2869,10 @@ func (m *redisMeta) doCompactChunk(inode Ino, indx uint32, origin []byte, ss []* _, err = tx.TxPipelined(ctx, func(pipe redis.Pipeliner) error { pipe.LTrim(ctx, key, int64(n), -1) - pipe.LPush(ctx, key, marshalSlice(pos, id, size, 0, size)) + for i := len(wslices); i > 0; i-- { + s := wslices[i-1] + pipe.LPush(ctx, key, marshalSlice(s.pos, s.id, s.size, s.off, s.len)) + } for i := skipped; i > 0; i-- { pipe.LPush(ctx, key, origin[(i-1)*sliceBytes:i*sliceBytes]) } @@ -2871,7 +2882,7 @@ func (m *redisMeta) doCompactChunk(inode Ino, indx uint32, origin []byte, ss []* pipe.HSet(ctx, m.delSlices(), fmt.Sprintf("%d_%d", id, time.Now().Unix()), delayed) } } else { - for i, s := range ss { + for i, s := range dslices { if s.id > 0 { rs[i] = pipe.HIncrBy(ctx, m.sliceRefs(), m.sliceKey(s.id, s.size), -1) } @@ -2896,7 +2907,7 @@ func (m *redisMeta) doCompactChunk(inode Ino, indx uint32, origin []byte, ss []* } else if st == 0 { m.cleanupZeroRef(m.sliceKey(id, size)) if delayed == nil { - for i, s := range ss { + for i, s := range dslices { if s.id > 0 && rs[i].Err() == nil && rs[i].Val() < 0 { m.deleteSlice(s.id, s.size) } diff --git a/pkg/meta/slice.go b/pkg/meta/slice.go index 3cab2af95e49..516ffb6e6222 100644 --- a/pkg/meta/slice.go +++ b/pkg/meta/slice.go @@ -154,60 +154,66 @@ func buildSlice(ss []*slice) []Slice { return chunk } -func interact(pos uint32, a Slice, b *slice) bool { - return pos+a.Len > b.pos && b.pos+b.len > pos -} - -func compactChunk(slices []*slice, skip int) (uint32, uint32, uint32, []Slice) { +func compactChunk(slices []*slice) (uint32, uint32, []*slice, []Slice) { var pos uint32 - ss := buildSlice(slices[skip:]) + ss := buildSlice(slices) if len(ss) > 0 && ss[0].Id == 0 && ss[0].Size > 0 { pos += ss[0].Len ss = ss[1:] } - var head, tail int -HEAD: - for ; head < len(ss)-1; head++ { - if ss[head].Id > 0 { + + var head, tail uint32 + trimmed := ss + for len(trimmed) > 1 { + if trimmed[0].Id == 0 { + head += trimmed[0].Len + trimmed = trimmed[1:] + } else { break } - for _, s := range slices[:skip] { - if interact(pos, ss[head], s) { // FIXME: shrink zero slice - break HEAD - } - } - pos += ss[head].Len } - ss = ss[head:] -TAIL: - for tail = len(ss); tail > 1; tail-- { - if ss[tail-1].Id > 0 { + for n := len(trimmed); n > 1; n-- { + if trimmed[n-1].Id == 0 { + tail += trimmed[n-1].Len + trimmed = trimmed[:n-1] + } else { break } - for _, s := range slices[:skip] { - if interact(pos, ss[tail-1], s) { - break TAIL - } - } } - ss = ss[:tail] - var write, delete uint32 - for _, c := range ss { - write += c.Len + if len(trimmed) == 1 && trimmed[0].Id == 0 { + head += trimmed[0].Len - 1 + trimmed[0].Len = 1 + } + var size, deleted uint32 + for _, c := range trimmed { + size += c.Len } - for _, s := range slices[skip:] { - if s.id > 0 { - delete += s.size + del := make(map[uint64]struct{}) + for _, s := range slices { + if _, ok := del[s.id]; ok || s.id == 0 { + continue } + del[s.id] = struct{}{} + deleted += s.size } - return write, delete, pos, ss + var wslices []*slice + if head > 0 { + wslices = append(wslices, &slice{pos: pos, len: head}) + pos += head + } + wslices = append(wslices, &slice{pos: pos, size: size, len: size}) + if tail > 0 { + wslices = append(wslices, &slice{pos: pos + size, len: tail}) + } + + return size, deleted, wslices, trimmed } func skipSome(slices []*slice) (int, int) { var head, tail int - write, delete, _, _ := compactChunk(slices, 0) + write, deleted, _, _ := compactChunk(slices) OUT: - for ; head < len(slices); head++ { + for ; head < len(slices)-1; head++ { var p uint32 ss := buildSlice(slices[head:]) if len(ss) > 0 && ss[0].Id == 0 && ss[0].Size > 0 { // padding @@ -215,30 +221,30 @@ OUT: ss = ss[1:] } for _, c := range ss { - if c.Id == 0 && c.Size > 0 && interact(p, c, slices[head]) { + if c.Id == 0 && c.Size > 0 && p+c.Len > slices[head].pos && slices[head].pos+slices[head].len > p { break OUT } p += c.Len } - write1, delete1, _, _ := compactChunk(slices, head+1) + write1, delete1, _, _ := compactChunk(slices[head+1:]) reduced := write - write1 // saved := delete - delete1 - if write < delete && (write1 == 0 || reduced < slices[head].len || reduced*5 < write || reduced < 2<<20) { + if write < deleted && (write1 == 0 || reduced < slices[head].len || reduced*5 < write || reduced < 2<<20) { break } write = write1 - delete = delete1 + deleted = delete1 } - for tail = len(slices); tail > head; tail-- { - write1, delete1, _, _ := compactChunk(slices[:tail-1], head) + for tail = len(slices); tail > head+1; tail-- { + write1, delete1, _, _ := compactChunk(slices[:tail-1]) reduced := write - write1 // saved := delete - delete1 - if write < delete && (write1 == 0 || reduced < slices[tail-1].len || reduced*5 < write || reduced < 2<<20) { + if write < deleted && (write1 == 0 || reduced < slices[tail-1].len || reduced*5 < write || reduced < 2<<20) { break } write = write1 - delete = delete1 + deleted = delete1 } return head, tail } diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index 9803051fe54d..80a68df75f89 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -2753,7 +2753,17 @@ func (m *dbMeta) doCleanupDelayedSlices(edge int64) (int, error) { return count, nil } -func (m *dbMeta) doCompactChunk(inode Ino, indx uint32, origin []byte, ss []*slice, skipped int, pos uint32, id uint64, size uint32, delayed []byte) syscall.Errno { +func (m *dbMeta) doCompactChunk(inode Ino, indx uint32, origin []byte, dslices, wslices []*slice, skipped int, delayed []byte) syscall.Errno { + var id uint64 + var size uint32 + var wbuf []byte + for _, s := range wslices { + wbuf = append(wbuf, marshalSlice(s.pos, s.id, s.size, s.off, s.len)...) + if s.id > 0 { + id, size = s.id, s.size + break + } + } st := errno(m.txn(func(s *xorm.Session) error { var c2 = chunk{Inode: inode, Indx: indx} _, err := s.ForUpdate().MustCols("indx").Get(&c2) @@ -2765,7 +2775,7 @@ func (m *dbMeta) doCompactChunk(inode Ino, indx uint32, origin []byte, ss []*sli return syscall.EINVAL } - c2.Slices = append(append(c2.Slices[:skipped*sliceBytes], marshalSlice(pos, id, size, 0, size)...), c2.Slices[len(origin):]...) + c2.Slices = append(append(c2.Slices[:skipped*sliceBytes], wbuf...), c2.Slices[len(origin):]...) if _, err := s.Where("Inode = ? AND indx = ?", inode, indx).Update(c2); err != nil { return err } @@ -2780,7 +2790,7 @@ func (m *dbMeta) doCompactChunk(inode Ino, indx uint32, origin []byte, ss []*sli } } } else { - for _, s_ := range ss { + for _, s_ := range dslices { if s_.id == 0 { continue } @@ -2813,7 +2823,7 @@ func (m *dbMeta) doCompactChunk(inode Ino, indx uint32, origin []byte, ss []*sli return mustInsert(s, &sliceRef{id, size, 0}) }) } else if st == 0 && delayed == nil { - for _, s := range ss { + for _, s := range dslices { if s.id == 0 { continue } diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go index 357bcfd1e97f..15c43043f3e9 100644 --- a/pkg/meta/tkv.go +++ b/pkg/meta/tkv.go @@ -2317,7 +2317,17 @@ func (m *kvMeta) doCleanupDelayedSlices(edge int64) (int, error) { return count, nil } -func (m *kvMeta) doCompactChunk(inode Ino, indx uint32, buf []byte, ss []*slice, skipped int, pos uint32, id uint64, size uint32, delayed []byte) syscall.Errno { +func (m *kvMeta) doCompactChunk(inode Ino, indx uint32, buf []byte, dslices, wslices []*slice, skipped int, delayed []byte) syscall.Errno { + var id uint64 + var size uint32 + var wbuf []byte + for _, s := range wslices { + wbuf = append(wbuf, marshalSlice(s.pos, s.id, s.size, s.off, s.len)...) + if s.id > 0 { + id, size = s.id, s.size + break + } + } st := errno(m.txn(func(tx *kvTxn) error { buf2 := tx.get(m.chunkKey(inode, indx)) if len(buf2) < len(buf) || !bytes.Equal(buf, buf2[:len(buf)]) { @@ -2325,7 +2335,7 @@ func (m *kvMeta) doCompactChunk(inode Ino, indx uint32, buf []byte, ss []*slice, return syscall.EINVAL } - buf2 = append(append(buf2[:skipped*sliceBytes], marshalSlice(pos, id, size, 0, size)...), buf2[len(buf):]...) + buf2 = append(append(buf2[:skipped*sliceBytes], wbuf...), buf2[len(buf):]...) tx.set(m.chunkKey(inode, indx), buf2) // create the key to tracking it tx.set(m.sliceKey(id, size), make([]byte, 8)) @@ -2334,7 +2344,7 @@ func (m *kvMeta) doCompactChunk(inode Ino, indx uint32, buf []byte, ss []*slice, tx.set(m.delSliceKey(time.Now().Unix(), id), delayed) } } else { - for _, s := range ss { + for _, s := range dslices { if s.id > 0 { tx.incrBy(m.sliceKey(s.id, s.size), -1) } @@ -2364,7 +2374,7 @@ func (m *kvMeta) doCompactChunk(inode Ino, indx uint32, buf []byte, ss []*slice, m.cleanupZeroRef(id, size) if delayed == nil { var refs int64 - for _, s := range ss { + for _, s := range dslices { if s.id > 0 && m.client.txn(func(tx *kvTxn) error { refs = tx.incrBy(m.sliceKey(s.id, s.size), 0) return nil From 78edeb89a4db9ecff8bc53cc0c56442ad68f8ca1 Mon Sep 17 00:00:00 2001 From: Sandy Xu Date: Wed, 11 Sep 2024 15:56:37 +0800 Subject: [PATCH 09/10] fix --- pkg/meta/base.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/meta/base.go b/pkg/meta/base.go index c96c701cfc40..7fd15e830ad8 100644 --- a/pkg/meta/base.go +++ b/pkg/meta/base.go @@ -2052,8 +2052,7 @@ func (m *baseMeta) compactChunk(inode Ino, indx uint32, once, force bool) { skipped, tail := skipSome(ss) ss = ss[:tail] compacted := ss[skipped:] - size, deleted, wslices, rslices := compactChunk(ss) - if len(compacted) < 2 || size == 0 { + if len(compacted) < 2 { return } @@ -2061,6 +2060,7 @@ func (m *baseMeta) compactChunk(inode Ino, indx uint32, once, force bool) { if st = m.NewSlice(Background, &id); st != 0 { return } + size, deleted, wslices, rslices := compactChunk(compacted) for _, s := range wslices { if s.size > 0 { s.id = id From 5d41e00ae7d0266b217d690e688b35696c099188 Mon Sep 17 00:00:00 2001 From: Davies Liu Date: Wed, 11 Sep 2024 21:36:35 +0800 Subject: [PATCH 10/10] review --- pkg/meta/base.go | 23 +++++++--- pkg/meta/slice.go | 107 ++++++++++++++++++---------------------------- 2 files changed, 58 insertions(+), 72 deletions(-) diff --git a/pkg/meta/base.go b/pkg/meta/base.go index 7fd15e830ad8..c67b2fdc7b78 100644 --- a/pkg/meta/base.go +++ b/pkg/meta/base.go @@ -2060,11 +2060,14 @@ func (m *baseMeta) compactChunk(inode Ino, indx uint32, once, force bool) { if st = m.NewSlice(Background, &id); st != 0 { return } - size, deleted, wslices, rslices := compactChunk(compacted) - for _, s := range wslices { - if s.size > 0 { - s.id = id - } + pos, headSize, size, tailSize, rslices := compactChunk(compacted) + logger.Infof("compact %d slices into %d, %d, %d %+v", len(ss), headSize, size, tailSize, rslices) + if len(rslices) < 1 { + return + } + var deleted uint32 + for _, s := range compacted { + deleted += s.size } logger.Debugf("compact %d:%d: skipped %d slices, will compact %d slices (%d bytes), write %d bytes", inode, indx, skipped, len(compacted), deleted, size) err := m.newMsg(CompactChunk, rslices, id) @@ -2089,6 +2092,14 @@ func (m *baseMeta) compactChunk(inode Ino, indx uint32, once, force bool) { for _, s := range ss { origin = append(origin, marshalSlice(s.pos, s.id, s.size, s.off, s.len)...) } + var wslices []*slice + if headSize > 0 { + wslices = append(wslices, &slice{pos: pos, len: headSize}) + } + wslices = append(wslices, &slice{id: id, size: size, pos: pos + headSize, len: size}) + if tailSize > 0 { + wslices = append(wslices, &slice{pos: pos + headSize + size, len: tailSize}) + } st = m.en.doCompactChunk(inode, indx, origin, compacted, wslices, skipped, dsbuf) if st == syscall.EINVAL { logger.Infof("compaction for %d:%d is wasted, delete slice %d (%d bytes)", inode, indx, id, size) @@ -2099,7 +2110,7 @@ func (m *baseMeta) compactChunk(inode Ino, indx uint32, once, force bool) { logger.Warnf("compact %d %d: %s", inode, indx, err) } - if force { + if force && (skipped > 0 || len(ss) > maxCompactSlices/2) { m.Lock() delete(m.compacting, k) m.Unlock() diff --git a/pkg/meta/slice.go b/pkg/meta/slice.go index 516ffb6e6222..9c8ddf594dd3 100644 --- a/pkg/meta/slice.go +++ b/pkg/meta/slice.go @@ -154,97 +154,72 @@ func buildSlice(ss []*slice) []Slice { return chunk } -func compactChunk(slices []*slice) (uint32, uint32, []*slice, []Slice) { - var pos uint32 +func compactChunk(slices []*slice) (uint32, uint32, uint32, uint32, []Slice) { + var pos uint32 = ChunkSize + for _, s := range slices { + if s.pos < pos { + pos = s.pos + if pos == 0 { + break + } + } + } ss := buildSlice(slices) - if len(ss) > 0 && ss[0].Id == 0 && ss[0].Size > 0 { - pos += ss[0].Len + if pos > 0 && len(ss) > 0 { + // remove left padding ss = ss[1:] } var head, tail uint32 trimmed := ss - for len(trimmed) > 1 { - if trimmed[0].Id == 0 { - head += trimmed[0].Len - trimmed = trimmed[1:] - } else { - break - } + for len(trimmed) > 0 && trimmed[0].Id == 0 { + head += trimmed[0].Len + trimmed = trimmed[1:] } - for n := len(trimmed); n > 1; n-- { - if trimmed[n-1].Id == 0 { - tail += trimmed[n-1].Len - trimmed = trimmed[:n-1] - } else { - break - } + for n := len(trimmed); n > 0 && trimmed[n-1].Id == 0; n-- { + tail += trimmed[n-1].Len + trimmed = trimmed[:n-1] } - if len(trimmed) == 1 && trimmed[0].Id == 0 { - head += trimmed[0].Len - 1 - trimmed[0].Len = 1 + if len(trimmed) == 0 { + tail = head - 1 + head = 0 + trimmed = []Slice{{Len: 1}} } - var size, deleted uint32 + var size uint32 for _, c := range trimmed { size += c.Len } - del := make(map[uint64]struct{}) - for _, s := range slices { - if _, ok := del[s.id]; ok || s.id == 0 { - continue - } - del[s.id] = struct{}{} - deleted += s.size - } - var wslices []*slice - if head > 0 { - wslices = append(wslices, &slice{pos: pos, len: head}) - pos += head + return pos, head, size, tail, trimmed +} + +func shouldSkip(s *slice, rest []*slice, lastWrite uint32) (bool, uint32) { + pos, head, write, tail, _ := compactChunk(rest) + if pos < s.pos+s.len && s.pos < pos+head+write+tail { + return false, 0 // overlap } - wslices = append(wslices, &slice{pos: pos, size: size, len: size}) - if tail > 0 { - wslices = append(wslices, &slice{pos: pos + size, len: tail}) + reduced := lastWrite - write + if write == 0 || reduced*5 < lastWrite || reduced < 2<<20 || reduced < s.size { + return false, 0 } - - return size, deleted, wslices, trimmed + return true, write } func skipSome(slices []*slice) (int, int) { var head, tail int - write, deleted, _, _ := compactChunk(slices) -OUT: + _, _, lastWrite, _, _ := compactChunk(slices) for ; head < len(slices)-1; head++ { - var p uint32 - ss := buildSlice(slices[head:]) - if len(ss) > 0 && ss[0].Id == 0 && ss[0].Size > 0 { // padding - p += ss[0].Len - ss = ss[1:] - } - for _, c := range ss { - if c.Id == 0 && c.Size > 0 && p+c.Len > slices[head].pos && slices[head].pos+slices[head].len > p { - break OUT - } - p += c.Len - } - - write1, delete1, _, _ := compactChunk(slices[head+1:]) - reduced := write - write1 - // saved := delete - delete1 - if write < deleted && (write1 == 0 || reduced < slices[head].len || reduced*5 < write || reduced < 2<<20) { + skip, write := shouldSkip(slices[head], slices[head+1:], lastWrite) + if !skip { break } - write = write1 - deleted = delete1 + lastWrite = write } for tail = len(slices); tail > head+1; tail-- { - write1, delete1, _, _ := compactChunk(slices[:tail-1]) - reduced := write - write1 - // saved := delete - delete1 - if write < deleted && (write1 == 0 || reduced < slices[tail-1].len || reduced*5 < write || reduced < 2<<20) { + skip, write := shouldSkip(slices[tail-1], slices[:tail-1], lastWrite) + if !skip { break } - write = write1 - deleted = delete1 + lastWrite = write } return head, tail }