file: reduce allocations, add benchmark to start performance monitoring

moov-io · Apr 11, 2024 · 8d83f72 · 8d83f72
1 parent 275aff4
commit 8d83f72
Show file tree

Hide file tree

Showing 3 changed files with 92 additions and 42 deletions.
diff --git a/pkg/file/file.go b/pkg/file/file.go
@@ -233,16 +233,30 @@ func NewReader(r io.Reader) *Reader {
 
 // scanRecord allows reader to split metro file by each record
 func scanRecord(data []byte, atEOF bool) (advance int, token []byte, err error) {
-
-	getStripedLength := func() int {
-		return len(bytes.ReplaceAll(bytes.ReplaceAll(data, []byte("\r\n"), nil), []byte("\n"), nil))
+	getStripedLength := func() (length int) {
+		// Count every byte except for carriage returns and newlines
+		for _, b := range data {
+			switch b {
+			case '\r', '\n':
+				continue
+			default:
+				length += 1
+			}
+		}
+		return length
 	}
 
 	getStripedData := func(size int) (int, []byte, error) {
+		var buffer bytes.Buffer
 		for i := size; i <= len(data); i++ {
-			converted := bytes.ReplaceAll(bytes.ReplaceAll(data[:i], []byte("\r\n"), nil), []byte("\n"), nil)
-			if len(converted) == size {
-				return i, converted, nil
+			buffer.Reset()
+			for _, b := range data[:i] {
+				if b != '\r' && b != '\n' {
+					buffer.WriteByte(b)
+				}
+			}
+			if buffer.Len() == size {
+				return i, buffer.Bytes(), nil
 			}
 		}
 		return 0, nil, io.ErrNoProgress

diff --git a/pkg/file/file_instance.go b/pkg/file/file_instance.go
@@ -5,11 +5,11 @@
 package file
 
 import (
-	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"reflect"
+	"slices"
 	"strings"
 	"unicode"
 
@@ -199,7 +199,9 @@ func (f *fileInstance) Validate() error {
 func (f *fileInstance) Parse(record []byte) error {
 
 	// remove new lines
-	record = bytes.ReplaceAll(bytes.ReplaceAll(record, []byte("\r\n"), nil), []byte("\n"), nil)
+	record = slices.DeleteFunc(record, func(b byte) bool {
+		return b == '\r' || b == '\n'
+	})
 
 	f.Bases = []lib.Record{}
 	offset := 0

diff --git a/pkg/file/file_test.go b/pkg/file/file_test.go
@@ -361,52 +361,86 @@ func (t *FileTest) TestWithUnknownFileType(c *check.C) {
 }
 
 func TestFile__Reader(t *testing.T) {
-
 	t.Run("Read with unpacked fixed file", func(t *testing.T) {
+		readUnpackedFixedFile(t)
+	})
 
-		fd, err := os.Open(filepath.Join("..", "..", "test", "testdata", "unpacked_fixed_file.dat"))
-		if err != nil {
-			t.Fatalf("Can not open local file: %s: \n", err)
-		}
-		defer fd.Close()
+	t.Run("Read with unpacked variable file", func(t *testing.T) {
+		readUnpackedVariableFile(t)
+	})
 
-		f, err := NewReader(fd).Read()
-		require.NoError(t, err)
+	t.Run("Read with packed file", func(t *testing.T) {
+		readPackedFile(t)
+	})
+}
 
-		// ensure we have a validated file structure
-		err = f.Validate()
-		require.NoError(t, err)
+func BenchmarkFile(b *testing.B) {
+	b.Run("Read with unpacked fixed file", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			readUnpackedFixedFile(b)
+		}
 	})
 
-	t.Run("Read with unpacked variable file", func(t *testing.T) {
+	b.Run("Read with unpacked variable file", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			readUnpackedVariableFile(b)
+		}
+	})
 
-		fd, err := os.Open(filepath.Join("..", "..", "test", "testdata", "unpacked_variable_file.dat"))
-		if err != nil {
-			t.Fatalf("Can not open local file: %s: \n", err)
+	b.Run("Read with packed file", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			readPackedFile(b)
 		}
-		defer fd.Close()
+	})
+}
 
-		f, err := NewReader(fd).Read()
-		require.NoError(t, err)
+func readUnpackedFixedFile(tb testing.TB) {
+	tb.Helper()
 
-		// ensure we have a validated file structure
-		err = f.Validate()
-		require.NoError(t, err)
-	})
+	fd, err := os.Open(filepath.Join("..", "..", "test", "testdata", "unpacked_fixed_file.dat"))
+	if err != nil {
+		tb.Fatalf("Can not open local file: %s: \n", err)
+	}
+	defer fd.Close()
 
-	t.Run("Read with packed file", func(t *testing.T) {
+	f, err := NewReader(fd).Read()
+	require.NoError(tb, err)
 
-		fd, err := os.Open(filepath.Join("..", "..", "test", "testdata", "packed_file.dat"))
-		if err != nil {
-			t.Fatalf("Can not open local file: %s: \n", err)
-		}
-		defer fd.Close()
+	// ensure we have a validated file structure
+	err = f.Validate()
+	require.NoError(tb, err)
+}
 
-		f, err := NewReader(fd).Read()
-		require.NoError(t, err)
+func readUnpackedVariableFile(tb testing.TB) {
+	tb.Helper()
 
-		// ensure we have a validated file structure
-		err = f.Validate()
-		require.NoError(t, err)
-	})
+	fd, err := os.Open(filepath.Join("..", "..", "test", "testdata", "unpacked_variable_file.dat"))
+	if err != nil {
+		tb.Fatalf("Can not open local file: %s: \n", err)
+	}
+	defer fd.Close()
+
+	f, err := NewReader(fd).Read()
+	require.NoError(tb, err)
+
+	// ensure we have a validated file structure
+	err = f.Validate()
+	require.NoError(tb, err)
+}
+
+func readPackedFile(tb testing.TB) {
+	tb.Helper()
+
+	fd, err := os.Open(filepath.Join("..", "..", "test", "testdata", "packed_file.dat"))
+	if err != nil {
+		tb.Fatalf("Can not open local file: %s: \n", err)
+	}
+	defer fd.Close()
+
+	f, err := NewReader(fd).Read()
+	require.NoError(tb, err)
+
+	// ensure we have a validated file structure
+	err = f.Validate()
+	require.NoError(tb, err)
 }