Add read/write of tags to fileset to restore tags, add fs index bootstrapping

glide.yaml

@@ -1,7 +1,7 @@
 package: github.com/m3db/m3db
 import:
 - package: github.com/m3db/m3x
-  version: 04f23ee000fdef43b2260cf99dd751e8b9cd3dca
+  version: 0ae10beff2f574adfc42a5b42ecf34e92533be50
   vcs: git
   subpackages:
   - checked

integration/disk_flush_helpers.go

@@ -140,9 +140,20 @@ func verifyForTime(
 		}
 		require.NoError(t, reader.Open(rOpts))
 		for i := 0; i < reader.Entries(); i++ {
-			id, data, _, err := reader.Read()
+			id, tagsIter, data, _, err := reader.Read()
 			require.NoError(t, err)
 
+			var tags ident.Tags
+			if tagsLen := tagsIter.Remaining(); tagsLen > 0 {
+				tags = make(ident.Tags, 0, tagsLen)
+				for tagsIter.Next() {
+					curr := tagsIter.Current()
+					tags = append(tags, ident.StringTag(curr.Name.String(), curr.Value.String()))
+				}
+				require.NoError(t, tagsIter.Err())
+				tagsIter.Close()
+			}
+
 			data.IncRef()
 
 			var datapoints []ts.Datapoint
@@ -157,6 +168,7 @@ func verifyForTime(
 
 			actual = append(actual, generate.Series{
 				ID:   id,
+				Tags: tags,
 				Data: datapoints,
 			})
 

integration/generate/types.go

@@ -22,6 +22,7 @@ type BlockConfig struct {
 // Series represents a generated series of data
 type Series struct {
 	ID   ident.ID
+	Tags ident.Tags
 	Data []ts.Datapoint
 }
 

integration/generate/writer.go

@@ -139,7 +139,8 @@ func writeToDisk(
 			data[0] = segment.Head
 			data[1] = segment.Tail
 			checksum := digest.SegmentChecksum(segment)
-			if err := writer.WriteAll(series.ID, data, checksum); err != nil {
+			err = writer.WriteAll(series.ID, series.Tags, data, checksum)
+			if err != nil {
 				return err
 			}
 		}

persist/fs/clone/cloner.go

@@ -1,3 +1,23 @@
+// Copyright (c) 2018 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
 package clone
 
 import (
@@ -62,16 +82,29 @@ func (c *cloner) Clone(src FileSetID, dest FileSetID, destBlocksize time.Duratio
 	}
 
 	for {
-		id, data, checksum, err := reader.Read()
+		id, tagsIter, data, checksum, err := reader.Read()
 		if err != nil {
 			if err == io.EOF {
 				break
 			}
 			return fmt.Errorf("unexpected error while reading data: %v", err)
 		}
 
+		var tags ident.Tags
+		if tagsLen := tagsIter.Remaining(); tagsLen > 0 {
+			tags = make(ident.Tags, 0, tagsLen)
+			for tagsIter.Next() {
+				curr := tagsIter.Current()
+				tags = append(tags, ident.StringTag(curr.Name.String(), curr.Value.String()))
+			}
+			if err := tagsIter.Err(); err != nil {
+				return fmt.Errorf("unable to decode tags: %v", err)
+			}
+			tagsIter.Close()
+		}
+
 		data.IncRef()
-		if err := writer.Write(id, data, checksum); err != nil {
+		if err := writer.Write(id, tags, data, checksum); err != nil {
 			return fmt.Errorf("unexpected error while writing data: %v", err)
 		}
 		data.DecRef()

persist/fs/clone/cloner_test.go

@@ -1,3 +1,23 @@
+// Copyright (c) 2018 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
 package clone
 
 import (
@@ -87,14 +107,32 @@ func TestCloner(t *testing.T) {
 	}
 	require.NoError(t, r2.Open(r2OpenOpts))
 	for {
-		t1, b1, c1, e1 := r1.Read()
-		t2, b2, c2, e2 := r2.Read()
+		t1, a1, b1, c1, e1 := r1.Read()
+		t2, a2, b2, c2, e2 := r2.Read()
 		if e1 == e2 && e1 == io.EOF {
 			break
 		}
+
+		require.NoError(t, e1)
+		require.NoError(t, e2)
+
 		b1.IncRef()
 		b2.IncRef()
 		require.Equal(t, t1.String(), t2.String())
+		require.Equal(t, a1.Remaining(), a2.Remaining())
+		numTags, numTagsMatched := a1.Remaining(), 0
+		for a1.Next() && a2.Next() {
+			tag0 := a1.Current()
+			tag1 := a2.Current()
+			require.Equal(t, tag0.Name.String(), tag1.Name.String())
+			require.Equal(t, tag0.Value.String(), tag1.Value.String())
+			numTagsMatched++
+		}
+		require.NoError(t, a1.Err())
+		require.NoError(t, a2.Err())
+		a1.Close()
+		a2.Close()
+		require.Equal(t, numTags, numTagsMatched)
 		require.Equal(t, b1.Bytes(), b2.Bytes())
 		require.Equal(t, c1, c2)
 		b1.DecRef()
@@ -121,8 +159,15 @@ func writeTestData(t *testing.T, bs time.Duration, src FileSetID, opts Options)
 	}
 	require.NoError(t, w.Open(writerOpts))
 	for i := 0; i < numTestSeries; i++ {
-		id := ident.StringID(fmt.Sprintf("testSeries.%d", i))
-		require.NoError(t, w.Write(id, testBytes, 1234))
+		id := ident.StringID(fmt.Sprintf("test-series.%d", i))
+		var tags ident.Tags
+		if i%2 == 0 {
+			tags = ident.Tags{
+				ident.StringTag("foo", "bar"),
+				ident.StringTag("qux", "qaz"),
+			}
+		}
+		require.NoError(t, w.Write(id, tags, testBytes, 1234))
 	}
 	require.NoError(t, w.Close())
 }

persist/fs/clone/options.go

@@ -1,3 +1,23 @@
+// Copyright (c) 2018 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
 package clone
 
 import (

persist/fs/clone/types.go

@@ -1,3 +1,23 @@
+// Copyright (c) 2018 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
 package clone
 
 import (