Use generic map to store entries

.excludecoverage

@@ -0,0 +1,7 @@
+_mock.go
+_gen.go
+_matcher.go
+generated/
+tools/
+vendor/
+integration/

.excludelint

@@ -1,3 +1,5 @@
 (vendor/)
 (generated/)
 (_mock.go)
+(_gen.go)
+(_string.go)

.excludemetalint

@@ -1,3 +1,4 @@
 vendor/
 generated/
 _mock.go
+_gen.go

.travis.yml

@@ -1,12 +1,19 @@
 language: go
 go:
-  - 1.8.3
-  - 1.9.2
+  - "1.9.x"
+  - "1.10.x"
+branches:
+  only:
+    - master
 install: make install-ci
-env:
- # Set higher timeouts and package name for travis
- - TEST_TIMEOUT_SCALE=20 PACKAGE=github.com/m3db/m3aggregator
 sudo: required
 dist: trusty
-script:
- - make all
+env:
+  global:
+    - TEST_TIMEOUT_SCALE=20 PACKAGE=github.com/m3db/m3aggregator
+  matrix:
+    - MAKE_TARGET="test-ci-unit"
+    - MAKE_TARGET="test-ci-integration"
+    - MAKE_TARGET="lint metalint"
+    - MAKE_TARGET="test-genny-all"
+script: "make $MAKE_TARGET"

Makefile

@@ -28,6 +28,8 @@ auto_gen             := .ci/auto-gen.sh
 license_dir          := .ci/uber-licence
 license_node_modules := $(license_dir)/node_modules
 
+include $(SELF_DIR)/generated-source-files.mk
+
 BUILD           := $(abspath ./bin)
 LINUX_AMD64_ENV := GOOS=linux GOARCH=amd64 CGO_ENABLED=0
 
@@ -139,7 +141,7 @@ clean:
 	@rm -f *.html *.xml *.out *.test
 
 .PHONY: all
-all: lint metalint test-ci-unit test-ci-integration m3aggregator
+all: lint metalint test-ci-unit test-ci-integration test-genny-all m3aggregator
 	@echo Made all successfully
 
 .DEFAULT_GOAL := all

aggregator/aggregator_test.go

@@ -283,7 +283,7 @@ func TestAggregatorAddMetricWithPoliciesListSuccessNoPlacementUpdate(t *testing.
 	agg.shardFn = func([]byte, int) uint32 { return 1 }
 	err := agg.AddMetricWithPoliciesList(testValidMetric, testPoliciesList)
 	require.NoError(t, err)
-	require.Equal(t, 1, len(agg.shards[1].metricMap.entries))
+	require.Equal(t, 1, agg.shards[1].metricMap.entries.Len())
 }
 
 func TestAggregatorAddMetricWithPoliciesListSuccessWithPlacementUpdate(t *testing.T) {
@@ -347,7 +347,7 @@ func TestAggregatorAddMetricWithPoliciesListSuccessWithPlacementUpdate(t *testin
 			require.Equal(t, expected.latestNanos, agg.shards[i].latestWriteableNanos)
 		}
 	}
-	require.Equal(t, 1, len(agg.shards[1].metricMap.entries))
+	require.Equal(t, 1, agg.shards[1].metricMap.entries.Len())
 	require.Equal(t, newPlacementCutoverNanos, agg.currPlacement.CutoverNanos())
 
 	for {

aggregator/entry_map_gen.go

@@ -0,0 +1,251 @@
+// This file was automatically generated by genny.
+// Any changes will be lost if this file is regenerated.
+// see https://github.com/mauricelam/genny
+
+package aggregator
+
+// Copyright (c) 2018 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+// entryMapHash is the hash for a given map entry, this is public to support
+// iterating over the map using a native Go for loop.
+type entryMapHash uint64
+
+// entryMapHashFn is the hash function to execute when hashing a key.
+type entryMapHashFn func(entryKey) entryMapHash
+
+// entryMapEqualsFn is the equals key function to execute when detecting equality of a key.
+type entryMapEqualsFn func(entryKey, entryKey) bool
+
+// entryMapCopyFn is the copy key function to execute when copying the key.
+type entryMapCopyFn func(entryKey) entryKey
+
+// entryMapFinalizeFn is the finalize key function to execute when finished with a key.
+type entryMapFinalizeFn func(entryKey)
+
+// entryMap uses the genny package to provide a generic hash map that can be specialized
+// by running the following command from this root of the repository:
+// ```
+// make hashmap-gen pkg=outpkg key_type=Type value_type=Type out_dir=/tmp
+// ```
+// Or if you would like to use bytes or ident.ID as keys you can use the
+// partially specialized maps to generate your own maps as well:
+// ```
+// make byteshashmap-gen pkg=outpkg value_type=Type out_dir=/tmp
+// make idhashmap-gen pkg=outpkg value_type=Type out_dir=/tmp
+// ```
+// This will output to stdout the generated source file to use for your map.
+// It uses linear probing by incrementing the number of the hash created when
+// hashing the identifier if there is a collision.
+// entryMap is a value type and not an interface to allow for less painful
+// upgrades when adding/removing methods, it is not likely to need mocking so
+// an interface would not be super useful either.
+type entryMap struct {
+	_entryMapOptions
+
+	// lookup uses hash of the identifier for the key and the MapEntry value
+	// wraps the value type and the key (used to ensure lookup is correct
+	// when dealing with collisions), we use uint64 for the hash partially
+	// because lookups of maps with uint64 keys has a fast path for Go.
+	lookup map[entryMapHash]entryMapEntry
+}
+
+// _entryMapOptions is a set of options used when creating an identifier map, it is kept
+// private so that implementers of the generated map can specify their own options
+// that partially fulfill these options.
+type _entryMapOptions struct {
+	// hash is the hash function to execute when hashing a key.
+	hash entryMapHashFn
+	// equals is the equals key function to execute when detecting equality.
+	equals entryMapEqualsFn
+	// copy is the copy key function to execute when copying the key.
+	copy entryMapCopyFn
+	// finalize is the finalize key function to execute when finished with a
+	// key, this is optional to specify.
+	finalize entryMapFinalizeFn
+	// initialSize is the initial size for the map, use zero to use Go's std map
+	// initial size and consequently is optional to specify.
+	initialSize int
+}
+
+// entryMapEntry is an entry in the map, this is public to support iterating
+// over the map using a native Go for loop.
+type entryMapEntry struct {
+	// key is used to check equality on lookups to resolve collisions
+	key _entryMapKey
+	// value type stored
+	value elementPtr
+}
+
+type _entryMapKey struct {
+	key      entryKey
+	finalize bool
+}
+
+// Key returns the map entry key.
+func (e entryMapEntry) Key() entryKey {
+	return e.key.key
+}
+
+// Value returns the map entry value.
+func (e entryMapEntry) Value() elementPtr {
+	return e.value
+}
+
+// _entryMapAlloc is a non-exported function so that when generating the source code
+// for the map you can supply a public constructor that sets the correct
+// hash, equals, copy, finalize options without users of the map needing to
+// implement them themselves.
+func _entryMapAlloc(opts _entryMapOptions) *entryMap {
+	m := &entryMap{_entryMapOptions: opts}
+	m.Reallocate()
+	return m
+}
+
+func (m *entryMap) newMapKey(k entryKey, opts _entryMapKeyOptions) _entryMapKey {
+	key := _entryMapKey{key: k, finalize: opts.finalizeKey}
+	if !opts.copyKey {
+		return key
+	}
+
+	key.key = m.copy(k)
+	return key
+}
+
+func (m *entryMap) removeMapKey(hash entryMapHash, key _entryMapKey) {
+	delete(m.lookup, hash)
+	if key.finalize {
+		m.finalize(key.key)
+	}
+}
+
+// Get returns a value in the map for an identifier if found.
+func (m *entryMap) Get(k entryKey) (elementPtr, bool) {
+	hash := m.hash(k)
+	for entry, ok := m.lookup[hash]; ok; entry, ok = m.lookup[hash] {
+		if m.equals(entry.key.key, k) {
+			return entry.value, true
+		}
+		// Linear probe to "next" to this entry (really a rehash)
+		hash++
+	}
+	var empty elementPtr
+	return empty, false
+}
+
+// Set will set the value for an identifier.
+func (m *entryMap) Set(k entryKey, v elementPtr) {
+	m.set(k, v, _entryMapKeyOptions{
+		copyKey:     true,
+		finalizeKey: m.finalize != nil,
+	})
+}
+
+// entryMapSetUnsafeOptions is a set of options to use when setting a value with
+// the SetUnsafe method.
+type entryMapSetUnsafeOptions struct {
+	NoCopyKey     bool
+	NoFinalizeKey bool
+}
+
+// SetUnsafe will set the value for an identifier with unsafe options for how
+// the map treats the key.
+func (m *entryMap) SetUnsafe(k entryKey, v elementPtr, opts entryMapSetUnsafeOptions) {
+	m.set(k, v, _entryMapKeyOptions{
+		copyKey:     !opts.NoCopyKey,
+		finalizeKey: !opts.NoFinalizeKey,
+	})
+}
+
+type _entryMapKeyOptions struct {
+	copyKey     bool
+	finalizeKey bool
+}
+
+func (m *entryMap) set(k entryKey, v elementPtr, opts _entryMapKeyOptions) {
+	hash := m.hash(k)
+	for entry, ok := m.lookup[hash]; ok; entry, ok = m.lookup[hash] {
+		if m.equals(entry.key.key, k) {
+			m.lookup[hash] = entryMapEntry{
+				key:   entry.key,
+				value: v,
+			}
+			return
+		}
+		// Linear probe to "next" to this entry (really a rehash)
+		hash++
+	}
+
+	m.lookup[hash] = entryMapEntry{
+		key:   m.newMapKey(k, opts),
+		value: v,
+	}
+}
+
+// Iter provides the underlying map to allow for using a native Go for loop
+// to iterate the map, however callers should only ever read and not write
+// the map.
+func (m *entryMap) Iter() map[entryMapHash]entryMapEntry {
+	return m.lookup
+}
+
+// Len returns the number of map entries in the map.
+func (m *entryMap) Len() int {
+	return len(m.lookup)
+}
+
+// Contains returns true if value exists for key, false otherwise, it is
+// shorthand for a call to Get that doesn't return the value.
+func (m *entryMap) Contains(k entryKey) bool {
+	_, ok := m.Get(k)
+	return ok
+}
+
+// Delete will remove a value set in the map for the specified key.
+func (m *entryMap) Delete(k entryKey) {
+	hash := m.hash(k)
+	for entry, ok := m.lookup[hash]; ok; entry, ok = m.lookup[hash] {
+		if m.equals(entry.key.key, k) {
+			m.removeMapKey(hash, entry.key)
+			return
+		}
+		// Linear probe to "next" to this entry (really a rehash)
+		hash++
+	}
+}
+
+// Reset will reset the map by simply deleting all keys to avoid
+// allocating a new map.
+func (m *entryMap) Reset() {
+	for hash, entry := range m.lookup {
+		m.removeMapKey(hash, entry.key)
+	}
+}
+
+// Reallocate will avoid deleting all keys and reallocate a new
+// map, this is useful if you believe you have a large map and
+// will not need to grow back to a similar size.
+func (m *entryMap) Reallocate() {
+	if m.initialSize > 0 {
+		m.lookup = make(map[entryMapHash]entryMapEntry, m.initialSize)
+	} else {
+		m.lookup = make(map[entryMapHash]entryMapEntry)
+	}
+}

hash/hash.go → aggregator/entry_new_map.go

@@ -18,19 +18,12 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 // THE SOFTWARE.
 
-// Package hash is a temporary measure used as in between while m3aggregator
-// is upgraded to use native source generated maps now accessible in m3x,
-// these types are missing from m3x when native source generated maps
-// were added.
-package hash
+package aggregator
 
-import "github.com/spaolacci/murmur3"
-
-// Hash128 is a 128-bit hash of an ID consisting of two unsigned 64-bit ints.
-type Hash128 [2]uint64
-
-// Murmur3Hash128 computes the 128-bit hash of an id.
-func Murmur3Hash128(data []byte) Hash128 {
-	h0, h1 := murmur3.Sum128(data)
-	return Hash128{h0, h1}
+func newEntryMap() *entryMap {
+	return _entryMapAlloc(_entryMapOptions{
+		hash:   func(k entryKey) entryMapHash { return entryMapHash(k.Hash()) },
+		equals: func(x, y entryKey) bool { return x.Equal(y) },
+		copy:   func(k entryKey) entryKey { return k.Clone() },
+	})
 }