Merge pull request #1 from m3db/prateek/initial-commit

.excludelint

@@ -0,0 +1,3 @@
+(vendor/)
+(mock_.*.go)
+(*.pb.go)

.gitignore

@@ -0,0 +1,59 @@
+### https://raw.github.com/github/gitignore/35c010258fc790ad769033e9ccfb1021178f2925/Go.gitignore
+
+# Glide vendor-ed deps
+vendor/
+
+# Binaries for programs and plugins
+*.exe
+*.dll
+*.so
+*.dylib
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
+.glide/
+
+
+### https://raw.github.com/github/gitignore/35c010258fc790ad769033e9ccfb1021178f2925/Global/VisualStudioCode.gitignore
+
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+
+### https://raw.github.com/github/gitignore/35c010258fc790ad769033e9ccfb1021178f2925/Global/macOS.gitignore
+
+*.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+

.travis.yml

@@ -0,0 +1,15 @@
+language: go
+go:
+  - 1.7
+install: make install-ci
+env:
+ # Set higher timeouts for Travis
+ - TEST_TIMEOUT_SCALE=20 PACKAGE=github.com/m3db/m3nsch
+sudo: required
+dist: trusty
+script:
+ - make lint
+ - make test-ci-unit
+ - make m3nsch_client
+ - make m3nsch_server
+

LICENSE.md

@@ -1,4 +1,4 @@
-Copyright (c) 2016 Uber Technologies, Inc.
+Copyright (c) 2017 Uber Technologies, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -16,4 +16,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+THE SOFTWARE.

Makefile

@@ -0,0 +1,80 @@
+SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
+include $(SELF_DIR)/.ci/common.mk
+
+SHELL=/bin/bash -o pipefail
+
+html_report := coverage.html
+test := .ci/test-cover.sh
+test_ci_integration := .ci/test-integration.sh
+convert-test-data := .ci/convert-test-data.sh
+coverfile := cover.out
+coverage_xml := coverage.xml
+junit_xml := junit.xml
+test_log := test.log
+lint_check := .ci/lint.sh
+
+BUILD := $(abspath ./out)
+LINUX_AMD64_ENV := GOOS=linux GOARCH=amd64 CGO_ENABLED=0
+VENDOR_ENV := GO15VENDOREXPERIMENT=1
+
+SERVICES := \
+	m3nsch_server \
+	m3nsch_client \
+
+setup:
+	mkdir -p $(BUILD)
+
+define SERVICE_RULES
+
+$(SERVICE): setup
+	@echo Building $(SERVICE)
+	$(VENDOR_ENV) go build -o $(BUILD)/$(SERVICE) ./$(SERVICE)/.
+
+$(SERVICE)-linux-amd64:
+	$(LINUX_AMD64_ENV) make $(SERVICE)
+
+endef
+
+services: $(SERVICES)
+services-linux-amd64:
+	$(LINUX_AMD64_ENV) make services
+
+$(foreach SERVICE,$(SERVICES),$(eval $(SERVICE_RULES)))
+
+lint:
+	@which golint > /dev/null || go get -u github.com/golang/lint/golint
+	$(VENDOR_ENV) $(lint_check)
+
+test-internal:
+	@which go-junit-report > /dev/null || go get -u github.com/sectioneight/go-junit-report
+	@$(VENDOR_ENV) $(test) $(coverfile) | tee $(test_log)
+
+test-xml: test-internal
+	go-junit-report < $(test_log) > $(junit_xml)
+	gocov convert $(coverfile) | gocov-xml > $(coverage_xml)
+	@$(convert-test-data) $(coverage_xml)
+	@rm $(coverfile) &> /dev/null
+
+test: test-internal
+	gocov convert $(coverfile) | gocov report
+
+testhtml: test-internal
+	gocov convert $(coverfile) | gocov-html > $(html_report) && open $(html_report)
+	@rm -f $(test_log) &> /dev/null
+
+test-ci-unit: test-internal
+	@which goveralls > /dev/null || go get -u -f github.com/mattn/goveralls
+	goveralls -coverprofile=$(coverfile) -service=travis-ci || echo -e "\x1b[31mCoveralls failed\x1b[m"
+
+test-ci-integration:
+	@$(VENDOR_ENV) $(test_ci_integration)
+
+clean:
+	echo Cleaning build artifacts...
+	go clean
+	rm -rf $(BUILD)
+	@rm -f *.html *.xml *.out *.test
+	echo
+
+.DEFAULT_GOAL := test
+.PHONY: test test-xml test-internal testhtml clean

README.md

@@ -0,0 +1,107 @@
+m3nsch
+======
+m3nsch (pronounced `mensch`) is a load testing tool for M3DB. It has two components:
+  - `m3nsch_server`: long lived process which does the load generation
+  - `m3nsch_client`: cli wrapper which controls the myriad `m3nsch_server(s)`
+
+A typical deploy will have multiple hosts, each running a single `m3nsch_server` instance,
+and a single `m3nsch_client` used to control them.
+
+### Build
+```
+$ make prod-m3nsch_server
+$ make prod-m3nsch_client
+$ cat <<EOF > server-conf.yaml
+server:
+  listenAddress: "0.0.0.0:12321"
+  debugAddress: "0.0.0.0:13441"
+  cpuFactor: 0.9
+
+metrics:
+  sampleRate: 0.1
+  m3:
+    hostPort: "<host-port-for-tally>"
+    service: "m3nsch"
+    includeHost: true
+    env: development
+
+m3nsch:
+  concurrency: 2000
+  numPointsPerDatum: 60
+
+# any additional configs you may have
+EOF
+```
+
+### Deploy
+(1) Transfer the `m3nsch_server` binary, and `server-conf.yaml` to all the hosts to be used to generate hosts, e.g. Ingesters
+
+(2) On each host from (1), kick of the server process by running:
+```
+./m3nsch_server -f server-conf.yaml
+```
+
+(3) Transfer `m3nsch_client` binary to a host with network connectivity to all the hosts.
+
+### Sample Usage
+```
+# set an env var containing the host endpoints seperated by commas
+$ export ENDPOINTS="host1:12321,host2:12321"
+
+# get the status of the various endpoints, make sure all are healthy
+$ ./m3nsch_client --endpoints $ENDPOINTS status
+
+# investigate the various options available during initialization
+$ ./m3nsch_client init --help
+...
+Flags:
+  -b, --basetime-offset duration   offset from current time to use for load, e.g. -2m, -30s (default -2m0s)
+  -c, --cardinality int            aggregate workload cardinality (default 10000)
+  -f, --force                      force initialization, stop any running workload
+  -i, --ingress-qps int            aggregate workload ingress qps (default 1000)
+  -p, --metric-prefix string       prefix added to each metric (default "m3nsch_")
+  -n, --namespace string           target namespace (default "testmetrics")
+  -v, --target-env string          target env for load test (default "test")
+  -z, --target-zone string         target zone for load test (default "sjc1")
+  -t, --token string               [required] unique identifier required for all subsequent interactions on this workload
+
+Global Flags:
+  -e, --endpoints stringSlice   host:port for each of the agent process endpoints
+
+# initialize the servers, set any workload parameters, target env/zone
+# the command below targets the production sjc1 m3db cluster with each `m3nsch_server` attempting to
+# sustain a load of 100K writes/s from a set of 1M metrics
+$ ./m3nsch_client --endpoints $ENDPOINTS init \
+  --token prateek-sample \
+  --target-env prod      \
+  --target-zone sjc1     \
+  --ingress-qps 100000   \
+  --cardinality 1000000  \
+
+# start the load generation
+$ ./m3nsch_client --endpoints $ENDPOINTS start
+
+# modifying the running load uses many of the same options as `init`
+$ ./m3nsch_client modify --help
+...
+Flags:
+  -b, --basetime-offset duration   offset from current time to use for load, e.g. -2m, -30s (default -2m0s)
+  -c, --cardinality int            aggregate workload cardinality (default 10000)
+  -i, --ingress-qps int            aggregate workload ingress qps (default 1000)
+  -p, --metric-prefix string       prefix added to each metric (default "m3nsch_")
+  -n, --namespace string           target namespace (default "testmetrics")
+
+Global Flags:
+  -e, --endpoints stringSlice   host:port for each of the agent process endpoints
+
+# the command below bumps up the workload on each `m3nsch_server`, to sustain
+# a load of 1M writes/s from a set of 10M metrics
+$ ./m3nsch_client --endpoints $ENDPOINTS
+  --ingress-qps 1000000   \
+  --cardinality 10000000  \
+
+# finally, stop the load
+$ ./m3nsch_client --endpoints $ENDPOINTS stop
+
+# probably want to teardown the running server processes on the various hosts
+```