Initial Commit

.excludelint

@@ -0,0 +1,3 @@
+(vendor/)
+(mock_.*.go)
+(*.pb.go)

.gitignore

@@ -0,0 +1,59 @@
+### https://raw.github.com/github/gitignore/35c010258fc790ad769033e9ccfb1021178f2925/Go.gitignore
+
+# Glide vendor-ed deps
+vendor/
+
+# Binaries for programs and plugins
+*.exe
+*.dll
+*.so
+*.dylib
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
+.glide/
+
+
+### https://raw.github.com/github/gitignore/35c010258fc790ad769033e9ccfb1021178f2925/Global/VisualStudioCode.gitignore
+
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+
+### https://raw.github.com/github/gitignore/35c010258fc790ad769033e9ccfb1021178f2925/Global/macOS.gitignore
+
+*.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+

.travis.yml

@@ -0,0 +1,15 @@
+language: go
+go:
+  - 1.7
+install: make install-ci
+env:
+ # Set higher timeouts for Travis
+ - TEST_TIMEOUT_SCALE=20 PACKAGE=github.com/m3db/m3nsch
+sudo: required
+dist: trusty
+script:
+ - make lint
+ - make test-ci-unit
+ - make m3nsch_client
+ - make m3nsch_server
+

Makefile

@@ -0,0 +1,80 @@
+SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST)))
+include $(SELF_DIR)/.ci/common.mk
+
+SHELL=/bin/bash -o pipefail
+
+html_report := coverage.html
+test := .ci/test-cover.sh
+test_ci_integration := .ci/test-integration.sh
+convert-test-data := .ci/convert-test-data.sh
+coverfile := cover.out
+coverage_xml := coverage.xml
+junit_xml := junit.xml
+test_log := test.log
+lint_check := .ci/lint.sh
+
+BUILD := $(abspath ./out)
+LINUX_AMD64_ENV := GOOS=linux GOARCH=amd64 CGO_ENABLED=0
+VENDOR_ENV := GO15VENDOREXPERIMENT=1
+
+SERVICES := \
+	m3nsch_server \
+	m3nsch_client \
+
+setup:
+	mkdir -p $(BUILD)
+
+define SERVICE_RULES
+
+$(SERVICE): setup
+	@echo Building $(SERVICE)
+	$(VENDOR_ENV) go build -o $(BUILD)/$(SERVICE) ./$(SERVICE)/.
+
+$(SERVICE)-linux-amd64:
+	$(LINUX_AMD64_ENV) make $(SERVICE)
+
+endef
+
+services: $(SERVICES)
+services-linux-amd64:
+	$(LINUX_AMD64_ENV) make services
+
+$(foreach SERVICE,$(SERVICES),$(eval $(SERVICE_RULES)))
+
+lint:
+	@which golint > /dev/null || go get -u github.com/golang/lint/golint
+	$(VENDOR_ENV) $(lint_check)
+
+test-internal:
+	@which go-junit-report > /dev/null || go get -u github.com/sectioneight/go-junit-report
+	@$(VENDOR_ENV) $(test) $(coverfile) | tee $(test_log)
+
+test-xml: test-internal
+	go-junit-report < $(test_log) > $(junit_xml)
+	gocov convert $(coverfile) | gocov-xml > $(coverage_xml)
+	@$(convert-test-data) $(coverage_xml)
+	@rm $(coverfile) &> /dev/null
+
+test: test-internal
+	gocov convert $(coverfile) | gocov report
+
+testhtml: test-internal
+	gocov convert $(coverfile) | gocov-html > $(html_report) && open $(html_report)
+	@rm -f $(test_log) &> /dev/null
+
+test-ci-unit: test-internal
+	@which goveralls > /dev/null || go get -u -f github.com/mattn/goveralls
+	goveralls -coverprofile=$(coverfile) -service=travis-ci || echo -e "\x1b[31mCoveralls failed\x1b[m"
+
+test-ci-integration:
+	@$(VENDOR_ENV) $(test_ci_integration)
+
+clean:
+	echo Cleaning build artifacts...
+	go clean
+	rm -rf $(BUILD)
+	@rm -f *.html *.xml *.out *.test
+	echo
+
+.DEFAULT_GOAL := test
+.PHONY: test test-xml test-internal testhtml clean

README.md

@@ -0,0 +1,107 @@
+m3nsch
+======
+m3nsch (pronounced `mensch`) is a load testing tool for M3DB. It has two components:
+  - `m3nsch_server`: long lived process which does the load generation
+  - `m3nsch_client`: cli wrapper which controls the myriad `m3nsch_server(s)`
+
+A typical deploy will have multiple hosts, each running a single `m3nsch_server` instance,
+and a single `m3nsch_client` used to control them.
+
+### Build
+```
+$ make prod-m3nsch_server
+$ make prod-m3nsch_client
+$ cat <<EOF > server-conf.yaml
+server:
+  listenAddress: "0.0.0.0:12321"
+  debugAddress: "0.0.0.0:13441"
+  cpuFactor: 0.9
+
+metrics:
+  sampleRate: 0.1
+  m3:
+    hostPort: "<host-port-for-tally>"
+    service: "m3nsch"
+    includeHost: true
+    env: development
+
+m3nsch:
+  concurrency: 2000
+  numPointsPerDatum: 60
+
+# any additional configs you may have
+EOF
+```
+
+### Deploy
+(1) Transfer the `m3nsch_server` binary, and `server-conf.yaml` to all the hosts to be used to generate hosts, e.g. Ingesters
+
+(2) On each host from (1), kick of the server process by running:
+```
+./m3nsch_server -f server-conf.yaml
+```
+
+(3) Transfer `m3nsch_client` binary to a host with network connectivity to all the hosts.
+
+### Sample Usage
+```
+# set an env var containing the host endpoints seperated by commas
+$ export ENDPOINTS="host1:12321,host2:12321"
+
+# get the status of the various endpoints, make sure all are healthy
+$ ./m3nsch_client --endpoints $ENDPOINTS status
+
+# investigate the various options available during initialization
+$ ./m3nsch_client init --help
+...
+Flags:
+  -b, --basetime-offset duration   offset from current time to use for load, e.g. -2m, -30s (default -2m0s)
+  -c, --cardinality int            aggregate workload cardinality (default 10000)
+  -f, --force                      force initialization, stop any running workload
+  -i, --ingress-qps int            aggregate workload ingress qps (default 1000)
+  -p, --metric-prefix string       prefix added to each metric (default "m3nsch_")
+  -n, --namespace string           target namespace (default "testmetrics")
+  -v, --target-env string          target env for load test (default "test")
+  -z, --target-zone string         target zone for load test (default "sjc1")
+  -t, --token string               [required] unique identifier required for all subsequent interactions on this workload
+
+Global Flags:
+  -e, --endpoints stringSlice   host:port for each of the agent process endpoints
+
+# initialize the servers, set any workload parameters, target env/zone
+# the command below targets the production sjc1 m3db cluster with each `m3nsch_server` attempting to
+# sustain a load of 100K writes/s from a set of 1M metrics
+$ ./m3nsch_client --endpoints $ENDPOINTS init \
+  --token prateek-sample \
+  --target-env prod      \
+  --target-zone sjc1     \
+  --ingress-qps 100000   \
+  --cardinality 1000000  \
+
+# start the load generation
+$ ./m3nsch_client --endpoints $ENDPOINTS start
+
+# modifying the running load uses many of the same options as `init`
+$ ./m3nsch_client modify --help
+...
+Flags:
+  -b, --basetime-offset duration   offset from current time to use for load, e.g. -2m, -30s (default -2m0s)
+  -c, --cardinality int            aggregate workload cardinality (default 10000)
+  -i, --ingress-qps int            aggregate workload ingress qps (default 1000)
+  -p, --metric-prefix string       prefix added to each metric (default "m3nsch_")
+  -n, --namespace string           target namespace (default "testmetrics")
+
+Global Flags:
+  -e, --endpoints stringSlice   host:port for each of the agent process endpoints
+
+# the command below bumps up the workload on each `m3nsch_server`, to sustain
+# a load of 1M writes/s from a set of 10M metrics
+$ ./m3nsch_client --endpoints $ENDPOINTS
+  --ingress-qps 1000000   \
+  --cardinality 10000000  \
+
+# finally, stop the load
+$ ./m3nsch_client --endpoints $ENDPOINTS stop
+
+# probably want to teardown the running server processes on the various hosts
+```