diff --git a/.ci/.gitmodules b/.ci/.gitmodules new file mode 100644 index 0000000..333b14d --- /dev/null +++ b/.ci/.gitmodules @@ -0,0 +1,3 @@ +[submodule "uber-licence"] + path = uber-licence + url = https://github.com/uber/uber-licence.git diff --git a/.ci/LICENSE.md b/.ci/LICENSE.md new file mode 100644 index 0000000..8765c9f --- /dev/null +++ b/.ci/LICENSE.md @@ -0,0 +1,19 @@ +Copyright (c) 2016 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/.ci/README.md b/.ci/README.md new file mode 100644 index 0000000..e50bc0e --- /dev/null +++ b/.ci/README.md @@ -0,0 +1,4 @@ +ci-scripts +========== + +This repository contains scripts used by sibling repositories for builds. It's intended to be used as a submodule, conventionally stored as a folder named `.ci` at the root level. diff --git a/.ci/auto-gen.sh b/.ci/auto-gen.sh new file mode 100755 index 0000000..ea79822 --- /dev/null +++ b/.ci/auto-gen.sh @@ -0,0 +1,111 @@ +#!/bin/bash +set -x +. "$(dirname $0)/variables.sh" + +autogen_clear() { + DIR="$1" + + FILES=${DIR}/* + for FILE in $(ls $FILES); do + if [ -d $FILE ]; then + autogen_subdir_clear $FILE + fi + done +} + +autogen_subdir_clear() { + DIR="$1" + + rm -f ${DIR}/*.go +} + +mocks_clear() { + local MOCK_PATTERN=$1 + for DIR in $SRC; + do + local MOCKS=${DIR}/$MOCK_PATTERN + if ls $MOCKS &> /dev/null; then + for FILE in $(ls $MOCKS); + do + rm $FILE + done + fi + done +} + +autogen_cleanup() { + DIR="$1" + + FILES=${DIR}/* + for FILE in $(ls $FILES); + do + if [ -d $FILE ]; then + autogen_cleanup $FILE + else + add_license $FILE $DIR + fi + done +} + +mocks_cleanup() { + local MOCK_PATTERN=$1 + for DIR in $SRC; + do + local MOCKS=${DIR}/${MOCK_PATTERN} + if ls $MOCKS &> /dev/null; then + for FILE in $(ls $MOCKS); + do + add_license $FILE $DIR + + # NB(xichen): there is an open issue (https://github.com/golang/mock/issues/30) + # with mockgen that causes the generated mock files to have vendored packages + # in the import list. For now we are working around it by removing the vendored + # path. Also sed -i'' does not work with BSD sed shipped with OS X, whereas + # sed -i '' doesn't work with GNU sed, so we work around it by redirecting to a + # temp file first and moving it back later. + sed "s|$VENDOR_PATH/||" $FILE > $FILE.tmp && mv $FILE.tmp $FILE + + # Strip GOPATH from the source file path + sed "s|Source: $GOPATH/src/\(.*\.go\)|Source: \1|" $FILE > $FILE.tmp && mv $FILE.tmp $FILE + done + fi + done +} + +add_license() { + FILE="$1" + DIR="$2" + + # Add uber license + PREV_PWD=$(pwd) + cd $DIR + $LICENSE_BIN --silent --file $(basename $FILE) + cd $PREV_PWD +} + +if [ $# -ne 2 ] || [ -z "$1" ] || [ -z "$2" ]; then + echo "usage: auto-gen.sh output_directory file_generation_rules_directory" + exit 1 +fi + +set -e + +. "$(dirname $0)/variables.sh" + +if [ "$2" = "generated/mocks" ]; then + # NOTE: m3db uses the *_mock.go convention, m3cluster uses mock_*.go + mocks_clear "*_mock.go" + mocks_clear "mock_*.go" +else + autogen_clear $1 +fi + +go generate $PACKAGE/$2 + +if [ "$2" = "generated/mocks" ]; then + # NOTE: m3db uses the *_mock.go convention, m3cluster uses mock_*.go + mocks_cleanup "*_mock.go" + mocks_cleanup "mock_*.go" +else + autogen_cleanup $1 +fi diff --git a/.ci/common.mk b/.ci/common.mk new file mode 100644 index 0000000..d5fd8c6 --- /dev/null +++ b/.ci/common.mk @@ -0,0 +1,11 @@ +install-vendor: install-glide + @echo Installing glide deps + glide --debug install + +install-glide: + @which glide > /dev/null || (go get -u github.com/Masterminds/glide && cd $(GOPATH)/src/github.com/Masterminds/glide && git checkout v0.12.3 && go install) + @glide -version > /dev/null || echo "Glide install failed" + +install-ci: + make install-vendor + diff --git a/.ci/convert-test-data.sh b/.ci/convert-test-data.sh new file mode 100755 index 0000000..e60a5ae --- /dev/null +++ b/.ci/convert-test-data.sh @@ -0,0 +1,4 @@ +#!/bin/bash +. "$(dirname $0)/variables.sh" + +sed -i'' -e "s|$PACKAGE/||g" "$1" diff --git a/.ci/gotestcover/LICENSE b/.ci/gotestcover/LICENSE new file mode 100755 index 0000000..210d800 --- /dev/null +++ b/.ci/gotestcover/LICENSE @@ -0,0 +1,7 @@ +Copyright (C) 2015 Pierre Durand + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/.ci/gotestcover/Makefile b/.ci/gotestcover/Makefile new file mode 100755 index 0000000..97e3b85 --- /dev/null +++ b/.ci/gotestcover/Makefile @@ -0,0 +1,19 @@ +#/bin/bash + +setup: + go get -u -v github.com/golang/lint/golint + go get -v -t ./... + +check: + gofmt -d . + go tool vet . + golint + +coverage: + gotestcover -coverprofile=coverage.txt github.com/pierrre/gotestcover + go tool cover -html=coverage.txt -o=coverage.html + +clean: + -rm coverage.txt + -rm coverage.html + gofmt -w . \ No newline at end of file diff --git a/.ci/gotestcover/README.md b/.ci/gotestcover/README.md new file mode 100755 index 0000000..5ee605a --- /dev/null +++ b/.ci/gotestcover/README.md @@ -0,0 +1,19 @@ +# Go test cover with multiple packages support + +## Features +- Coverage profile with multiple packages (`go test` doesn't support that) + +## Install +`go get github.com/pierrre/gotestcover` + +## Usage +```sh +gotestcover -coverprofile=cover.out mypackage +go tool cover -html=cover.out -o=cover.html +``` + +Run on multiple package with: +- `package1 package2` +- `package/...` + +Some `go test / build` flags are available. diff --git a/.ci/gotestcover/gotestcover.go b/.ci/gotestcover/gotestcover.go new file mode 100755 index 0000000..8a914b9 --- /dev/null +++ b/.ci/gotestcover/gotestcover.go @@ -0,0 +1,282 @@ +// Package gotestcover provides multiple packages support for Go test cover. +package main + +import ( + "bufio" + "bytes" + "flag" + "fmt" + "io/ioutil" + "os" + "os/exec" + "runtime" + "strings" + "sync" +) + +var ( + // go build + flagA bool + flagX bool + flagRace bool + flagTags string + + // go test + flagV bool + flagCount int + flagCPU string + flagParallel string + flagRun string + flagShort bool + flagTimeout string + flagCoverMode string + flagCoverProfile string + + // custom + flagParallelPackages = runtime.GOMAXPROCS(0) + + // GAE/Go + flagGoogleAppEngine bool +) + +func main() { + err := run() + if err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func run() error { + err := parseFlags() + if err != nil { + return err + } + pkgArgs, flagArgs := parseArgs() + pkgs, err := resolvePackages(pkgArgs) + if err != nil { + return err + } + cov, failed := runAllPackageTests(pkgs, flagArgs, func(out string) { + fmt.Print(out) + }) + err = writeCoverProfile(cov) + if err != nil { + return err + } + if failed { + return fmt.Errorf("test failed") + } + return nil +} + +func parseFlags() error { + flag.BoolVar(&flagA, "a", flagA, "see 'go build' help") + flag.BoolVar(&flagX, "x", flagX, "see 'go build' help") + flag.BoolVar(&flagRace, "race", flagRace, "see 'go build' help") + flag.StringVar(&flagTags, "tags", flagTags, "see 'go build' help") + + flag.BoolVar(&flagV, "v", flagV, "see 'go test' help") + flag.IntVar(&flagCount, "count", flagCount, "see 'go test' help") + flag.StringVar(&flagCPU, "cpu", flagCPU, "see 'go test' help") + flag.StringVar(&flagParallel, "parallel", flagParallel, "see 'go test' help") + flag.StringVar(&flagRun, "run", flagRun, "see 'go test' help") + flag.BoolVar(&flagShort, "short", flagShort, "see 'go test' help") + flag.StringVar(&flagTimeout, "timeout", flagTimeout, "see 'go test' help") + flag.StringVar(&flagCoverMode, "covermode", flagCoverMode, "see 'go test' help") + flag.StringVar(&flagCoverProfile, "coverprofile", flagCoverProfile, "see 'go test' help") + + flag.IntVar(&flagParallelPackages, "parallelpackages", flagParallelPackages, "Number of package test run in parallel") + + flag.BoolVar(&flagGoogleAppEngine, "gae", flagGoogleAppEngine, "Bool of Command exec in GAE/Go") + + flag.Parse() + if flagCoverProfile == "" { + return fmt.Errorf("flag coverprofile must be set") + } + if flagParallelPackages < 1 { + return fmt.Errorf("flag parallelpackages must be greater than or equal to 1") + } + return nil +} + +func parseArgs() (pkgArgs, flagArgs []string) { + args := flag.Args() + for i, a := range args { + if strings.HasPrefix(a, "-") { + return args[:i], args[i:] + } + } + return args, nil +} + +func resolvePackages(pkgArgs []string) ([]string, error) { + cmdArgs := []string{"list"} + cmdArgs = append(cmdArgs, pkgArgs...) + cmdOut, err := runGoCommand(cmdArgs...) + if err != nil { + return nil, err + } + var pkgs []string + sc := bufio.NewScanner(bytes.NewReader(cmdOut)) + for sc.Scan() { + pkgs = append(pkgs, sc.Text()) + } + return pkgs, nil +} + +func runAllPackageTests(pkgs []string, flgs []string, pf func(string)) ([]byte, bool) { + pkgch := make(chan string) + type res struct { + out string + cov []byte + err error + } + resch := make(chan res) + wg := new(sync.WaitGroup) + wg.Add(flagParallelPackages) + go func() { + for _, pkg := range pkgs { + pkgch <- pkg + } + close(pkgch) + wg.Wait() + close(resch) + }() + for i := 0; i < flagParallelPackages; i++ { + go func() { + for p := range pkgch { + out, cov, err := runPackageTests(p, flgs) + resch <- res{ + out: out, + cov: cov, + err: err, + } + } + wg.Done() + }() + } + failed := false + var cov []byte + for r := range resch { + if r.err == nil { + pf(r.out) + cov = append(cov, r.cov...) + } else { + pf(r.err.Error()) + failed = true + } + } + return cov, failed +} + +func runPackageTests(pkg string, flgs []string) (out string, cov []byte, err error) { + coverFile, err := ioutil.TempFile("", "gotestcover-") + if err != nil { + return "", nil, err + } + defer os.Remove(coverFile.Name()) + defer coverFile.Close() + var args []string + args = append(args, "test") + + if flagA { + args = append(args, "-a") + } + if flagX { + args = append(args, "-x") + } + if flagRace { + args = append(args, "-race") + } + if flagTags != "" { + args = append(args, "-tags", flagTags) + } + + if flagV { + args = append(args, "-v") + } + if flagCount != 0 { + args = append(args, "-count", fmt.Sprint(flagCount)) + } + if flagCPU != "" { + args = append(args, "-cpu", flagCPU) + } + if flagParallel != "" { + args = append(args, "-parallel", flagParallel) + } + if flagRun != "" { + args = append(args, "-run", flagRun) + } + if flagShort { + args = append(args, "-short") + } + if flagTimeout != "" { + args = append(args, "-timeout", flagTimeout) + } + args = append(args, "-cover") + if flagCoverMode != "" { + args = append(args, "-covermode", flagCoverMode) + } + args = append(args, "-coverprofile", coverFile.Name()) + + args = append(args, pkg) + + args = append(args, flgs...) + + cmdOut, err := runGoCommand(args...) + if err != nil { + return "", nil, err + } + cov, err = ioutil.ReadAll(coverFile) + if err != nil { + return "", nil, err + } + cov = removeFirstLine(cov) + return string(cmdOut), cov, nil +} + +func writeCoverProfile(cov []byte) error { + if len(cov) == 0 { + return nil + } + buf := new(bytes.Buffer) + mode := flagCoverMode + if mode == "" { + if flagRace { + mode = "atomic" + } else { + mode = "set" + } + } + fmt.Fprintf(buf, "mode: %s\n", mode) + buf.Write(cov) + return ioutil.WriteFile(flagCoverProfile, buf.Bytes(), os.FileMode(0644)) +} + +func runGoCommand(args ...string) ([]byte, error) { + goCmd := "go" + if flagGoogleAppEngine { + goCmd = "goapp" + } + cmd := exec.Command(goCmd, args...) + out, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("command %s: %s\n%s", cmd.Args, err, out) + } + return out, nil +} + +func removeFirstLine(b []byte) []byte { + out := new(bytes.Buffer) + sc := bufio.NewScanner(bytes.NewReader(b)) + firstLine := true + for sc.Scan() { + if firstLine { + firstLine = false + continue + } + fmt.Fprintf(out, "%s\n", sc.Bytes()) + } + return out.Bytes() +} diff --git a/.ci/gotestcover/gotestcover_test.go b/.ci/gotestcover/gotestcover_test.go new file mode 100755 index 0000000..3a662d6 --- /dev/null +++ b/.ci/gotestcover/gotestcover_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "os" + "testing" +) + +func TestParseFlags(t *testing.T) { + os.Args = []string{"gotestcover", + "-v", + "-a", + "-x", + "-tags=foobar", + "-race", + "-cpu=4", + "-parallel=2", + "-run=abc", + "-short", + "-timeout=15s", + "-covermode=atomic", + "-parallelpackages=2", + "-coverprofile=cover.out", + "-gae", + } + + err := parseFlags() + + if err != nil { + t.Fatal(err) + } + + if !flagV { + t.Errorf("flagV should be set to true") + } + + if !flagA { + t.Errorf("flagA should be set to true") + } + + if !flagX { + t.Errorf("flagX should be set to true") + } + + if flagTags != "foobar" { + t.Errorf("flagCPU is not equal to foobar, got %s", flagTags) + } + + if !flagRace { + t.Errorf("flagRace should be set to true") + } + + if flagCPU != "4" { + t.Errorf("flagCPU is not equal to 4, got %s", flagCPU) + } + + if flagParallel != "2" { + t.Errorf("flagParallel is not equal to 2, got %s", flagParallel) + } + + if flagRun != "abc" { + t.Errorf("flagRun is not equal to 'abc', got %s", flagRun) + } + + if !flagShort { + t.Errorf("flagShort should be set to true") + } + + if flagTimeout != "15s" { + t.Errorf("flagTimeout is not equal to '15s', got %s", flagTimeout) + } + + if flagCoverMode != "atomic" { + t.Errorf("flagCoverMode is not equal to 'atomic', got %s", flagCoverMode) + } + + if flagParallelPackages != 2 { + t.Errorf("flagParallelPackages is not equal to '2', got %d", flagParallelPackages) + } + + if flagCoverProfile != "cover.out" { + t.Errorf("flagCoverProfile is not equal to 'cover.out', got %s", flagCoverProfile) + } + + if !flagGoogleAppEngine { + t.Errorf("flagGoogleAppEngine should be set to true") + } +} diff --git a/.ci/lint.sh b/.ci/lint.sh new file mode 100755 index 0000000..33f31bc --- /dev/null +++ b/.ci/lint.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +! golint ./... | egrep -v $(cat .excludelint | sed -e 's/^/ -e /' | sed -e 's/(//' | sed -e 's/)//' | tr -d '\n') diff --git a/.ci/proto-gen.sh b/.ci/proto-gen.sh new file mode 100755 index 0000000..1677e9f --- /dev/null +++ b/.ci/proto-gen.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +PROTO_SRC=$1 +for i in "$PROTO_SRC"/*; do + if ! [ -d $i ]; then + continue + fi + + if ls $i/*.proto > /dev/null 2>&1; then + echo "generating from $i" + protoc -I$i --go_out=$i $i/*.proto + fi +done diff --git a/.ci/test-cover.sh b/.ci/test-cover.sh new file mode 100755 index 0000000..aae4c17 --- /dev/null +++ b/.ci/test-cover.sh @@ -0,0 +1,35 @@ +#!/bin/bash +. "$(dirname $0)/variables.sh" + +set -e + +TARGET=${1:-profile.cov} +LOG=${2:-test.log} + +rm $TARGET &>/dev/null || true +echo "mode: count" > $TARGET +echo "" > $LOG + +DIRS="" +for DIR in $SRC; +do + if ls $DIR/*_test.go &> /dev/null; then + DIRS="$DIRS $DIR" + fi +done + +if [ "$NPROC" = "" ]; then + NPROC=$(getconf _NPROCESSORS_ONLN) +fi + +echo "test-cover begin: concurrency $NPROC" +go run .ci/gotestcover/gotestcover.go -race -covermode=atomic -coverprofile=profile.tmp -v -parallelpackages $NPROC $DIRS | tee $LOG + +TEST_EXIT=${PIPESTATUS[0]} + +cat profile.tmp | grep -v "_mock.go" > $TARGET + +find . -not -path '*/vendor/*' | grep \\.tmp$ | xargs -I{} rm {} +echo "test-cover result: $TEST_EXIT" + +exit $TEST_EXIT diff --git a/.ci/test-integration.sh b/.ci/test-integration.sh new file mode 100755 index 0000000..0c3caa6 --- /dev/null +++ b/.ci/test-integration.sh @@ -0,0 +1,26 @@ +#!/bin/bash +. "$(dirname $0)/variables.sh" + +set -e + +TAGS="integration" +DIR="integration" + +# compile the integration test binary +go test -test.c -test.tags=${TAGS} ./${DIR} + +# list the tests +TESTS=$(./integration.test -test.v -test.short | grep RUN | tr -s " " | cut -d ' ' -f 3) + +# execute tests one by one for isolation +for TEST in $TESTS; do + ./integration.test -test.v -test.run $TEST ./integration + TEST_EXIT=$? + if [ "$TEST_EXIT" != "0" ]; then + echo "$TEST failed" + exit $TEST_EXIT + fi + sleep 0.1 +done + +echo "PASS all integrations tests" diff --git a/.ci/test-one-integration.sh b/.ci/test-one-integration.sh new file mode 100755 index 0000000..a590b35 --- /dev/null +++ b/.ci/test-one-integration.sh @@ -0,0 +1,9 @@ +#!/bin/bash +. "$(dirname $0)/variables.sh" + +set -e + +TAGS="integration" +DIR="integration" + +go test -test.tags=${TAGS} -test.v -test.run $1 ./${DIR} diff --git a/.ci/variables.sh b/.ci/variables.sh new file mode 100755 index 0000000..64daf6d --- /dev/null +++ b/.ci/variables.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# set PACKAGE in .travis.yml +export VENDOR_PATH=$PACKAGE/vendor +export LICENSE_BIN=$GOPATH/src/$PACKAGE/.ci/uber-licence/bin/licence +export GO15VENDOREXPERIMENT=1 +export SRC=$(find ./ -maxdepth 10 -not -path '*/.git*' -not -path '*/.ci*' -not -path '*/_*' -not -path '*/vendor/*' -type d) diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..bb1ae67 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule ".ci"] + path = .ci + url = git@github.com:m3db/ci-scripts diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..721b703 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,19 @@ +Copyright (c) 2016 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. \ No newline at end of file