diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a42d39e --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +/.idea +/build +.DS_Store +/saltbeat +/saltbeat.test +*.pyc +*.iml +vendor diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..c026c29 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,43 @@ +sudo: required +dist: trusty +services: + - docker + +language: go + +go: + - 1.6 + +os: + - linux + - osx + +env: + matrix: + - TARGETS="check" + - TARGETS="-C saltbeat testsuite" + + global: + # Cross-compile for amd64 only to speed up testing. + - GOX_FLAGS="-arch amd64" + +addons: + apt: + packages: + - python-virtualenv + +before_install: + # Redo the travis setup but with the elastic/libbeat path. This is needed so the package path is correct + - mkdir -p $HOME/gopath/src/github.com/elastic/beats/ + - rsync -az ${TRAVIS_BUILD_DIR}/ $HOME/gopath/src/github.com/elastic/beats/ + - export TRAVIS_BUILD_DIR=$HOME/gopath/src/github.com/elastic/beats/ + - cd $HOME/gopath/src/github.com/elastic/beats/ + +install: + - true + +script: + - make $TARGETS + +after_success: + # Copy full.cov to coverage.txt because codecov.io requires this file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e69de29 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..6868067 --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2016 Martin Hoefling + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fa4fb97 --- /dev/null +++ b/Makefile @@ -0,0 +1,42 @@ +BEATNAME=saltbeat +BEAT_DIR=github.com/martinhoefling +SYSTEM_TESTS=false +TEST_ENVIRONMENT=false +ES_BEATS=./vendor/github.com/elastic/beats +GOPACKAGES=$(shell glide novendor) +PREFIX?=. + +# Path to the libbeat Makefile +-include $(ES_BEATS)/libbeat/scripts/Makefile + +.PHONY: init +init: + glide update --no-recursive + make update + git init + +.PHONY: commit +commit: + git add README.md CONTRIBUTING.md + git commit -m "Initial commit" + git add LICENSE + git commit -m "Add the LICENSE" + git add .gitignore + git commit -m "Add git settings" + git add . + git reset -- .travis.yml + git commit -m "Add saltbeat" + git add .travis.yml + git commit -m "Add Travis CI" + +.PHONY: update-deps +update-deps: + glide update --no-recursive --strip-vcs + +# This is called by the beats packer before building starts +.PHONY: before-build +before-build: + +# Create binary packages for the beat +pack: create-packer + cd dev-tools/packer; make deps images saltbeat diff --git a/README.md b/README.md new file mode 100644 index 0000000..07d45a9 --- /dev/null +++ b/README.md @@ -0,0 +1,131 @@ +# Saltbeat + +Welcome to Saltbeat. + +Ensure that this folder is at the following location: +`${GOPATH}/github.com/martinhoefling` + +## Getting Started with Saltbeat + +### Requirements + +* [Golang](https://golang.org/dl/) 1.6 +* [Glide](https://github.com/Masterminds/glide) >= 0.10.0 + +### Init Project +To get running with Saltbeat, run the following command: + +``` +make init +``` + +To commit the first version before you modify it, run: + +``` +make commit +``` + +It will create a clean git history for each major step. Note that you can always rewrite the history if you wish before pushing your changes. + +To push Saltbeat in the git repository, run the following commands: + +``` +git remote set-url origin https://github.com/martinhoefling/saltbeat +git push origin master +``` + +For further development, check out the [beat developer guide](https://www.elastic.co/guide/en/beats/libbeat/current/new-beat.html). + +### Build + +To build the binary for Saltbeat run the command below. This will generate a binary +in the same directory with the name saltbeat. + +``` +make +``` + + +### Run + +To run Saltbeat with debugging output enabled, run: + +``` +./saltbeat -c saltbeat.yml -e -d "*" +``` + + +### Test + +To test Saltbeat, run the following command: + +``` +make testsuite +``` + +alternatively: +``` +make unit-tests +make system-tests +make integration-tests +make coverage-report +``` + +The test coverage is reported in the folder `./build/coverage/` + + +### Package + +To be able to package Saltbeat the requirements are as follows: + + * [Docker Environment](https://docs.docker.com/engine/installation/) >= 1.10 + * $GOPATH/bin must be part of $PATH: `export PATH=${PATH}:${GOPATH}/bin` + +To cross-compile and package Saltbeat for all supported platforms, run the following commands: + +``` +cd dev-tools/packer +make deps +make images +make +``` + +### Update + +Each beat has a template for the mapping in elasticsearch and a documentation for the fields +which is automatically generated based on `etc/fields.yml`. +To generate etc/saltbeat.template.json and etc/saltbeat.asciidoc + +``` +make update +``` + + +### Cleanup + +To clean Saltbeat source code, run the following commands: + +``` +make fmt +make simplify +``` + +To clean up the build directory and generated artifacts, run: + +``` +make clean +``` + + +### Clone + +To clone Saltbeat from the git repository, run the following commands: + +``` +mkdir -p ${GOPATH}/github.com/martinhoefling +cd ${GOPATH}/github.com/martinhoefling +git clone https://github.com/martinhoefling/saltbeat +``` + + +For further development, check out the [beat developer guide](https://www.elastic.co/guide/en/beats/libbeat/current/new-beat.html). diff --git a/beater/saltbeat.go b/beater/saltbeat.go new file mode 100644 index 0000000..659f376 --- /dev/null +++ b/beater/saltbeat.go @@ -0,0 +1,153 @@ +package beater + +import ( + "fmt" + "time" + + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/publisher" + + "bytes" + "net" + "reflect" + + "github.com/martinhoefling/saltbeat/config" + "github.com/ugorji/go/codec" +) + +type Saltbeat struct { + beatConfig *config.Config + done chan struct{} + messages chan map[string]interface{} + socketConnection *net.UnixConn + client publisher.Client +} + +// Creates beater +func New() *Saltbeat { + logp.Debug("beater", "Creating new beater") + return &Saltbeat{ + done: make(chan struct{}), + messages: make(chan map[string]interface{}), + } +} + +/// *** Beater interface methods ***/// + +func (bt *Saltbeat) Config(b *beat.Beat) error { + logp.Debug("beater", "Configuring beater") + + // Load beater beatConfig + err := b.RawConfig.Unpack(&bt.beatConfig) + if err != nil { + return fmt.Errorf("Error reading config file: %v", err) + } + + return nil +} + +func (bt *Saltbeat) Setup(b *beat.Beat) error { + logp.Debug("beater", "Setting up beater") + // Setting default period if not set + if bt.beatConfig.Saltbeat.MasterEventPub == "" { + bt.beatConfig.Saltbeat.MasterEventPub = "/var/run/salt/master/master_event_pub.ipc" + } + bt.client = b.Publisher.Connect() + + var err error + logp.Info("Opening socket %s", bt.beatConfig.Saltbeat.MasterEventPub) + bt.socketConnection, err = net.DialUnix("unix", nil, &net.UnixAddr{bt.beatConfig.Saltbeat.MasterEventPub, "unix"}) + if err != nil { + return err + } + err = bt.socketConnection.CloseWrite() + if err != nil { + return err + } + + go func() { + var err error + var handle codec.MsgpackHandle + handle.MapType = reflect.TypeOf(map[string]interface{}(nil)) + for { + logp.Debug("message", "Waiting for message") + message_decoder := codec.NewDecoder(bt.socketConnection, &handle) + var message map[string]interface{} + err = message_decoder.Decode(&message) + if err != nil { + logp.WTF(err.Error()) + } + logp.Debug("message", "Message read") + bt.messages <- message + } + }() + + return nil +} + +func parseMessage(handle codec.MsgpackHandle, message map[string]interface{}) (string, map[string]interface{}) { + body := message["body"].([]byte) + newline := byte(10) + splitted := bytes.SplitN(body, []byte{newline, newline}, 2) + + tag := string(splitted[0]) + logp.Debug("message", "Message tag is %s", tag) + + payload_bytes := splitted[1] + payload_decoder := codec.NewDecoderBytes(payload_bytes, &handle) + + var payload map[string]interface{} + err := payload_decoder.Decode(&payload) + if err != nil { + logp.WTF(err.Error()) + } + + logp.Debug("message", "Decoded payload is %s", payload) + return tag, payload +} + +func (bt *Saltbeat) Run(b *beat.Beat) error { + logp.Info("saltbeat is running! Hit CTRL-C to stop it.") + + var err error + var handle codec.MsgpackHandle + handle.MapType = reflect.TypeOf(map[string]interface{}(nil)) + handle.RawToString = true + + for { + select { + case <-bt.done: + return nil + case message := <-bt.messages: + tag, payload := parseMessage(handle, message) + logp.Debug("publish", "Publishing event") + + event := common.MapStr{ + "@timestamp": common.Time(time.Now()), + "type": b.Name, + "tag": tag, + "payload": payload, + } + + ok := bt.client.PublishEvent(event) + if !ok { + logp.Debug("publish", "Cannot publish event") + logp.WTF(err.Error()) + } + logp.Debug("publish", "Published") + } + } +} + +func (bt *Saltbeat) Cleanup(b *beat.Beat) error { + logp.Info("Closing socket %s", bt.beatConfig.Saltbeat.MasterEventPub) + bt.socketConnection.Close() + return nil +} + +func (bt *Saltbeat) Stop() { + close(bt.done) + close(bt.messages) +} diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..f7cdffd --- /dev/null +++ b/config/config.go @@ -0,0 +1,12 @@ +// Config is put into a different package to prevent cyclic imports in case +// it is needed in several locations + +package config + +type Config struct { + Saltbeat SaltbeatConfig +} + +type SaltbeatConfig struct { + MasterEventPub string `config:"master_event_pub"` +} diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..d177de3 --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,3 @@ +// +build !integration + +package config diff --git a/dev-tools/packer/Makefile b/dev-tools/packer/Makefile new file mode 100644 index 0000000..0e0114e --- /dev/null +++ b/dev-tools/packer/Makefile @@ -0,0 +1,34 @@ +BUILDID=$(shell git rev-parse HEAD) + +.PHONY: all +all: saltbeat/deb saltbeat/rpm saltbeat/darwin saltbeat/win saltbeat/bin \ + build/upload/build_id.txt + +ES_BEATS=../../vendor/github.com/elastic/beats +include $(ES_BEATS)/dev-tools/packer/scripts/Makefile + + +.PHONY: saltbeat +saltbeat: build/upload + # cross compile on ubuntu + docker run --rm \ + -v $(abspath build):/build \ + -v $(abspath $(ES_BEATS)/dev-tools/packer/xgo-scripts):/scripts \ + -v $(abspath ../..):/source \ + -e PACK=$@ \ + -e BEFORE_BUILD=before_build.sh \ + -e SOURCE=/source \ + -e BUILDID=${BUILDID} \ + tudorg/beats-builder \ + github.com/martinhoefling/saltbeat + # linux builds on debian 6 + docker run --rm \ + -v $(abspath build):/build \ + -v $(abspath $(ES_BEATS)/dev-tools/packer/xgo-scripts):/scripts \ + -v $(abspath ../..):/source \ + -e PACK=$@ \ + -e BEFORE_BUILD=before_build.sh \ + -e SOURCE=/source \ + -e BUILDID=${BUILDID} \ + tudorg/beats-builder-deb6 \ + github.com/martinhoefling/saltbeat diff --git a/dev-tools/packer/beats/saltbeat.yml b/dev-tools/packer/beats/saltbeat.yml new file mode 100644 index 0000000..b21468a --- /dev/null +++ b/dev-tools/packer/beats/saltbeat.yml @@ -0,0 +1,4 @@ +beat_name: saltbeat +beat_url: 'https://github.com/martinhoefling/saltbeat' +beat_repo: 'github.com/martinhoefling/saltbeat' +beat_description: Sends salt master events to Logstash or directly to Elasticsearch. diff --git a/dev-tools/packer/version.yml b/dev-tools/packer/version.yml new file mode 100644 index 0000000..71e3fd1 --- /dev/null +++ b/dev-tools/packer/version.yml @@ -0,0 +1 @@ +version: "1.0.0" diff --git a/docs/fields.asciidoc b/docs/fields.asciidoc new file mode 100644 index 0000000..dadac73 --- /dev/null +++ b/docs/fields.asciidoc @@ -0,0 +1,55 @@ + +//// +This file is generated! See etc/fields.yml and scripts/generate_field_docs.py +//// + +[[exported-fields]] +== Exported Fields + +This document describes the fields that are exported by Saltbeat. They are +grouped in the following categories: + +* <> + +[[exported-fields-beat]] +=== Beat Fields + +Contains common beat fields available in all event types. + + + +==== beat.name + +The name of the Beat sending the log messages. If the shipper name is set in the configuration file, then that value is used. If it is not set, the hostname is used. + + +==== beat.hostname + +The hostname as returned by the operating system on which the Beat is running. + + +==== @timestamp + +type: date + +example: 2015-01-24 14:06:05.071000 + +format: YYYY-MM-DDTHH:MM:SS.milliZ + +required: True + +The timestamp when the event log record was generated. + + +==== tags + +Arbitrary tags that can be set per Beat and per transaction type. + + +==== fields + +type: dict + +Contains user configurable fields. + + diff --git a/docs/index.asciidoc b/docs/index.asciidoc new file mode 100644 index 0000000..c6e7306 --- /dev/null +++ b/docs/index.asciidoc @@ -0,0 +1,8 @@ += Filebeat Reference +:libbeat: http://www.elastic.co/guide/en/beats/libbeat/master +:elasticsearch: https://www.elastic.co/guide/en/elasticsearch/reference/master/ +:version: 1.0.0 +:beatname_lc: saltbeat +:beatname_uc: Saltbeat + +include::./fields.asciidoc[] diff --git a/etc/beat.yml b/etc/beat.yml new file mode 100644 index 0000000..e6f4a6f --- /dev/null +++ b/etc/beat.yml @@ -0,0 +1,7 @@ +################### Saltbeat Configuration Example ######################### + +############################# Saltbeat ###################################### + +saltbeat: + # Path to salt master event bus + master_event_pub: /var/run/salt/master/master_event_pub.ipc diff --git a/etc/fields.yml b/etc/fields.yml new file mode 100644 index 0000000..2ac6f4b --- /dev/null +++ b/etc/fields.yml @@ -0,0 +1,51 @@ +defaults: + type: keyword + required: false + index: true + +env: + type: group + description: > + Contains common fields available in all event types. + fields: + - type: group + name: beat + fields: + - name: name + description: > + The name of the Beat sending the log messages. If the shipper name is set + in the configuration file, then that value is used. If it is not set, + the hostname is used. + - name: hostname + description: > + The hostname as returned by the operating system on which the Beat is + running. + - name: "@timestamp" + type: date + required: true + format: YYYY-MM-DDTHH:MM:SS.milliZ + example: 2015-01-24T14:06:05.071Z + description: > + The timestamp when the log line was read. The precision is in + milliseconds. The timezone is UTC. + - name: type + required: true + description: > + The type is always saltbeat for messages generated from this beat + +saltbeat: + fields: + - name: tag + type: string + required: true + description: > + Event Tag, e.g. salt/job/20160708193549773070/ret/testminion + - name: payload + type: dict + required: true + description: > + The event payload as json object + +sections: + - ["env", "Common"] + - ["saltbeat", "Saltbeat"] diff --git a/glide.yaml b/glide.yaml new file mode 100644 index 0000000..38f37af --- /dev/null +++ b/glide.yaml @@ -0,0 +1,8 @@ +package: github.com/martinhoefling/saltbeat +import: +- package: github.com/elastic/beats + subpackages: + - libbeat/beat + - libbeat/common + - libbeat/logp + - libbeat/publisher diff --git a/main.go b/main.go new file mode 100644 index 0000000..ca8e876 --- /dev/null +++ b/main.go @@ -0,0 +1,16 @@ +package main + +import ( + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/logp" + "github.com/martinhoefling/saltbeat/beater" + "os" +) + +func main() { + logp.Debug("main", "Starting saltbeat") + err := beat.Run("saltbeat", "", beater.New()) + if err != nil { + os.Exit(1) + } +} diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..a4e2de4 --- /dev/null +++ b/main_test.go @@ -0,0 +1,22 @@ +package main + +// This file is mandatory as otherwise the saltbeat.test binary is not generated correctly. + +import ( + "flag" + "testing" +) + +var systemTest *bool + +func init() { + systemTest = flag.Bool("systemTest", false, "Set to true when running system tests") +} + +// Test started when the test binary is started. Only calls main. +func TestSystem(t *testing.T) { + + if *systemTest { + main() + } +} diff --git a/saltbeat.full.yml b/saltbeat.full.yml new file mode 100644 index 0000000..a205b0d --- /dev/null +++ b/saltbeat.full.yml @@ -0,0 +1,499 @@ + +#================================ General ===================================== + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. Fields can be scalar values, arrays, dictionaries, or any nested +# combination of these. +#fields: +# env: staging + +# If this option is set to true, the custom fields are stored as top-level +# fields in the output document instead of being grouped under a fields +# sub-dictionary. Default is false. +#fields_under_root: false + +# Uncomment the following if you want to ignore transactions created +# by the server on which the shipper is installed. This option is useful +# to remove duplicates if shippers are installed on multiple servers. +#ignore_outgoing: true + +# How often (in seconds) shippers are publishing their IPs to the topology map. +# The default is 10 seconds. +#refresh_topology_freq: 10 + +# Expiration time (in seconds) of the IPs published by a shipper to the topology map. +# All the IPs will be deleted afterwards. Note, that the value must be higher than +# refresh_topology_freq. The default is 15 seconds. +#topology_expire: 15 + +# Internal queue size for single events in processing pipeline +#queue_size: 1000 + +# Sets the maximum number of CPUs that can be executing simultaneously. The +# default is the number of logical CPUs available in the system. +#max_procs: + +#================================ Filters ===================================== + +# This section defines a list of filtering rules that are applied one by one +# starting with the exported event: +# +# event -> filter1 -> event1 -> filter2 ->event2 ... +# +# Supported actions: drop_fields, drop_event, include_fields +# +# For example, the following filter configuration uses multiple actions to keep +# the fields that contain CPU load percentages, but remove the fields that +# contain CPU ticks values: +# +#filters: +#- include_fields: +# fields: ["cpu"] +#- drop_fields: +# fields: ["cpu.user", "cpu.system"] +# +# The following example drops the events that have the HTTP response code 200: +# +#filters: +#- drop_event: +# equals: +# http.code: 200 +# + +#================================ Outputs ===================================== + +# Configure what outputs to use when sending the data collected by the beat. +# Multiple outputs may be used. + +#-------------------------- Elasticsearch output ------------------------------ +output.elasticsearch: + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (http and 9200) + # In case you specify and additional path, the scheme is required: http://localhost:9200/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 + hosts: ["localhost:9200"] + + # Set gzip compression level. + #compression_level: 0 + + # Optional protocol and basic auth credentials. + #protocol: "https" + #username: "admin" + #password: "s3cr3t" + + # Dictionary of HTTP parameters to pass within the url with index operations. + #parameters: + #param1: value1 + #param2: value2 + + # Number of workers per Elasticsearch host. + #worker: 1 + + # Optional index name. The default is "saltbeat" and generates + # [saltbeat-]YYYY.MM.DD keys. + #index: "saltbeat" + + # Optional HTTP Path + #path: "/elasticsearch" + + # Proxy server url + #proxy_url: http://proxy:3128 + + # The number of times a particular Elasticsearch index operation is attempted. If + # the indexing operation doesn't succeed after this many retries, the events are + # dropped. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Elasticsearch bulk API index request. + # The default is 50. + #bulk_max_size: 50 + + # Configure http request timeout before failing an request to Elasticsearch. + #timeout: 90 + + # The number of seconds to wait for new events between two bulk API index requests. + # If `bulk_max_size` is reached before this interval expires, addition bulk index + # requests are made. + #flush_interval: 1s + + # Boolean that sets if the topology is kept in Elasticsearch. The default is + # false. This option makes sense only for Packetbeat. + #save_topology: false + + # The time to live in seconds for the topology information that is stored in + # Elasticsearch. The default is 15 seconds. + #topology_expire: 15 + + # A template is used to set the mapping in Elasticsearch + # By default template loading is enabled and the template is loaded. + # These settings can be adjusted to load your own template or overwrite existing ones + + # Template name. By default the template name is saltbeat. + template.name: "saltbeat" + + # Path to template file + template.path: "saltbeat.template.json" + + # Overwrite existing template + template.overwrite: false + + # TLS configuration. By default is off. + # List of root certificates for HTTPS server verifications + #tls.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for TLS client authentication + #tls.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #tls.certificate_key: "/etc/pki/client/cert.key" + + # Controls whether the client verifies server certificates and host name. + # If insecure is set to true, all server host names and certificates will be + # accepted. In this mode TLS based connections are susceptible to + # man-in-the-middle attacks. Use only for testing. + #tls.insecure: true + + # Configure cipher suites to be used for TLS connections + #tls.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #tls.curve_types: [] + + # Configure minimum TLS version allowed for connection to logstash + #tls.min_version: 1.0 + + # Configure maximum TLS version allowed for connection to logstash + #tls.max_version: 1.2 + + +#----------------------------- Logstash output -------------------------------- +#output.logstash: + # The Logstash hosts + #hosts: ["localhost:5044"] + + # Number of workers per Logstash host. + #worker: 1 + + # Set gzip compression level. + #compression_level: 3 + + # Optional load balance the events between the Logstash hosts + #loadbalance: true + + # Number of batches to be send asynchronously to logstash while processing + # new batches. + #pipelining: 0 + + # Optional index name. The default index name is set to name of the beat + # in all lowercase. + #index: saltbeat + + # SOCKS5 proxy server URL + #proxy_url: socks5://user:password@socks5-server:2233 + + # Resolve names locally when using a proxy server. Defaults to false. + #proxy_use_local_resolver: false + + # Optional TLS configuration options. TLS is off by default. + # List of root certificates for HTTPS server verifications + #tls.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for TLS client authentication + #tls.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #tls.certificate_key: "/etc/pki/client/cert.key" + + # Controls whether the client verifies server certificates and host name. + # If insecure is set to true, all server host names and certificates will be + # accepted. In this mode TLS based connections are susceptible to + # man-in-the-middle attacks. Use only for testing. + #tls.insecure: true + + # Configure cipher suites to be used for TLS connections + #tls.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #tls.curve_types: [] + +#------------------------------- Kafka output --------------------------------- +#output.kafka: + # The list of Kafka broker addresses from where to fetch the cluster metadata. + # The cluster metadata contain the actual Kafka brokers events are published + # to. + #hosts: ["localhost:9092"] + + # The Kafka topic used for produced events. If use_type is set to true, the + # topic will not be used. + #topic: beats + + # Set Kafka topic by event type. If use_type is false, the topic option must + # be configured. The default is false. + #use_type: false + + # The number of concurrent load-balanced Kafka output workers. + #worker: 1 + + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat, ignore the max_retries setting and retry until + # all events are published. Set max_retries to a value less than 0 to retry + # until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Kafka request. The default + # is 2048. + #bulk_max_size: 2048 + + # The number of seconds to wait for responses from the Kafka brokers before + # timing out. The default is 30s. + #timeout: 30s + + # The maximum duration a broker will wait for number of required ACKs. The + # default is 10s. + #broker_timeout: 10s + + # The number of messages buffered for each Kafka broker. The default is 256. + #channel_buffer_size: 256 + + # The keep-alive period for an active network connection. If 0s, keep-alives + # are disabled. The default is 0 seconds. + #keep_alive: 0 + + # Sets the output compression codec. Must be one of none, snappy and gzip. The + # default is gzip. + #compression: gzip + + # The maximum permitted size of JSON-encoded messages. Bigger messages will be + # dropped. The default value is 1000000 (bytes). This value should be equal to + # or less than the broker's message.max.bytes. + #max_message_bytes: 1000000 + + # The ACK reliability level required from broker. 0=no response, 1=wait for + # local commit, -1=wait for all replicas to commit. The default is 1. Note: + # If set to 0, no ACKs are returned by Kafka. Messages might be lost silently + # on error. + #required_acks: 1 + + # The number of seconds to wait for new events between two producer API calls. + #flush_interval: 1s + + # The configurable ClientID used for logging, debugging, and auditing + # purposes. The default is "beats". + #client_id: beats + + # Optional TLS configuration options. TLS is off by default. + # List of root certificates for HTTPS server verifications + #tls.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for TLS client authentication + #tls.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #tls.certificate_key: "/etc/pki/client/cert.key" + + # Controls whether the client verifies server certificates and host name. + # If insecure is set to true, all server host names and certificates will be + # accepted. In this mode TLS based connections are susceptible to + # man-in-the-middle attacks. Use only for testing. + #tls.insecure: true + + # Configure cipher suites to be used for TLS connections + #tls.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #tls.curve_types: [] + +#------------------------------- Redis output --------------------------------- +#output.redis: + # The list of Redis servers to connect to. If load balancing is enabled, the + # events are distributed to the servers in the list. If one server becomes + # unreachable, the events are distributed to the reachable servers only. + #hosts: ["localhost:6379"] + + # The Redis port to use if hosts does not contain a port number. The default + # is 6379. + #port: 6379 + + # The name of the Redis list or channel the events are published to. The + # default is saltbeat. + #index: saltbeat + + # The password to authenticate with. The default is no authentication. + #password: + + # The Redis database number where the events are published. The default is 0. + #db: 0 + + # The Redis data type to use for publishing events. If the data type is list, + # the Redis RPUSH command is used. If the data type is channel, the Redis + # PUBLISH command is used. The default value is list. + #datetype: list + + # The Redis host to connect to when using topology map support. Topology map + # support is disabled if this option is not set. + #host_topology: + + # The password to use for authenticating with the Redis topology server. The + # default is no authentication. + #password_topology: + + # The Redis database number where the topology information is stored. The + # default is 1. + #db_topology: 1 + + # The number of workers to use for each host configured to publish events to + # Redis. Use this setting along with the loadbalance option. For example, if + # you have 2 hosts and 3 workers, in total 6 workers are started (3 for each + # host). + #worker: 1 + + # If set to true and multiple hosts or workers are configured, the output + # plugin load balances published events onto all Redis hosts. If set to false, + # the output plugin sends all events to only one host (determined at random) + # and will switch to another host if the currently selected one becomes + # unreachable. The default value is true. + #loadbalance: true + + # The Redis connection timeout in seconds. The default is 5 seconds. + #timeout: 5s + + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat, ignore the max_retries setting and retry until + # all events are published. Set max_retries to a value less than 0 to retry + # until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Redis request or pipeline. + # The default is 2048. + #bulk_max_size: 2048 + + # The URL of the SOCKS5 proxy to use when connecting to the Redis servers. The + # value must be a URL with a scheme of socks5://. + #proxy_url: + + # This option determines whether Redis hostnames are resolved locally when + # using a proxy. The default value is false, which means that name resolution + # occurs on the proxy server. + #proxy_use_local_resolver: false + + # Optional TLS configuration options. TLS is off by default. + # List of root certificates for HTTPS server verifications + #tls.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for TLS client authentication + #tls.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #tls.certificate_key: "/etc/pki/client/cert.key" + + # Controls whether the client verifies server certificates and host name. + # If insecure is set to true, all server host names and certificates will be + # accepted. In this mode TLS based connections are susceptible to + # man-in-the-middle attacks. Use only for testing. + #tls.insecure: true + + # Configure cipher suites to be used for TLS connections + #tls.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #tls.curve_types: [] + +#------------------------------- File output ---------------------------------- +#output.file: + # Path to the directory where to save the generated files. The option is + # mandatory. + #path: "/tmp/saltbeat" + + # Name of the generated files. The default is `saltbeat` and it generates + # files: `saltbeat`, `saltbeat.1`, `saltbeat.2`, etc. + #filename: saltbeat + + # Maximum size in kilobytes of each file. When this size is reached, and on + # every saltbeat restart, the files are rotated. The default value is 10240 + # kB. + #rotate_every_kb: 10000 + + # Maximum number of files under path. When this number of files is reached, + # the oldest file is deleted and the rest are shifted from last to first. The + # default is 7 files. + #number_of_files: 7 + + +#----------------------------- Console output --------------------------------- +#output.console: + # Pretty print json event + #pretty: false + +#================================= Paths ====================================== + +# The home path for the saltbeat installation. This is the default base path +# for all other path settings and for miscellaneous files that come with the +# distribution (for example, the sample dashboards). +# If not set by a CLI flag or in the configuration file, the default for the +# home path is the location of the binary. +#path.home: + +# The configuration path for the saltbeat installation. This is the default +# base path for configuration files, including the main YAML configuration file +# and the Elasticsearch template file. If not set by a CLI flag or in the +# configuration file, the default for the configuration path is the home path. +#path.config: + +# The data path for the saltbeat installation. This is the default base path +# for all the files in which saltbeat needs to store its data. If not set by a +# CLI flag or in the configuration file, the default for the data path is a data +# subdirectory inside the home path. +#path.data: + +# The logs path for a saltbeat installation. This is the default location for +# the Beat's log files. If not set by a CLI flag or in the configuration file, +# the default for the logs path is a logs subdirectory inside the home path. +#path.logs: + +#================================ Logging ===================================== +# There are three options for the log output: syslog, file, stderr. +# Under Windows systems, the log files are per default sent to the file output, +# under all other system per default to syslog. + +# Sets log level. The default log level is error. +# Available log levels are: critical, error, warning, info, debug +#logging.level: error + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are beat, publish, service +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: true + +# Logging to rotating files files. Set logging.to_files to false to disable logging to +# files. +logging.to_files: true +logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/mybeat + + # The name of the files where the logs are written to. + #name: mybeat + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + diff --git a/saltbeat.template-es2x.json b/saltbeat.template-es2x.json new file mode 100644 index 0000000..a9fa1b4 --- /dev/null +++ b/saltbeat.template-es2x.json @@ -0,0 +1,53 @@ +{ + "mappings": { + "_default_": { + "_all": { + "norms": { + "enabled": false + } + }, + "dynamic_templates": [ + { + "fields": { + "mapping": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "match_mapping_type": "string", + "path_match": "fields.*" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + }, + "name": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + }, + "tags": { + "ignore_above": 1024, + "index": "not_analyzed", + "type": "string" + } + } + } + }, + "order": 0, + "settings": { + "index.refresh_interval": "5s" + }, + "template": "saltbeat-*" +} \ No newline at end of file diff --git a/saltbeat.template.json b/saltbeat.template.json new file mode 100644 index 0000000..832c987 --- /dev/null +++ b/saltbeat.template.json @@ -0,0 +1,47 @@ +{ + "mappings": { + "_default_": { + "_all": { + "norms": false + }, + "dynamic_templates": [ + { + "fields": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string", + "path_match": "fields.*" + } + } + ], + "properties": { + "@timestamp": { + "type": "date" + }, + "beat": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "tags": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + }, + "order": 0, + "settings": { + "index.refresh_interval": "5s" + }, + "template": "saltbeat-*" +} \ No newline at end of file diff --git a/saltbeat.yml b/saltbeat.yml new file mode 100644 index 0000000..9c592a4 --- /dev/null +++ b/saltbeat.yml @@ -0,0 +1,62 @@ +################### Saltbeat Configuration Example ######################### + +############################# Saltbeat ###################################### + +saltbeat: + # Path to salt master event bus + master_event_pub: /var/run/salt/master/master_event_pub.ipc + +#================================ General ===================================== + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. +#fields: +# env: staging + +#================================ Outputs ===================================== + +# Configure what outputs to use when sending the data collected by the beat. +# Multiple outputs may be used. + +#-------------------------- Elasticsearch output ------------------------------ +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["localhost:9200"] + + # Template name. By default the template name is saltbeat. + template.name: "saltbeat" + + # Path to template file + template.path: "saltbeat.template.json" + + # Overwrite existing template + template.overwrite: false + +#----------------------------- Logstash output -------------------------------- +#output.logstash: + # The Logstash hosts + #hosts: ["localhost:5044"] + + # Optional TLS. By default is off. + # List of root certificates for HTTPS server verifications + #tls.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for TLS client authentication + #tls.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #tls.certificate_key: "/etc/pki/client/cert.key" + +#================================ Logging ===================================== + +# Sets log level. The default log level is error. +# Available log levels are: critical, error, warning, info, debug +#logging.level: debug diff --git a/tests/system/config/saltbeat.yml.j2 b/tests/system/config/saltbeat.yml.j2 new file mode 100644 index 0000000..9c82b2f --- /dev/null +++ b/tests/system/config/saltbeat.yml.j2 @@ -0,0 +1,83 @@ +################### Beat Configuration ######################### + + + +############################# Output ########################################## + +# Configure what outputs to use when sending the data collected by the beat. +# You can enable one or multiple outputs by setting enabled option to true. +output: + + ### File as output + file: + # Enabling file output + enabled: true + + # Path to the directory where to save the generated files. The option is mandatory. + path: {{ output_file_path|default(beat.working_dir + "/output") }} + + + # Name of the generated files. The default is `saltbeat` and it generates + # files: `saltbeat`, `saltbeat.1`, `saltbeat.2`, etc. + filename: "{{ output_file_filename|default("saltbeat") }}" + + # Maximum size in kilobytes of each file. When this size is reached, the files are + # rotated. The default value is 10 MB. + #rotate_every_kb: 10000 + + # Maximum number of files under path. When this number of files is reached, the + # oldest file is deleted and the rest are shifted from last to first. The default + # is 7 files. + #number_of_files: 7 + + + +############################# Shipper ######################################### + +shipper: + # The name of the shipper that publishes the network data. It can be used to group + # all the transactions sent by a single shipper in the web interface. + # If this options is not defined, the hostname is used. + #name: + + # The tags of the shipper are included in their own field with each + # transaction published. Tags make it easy to group servers by different + # logical properties. + #tags: ["service-X", "web-tier"] + + # Uncomment the following if you want to ignore transactions created + # by the server on which the shipper is installed. This option is useful + # to remove duplicates if shippers are installed on multiple servers. + #ignore_outgoing: true + + +############################# Logging ######################################### + +#logging: + # Send all logging output to syslog. On Windows default is false, otherwise + # default is true. + #to_syslog: true + + # Write all logging output to files. Beats automatically rotate files if configurable + # limit is reached. + #to_files: false + + # Enable debug output for selected components. + #selectors: [] + + # Set log level + #level: error + + #files: + # The directory where the log files will written to. + #path: /var/log/saltbeat + + # The name of the files where the logs are written to. + #name: saltbeat + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 diff --git a/tests/system/requirements.txt b/tests/system/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/tests/system/saltbeat.py b/tests/system/saltbeat.py new file mode 100644 index 0000000..be8dba2 --- /dev/null +++ b/tests/system/saltbeat.py @@ -0,0 +1,11 @@ +import sys +sys.path.append('../../vendor/github.com/elastic/beats/libbeat/tests/system') +from beat.beat import TestCase + +class BaseTest(TestCase): + + @classmethod + def setUpClass(self): + self.beat_name = "saltbeat" + self.build_path = "../../build/system-tests/" + self.beat_path = "../../saltbeat.test" diff --git a/tests/system/test_base.py b/tests/system/test_base.py new file mode 100644 index 0000000..4484412 --- /dev/null +++ b/tests/system/test_base.py @@ -0,0 +1,19 @@ +from saltbeat import BaseTest + +import os + + +class Test(BaseTest): + + def test_base(self): + """ + Basic test with exiting Saltbeat normally + """ + self.render_config_template( + path=os.path.abspath(self.working_dir) + "/log/*" + ) + + saltbeat_proc = self.start_beat() + self.wait_until( lambda: self.log_contains("saltbeat is running")) + exit_code = saltbeat_proc.kill_and_wait() + assert exit_code == 0