diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1ac9931 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +account.json +__pycache__/ +venv/ +.vscode/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..5ab3809 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,23 @@ +# Contributing + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to https://cla.developers.google.com/ to see your +current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it again. + +## Code reviews +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult GitHub Help for more +information on using pull requests. + +## Community Guidelines +This project follows +[Google's Open Source Community Guidelines](CODE-OF-CONDUCT.md). diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e5fc621 --- /dev/null +++ b/Makefile @@ -0,0 +1,60 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Make will use bash instead of sh +SHELL := /usr/bin/env bash + +# All is the first target in the file so it will get picked up when you just run 'make' on its own +all: check_shell check_python check_golang check_terraform check_docker check_base_files check_headers check_trailing_whitespace + +# The .PHONY directive tells make that this isn't a real target and so +# the presence of a file named 'check_shell' won't cause this target to stop +# working +.PHONY: check_shell +check_shell: + @source test/make.sh && check_shell + +.PHONY: check_python +check_python: + @source test/make.sh && check_python + +.PHONY: check_golang +check_golang: + @source test/make.sh && golang + +.PHONY: check_terraform +check_terraform: + @source test/make.sh && check_terraform + +.PHONY: check_docker +check_docker: + @source test/make.sh && docker + +.PHONY: check_base_files +check_base_files: + @source test/make.sh && basefiles + +.PHONY: check_shebangs +check_shebangs: + @source test/make.sh && check_bash + +.PHONY: check_trailing_whitespace +check_trailing_whitespace: + @source test/make.sh && check_trailing_whitespace + +.PHONY: check_headers +check_headers: + @echo "Checking file headers" + @python test/verify_boilerplate.py + diff --git a/README.md b/README.md new file mode 100644 index 0000000..f6bc7b5 --- /dev/null +++ b/README.md @@ -0,0 +1,425 @@ +# Tracing with Stackdriver on Kubernetes Engine + +* [Introduction](#introduction) +* [Architecture](#architecture) +* [Prerequisites](#prerequisites) + * [Enable GCP APIs](#enable-gcp-apis) + * [Install Cloud SDK](#install-cloud-sdk) + * [Install Terraform](#install-terraform) + * [Configure Authentication](#configure-authentication) +* [Deployment](#deployment) + * [Introduction to Terraform](#introduction-to-terraform) + * [Running Terraform](#running-terraform) + * [Deploying the Demo Application](#deploying-the-demo-application) +* [Validation](#validation) + * [Generating Telemetry Data](#generating-telemetry-data) + * [Examining Traces](#examining-traces) + * [Pulling Pub/Sub Messages](#pulling-pubsub-messages) + * [Monitoring and Logging](#monitoring-and-logging) +* [Teardown](#teardown) +* [Troubleshooting](#troubleshooting) +* [Relevant Materials](#relevant-materials) + * [Kubernetes](#kubernetes) + * [OpenCensus](#opencensus) + * [Spring Sleuth](#spring-sleuth) + * [Stackdriver](#stackdriver) + * [Terraform](#terraform) + * [Zipkin](#zipkin) + +## Introduction + +When supporting a production system that services HTTP requests or provides an +API, it is important to measure the latency of your endpoints to detect when a +system's performance is not operating within specification. In monolithic +systems this single latency measure may be useful to detect and diagnose +deteriorating behavior. With modern microservice architectures, however, this +becomes much more difficult because a single request may result in numerous +additional requests to other systems before the request can be fully handled. +Deteriorating performance in an underlying system may impact all other systems +that rely on it. While latency can be measured at each service endpoint it can +be difficult to correlate slow behavior in the public endpoint with a +particular sub-service that is misbehaving. + +Enter distributed tracing. Distributed tracing uses metadata passed along with +requests to correlate requests across service tiers. By collecting telemetry +data from all the services in a microservice architecture and propagating a +_trace id_ from an initial request to all subsidiary requests, developers can +much more easily identify which service is causing slowdowns affecting the rest +of the system. + +[Google Cloud Platform (GCP)](https://cloud.google.com/) provides the +[Stackdriver](https://cloud.google.com/stackdriver/) suite of products to +handle logging, monitoring, and distributed tracing. This document will discuss +the latter feature and provide a distributed tracing demo that can service as a +basis for your own applications. The demo will be deployed to +[Kubernetes Engine](https://cloud.google.com/kubernetes-engine/) and will +demonstrate a multi-tier architecture implementing distributed tracing. It will +also take advantage of [Terraform](#terraform) to build out necessary +infrastructure. + +This demo requires an active GCP account. You can sign up for a +[free account](https://cloud.google.com/) if you don't already have one. It +will come with free credits for beginning your GCP experience. + +This demo is designed for MacOS and Linux systems but users can alternatively +use the [Google Cloud Shell](https://cloud.google.com/shell/docs/) entirely in +the cloud without requiring a local system. + +## Architecture + +This demonstration application will begin by deploying a Kubernetes Engine +cluster. To this cluster will be deployed a simple web application fronted by a +load balancer. The web app will publish messages provided by the user to a +[Cloud Pub/Sub](https://cloud.google.com/pubsub/docs/overview) topic. The +application is instrumented such that HTTP requests to it will result in the +creation of a trace whose context will be propagated to the Cloud Pub/Sub +publish API request. The correlated telemetry data from these requests will be +available in the Stackdriver Trace Console. + +![](images/architecture.png) + +## Prerequisites + +The steps described in this document require the installation of several tools +and the proper configuration of authentication to allow them to access your +GCP resources. + +### Enable GCP APIs + +The following APIs need to be enabled: +* Kubernetes Engine API +* Stackdriver Trace API + +A script is provided in the /scripts folder named **enable-apis.sh** that will +enable these three API's. Follow these steps to execute the script: +1. In the GCP console, change to the project you want to enable the API's for. +2. Click on the **Activate Cloud Shell Console** Visit the **APIs & Services** + section of the GCP Console. +3. Upload the **enable-apis.sh** script in the **Cloud Shell** window. +4. Execute the script. + + +### Install Cloud SDK + +The Google Cloud SDK is used to interact with your GCP resources. +[Installation instructions](https://cloud.google.com/sdk/downloads) for +multiple platforms are available online. + +The Google Cloud SDK provides a `gcloud` command that is used to interact with +GCP via its APIs. The base installation only includes some of the possible +components that are available. The following command will display available +components: + +```console +gcloud components list +``` + +This demo also requires `kubectl` so you will need to +[install it](https://kubernetes.io/docs/tasks/tools/install-kubectl/) as well. + +### Install Terraform + +Terraform is used to automate the manipulation of cloud infrastructure. Its +[installation instructions](https://www.terraform.io/intro/getting-started/install.html) +are also available online. + +### Configure Authentication + +In order to interact with GCP from your system you will need to authenticate: + +```console +gcloud auth application-default login +``` + + +## Deployment + +### Introduction to Terraform + +Following the principles of +[infrastructure as code](https://en.wikipedia.org/wiki/Infrastructure_as_Code) +and +[immutable infrastructure](https://www.oreilly.com/ideas/an-introduction-to-immutable-infrastructure), +Terraform supports the writing of declarative descriptions of the desired state +of infrastructure. When the descriptor is applied, Terraform uses GCP APIs to +provision and update resources to match. Terraform compares the desired state +with the current state so incremental changes can be made without deleting +everything and starting over. For instance, Terraform can build out GCP +projects and compute instances, etc., even set up a Kubernetes Engine cluster +and deploy applications to it. When requirements change, the descriptor can be +updated and Terraform will adjust the cloud infrastructure accordingly. + +This example will start up a Kubernetes Engine cluster using Terraform. Then +you will use Kubernetes commands to deploy a demo application to the cluster. +By default, Kubernetes Engine clusters in GCP are launched with a +pre-configured [Fluentd](https://www.fluentd.org/)-based collector that +forwards logging events for the cluster to Stackdriver. Interacting with the +demo app will produce trace events that are visible in the +[Stackdriver Trace UI](https://console.cloud.google.com/traces). + +### Running Terraform + +There are three Terraform files provided with this example. The first one, +`main.tf`, is the starting point for Terraform. It describes the features that +will be used, the resources that will be manipulated, and the outputs that will +result. The second file is `provider.tf`, which indicates which cloud provider +and version will be the target of the Terraform commands--in this case GCP. The +final file is `variables.tf`, which contains a list of variables that are used +as inputs into Terraform. Any variables referenced in the `main.tf` that do not +have defaults configured in `variables.tf` will result in prompts to the user +at runtime. + +Given that authentication was [configured](#configure-authentication) above, we +are now ready to deploy the infrastructure. Run the following command to do the +deploy: + +```console +./deploy.sh +``` + +This script will manage the deployment by doing the following things: + +1. Generate the Terraform variable values using the `generate-tfvars.sh` script +2. Ensure Terraform is initialized using `terraform init` +3. Deploy the Terraform resources using `terraform apply` +4. Deploy the Kubernetes resources using + `kubectl apply -f ./tracing-demo-deployment.yaml` + +You will need to enter any variables again that don't have defaults provided. +If no errors are displayed then after a few minutes you should see your +Kubernetes Engine cluster in the +[GCP Console](https://console.cloud.google.com/kubernetes) with the sample +application deployed. + +At this point you should find a Kubernetes cluster has been deployed to GCP and +a Pub/Sub topic will have be been created as well a subscription to the topic. +The final thing you should see is a Kubernetes deployment in the +[Workload](https://console.cloud.google.com/kubernetes/workload) tab of the +Kubernetes Engine Console + +Once the app has been deployed, it can be viewed in the +[Workload](https://console.cloud.google.com/kubernetes/workload) tab of the +Kubernetes Engine Console. You can also see the load balancer that was created +for the application in the +[Services](https://console.cloud.google.com/kubernetes/discovery) section of +the console. + + +## Validation + +### Generating Telemetry Data + +Once the demo application is deployed, go to +https://console.cloud.google.com/kubernetes/discovery and if it isn't already +selected select the project used for this demo with the combobox in the +upper-left corner. You should see a list of your exposed services: + +![](images/services-ui.png) + +Clicking on the endpoint listed next to the `tracing-demo` load balancer will +open the demo app web page in a new tab of your browser. Note that your IP +address will likely be different from the example above. The page displayed is +simple: + +![](images/hello-world-browser.png) + +To the url, add the string: `?string=CustomMessage` and see that the message is +displayed: + +![](images/custom-message-browser.png) + +As you can see, if a `string` parameter is not provided it uses a default value +of `Hello World`. The app itself isn't very interesting. We are simply using it +to generate trace telemetry data. + +### Examining Traces + +Navigate to https://console.cloud.google.com/traces/traces and select your +project from the combobox in the upper left-hand corner of your screen if it is +not already selected. You should see a chart displaying trace events plotted on +a timeline with latency as the vertical metric. + +![](images/trace-ui.png) + +Click on one of the dots on the chart. You will see a chart with two bars, the +top one longer than the bottom one: + +![](images/span-ui-full.png) + +The top bar is known as the _root span_ and represents the duration of the HTTP +request, from the moment the first byte arrives until the moment the last byte +of the response is sent. The bottom bar represents the duration of the request +made when sending the message to the Pub/Sub topic. + +Since the handling of the HTTP request is blocked by the call to the Pub/Sub +API it is clear that the vast majority of the time spent within the HTTP +request is taken up by the Pub/Sub interaction. This demonstrates that by +instrumenting each tier of your application you can easily identify where the +bottlenecks are. + +### Pulling Pub/Sub Messages + +As described in the [Architecture](#architecture) section of this document, +messages from the demo app are published to a Pub/Sub topic. These messages can +be consumed from the topic using the `gcloud` CLI: + +```console +gcloud pubsub subscriptions pull --auto-ack --limit 10 tracing-demo-cli +┌─────────────┬─────────────────┬────────────┐ +│ DATA │ MESSAGE_ID │ ATTRIBUTES │ +├─────────────┼─────────────────┼────────────┤ +│ Hello World │ 121487601177521 │ │ +└─────────────┴─────────────────┴────────────┘ +``` + +Pulling the messages from the topic has no impact on tracing. This section +simply provides a consumer of the messages for verification purposes. + +### Monitoring and Logging + +Stackdriver monitoring and logging are not the subject of this demo, but it is +worth noting that the application you deployed will publish logs to +[Stackdriver Logging](https://console.cloud.google.com/logs/viewer) and +metrics to [Stackdriver Monitoring](https://app.google.stackdriver.com/) + +For instance, see https://app.google.stackdriver.com/metrics-explorer and +select the `GKE Container` Resource Type and the `CPU usage` metric. You should +see a chart of this metric plotted for different pods running in the cluster: + +![](images/metrics.png) + +As well, logs can be seen at https://console.cloud.google.com/logs/viewer: + +![](images/logs.png) + +You may need to click the `Load older logs` button to display published logs. + +## Teardown + +When you are finished with this example you will want to clean up the resources +that were created so that you avoid accruing charges: + +```console +terraform destroy +``` + +Since Terraform tracks the resources it created it is able to tear down the +cluster, the Pub/Sub topic, and the Pub/Sub subscription. + +## Troubleshooting + +1. You may run into an issue where an API has not been enabled for your + project. If you receive such an error the necessary API name may be listed. You + can enable it with: + + ```console + gcloud services list + ``` + + where `` is the identified service name. If the disabled API is + not provided you can see a list of those available with the following command: + + ```console + gcloud services list --available + ``` + +Identify the necessary service and enable it with the above command. Keep in +mind the +[principle of least privilege](https://en.wikipedia.org/wiki/Principle_of_least_privilege) +and limit access to those strictly necessary to run your applications. +2. A number of possible errors can be diagnosed using the `kubectl` command. + For instance, a deployment can be shown: + ```console + kubectl get deployment tracing-demo + NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE + tracing-demo 1 1 1 1 2h + ``` + + More details can be shown with `describe`: + + ```console + kubectl describe deployment tracing-demo + ... + ``` + + This command will show a list of deployed pods: + + ```console + kubectl get pod + NAME READY STATUS RESTARTS AGE + tracing-demo-59cc7988fc-h5w27 1/1 Running 0 2h + ``` + Again, details of the pod can be seen with `describe`: + ```console + kubectl describe pod tracing-demo + ... + ``` + + Once the pod name is known logs can be viewed locally: + + ```console + kubectl logs tracing-demo-59cc7988fc-h5w27 + 10.60.0.1 - - [22/Jun/2018:19:42:23 +0000] "HEAD / HTTP/1.0" 200 - "-" "-" + Publishing string: Hello World + 10.60.0.1 - - [22/Jun/2018:19:42:23 +0000] "GET / HTTP/1.1" 200 669 "-" "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36" + ``` +3. The install script fails with a `Permission denied` when running Terraform. + The credentials that Terraform is using do not provide the necessary + permissions to create resources in the selected projects. Ensure that the + account listed in `gcloud config list` has necessary permissions to create + resources. If it does, regenerate the application default credentials using + `gcloud auth application-default login`. + +## Relevant Materials + +The following are some relevant materials for further investigation: + +### Kubernetes +[Kubernetes](https://kubernetes.io/) is a container orchestration platform +popular in microservice architectures. GCP provides a managed version of +Kubernetes called [Kubernetes Engine](https://cloud.google.com/kubernetes-engine/). + +### OpenCensus +[OpenCensus](https://opencensus.io/) provides standardized libraries for +capturing and publishing trace telemetry data. Libraries are provided for a +number of popular languages and numerous trace platforms are supported, +including [Stackdriver](#stackdriver) and [Zipkin](#zipkin). The demo this +document describes uses OpenCensus to publish telemetry data to Stackdriver. + +### Spring Sleuth +[Spring Sleuth](https://cloud.spring.io/spring-cloud-sleuth/) provides +instrumentation of Java applications that use the popular +[Spring](https://spring.io/) framework. Spring Sleuth supports an abstraction +over distributed trace telemetry collectors so developers can seamlessly switch +between [Zipkin](#zipkin), [Stackdriver](#stackdriver), and other trace engines. + +### Stackdriver +[Stackdriver](https://cloud.google.com/stackdriver/) is a suite of tools in GCP +providing logging, monitoring, tracing, and related features. This document and +demo are particularly concerned with the +[Stackdriver Trace](https://cloud.google.com/trace/) feature it provides. + +### Terraform +[Terraform](https://www.terraform.io/) is a declarative +[infrastructure as code](https://en.wikipedia.org/wiki/Infrastructure_as_Code) +tool that enables configuration files to be used to automate the deployment and +evolution of infrastructure in the cloud. + +### Zipkin +[Zipkin](https://zipkin.io/) is a distributed tracing tool that helped +popularize the practice. Many frameworks provide Zipkin +[integration](https://zipkin.io/pages/existing_instrumentations.html) and those +that don't can [implement their own](https://zipkin.io/pages/instrumenting). + +Applications that are already instrumented for Zipkin can have their Zipkin +telemetry data adapted to Stackdriver events through the use of a +[Zipkin Collector](https://cloud.google.com/trace/docs/zipkin). It can be +deployed to Kubernetes Engine: + +```console +kubectl run stackdriver-zipkin --image=gcr.io/stackdriver-trace-docker/zipkin-collector --expose --port=9411 +``` + +This will deploy the collector to the well known Zipkin port `9411` and +applications looking for it on their local port will find it indistinguishable +from a Zipkin server but telemetry data will appear in Stackdriver Trace. diff --git a/demo-app/Dockerfile b/demo-app/Dockerfile new file mode 100644 index 0000000..77edec2 --- /dev/null +++ b/demo-app/Dockerfile @@ -0,0 +1,29 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM python:3.6.4-stretch + +RUN apt-get update \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /app/ + +WORKDIR /app +RUN pip install -r requirements.txt + +COPY templates/ /app/templates/ +COPY *.py /app/ + +CMD gunicorn --access-logfile - --error-logfile - -w 4 -b 0.0.0.0:8080 app:app \ No newline at end of file diff --git a/demo-app/app.py b/demo-app/app.py new file mode 100644 index 0000000..53d53d6 --- /dev/null +++ b/demo-app/app.py @@ -0,0 +1,75 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +A sample app demonstrating Stackdriver Trace +""" +import google.auth +from google.cloud import pubsub_v1 + +from flask import Flask, render_template, request + +from opencensus.trace import execution_context +from opencensus.trace.exporters import stackdriver_exporter +from opencensus.trace.exporters.transports import background_thread +from opencensus.trace.ext.flask.flask_middleware import FlaskMiddleware +from opencensus.trace.propagation import google_cloud_format +from opencensus.trace.samplers import always_on + +app = Flask(__name__) + +topic_name = 'tracing-demo' + +# Configure Tracing +exporter = stackdriver_exporter.StackdriverExporter( + transport=background_thread.BackgroundThreadTransport) +propagator = google_cloud_format.GoogleCloudFormatPropagator() +sampler = always_on.AlwaysOnSampler() +blacklist_paths = ['favicon.ico'] + +# Instrument Flask to do tracing automatically +middleware = FlaskMiddleware( + app, + exporter=exporter, + propagator=propagator, + sampler=sampler, + blacklist_paths=blacklist_paths) + +# Create Pub/Sub client +# Messages sent with the HTTP request will be published to Cloud Pub/Sub +publisher = pubsub_v1.PublisherClient() +_, project_id = google.auth.default() +topic_path = publisher.topic_path(project_id, topic_name) + + +@app.route('/') +def template_test(): + """ + Handle the root path for this app. Renders a simple web page displaying a + message. The default message is Hello World but this can be overridden by + the use of a parameter: ?string=Test + """ + + tracer = execution_context.get_opencensus_tracer() + + # Trace Pub/Sub call using Context Manager + with tracer.start_span() as pubsub_span: + pubsub_span.name = '[{}]{}'.format('publish', 'Pub/Sub') + pubsub_span.add_attribute('Topic Path', topic_path) + + string = request.args.get('string') + string = string if string else 'Hello World' + print('Publishing string: %s' % string) + publisher.publish(topic_path, data=string.encode('utf-8')).result() + + return render_template('template.html', my_string=string) diff --git a/demo-app/cloudbuild.yaml b/demo-app/cloudbuild.yaml new file mode 100644 index 0000000..7c0d8e8 --- /dev/null +++ b/demo-app/cloudbuild.yaml @@ -0,0 +1,20 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +steps: +- name: 'gcr.io/cloud-builders/docker' + args: [ 'build', '-t', 'gcr.io/$PROJECT_ID/tracing-demo:1.0.0', '.'] +images: +- 'gcr.io/$PROJECT_ID/tracing-demo:1.0.0' + diff --git a/demo-app/requirements.txt b/demo-app/requirements.txt new file mode 100644 index 0000000..e0a2b88 --- /dev/null +++ b/demo-app/requirements.txt @@ -0,0 +1,6 @@ +google-cloud-monitoring==0.29.0 +google-cloud-pubsub==0.35.4 +google-cloud-trace==0.19.0 +gunicorn==19.8.1 +opencensus==0.1.5 +Flask==1.0.2 diff --git a/demo-app/templates/template.html b/demo-app/templates/template.html new file mode 100644 index 0000000..10c762d --- /dev/null +++ b/demo-app/templates/template.html @@ -0,0 +1,21 @@ + + + + Stackdriver Trace Example + + + + + +
+

Message: {{my_string}}

+
+ + + + \ No newline at end of file diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..36a13d1 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,29 @@ +#! /usr/bin/env bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +REPO_ROOT="$(git rev-parse --show-toplevel)" + +# Generate the variables to be used by Terraform +"$REPO_ROOT/generate-tfvars.sh" + +terraform init +terraform apply + +# Deploy the Kubernetes resources to the created cluster +kubectl apply -f "$REPO_ROOT/tracing-demo-deployment.yaml" + diff --git a/generate-tfvars.sh b/generate-tfvars.sh new file mode 100755 index 0000000..b6532d9 --- /dev/null +++ b/generate-tfvars.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# "---------------------------------------------------------" +# "- -" +# "- Helper script to generate terraform variables -" +# "- file based on glcoud defaults. -" +# "- -" +# "---------------------------------------------------------" + +# Stop immediately if something goes wrong +set -euo pipefail + +# This script should be run from directory that contains the terraform directory. +# The purpose is to populate defaults for subsequent terraform commands. + +# git is required for this tutorial +command -v git >/dev/null 2>&1 || { \ + echo >&2 "I require git but it's not installed. Aborting."; exit 1; } + +# gcloud is required for this tutorial +command -v gcloud >/dev/null 2>&1 || { \ + echo >&2 "I require gcloud but it's not installed. Aborting."; exit 1; } + + +# gcloud config holds values related to your environment. If you already +# defined a default zone we will retrieve it and use it +ZONE=$(gcloud config get-value compute/zone) +if [[ -z "${ZONE}" ]]; then + echo "https://cloud.google.com/compute/docs/regions-zones/changing-default-zone-region" 1>&2 + echo "gcloud cli must be configured with a default zone." 1>&2 + echo "run 'gcloud config set compute/zone ZONE'." 1>&2 + echo "replace 'ZONE' with the zone name like us-west1-a." 1>&2 + exit 1; +fi + +# gcloud config holds values related to your environment. If you already +# defined a default project we will retrieve it and use it +PROJECT=$(gcloud config get-value core/project) +if [[ -z "${PROJECT}" ]]; then + echo "gcloud cli must be configured with a default project." 1>&2 + echo "run 'gcloud config set core/project PROJECT'." 1>&2 + echo "replace 'PROJECT' with the project name." 1>&2 + exit 1; +fi + + +# Use git to find the top-level directory and confirm +# by looking for the 'terraform' directory +PROJECT_DIR=$(git rev-parse --show-toplevel) +if [[ -d "./terraform" ]]; then + PROJECT_DIR=$(pwd) +fi +if [[ -z "${PROJECT_DIR}" ]]; then + echo "Could not identify project base directory." 1>&2 + echo "Please re-run from a project directory and ensure" 1>&2 + echo "the .git directory exists." 1>&2 + exit 1; +fi + + +( +cd "${PROJECT_DIR}" + +TFVARS_FILE="./terraform.tfvars" + +# We don't want to overwrite a pre-existing tfvars file +if [[ -f "${TFVARS_FILE}" ]] +then + echo "${TFVARS_FILE} already exists." 1>&2 + echo "Please remove or rename before regenerating." 1>&2 + exit 1; +else +# Write out all the values we gathered into a tfvars file so you don't +# have to enter the values manually + cat < "${TFVARS_FILE}" +project="${PROJECT}" +zone="${ZONE}" +EOF +fi +) + diff --git a/images/analysis-report.png b/images/analysis-report.png new file mode 100644 index 0000000..27f2766 Binary files /dev/null and b/images/analysis-report.png differ diff --git a/images/architecture.png b/images/architecture.png new file mode 100644 index 0000000..fae31fa Binary files /dev/null and b/images/architecture.png differ diff --git a/images/custom-message-browser.png b/images/custom-message-browser.png new file mode 100644 index 0000000..493928c Binary files /dev/null and b/images/custom-message-browser.png differ diff --git a/images/hello-world-browser.png b/images/hello-world-browser.png new file mode 100644 index 0000000..29b081f Binary files /dev/null and b/images/hello-world-browser.png differ diff --git a/images/logs.png b/images/logs.png new file mode 100644 index 0000000..e0a5b42 Binary files /dev/null and b/images/logs.png differ diff --git a/images/metrics.png b/images/metrics.png new file mode 100644 index 0000000..e015c5c Binary files /dev/null and b/images/metrics.png differ diff --git a/images/services-ui.png b/images/services-ui.png new file mode 100644 index 0000000..294b2b3 Binary files /dev/null and b/images/services-ui.png differ diff --git a/images/span-ui-full.png b/images/span-ui-full.png new file mode 100644 index 0000000..18c4474 Binary files /dev/null and b/images/span-ui-full.png differ diff --git a/images/span-ui.png b/images/span-ui.png new file mode 100644 index 0000000..70f707e Binary files /dev/null and b/images/span-ui.png differ diff --git a/images/trace-ui.png b/images/trace-ui.png new file mode 100644 index 0000000..ea7a78a Binary files /dev/null and b/images/trace-ui.png differ diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..b56a05a --- /dev/null +++ b/main.tf @@ -0,0 +1,66 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + +// Provides access to available Google Container Engine versions in a zone for a given project. +// https://www.terraform.io/docs/providers/google/d/google_container_engine_versions.html +data "google_container_engine_versions" "on-prem" { + zone = "${var.zone}" + project = "${var.project}" +} + +// https://www.terraform.io/docs/providers/google/d/google_container_cluster.html +// Create the primary cluster for this project. + +// Create the GKE Cluster +resource "google_container_cluster" "primary" { + name = "tracing-demo-space" + zone = "${var.zone}" + initial_node_count = 1 + min_master_version = "${data.google_container_engine_versions.on-prem.latest_master_version}" + + // Scopes were a pre-IAM method of giving instances API access + // They are still around we need to give our cluster nodes + // access to PubSub and Tracing as well as the standard scopes + node_config { + oauth_scopes = [ + "https://www.googleapis.com/auth/compute", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/pubsub", + "https://www.googleapis.com/auth/trace.append", + ] + } + + // Here we use gcloud to gather authentication information about our new cluster and write that + // information to kubectls config file + provisioner "local-exec" { + command = "gcloud container clusters get-credentials ${google_container_cluster.primary.name} --zone ${google_container_cluster.primary.zone} --project ${var.project}" + } +} + +// Creates a Cloud Pub/Sub Topic +resource "google_pubsub_topic" "tracing-demo-topic" { + name = "tracing-demo" +} + +// Creates a Cloud Pub/Sub Subscription +// You need a subscription to pull messages from a topic +resource "google_pubsub_subscription" "tracing-demo-subscription" { + name = "tracing-demo-cli" + topic = "${google_pubsub_topic.tracing-demo-topic.name}" +} diff --git a/provider.tf b/provider.tf new file mode 100644 index 0000000..43dd0a3 --- /dev/null +++ b/provider.tf @@ -0,0 +1,22 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + +// Identifies allowable version range for Terraform Google Provider +provider "google" { + project = "${var.project}" + version = "~> 1.13" +} diff --git a/scripts/enable-apis.sh b/scripts/enable-apis.sh new file mode 100755 index 0000000..3eb8823 --- /dev/null +++ b/scripts/enable-apis.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script checks to make sure that the pre-requisite APIs are enabled. + +# Enable the Kubernetes Engine API +gcloud services enable container.googleapis.com +gcloud services enable cloudtrace.googleapis.com diff --git a/test/boilerplate/boilerplate.Dockerfile.txt b/test/boilerplate/boilerplate.Dockerfile.txt new file mode 100644 index 0000000..b0c7da3 --- /dev/null +++ b/test/boilerplate/boilerplate.Dockerfile.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.Makefile.txt b/test/boilerplate/boilerplate.Makefile.txt new file mode 100644 index 0000000..b0c7da3 --- /dev/null +++ b/test/boilerplate/boilerplate.Makefile.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.go.txt b/test/boilerplate/boilerplate.go.txt new file mode 100644 index 0000000..557e16f --- /dev/null +++ b/test/boilerplate/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/test/boilerplate/boilerplate.py.txt b/test/boilerplate/boilerplate.py.txt new file mode 100644 index 0000000..b0c7da3 --- /dev/null +++ b/test/boilerplate/boilerplate.py.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.sh.txt b/test/boilerplate/boilerplate.sh.txt new file mode 100644 index 0000000..b0c7da3 --- /dev/null +++ b/test/boilerplate/boilerplate.sh.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/boilerplate/boilerplate.tf.txt b/test/boilerplate/boilerplate.tf.txt new file mode 100644 index 0000000..557e16f --- /dev/null +++ b/test/boilerplate/boilerplate.tf.txt @@ -0,0 +1,15 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/test/boilerplate/boilerplate.xml.txt b/test/boilerplate/boilerplate.xml.txt new file mode 100644 index 0000000..3d98cdc --- /dev/null +++ b/test/boilerplate/boilerplate.xml.txt @@ -0,0 +1,15 @@ + diff --git a/test/boilerplate/boilerplate.yaml.txt b/test/boilerplate/boilerplate.yaml.txt new file mode 100644 index 0000000..b0c7da3 --- /dev/null +++ b/test/boilerplate/boilerplate.yaml.txt @@ -0,0 +1,13 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/test/make.sh b/test/make.sh new file mode 100755 index 0000000..adafbbd --- /dev/null +++ b/test/make.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This function checks to make sure that every +# shebang has a '- e' flag, which causes it +# to exit on error +function check_bash() { +find . -name "*.sh" | while IFS= read -d '' -r file; +do + if [[ "$file" != *"bash -e"* ]]; + then + echo "$file is missing shebang with -e"; + exit 1; + fi; +done; +} + +# This function makes sure that the required files for +# releasing to OSS are present +function basefiles() { + echo "Checking for required files" + test -f CONTRIBUTING.md || echo "Missing CONTRIBUTING.md" + test -f LICENSE || echo "Missing LICENSE" + test -f README.md || echo "Missing README.md" +} + +# This function runs the hadolint linter on +# every file named 'Dockerfile' +function docker() { + echo "Running hadolint on Dockerfiles" + find . -name "Dockerfile" -exec hadolint {} \; +} + +# This function runs 'terraform validate' against all +# files ending in '.tf' +function check_terraform() { + echo "Running terraform validate" + #shellcheck disable=SC2156 + find . -name "*.tf" -exec bash -c 'terraform validate --check-variables=false $(dirname "{}")' \; +} + +# This function runs 'go fmt' and 'go vet' on eery file +# that ends in '.go' +function golang() { + echo "Running go fmt and go vet" + find . -name "*.go" -exec go fmt {} \; + find . -name "*.go" -exec go vet {} \; +} + +# This function runs the flake8 linter on every file +# ending in '.py' +function check_python() { + echo "Running flake8" + find . -name "*.py" -exec flake8 {} \; +} + +# This function runs the shellcheck linter on every +# file ending in '.sh' +function check_shell() { + echo "Running shellcheck" + find . -name "*.sh" -exec shellcheck -x {} \; +} + +# This function makes sure that there is no trailing whitespace +# in any files in the project. +# There are some exclusions +function check_trailing_whitespace() { + echo "The following lines have trailing whitespace" + grep -r '[[:blank:]]$' --exclude-dir=".terraform" --exclude="*.png" --exclude-dir=".git" . + rc=$? + if [ $rc = 0 ]; then + exit 1 + fi +} diff --git a/test/verify_boilerplate.py b/test/verify_boilerplate.py new file mode 100644 index 0000000..a632fde --- /dev/null +++ b/test/verify_boilerplate.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python + +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Verifies that all source files contain the necessary copyright boilerplate +# snippet. +# This is based on existing work +# https://github.com/kubernetes/test-infra/blob/master/hack +# /verify_boilerplate.py +from __future__ import print_function +import argparse +import glob +import os +import re +import sys + + +def get_args(): + """Parses command line arguments. + + Configures and runs argparse.ArgumentParser to extract command line + arguments. + + Returns: + An argparse.Namespace containing the arguments parsed from the + command line + """ + parser = argparse.ArgumentParser() + parser.add_argument("filenames", + help="list of files to check, " + "all files if unspecified", + nargs='*') + rootdir = os.path.dirname(__file__) + "/../" + rootdir = os.path.abspath(rootdir) + parser.add_argument( + "--rootdir", + default=rootdir, + help="root directory to examine") + + default_boilerplate_dir = os.path.join(rootdir, "test/boilerplate") + parser.add_argument("--boilerplate-dir", default=default_boilerplate_dir) + return parser.parse_args() + + +def get_refs(ARGS): + """Converts the directory of boilerplate files into a map keyed by file + extension. + + Reads each boilerplate file's contents into an array, then adds that array + to a map keyed by the file extension. + + Returns: + A map of boilerplate lines, keyed by file extension. For example, + boilerplate.py.txt would result in the k,v pair {".py": py_lines} where + py_lines is an array containing each line of the file. + """ + refs = {} + + # Find and iterate over the absolute path for each boilerplate template + for path in glob.glob(os.path.join( + ARGS.boilerplate_dir, + "boilerplate.*.txt")): + extension = os.path.basename(path).split(".")[1] + ref_file = open(path, 'r') + ref = ref_file.read().splitlines() + ref_file.close() + refs[extension] = ref + return refs + + +# pylint: disable=too-many-locals +def has_valid_header(filename, refs, regexs): + """Test whether a file has the correct boilerplate header. + + Tests each file against the boilerplate stored in refs for that file type + (based on extension), or by the entire filename (eg Dockerfile, Makefile). + Some heuristics are applied to remove build tags and shebangs, but little + variance in header formatting is tolerated. + + Args: + filename: A string containing the name of the file to test + refs: A map of boilerplate headers, keyed by file extension + regexs: a map of compiled regex objects used in verifying boilerplate + + Returns: + True if the file has the correct boilerplate header, otherwise returns + False. + """ + try: + with open(filename, 'r') as fp: # pylint: disable=invalid-name + data = fp.read() + except IOError: + return False + basename = os.path.basename(filename) + extension = get_file_extension(filename) + if extension: + ref = refs[extension] + else: + ref = refs[basename] + # remove build tags from the top of Go files + if extension == "go": + con = regexs["go_build_constraints"] + (data, found) = con.subn("", data, 1) + # remove shebang + elif extension == "sh" or extension == "py": + she = regexs["shebang"] + (data, found) = she.subn("", data, 1) + data = data.splitlines() + # if our test file is smaller than the reference it surely fails! + if len(ref) > len(data): + return False + # trim our file to the same number of lines as the reference file + data = data[:len(ref)] + year = regexs["year"] + for datum in data: + if year.search(datum): + return False + + # if we don't match the reference at this point, fail + if ref != data: + return False + return True + + +def get_file_extension(filename): + """Extracts the extension part of a filename. + + Identifies the extension as everything after the last period in filename. + + Args: + filename: string containing the filename + + Returns: + A string containing the extension in lowercase + """ + return os.path.splitext(filename)[1].split(".")[-1].lower() + + +# These directories will be omitted from header checks +SKIPPED_DIRS = [ + 'Godeps', 'third_party', '_gopath', '_output', + '.git', 'vendor', '__init__.py', 'node_modules' +] + + +def normalize_files(files): + """Extracts the files that require boilerplate checking from the files + argument. + + A new list will be built. Each path from the original files argument will + be added unless it is within one of SKIPPED_DIRS. All relative paths will + be converted to absolute paths by prepending the root_dir path parsed from + the command line, or its default value. + + Args: + files: a list of file path strings + + Returns: + A modified copy of the files list where any any path in a skipped + directory is removed, and all paths have been made absolute. + """ + newfiles = [] + for pathname in files: + if any(x in pathname for x in SKIPPED_DIRS): + continue + newfiles.append(pathname) + for idx, pathname in enumerate(newfiles): + if not os.path.isabs(pathname): + newfiles[idx] = os.path.join(ARGS.rootdir, pathname) + return newfiles + + +def get_files(extensions, ARGS): + """Generates a list of paths whose boilerplate should be verified. + + If a list of file names has been provided on the command line, it will be + treated as the initial set to search. Otherwise, all paths within rootdir + will be discovered and used as the initial set. + + Once the initial set of files is identified, it is normalized via + normalize_files() and further stripped of any file name whose extension is + not in extensions. + + Args: + extensions: a list of file extensions indicating which file types + should have their boilerplate verified + + Returns: + A list of absolute file paths + """ + files = [] + if ARGS.filenames: + files = ARGS.filenames + else: + for root, dirs, walkfiles in os.walk(ARGS.rootdir): + # don't visit certain dirs. This is just a performance improvement + # as we would prune these later in normalize_files(). But doing it + # cuts down the amount of filesystem walking we do and cuts down + # the size of the file list + for dpath in SKIPPED_DIRS: + if dpath in dirs: + dirs.remove(dpath) + for name in walkfiles: + pathname = os.path.join(root, name) + files.append(pathname) + files = normalize_files(files) + outfiles = [] + for pathname in files: + basename = os.path.basename(pathname) + extension = get_file_extension(pathname) + if extension in extensions or basename in extensions: + outfiles.append(pathname) + return outfiles + + +def get_regexs(): + """Builds a map of regular expressions used in boilerplate validation. + + There are two scenarios where these regexes are used. The first is in + validating the date referenced is the boilerplate, by ensuring it is an + acceptable year. The second is in identifying non-boilerplate elements, + like shebangs and compiler hints that should be ignored when validating + headers. + + Returns: + A map of compiled regular expression objects, keyed by mnemonic. + """ + regexs = {} + # Search for "YEAR" which exists in the boilerplate, but shouldn't in the + # real thing + regexs["year"] = re.compile('YEAR') + # dates can be 2014, 2015, 2016 or 2017, company holder names can be + # anything + regexs["date"] = re.compile('(2014|2015|2016|2017|2018)') + # strip // +build \n\n build constraints + regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", + re.MULTILINE) + # strip #!.* from shell/python scripts + regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) + return regexs + + +def main(args): + """Identifies and verifies files that should have the desired boilerplate. + + Retrieves the lists of files to be validated and tests each one in turn. + If all files contain correct boilerplate, this function terminates + normally. Otherwise it prints the name of each non-conforming file and + exists with a non-zero status code. + """ + regexs = get_regexs() + refs = get_refs(args) + filenames = get_files(refs.keys(), args) + nonconforming_files = [] + for filename in filenames: + if not has_valid_header(filename, refs, regexs): + nonconforming_files.append(filename) + if nonconforming_files: + print('%d files have incorrect boilerplate headers:' % len( + nonconforming_files)) + for filename in sorted(nonconforming_files): + print(os.path.relpath(filename, args.rootdir)) + sys.exit(1) + + +if __name__ == "__main__": + ARGS = get_args() + main(ARGS) diff --git a/tracing-demo-deployment.yaml b/tracing-demo-deployment.yaml new file mode 100644 index 0000000..d1b9416 --- /dev/null +++ b/tracing-demo-deployment.yaml @@ -0,0 +1,48 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tracing-demo + labels: + app: tracing-demo +spec: + replicas: 1 + selector: + matchLabels: + app: tracing-demo + template: + metadata: + labels: + app: tracing-demo + spec: + containers: + - name: tracing-demo-container + image: gcr.io/pso-examples/tracing-demo:1.0.0 + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: tracing-demo +spec: + selector: + app: tracing-demo + ports: + - protocol: TCP + port: 80 + targetPort: 8080 + type: LoadBalancer diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..d733a06 --- /dev/null +++ b/variables.tf @@ -0,0 +1,26 @@ +/* +Copyright 2018 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + +variable "zone" { + description = "The zone in which to create the Kubernetes cluster. Must match the region" + type = "string" +} + +variable "project" { + description = "the project for this network" + type = "string" +}