From aa8f0ea1000488f2df4ddf2f86aeccac8bd066ae Mon Sep 17 00:00:00 2001 From: Juan Hernandez Date: Mon, 26 Jun 2017 16:21:28 +0200 Subject: [PATCH] Embed project configuration Currently the tools that build or deploy the project need to be executed in a directory that contains the 'project.conf' file, as well as the rest of the source files of the project. That complicates things for users that just want to download and test the containers, as they need to download the complete set of files. In order to simplify that this patch changes the tool so that all the required files are embedded within the binary of the tool. The tool will now check if the 'project.conf' file is available. If it isn't it will extract the embedded files to a temporary directory and use them. This means that the user will only need to download the 'ovc' binary and then run 'ovc deploy'. Change-Id: I6fec0b5aaa432030f9ce6ef142a06f0ab8ec781d Signed-off-by: Juan Hernandez --- .gitignore | 1 + Makefile | 13 ++- tools/src/ovc/main.go | 98 ++++++++++++++++- tools/src/ovc/scripts/embed.go | 191 +++++++++++++++++++++++++++++++++ 4 files changed, 299 insertions(+), 4 deletions(-) create mode 100644 tools/src/ovc/scripts/embed.go diff --git a/.gitignore b/.gitignore index cac95cb..5ce7e5e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ /exported-artifacts/ /tools/bin/ /tools/pkg/ +/tools/src/ovc/embedded.go vendor/ diff --git a/Makefile b/Makefile index 9f03a2f..dd7efa3 100644 --- a/Makefile +++ b/Makefile @@ -47,12 +47,23 @@ $(GLIDE_BINARY): export PATH; \ curl https://glide.sh/get | sh +# The sources of the tool are the .go files, but also the image +# specifications and the OpenShift manifests, as they are embedded +# within the binary: +TOOL_SOURCES=\ + project.conf \ + $(shell find tools/src -type f -name '*.go') \ + $(shell find image-specifications -type f) \ + $(shell find os-manifests -type f) \ + $(NULL) + # Rule to build the tool from its source code: -$(TOOL_BINARY): $(GLIDE_BINARY) $(shell find tools/src -type f) +$(TOOL_BINARY): $(GLIDE_BINARY) $(TOOL_SOURCES) GOPATH="$(ROOT)"; \ export GOPATH; \ pushd $$(dirname $(GLIDE_PROJECT)); \ $(GLIDE_BINARY) install && \ + $(GO_BINARY) generate && \ $(GO_BINARY) build -o $@ *.go || \ exit 1; \ popd \ diff --git a/tools/src/ovc/main.go b/tools/src/ovc/main.go index e23bd95..d73cbd6 100644 --- a/tools/src/ovc/main.go +++ b/tools/src/ovc/main.go @@ -14,12 +14,23 @@ See the License for the specific language governing permissions and limitations under the License. */ +// This is used to embed the project configuration into the binary of the tool, +// so that during run-time there is no need to have both the binary and the +// configuration files. +// +//go:generate go run scripts/embed.go -directory ../../.. -output tools/src/ovc/embedded.go project.conf image-specifications os-manifests + package main // This tool loads and builds all the images. import ( + "archive/tar" + "bytes" + "compress/gzip" "fmt" + "io" + "io/ioutil" "os" "path/filepath" @@ -41,6 +52,10 @@ var tools = map[string]ToolFunc{ "save": saveTool, } +// The name of the project file. +// +const conf = "project.conf" + func main() { // Get the name of the tool: if len(os.Args) < 2 { @@ -67,10 +82,29 @@ func run(name string, tool ToolFunc) int { log.Info("Log file is '%s'", log.Path()) defer log.Close() + // Check if the project file exists. If doesn't exist then we + // need extract it, together with the rest of the source files + // of the project, from the embedded data. + file, _ := filepath.Abs(conf) + if _, err := os.Stat(file); os.IsNotExist(err) { + log.Info("Extracting project") + tmp, err := ioutil.TempDir("", "project") + if err != nil { + log.Error("Can't create temporary directory for project: %s", err) + os.Exit(1) + } + defer os.RemoveAll(tmp) + err = extractData(embedded, tmp) + if err != nil { + log.Error("Can't extract project: %s", err) + os.Exit(1) + } + file = filepath.Join(tmp, conf) + } + // Load the project: - path, _ := filepath.Abs("project.conf") - log.Info("Loading project file '%s'", path) - project, err := build.LoadProject(path) + log.Info("Loading project file '%s'", file) + project, err := build.LoadProject(file) if err != nil { log.Error("%s", err) return 1 @@ -89,3 +123,61 @@ func run(name string, tool ToolFunc) int { return 0 } } + +func extractData(data []byte, dir string) error { + // Open the data archive: + buffer := bytes.NewReader(data) + expand, err := gzip.NewReader(buffer) + if err != nil { + return err + } + archive := tar.NewReader(expand) + + // Iterate through the entries of the archive and extract them + // to the output directory: + for { + header, err := archive.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + switch header.Typeflag { + case tar.TypeReg: + err = extractFile(archive, header, dir) + case tar.TypeDir: + err = extractDir(archive, header, dir) + default: + err = nil + } + if err != nil { + return err + } + } + + return nil +} + +func extractFile(archive *tar.Reader, header *tar.Header, dir string) error { + // Create the file: + path := filepath.Join(dir, header.Name) + info := header.FileInfo() + log.Debug("Extracting file '%s' to '%s'", header.Name, path) + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode().Perm()) + if err != nil { + return err + } + defer file.Close() + + // Copy the contents: + _, err = io.Copy(file, archive) + return err +} + +func extractDir(archive *tar.Reader, header *tar.Header, dir string) error { + path := filepath.Join(dir, header.Name) + info := header.FileInfo() + log.Debug("Extracting directory '%s' to '%s'", header.Name, path) + return os.Mkdir(path, info.Mode().Perm()) +} diff --git a/tools/src/ovc/scripts/embed.go b/tools/src/ovc/scripts/embed.go new file mode 100644 index 0000000..dc6c3f3 --- /dev/null +++ b/tools/src/ovc/scripts/embed.go @@ -0,0 +1,191 @@ +/* +Copyright (c) 2017 Red Hat, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +// This file is intended to be used as an script. It takes the files +// given in the command line, creates a compresses tarball, and +// generates a Go source file that contains a data variable with the +// contents of that tarball. For example, to following command line will +// generate an embedded.go file that contains the the project.conf file +// and os-manifests directory: +// +// go run embed.go -output embedded.go project.conf os-manifests +// +// The generated embedded.go file can then be added to the project, and +// used to extract the embedded data during run-time. + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "flag" + "fmt" + "io" + "log" + "os" + "path/filepath" +) + +func main() { + // Option for the directory where the script should change to before + // doing anything else: + var directoryFlag string + flag.StringVar( + &directoryFlag, + "directory", + "", + "change to `directory` before doing anything else", + ) + + // Option for the name of the output file: + var outputFlag string + flag.StringVar( + &outputFlag, + "output", + "embedded.go", + "name of the output `file`", + ) + + // Prepare the usage message: + flag.Usage = func() { + fmt.Fprintf( + os.Stderr, + "Usage: %s [OPTIONS] FILE ...\n", + filepath.Base(os.Args[0]), + ) + flag.PrintDefaults() + } + + // Parse the command line: + flag.Parse() + + // Check that there is at least one file to add: + paths := flag.Args() + if len(paths) == 0 { + flag.Usage() + os.Exit(1) + } + + // Change directory: + if directoryFlag != "" { + err := os.Chdir(directoryFlag) + if err != nil { + fmt.Fprintf( + os.Stderr, + "Can't change to directory '%s': %s\n", + directoryFlag, + err, + ) + os.Exit(1) + } + } + + // Create a tar archive writer that will compress and write the + // result to an in-memory buffer: + buffer := new(bytes.Buffer) + compress := gzip.NewWriter(buffer) + archive := tar.NewWriter(compress) + + // Add paths to the tar archive: + for _, path := range paths { + addTree(archive, path) + } + + // Close the tarball and the gzip stream: + if err := archive.Close(); err != nil { + log.Fatalln(err) + } + if err := compress.Close(); err != nil { + log.Fatalln(err) + } + + // Create the output file: + outputFile, err := os.Create(outputFlag) + if err != nil { + fmt.Fprintf( + os.Stderr, + "Can't create output file '%s': %s\n", + outputFlag, + err, + ) + os.Exit(1) + } + + // Generate the Go source code that contains the compressed + // tarball: + fmt.Fprintf(outputFile, "package main\n") + fmt.Fprintf(outputFile, "\n") + fmt.Fprintf(outputFile, "var embedded = []byte{\n") + for _, datum := range buffer.Bytes() { + fmt.Fprintf(outputFile, "\t0x%02x,\n", datum) + } + fmt.Fprintf(outputFile, "}\n") + + // Close the output file: + outputFile.Close() +} + +func addTree(archive *tar.Writer, base string) error { + return filepath.Walk(base, func(path string, info os.FileInfo, err error) error { + // Stop inmediately if something fails when trying to + // walk the directory: + if err != nil { + return err + } + + // Add an entry to the tarball: + switch { + case info.Mode().IsRegular(): + return addFile(archive, path, info) + case info.Mode().IsDir(): + return addDir(archive, path, info) + default: + return nil + } + }) +} + +func addFile(archive *tar.Writer, path string, info os.FileInfo) error { + // Add the header: + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + header.Name = path + err = archive.WriteHeader(header) + if err != nil { + return err + } + + // Add the content: + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(archive, file) + return err +} + +func addDir(archive *tar.Writer, path string, info os.FileInfo) error { + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + header.Name = path + "/" + return archive.WriteHeader(header) +}