✨ Fix some exporter bugs, show realtime command stdout/stderr

exporter_custom.go

@@ -1,6 +1,7 @@
 package ortfodb
 
 import (
+	"bufio"
 	"fmt"
 	"os/exec"
 	"strings"
@@ -55,30 +56,54 @@ func (e *CustomExporter) runCommands(ctx *RunContext, verbose bool, commands []E
 				ExporterLogCustom(e, "Running", "yellow", commandline)
 			}
 
-			var stderrBuf strings.Builder
-			var stdoutBuf strings.Builder
-
 			proc := exec.Command("bash", "-c", commandline)
 			proc.Dir = e.cwd
-			proc.Stderr = &stderrBuf
-			proc.Stdout = &stdoutBuf
-			err := proc.Run()
+			stderr, _ := proc.StderrPipe()
+			stdout, _ := proc.StdoutPipe()
+			err := proc.Start()
+			if err != nil {
+				return fmt.Errorf("while starting command %q: %w", commandline, err)
+			}
 
-			stdout := strings.TrimSpace(stdoutBuf.String())
-			stderr := strings.TrimSpace(stderrBuf.String())
+			outputBuffer := new(strings.Builder)
+			outputChannel := make(chan string)
 
-			if stdout != "" {
-				ExporterLogCustom(e, ">", "blue", stdout)
-			}
-			if stderr != "" {
-				if !verbose {
-					ExporterLogCustom(e, "Error", "red", "While running %s\n%s", commandline, stderr)
-				} else {
-					ExporterLogCustom(e, "!", "red", stderr)
+			// Goroutine to read from stdout and send lines to the output channel
+			go func() {
+				scanner := bufio.NewScanner(stdout)
+				for scanner.Scan() {
+					outputChannel <- scanner.Text()
 				}
-			}
+			}()
 
-			if err != nil {
+			// Goroutine to read from stderr and send lines to the output channel
+			go func() {
+				scanner := bufio.NewScanner(stderr)
+				for scanner.Scan() {
+					outputChannel <- scanner.Text()
+				}
+			}()
+
+			linesPrinterCount := 0
+
+			go func() {
+				for line := range outputChannel {
+					if linesPrinterCount > 5 {
+						// Clear the line fives lines after the first output
+						fmt.Print("\033[5A\033[K")
+					}
+					outputBuffer.WriteString(line + "\n")
+					ExporterLogCustomNoFormatting(e, ">", "blue", line)
+					if linesPrinterCount > 5 {
+						// Go back to last line
+						fmt.Print("\033[5B")
+					}
+					linesPrinterCount++
+				}
+			}()
+
+			if err = proc.Wait(); err != nil {
+				ExporterLogCustomNoFormatting(e, "Error", "red", fmt.Sprintf("While running %s\n%s", commandline, outputBuffer.String()))
 				return fmt.Errorf("while running %s: %w", commandline, err)
 			}
 		} else {

exporters.go

@@ -86,25 +86,32 @@ func (ctx *RunContext) FindExporter(name string) (Exporter, error) {
 	}
 	for _, builtinName := range BuiltinYAMLExporters {
 		if builtinName == name {
-			return DownloadExporter(name, fmt.Sprintf("https://raw.githubusercontent.com/ortfo/db/main/exporters/%s.yaml", name), ctx.Config.Exporters[name])
+			return ctx.DownloadExporter(name, fmt.Sprintf("https://raw.githubusercontent.com/ortfo/db/main/exporters/%s.yaml", name), ctx.Config.Exporters[name])
 		}
 	}
-	if strings.HasPrefix(name, "./") {
-		rawManifest, err := readFile(filepath.Join(filepath.Dir(ctx.Flags.Config), name))
+	if strings.HasPrefix(name, "./") || strings.HasPrefix(name, "/") {
+		var manifestPath string
+		if filepath.IsAbs(name) {
+			manifestPath = name
+		} else {
+			manifestPath = filepath.Join(filepath.Dir(ctx.Flags.Config), name)
+		}
+
+		rawManifest, err := readFile(manifestPath)
 		if err != nil {
 			return &CustomExporter{}, fmt.Errorf("while reading local manifest file at %s: %w", name, err)
 		}
-		return LoadExporter(name, rawManifest, ctx.Config.Exporters[name])
+		return ctx.LoadExporter(name, rawManifest, ctx.Config.Exporters[name])
 	} else if isValidURL(ensureHttpPrefix(name)) {
 		url := ensureHttpPrefix(name)
 		ctx.LogDebug("No builtin exporter named %s, attempting download since %s looks like an URL…", name, url)
-		return DownloadExporter(name, url, ctx.Config.Exporters[name])
+		return ctx.DownloadExporter(name, url, ctx.Config.Exporters[name])
 	}
 	return nil, fmt.Errorf("no exporter named %s", name)
 }
 
 // LoadExporter loads an exporter from a manifest YAML file's contents.
-func LoadExporter(name string, manifestRaw string, config map[string]any) (*CustomExporter, error) {
+func (ctx *RunContext) LoadExporter(name string, manifestRaw string, config map[string]any) (*CustomExporter, error) {
 	var manifest ExporterManifest
 	err := yaml.Unmarshal([]byte(manifestRaw), &manifest)
 	if err != nil {
@@ -125,7 +132,7 @@ func LoadExporter(name string, manifestRaw string, config map[string]any) (*Cust
 
 	cwd := "."
 	if !isValidURL(name) && fileExists(name) {
-		cwd = filepath.Dir(name)
+		cwd = filepath.Dir(ctx.Config.source)
 	}
 
 	exporter := CustomExporter{
@@ -140,14 +147,14 @@ func LoadExporter(name string, manifestRaw string, config map[string]any) (*Cust
 }
 
 // DownloadExporter loads an exporter from a URL.
-func DownloadExporter(name string, url string, config map[string]any) (*CustomExporter, error) {
+func (ctx *RunContext) DownloadExporter(name string, url string, config map[string]any) (*CustomExporter, error) {
 	LogCustom("Installing", "cyan", "exporter at %s", url)
 	manifestRaw, err := downloadFile(url)
 	if err != nil {
 		return &CustomExporter{}, fmt.Errorf("while downloading exporter manifest file: %w", err)
 	}
 
-	exporter, err := LoadExporter(name, manifestRaw, config)
+	exporter, err := ctx.LoadExporter(name, manifestRaw, config)
 	if err != nil {
 		return &CustomExporter{}, err
 	}

ui.go

@@ -32,8 +32,25 @@ func ExporterLogCustom(exporter Exporter, verb string, color string, message str
 	}
 }
 
+func ExporterLogCustomNoFormatting(exporter Exporter, verb string, color string, message string) {
+	if os.Getenv("DEBUG") == "1" {
+		LogCustomNoFormatting(verb, color, colorstring.Color("[dim][bold](from exporter "+exporter.Name()+")[reset] ")+message)
+	} else {
+		LogCustomNoFormatting(verb, color, message)
+	}
+}
+
 func LogCustom(verb string, color string, message string, fmtArgs ...interface{}) {
-	fmt.Fprintln(logWriter(), colorstring.Color(fmt.Sprintf("[bold][%s]%15s[reset] %s", color, verb, indentSubsequent(15+1, fmt.Sprintf(message, fmtArgs...)))))
+	LogCustomNoFormatting(verb, color, colorstring.Color(fmt.Sprintf(message, fmtArgs...)))
+}
+
+func LogCustomNoFormatting(verb string, color string, message string) {
+	fmt.Fprintln(
+		logWriter(),
+		colorstring.Color(fmt.Sprintf("[bold][%s]%15s[reset]", color, verb))+
+			" "+
+			indentSubsequent(15+1, message),
+	)
 }
 
 // DisplayValidationErrors takes in a slice of json schema validation errors and displays them nicely to in the terminal.