diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3691d9c..a00af99 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,11 +33,11 @@ jobs: steps: - uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: 1.21 - uses: actions/checkout@v3 - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version - version: v1.50.1 - args: --timeout 2m --disable-all -E gofmt + version: v1.54 + # args: --timeout 2m diff --git a/.golangci.yml b/.golangci.yml index d9a5733..0bbd141 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -23,6 +23,5 @@ linters: - goimports - misspell - typecheck - - golint - gosimple - govet \ No newline at end of file diff --git a/cmd/graphplan/main.go b/cmd/graphplan/main.go new file mode 100644 index 0000000..3b8843e --- /dev/null +++ b/cmd/graphplan/main.go @@ -0,0 +1,59 @@ +package graphplan + +import ( + "log" + "path/filepath" + + "github.com/bmeg/sifter/graphplan" + "github.com/bmeg/sifter/playbook" + "github.com/spf13/cobra" +) + +var outScriptDir = "" +var outDataDir = "./" + +// Cmd is the declaration of the command line +var Cmd = &cobra.Command{ + Use: "graph-plan", + Short: "Scan directory to plan operations", + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + + scriptPath, _ := filepath.Abs(args[0]) + + /* + if outScriptDir != "" { + baseDir, _ = filepath.Abs(outScriptDir) + } else if len(args) > 1 { + return fmt.Errorf("for multiple input directories, based dir must be defined") + } + + _ = baseDir + */ + outScriptDir, _ = filepath.Abs(outScriptDir) + outDataDir, _ = filepath.Abs(outDataDir) + + outDataDir, _ = filepath.Rel(outScriptDir, outDataDir) + + pb := playbook.Playbook{} + + if sifterErr := playbook.ParseFile(scriptPath, &pb); sifterErr == nil { + if len(pb.Pipelines) > 0 || len(pb.Inputs) > 0 { + err := graphplan.NewGraphBuild( + &pb, outScriptDir, outDataDir, + ) + if err != nil { + log.Printf("Error: %s\n", err) + } + } + } + + return nil + }, +} + +func init() { + flags := Cmd.Flags() + flags.StringVarP(&outScriptDir, "dir", "C", outScriptDir, "Change Directory for script base") + flags.StringVarP(&outDataDir, "out", "o", outDataDir, "Change output Directory") +} diff --git a/cmd/root.go b/cmd/root.go index d23161c..2e889eb 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -3,8 +3,10 @@ package cmd import ( "os" + "github.com/bmeg/sifter/cmd/graphplan" "github.com/bmeg/sifter/cmd/inspect" "github.com/bmeg/sifter/cmd/run" + "github.com/bmeg/sifter/cmd/scan" "github.com/spf13/cobra" ) @@ -18,6 +20,8 @@ var RootCmd = &cobra.Command{ func init() { RootCmd.AddCommand(run.Cmd) RootCmd.AddCommand(inspect.Cmd) + RootCmd.AddCommand(graphplan.Cmd) + RootCmd.AddCommand(scan.Cmd) } var genBashCompletionCmd = &cobra.Command{ diff --git a/cmd/run/run.go b/cmd/run/run.go index 574fdb5..07ad0b9 100644 --- a/cmd/run/run.go +++ b/cmd/run/run.go @@ -21,7 +21,7 @@ func ExecuteFile(playFile string, workDir string, outDir string, inputs map[stri a, _ := filepath.Abs(playFile) baseDir := filepath.Dir(a) log.Printf("basedir: %s", baseDir) - log.Printf("playbook: %s", pb) + log.Printf("playbook: %#v", pb) return Execute(pb, baseDir, workDir, outDir, inputs) } diff --git a/cmd/scan/main.go b/cmd/scan/main.go new file mode 100644 index 0000000..f1fbe48 --- /dev/null +++ b/cmd/scan/main.go @@ -0,0 +1,217 @@ +package scan + +import ( + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/bmeg/sifter/playbook" + "github.com/bmeg/sifter/task" + "github.com/spf13/cobra" +) + +var jsonOut = false +var objectsOnly = false +var baseDir = "" + +type Entry struct { + ObjectType string `json:"objectType"` + SifterFile string `json:"sifterFile"` + Outfile string `json:"outFile"` +} + +var ObjectCommand = &cobra.Command{ + Use: "objects", + Short: "Scan for outputs", + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + + scanDir := args[0] + + outputs := []Entry{} + + PathWalker(scanDir, func(pb *playbook.Playbook) { + for pname, p := range pb.Pipelines { + emitName := "" + for _, s := range p { + if s.Emit != nil { + emitName = s.Emit.Name + } + } + if emitName != "" { + for _, s := range p { + outdir := pb.GetDefaultOutDir() + outname := fmt.Sprintf("%s.%s.%s.json.gz", pb.Name, pname, emitName) + outpath := filepath.Join(outdir, outname) + o := Entry{SifterFile: pb.GetPath(), Outfile: outpath} + if s.ObjectValidate != nil { + //outpath, _ = filepath.Rel(baseDir, outpath) + //fmt.Printf("%s\t%s\n", s.ObjectValidate.Title, outpath) + o.ObjectType = s.ObjectValidate.Title + } + if objectsOnly { + if o.ObjectType != "" { + outputs = append(outputs, o) + } + } else { + outputs = append(outputs, o) + } + } + } + } + }) + + if jsonOut { + j := json.NewEncoder(os.Stdout) + j.SetIndent("", " ") + j.Encode(outputs) + } else { + for _, i := range outputs { + fmt.Printf("%s\t%s\n", i.ObjectType, i.Outfile) + } + } + + return nil + + }, +} + +type ScriptEntry struct { + Name string `json:"name"` + Path string `json:"path"` + Inputs []string `json:"inputs"` + Outputs []string `json:"outputs"` +} + +func removeDuplicates(s []string) []string { + t := map[string]bool{} + + for _, i := range s { + t[i] = true + } + out := []string{} + for k := range t { + out = append(out, k) + } + return out +} + +func relPathArray(basedir string, paths []string) []string { + out := []string{} + for _, i := range paths { + if o, err := filepath.Rel(baseDir, i); err == nil { + out = append(out, o) + } + } + return out +} + +var ScriptCommand = &cobra.Command{ + Use: "scripts", + Short: "Scan for scripts", + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + + scanDir := args[0] + + scripts := []ScriptEntry{} + + if baseDir == "" { + baseDir, _ = os.Getwd() + } + baseDir, _ = filepath.Abs(baseDir) + //fmt.Printf("basedir: %s\n", baseDir) + + userInputs := map[string]string{} + + PathWalker(scanDir, func(pb *playbook.Playbook) { + path := pb.GetPath() + scriptDir := filepath.Dir(path) + + config, _ := pb.PrepConfig(userInputs, baseDir) + + task := task.NewTask(pb.Name, scriptDir, baseDir, pb.GetDefaultOutDir(), config) + sourcePath, _ := filepath.Abs(path) + + cmdPath, _ := filepath.Rel(baseDir, sourcePath) + + inputs := []string{} + outputs := []string{} + for _, p := range pb.GetConfigFields() { + if p.IsDir() || p.IsFile() { + inputs = append(inputs, config[p.Name]) + } + } + //inputs = append(inputs, sourcePath) + + sinks, _ := pb.GetOutputs(task) + for _, v := range sinks { + outputs = append(outputs, v...) + } + + emitters, _ := pb.GetEmitters(task) + for _, v := range emitters { + outputs = append(outputs, v) + } + + //for _, e := range pb.Inputs { + //} + + s := ScriptEntry{ + Path: cmdPath, + Name: pb.Name, + Outputs: relPathArray(baseDir, removeDuplicates(outputs)), + Inputs: relPathArray(baseDir, removeDuplicates(inputs)), + } + scripts = append(scripts, s) + }) + + if jsonOut { + e := json.NewEncoder(os.Stdout) + e.SetIndent("", " ") + e.Encode(scripts) + } else { + for _, i := range scripts { + fmt.Printf("%s\n", i) + } + } + + return nil + }, +} + +// Cmd is the declaration of the command line +var Cmd = &cobra.Command{ + Use: "scan", + Short: "Scan for scripts or objects", +} + +func init() { + Cmd.AddCommand(ObjectCommand) + Cmd.AddCommand(ScriptCommand) + + objFlags := ObjectCommand.Flags() + objFlags.BoolVarP(&objectsOnly, "objects", "s", objectsOnly, "Objects Only") + objFlags.BoolVarP(&jsonOut, "json", "j", jsonOut, "Output JSON") + + scriptFlags := ScriptCommand.Flags() + scriptFlags.StringVarP(&baseDir, "base", "b", baseDir, "Base Dir") + scriptFlags.BoolVarP(&jsonOut, "json", "j", jsonOut, "Output JSON") + +} + +func PathWalker(baseDir string, userFunc func(*playbook.Playbook)) { + filepath.Walk(baseDir, + func(path string, info fs.FileInfo, err error) error { + if strings.HasSuffix(path, ".yaml") { + pb := playbook.Playbook{} + if parseErr := playbook.ParseFile(path, &pb); parseErr == nil { + userFunc(&pb) + } + } + return nil + }) +} diff --git a/graphplan/build_template.go b/graphplan/build_template.go new file mode 100644 index 0000000..5c5e2a1 --- /dev/null +++ b/graphplan/build_template.go @@ -0,0 +1,137 @@ +package graphplan + +import ( + "fmt" + "log" + "os" + "path/filepath" + "text/template" + + "github.com/bmeg/sifter/evaluate" + "github.com/bmeg/sifter/playbook" + "github.com/bmeg/sifter/task" +) + +type ObjectConvertStep struct { + Name string + Input string + Class string + Schema string +} + +type GraphBuildStep struct { + Name string + Outdir string + Objects []ObjectConvertStep +} + +var graphScript string = ` + +name: {{.Name}} +class: sifter + +outdir: {{.Outdir}} + +config: +{{range .Objects}} + {{.Name}}: {{.Input}} + {{.Name}}Schema: {{.Schema}} +{{end}} + +inputs: +{{range .Objects}} + {{.Name}}: + jsonLoad: + input: "{{ "{{config." }}{{.Name}}{{"}}"}}" +{{end}} + +pipelines: +{{range .Objects}} + {{.Name}}-graph: + - from: {{.Name}} + - graphBuild: + schema: "{{ "{{config."}}{{.Name}}Schema{{ "}}" }}" + title: {{.Class}} +{{end}} +` + +func contains(n string, c []string) bool { + for _, c := range c { + if n == c { + return true + } + } + return false +} + +func uniqueName(name string, used []string) string { + if !contains(name, used) { + return name + } + for i := 1; ; i++ { + f := fmt.Sprintf("%s_%d", name, i) + if !contains(f, used) { + return f + } + } +} + +func NewGraphBuild(pb *playbook.Playbook, scriptOutDir, dataDir string) error { + userInputs := map[string]string{} + localInputs, _ := pb.PrepConfig(userInputs, filepath.Dir(pb.GetPath())) + + task := task.NewTask(pb.Name, filepath.Dir(pb.GetPath()), filepath.Dir(pb.GetPath()), pb.GetDefaultOutDir(), localInputs) + + convertName := fmt.Sprintf("%s-graph", pb.Name) + + gb := GraphBuildStep{Name: convertName, Objects: []ObjectConvertStep{}, Outdir: dataDir} + + for pname, p := range pb.Pipelines { + emitName := "" + for _, s := range p { + if s.Emit != nil { + emitName = s.Emit.Name + } + } + if emitName != "" { + for _, s := range p { + if s.ObjectValidate != nil { + schema, _ := evaluate.ExpressionString(s.ObjectValidate.Schema, task.GetConfig(), map[string]any{}) + outdir := pb.GetDefaultOutDir() + outname := fmt.Sprintf("%s.%s.%s.json.gz", pb.Name, pname, emitName) + + outpath := filepath.Join(outdir, outname) + outpath, _ = filepath.Rel(scriptOutDir, outpath) + + schemaPath, _ := filepath.Rel(scriptOutDir, schema) + + _ = schemaPath + + objCreate := ObjectConvertStep{Name: pname, Input: outpath, Class: s.ObjectValidate.Title, Schema: schemaPath} + gb.Objects = append(gb.Objects, objCreate) + + } + } + } + } + + if len(gb.Objects) > 0 { + log.Printf("Found %d objects", len(gb.Objects)) + tmpl, err := template.New("graphscript").Parse(graphScript) + if err != nil { + panic(err) + } + + outfile, err := os.Create(filepath.Join(scriptOutDir, fmt.Sprintf("%s.yaml", pb.Name))) + if err != nil { + fmt.Printf("Error: %s\n", err) + } + + err = tmpl.Execute(outfile, gb) + outfile.Close() + if err != nil { + fmt.Printf("Error: %s\n", err) + } + } + return nil +} diff --git a/task/task.go b/task/task.go index da33da2..721aeb4 100644 --- a/task/task.go +++ b/task/task.go @@ -107,10 +107,10 @@ func (m *Task) BaseDir() string { func (m *Task) Emit(n string, e map[string]interface{}, useName bool) error { - new_name := m.GetName() + "." + n + newName := m.GetName() + "." + n if useName { temp := strings.Split(n, ".") - new_name = temp[len(temp)-1] + newName = temp[len(temp)-1] } - return m.Emitter.Emit(new_name, e, useName) + return m.Emitter.Emit(newName, e, useName) } diff --git a/transform/object_validate.go b/transform/object_validate.go index 28ee452..08838cc 100644 --- a/transform/object_validate.go +++ b/transform/object_validate.go @@ -10,7 +10,7 @@ import ( "github.com/bmeg/sifter/evaluate" "github.com/bmeg/sifter/task" "github.com/santhosh-tekuri/jsonschema/v5" - _ "github.com/santhosh-tekuri/jsonschema/v5/httploader" + _ "github.com/santhosh-tekuri/jsonschema/v5/httploader" // setup the httploader for the jsonschema checker ) type ObjectValidateStep struct {