diff --git a/models/endings_linux.go b/models/endings_linux.go new file mode 100644 index 0000000..f430509 --- /dev/null +++ b/models/endings_linux.go @@ -0,0 +1,6 @@ +package models + +// NormalizeEndingsNative normalize file line endings to the current OS's endings +func NormalizeEndingsNative(content []byte) ([]byte, error) { + return NormalizeEndingsUnix(content) +} diff --git a/models/endings_windows.go b/models/endings_windows.go new file mode 100644 index 0000000..5f6b041 --- /dev/null +++ b/models/endings_windows.go @@ -0,0 +1,6 @@ +package models + +// NormalizeEndingsNative normalize file line endings to the current OS's endings +func NormalizeEndingsNative(content []byte) ([]byte, error) { + return NormalizeEndingsWindows(content) +} diff --git a/models/files.go b/models/files.go index 71dea12..3b30da7 100644 --- a/models/files.go +++ b/models/files.go @@ -1,8 +1,10 @@ package models import ( + "bytes" "fmt" "path/filepath" + "runtime" "strings" "github.com/natsukagami/kjudge/db" @@ -10,6 +12,47 @@ import ( "github.com/pkg/errors" ) +// NormalizeEndingsUnix normalize file line endings to LF +func NormalizeEndingsUnix(content []byte) ([]byte, error) { + return bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n")), nil +} + +// NormalizeEndingsWindows normalize file line endings to CRLF +// and throws if there is LF and CRLF mixed together +func NormalizeEndingsWindows(content []byte) ([]byte, error) { + lf := bytes.Count(content, []byte("\n")) + crlf := bytes.Count(content, []byte("\r\n")) + if crlf == lf { + return content, nil + } + var err error = nil + if crlf != 0 { + err = errors.Errorf("number of crlf and lf (%v, %v) does not match", crlf, lf) + } + return bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n")), err +} + +// NormalizeEndings normalize file line endings to the target OS's endings +// target accepts "windows" or "linux". Returns error if OS is not supported +// or there is LF and CRLF mixed together +func NormalizeEndings(content []byte, target string) ([]byte, error) { + switch target { + case "windows": + return NormalizeEndingsWindows(content) + case "linux": + return NormalizeEndingsUnix(content) + default: + return nil, errors.Errorf("%s not supported for line ending conversion", runtime.GOOS) + } +} + +// IsTextFile applies heuristics to determine +// whether specified filename is a text file +func IsTextFile(filename string) bool { + ext := filepath.Ext(filename) + return !(ext == "" || ext == "exe" || ext == "pdf" || ext == "zip") +} + // GetFileWithName returns a file with a given name. func GetFileWithName(db db.DBContext, problemID int, filename string) (*File, error) { var f File diff --git a/models/generate/main.go b/models/generate/main.go index 81ff7ae..33fd352 100644 --- a/models/generate/main.go +++ b/models/generate/main.go @@ -26,6 +26,9 @@ type TomlTables map[string]TomlTable // SnakeToGocase translates snake case to go-case. // If export is true, the returned value has the first character in uppercase. func SnakeToGocase(s string, export bool) string { + s, explicit_private := RipPrivate(s) + export = export && !explicit_private + parts := strings.Split(s, "_") result := strings.Builder{} for i, part := range parts { @@ -42,6 +45,14 @@ func SnakeToGocase(s string, export bool) string { return result.String() } +// RipPrivate removes the "__" prefix from a string. +func RipPrivate(s string) (string, bool) { + if strings.HasPrefix(s, "__") { + return s[len("__"):], true + } + return s, false +} + var t = template.New("main") func init() { @@ -54,6 +65,7 @@ func init() { "args": JoinArguments, "marks": Marks, "fkey": ForeignKey, + "rppriv": func(s string) string { s, _ = RipPrivate(s); return s }, }) } @@ -80,6 +92,7 @@ func JoinCondition(keys map[string]string, sep string) string { s.WriteString(sep) } first = false + key, _ = RipPrivate(key) s.WriteString(key + " = ?") } return s.String() @@ -99,10 +112,13 @@ func sortKeys(k map[string]string) []string { func JoinArguments(keys map[string]string, structName string) string { var s []string for _, key := range sortKeys(keys) { + key, private := RipPrivate(key) if structName == "" { s = append(s, SnakeToGocase(key, false)) } else if structName == "-" { s = append(s, key) + } else if private { + s = append(s, structName+"."+SnakeToGocase(key, true)+"()") } else { s = append(s, structName+"."+SnakeToGocase(key, true)) } @@ -196,7 +212,7 @@ const TableTemplate = ` // {{$name}} is the struct generated from table "{{.Name}}". type {{$name}} struct { {{- range $field, $type := .Fields}} - {{$field | field}} {{$type}} {{$tick}}db:"{{$field}}"{{$tick}} + {{$field | field}} {{$type}} {{$tick}}db:"{{$field | rppriv}}"{{$tick}} {{- end}} } diff --git a/models/models.toml b/models/models.toml index 0b8a8ae..bed861c 100644 --- a/models/models.toml +++ b/models/models.toml @@ -90,7 +90,7 @@ _order_by = "priority DESC, id ASC" id = "int" problem_id = "int" filename = "string" -content = "[]byte" +__content = "[]byte" public = "bool" [announcements] diff --git a/server/admin/problem.go b/server/admin/problem.go index c9ef2a6..ca21a39 100644 --- a/server/admin/problem.go +++ b/server/admin/problem.go @@ -203,6 +203,14 @@ func (g *Group) ProblemAddFile(c echo.Context) error { if rename != "" && len(files) == 1 { files[0].Filename = rename } + for _, file := range files { + if models.IsTextFile(file.Filename) { + file.Content, err = models.NormalizeEndingsUnix(file.Content) + if err != nil { + return err + } + } + } if err := ctx.Problem.WriteFiles(g.db, files); err != nil { return httperr.BadRequestf("cannot write files: %v", err) } diff --git a/server/admin/test_groups.go b/server/admin/test_groups.go index 1e4c25b..edcb385 100644 --- a/server/admin/test_groups.go +++ b/server/admin/test_groups.go @@ -105,10 +105,18 @@ func (g *Group) TestGroupUploadSingle(c echo.Context) error { if err != nil { return err } + input, err = models.NormalizeEndingsUnix(input) + if err != nil { + return err + } output, err := readFromForm("output", mp) if err != nil { return err } + output, err = models.NormalizeEndingsUnix(output) + if err != nil { + return err + } // Make the test test := &models.Test{ TestGroupID: tg.ID, @@ -150,7 +158,7 @@ func (g *Group) TestGroupUploadMultiple(c echo.Context) error { if err != nil { return httperr.BadRequestf("cannot unpack tests: %v", err) } - if err := tg.WriteTests(tx, tests, override); err != nil { + if err := tg.WriteTestsNormalized(tx, tests, override); err != nil { return httperr.BadRequestf("Cannot write tests: %v", err) } if err := tx.Commit(); err != nil { @@ -241,10 +249,10 @@ func (g *Group) TestGroupRejudgePost(c echo.Context) error { return c.Redirect(http.StatusSeeOther, fmt.Sprintf("/admin/problems/%d/submissions", tg.ProblemID)) } -// WriteTests writes the given set of tests into the Database. -// If override is set, all tests in the test group gets deleted first. -// The LazyTests are STILL invalid models.Tests. DO NOT USE. -func (r *TestGroupCtx) WriteTests(db db.DBContext, tests []*tests.LazyTest, override bool) error { +// WriteTestsNormalized normalizes line endings and writes the given set of +// tests into the Database. If override is set, all tests in the test group +// gets deleted first. The LazyTests are STILL invalid models.Tests. DO NOT USE. +func (r *TestGroupCtx) WriteTestsNormalized(db db.DBContext, tests []*tests.LazyTest, override bool) error { for _, test := range tests { test.TestGroupID = r.ID if err := test.Verify(); err != nil { @@ -261,10 +269,18 @@ func (r *TestGroupCtx) WriteTests(db db.DBContext, tests []*tests.LazyTest, over if err != nil { return errors.Wrapf(err, "test %v input", test.Name) } + input, err = models.NormalizeEndingsUnix(input) + if err != nil { + return errors.Wrapf(err, "test %v input", test.Name) + } output, err := readZip(test.Output) if err != nil { return errors.Wrapf(err, "test %v output", test.Name) } + output, err = models.NormalizeEndingsUnix(output) + if err != nil { + return errors.Wrapf(err, "test %v output", test.Name) + } if _, err := db.Exec( "INSERT INTO tests(name, test_group_id, input, output) VALUES (?, ?, ?, ?)", test.Name, diff --git a/server/contests/problem.go b/server/contests/problem.go index 7092f0e..477bd86 100644 --- a/server/contests/problem.go +++ b/server/contests/problem.go @@ -158,6 +158,14 @@ func (g *Group) SubmitPost(c echo.Context) error { if err != nil { return errors.WithStack(err) } + + // Submitted files can be executable + if models.IsTextFile(file.Filename) { + source, err = models.NormalizeEndingsUnix(source) + if err != nil { + return err + } + } sub := models.Submission{ ProblemID: ctx.Problem.ID, UserID: ctx.Me.ID,