From 6d2e9f5f8f9837c00940641fa137a359a9dd01e2 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 4 Jun 2023 10:27:09 +0700 Subject: [PATCH 01/12] Functions to normalize endings --- models/files.go | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/models/files.go b/models/files.go index 71dea12c..9214e76e 100644 --- a/models/files.go +++ b/models/files.go @@ -1,8 +1,10 @@ package models import ( + "bytes" "fmt" "path/filepath" + "runtime" "strings" "github.com/natsukagami/kjudge/db" @@ -10,6 +12,41 @@ import ( "github.com/pkg/errors" ) +func crlftoLF(content []byte) ([]byte, error) { + return bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n")), nil +} + +func lftoCRLF(content []byte) ([]byte, error) { + lf := bytes.Count(content, []byte("\n")) + crlf := bytes.Count(content, []byte("\r\n")) + if crlf == 0 { + return bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n")), nil + } + if crlf == lf { + return content, nil + } + return nil, errors.Errorf("number of crlf and lf (%v, %v) does not match", crlf, lf) +} + +// NormalizeEndings normalize file line endings to the target OS's endings +// target accepts "windows" or "linux" +func NormalizeEndings(content []byte, target string) ([]byte, error) { + switch (target){ + case "windows": + return lftoCRLF(content) + case "linux": + return crlftoLF(content) + default: + return nil, errors.Errorf("%s not supported for line ending conversion", runtime.GOOS) + } +} + +// NormalizeEndings normalize file line endings to the current OS's endings +// target accepts "windows" or "linux" +func NormalizeEndingsNative(content []byte) ([]byte, error) { + return NormalizeEndings(content, runtime.GOOS) +} + // GetFileWithName returns a file with a given name. func GetFileWithName(db db.DBContext, problemID int, filename string) (*File, error) { var f File From bf607a45d77a0b7e1718093dccdfb1d0f0a69f3a Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 4 Jun 2023 10:40:22 +0700 Subject: [PATCH 02/12] IsTextFile --- models/files.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/models/files.go b/models/files.go index 9214e76e..806a412d 100644 --- a/models/files.go +++ b/models/files.go @@ -47,6 +47,13 @@ func NormalizeEndingsNative(content []byte) ([]byte, error) { return NormalizeEndings(content, runtime.GOOS) } +// IsTextFile applies heuristics to determine +// whether specified filename is a text file +func IsTextFile(filename string) bool { + ext := filepath.Ext(filename) + return !(ext == "" || ext == "exe" || ext == "pdf") +} + // GetFileWithName returns a file with a given name. func GetFileWithName(db db.DBContext, problemID int, filename string) (*File, error) { var f File From 3461adba88ae2ed80a29bcfd15c95d659aabe258 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 4 Jun 2023 10:48:37 +0700 Subject: [PATCH 03/12] Tries to convert to LF even if file endings are mixed --- models/files.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/models/files.go b/models/files.go index 806a412d..a57576d2 100644 --- a/models/files.go +++ b/models/files.go @@ -19,17 +19,19 @@ func crlftoLF(content []byte) ([]byte, error) { func lftoCRLF(content []byte) ([]byte, error) { lf := bytes.Count(content, []byte("\n")) crlf := bytes.Count(content, []byte("\r\n")) - if crlf == 0 { - return bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n")), nil - } if crlf == lf { return content, nil } - return nil, errors.Errorf("number of crlf and lf (%v, %v) does not match", crlf, lf) + var err error = nil + if crlf != 0 { + err = errors.Errorf("number of crlf and lf (%v, %v) does not match", crlf, lf) + } + return bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n")), err } // NormalizeEndings normalize file line endings to the target OS's endings -// target accepts "windows" or "linux" +// target accepts "windows" or "linux". Returns error if OS is not supported +// or there is LF and CRLF mixed together func NormalizeEndings(content []byte, target string) ([]byte, error) { switch (target){ case "windows": From bac1aa82b39514a7bbacc8fbddee4fce316c047a Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 4 Jun 2023 10:55:09 +0700 Subject: [PATCH 04/12] Normalize problem files --- server/admin/problem.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/server/admin/problem.go b/server/admin/problem.go index c9ef2a64..807ad137 100644 --- a/server/admin/problem.go +++ b/server/admin/problem.go @@ -4,6 +4,7 @@ import ( "database/sql" "fmt" "io" + "log" "net/http" "strconv" @@ -203,6 +204,14 @@ func (g *Group) ProblemAddFile(c echo.Context) error { if rename != "" && len(files) == 1 { files[0].Filename = rename } + for _, file := range files { + if models.IsTextFile(file.Filename) { + file.Content, err = models.NormalizeEndingsNative(file.Content) + if err != nil { + log.Printf("%v while processing %v", err, file.Filename) + } + } + } if err := ctx.Problem.WriteFiles(g.db, files); err != nil { return httperr.BadRequestf("cannot write files: %v", err) } From 9e7ae7a8fc69a4cfe885e04cb838dda31f0af0a3 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 4 Jun 2023 11:27:06 +0700 Subject: [PATCH 05/12] Normalize group test --- server/admin/problem.go | 3 +-- server/admin/test_groups.go | 26 +++++++++++++++++++++----- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/server/admin/problem.go b/server/admin/problem.go index 807ad137..05b88622 100644 --- a/server/admin/problem.go +++ b/server/admin/problem.go @@ -4,7 +4,6 @@ import ( "database/sql" "fmt" "io" - "log" "net/http" "strconv" @@ -208,7 +207,7 @@ func (g *Group) ProblemAddFile(c echo.Context) error { if models.IsTextFile(file.Filename) { file.Content, err = models.NormalizeEndingsNative(file.Content) if err != nil { - log.Printf("%v while processing %v", err, file.Filename) + return err } } } diff --git a/server/admin/test_groups.go b/server/admin/test_groups.go index 1e4c25b4..03dae226 100644 --- a/server/admin/test_groups.go +++ b/server/admin/test_groups.go @@ -105,10 +105,18 @@ func (g *Group) TestGroupUploadSingle(c echo.Context) error { if err != nil { return err } + input, err = models.NormalizeEndingsNative(input) + if err != nil { + return err + } output, err := readFromForm("output", mp) if err != nil { return err } + output, err = models.NormalizeEndingsNative(output) + if err != nil { + return err + } // Make the test test := &models.Test{ TestGroupID: tg.ID, @@ -150,7 +158,7 @@ func (g *Group) TestGroupUploadMultiple(c echo.Context) error { if err != nil { return httperr.BadRequestf("cannot unpack tests: %v", err) } - if err := tg.WriteTests(tx, tests, override); err != nil { + if err := tg.WriteTestsNormalized(tx, tests, override); err != nil { return httperr.BadRequestf("Cannot write tests: %v", err) } if err := tx.Commit(); err != nil { @@ -241,10 +249,10 @@ func (g *Group) TestGroupRejudgePost(c echo.Context) error { return c.Redirect(http.StatusSeeOther, fmt.Sprintf("/admin/problems/%d/submissions", tg.ProblemID)) } -// WriteTests writes the given set of tests into the Database. -// If override is set, all tests in the test group gets deleted first. -// The LazyTests are STILL invalid models.Tests. DO NOT USE. -func (r *TestGroupCtx) WriteTests(db db.DBContext, tests []*tests.LazyTest, override bool) error { +// WriteTestsNormalized normalizes line endings and writes the given set of +// tests into the Database. If override is set, all tests in the test group +// gets deleted first. The LazyTests are STILL invalid models.Tests. DO NOT USE. +func (r *TestGroupCtx) WriteTestsNormalized(db db.DBContext, tests []*tests.LazyTest, override bool) error { for _, test := range tests { test.TestGroupID = r.ID if err := test.Verify(); err != nil { @@ -261,10 +269,18 @@ func (r *TestGroupCtx) WriteTests(db db.DBContext, tests []*tests.LazyTest, over if err != nil { return errors.Wrapf(err, "test %v input", test.Name) } + input, err = models.NormalizeEndingsNative(input) + if err != nil { + return errors.Wrapf(err, "test %v input", test.Name) + } output, err := readZip(test.Output) if err != nil { return errors.Wrapf(err, "test %v output", test.Name) } + output, err = models.NormalizeEndingsNative(output) + if err != nil { + return errors.Wrapf(err, "test %v output", test.Name) + } if _, err := db.Exec( "INSERT INTO tests(name, test_group_id, input, output) VALUES (?, ?, ?, ?)", test.Name, From 6ca209847d87a69ec5de5c9b888350188bb3c9fe Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 4 Jun 2023 11:32:15 +0700 Subject: [PATCH 06/12] Normalize submissions --- server/contests/problem.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/contests/problem.go b/server/contests/problem.go index 7092f0e4..b69ea95a 100644 --- a/server/contests/problem.go +++ b/server/contests/problem.go @@ -158,6 +158,14 @@ func (g *Group) SubmitPost(c echo.Context) error { if err != nil { return errors.WithStack(err) } + + // Submitted files can be executable + if models.IsTextFile(file.Filename){ + source, err = models.NormalizeEndingsNative(source) + if err != nil { + return err + } + } sub := models.Submission{ ProblemID: ctx.Problem.ID, UserID: ctx.Me.ID, From 9caa71744e24cc0ff04d280e6ac29da2beb550a7 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 4 Jun 2023 11:45:58 +0700 Subject: [PATCH 07/12] No line endings conversion for zip files --- models/files.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/files.go b/models/files.go index a57576d2..0a3416ad 100644 --- a/models/files.go +++ b/models/files.go @@ -53,7 +53,7 @@ func NormalizeEndingsNative(content []byte) ([]byte, error) { // whether specified filename is a text file func IsTextFile(filename string) bool { ext := filepath.Ext(filename) - return !(ext == "" || ext == "exe" || ext == "pdf") + return !(ext == "" || ext == "exe" || ext == "pdf" || ext == "zip") } // GetFileWithName returns a file with a given name. From 217aef42640c5cf6b95b73018ad6abccb1881dfc Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Tue, 6 Jun 2023 21:01:24 +0700 Subject: [PATCH 08/12] Run gofmt --- models/files.go | 2 +- server/contests/problem.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/files.go b/models/files.go index 0a3416ad..af58589e 100644 --- a/models/files.go +++ b/models/files.go @@ -33,7 +33,7 @@ func lftoCRLF(content []byte) ([]byte, error) { // target accepts "windows" or "linux". Returns error if OS is not supported // or there is LF and CRLF mixed together func NormalizeEndings(content []byte, target string) ([]byte, error) { - switch (target){ + switch target { case "windows": return lftoCRLF(content) case "linux": diff --git a/server/contests/problem.go b/server/contests/problem.go index b69ea95a..a4ac5512 100644 --- a/server/contests/problem.go +++ b/server/contests/problem.go @@ -160,7 +160,7 @@ func (g *Group) SubmitPost(c echo.Context) error { } // Submitted files can be executable - if models.IsTextFile(file.Filename){ + if models.IsTextFile(file.Filename) { source, err = models.NormalizeEndingsNative(source) if err != nil { return err From d74e63bc42ab80af48c96c8c38046a5bc14a7c42 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Tue, 27 Jun 2023 16:55:24 +0700 Subject: [PATCH 09/12] Conditional compilation --- models/endings_linux.go | 6 ++++++ models/endings_windows.go | 6 ++++++ models/files.go | 6 ------ 3 files changed, 12 insertions(+), 6 deletions(-) create mode 100644 models/endings_linux.go create mode 100644 models/endings_windows.go diff --git a/models/endings_linux.go b/models/endings_linux.go new file mode 100644 index 00000000..3e5b521e --- /dev/null +++ b/models/endings_linux.go @@ -0,0 +1,6 @@ +package models + +// NormalizeEndingsNative normalize file line endings to the current OS's endings +func NormalizeEndingsNative(content []byte) ([]byte, error) { + return crlftoLF(content) +} diff --git a/models/endings_windows.go b/models/endings_windows.go new file mode 100644 index 00000000..9579cebd --- /dev/null +++ b/models/endings_windows.go @@ -0,0 +1,6 @@ +package models + +// NormalizeEndingsNative normalize file line endings to the current OS's endings +func NormalizeEndingsNative(content []byte) ([]byte, error) { + return lftoCRLF(content) +} diff --git a/models/files.go b/models/files.go index af58589e..73bdc40c 100644 --- a/models/files.go +++ b/models/files.go @@ -43,12 +43,6 @@ func NormalizeEndings(content []byte, target string) ([]byte, error) { } } -// NormalizeEndings normalize file line endings to the current OS's endings -// target accepts "windows" or "linux" -func NormalizeEndingsNative(content []byte) ([]byte, error) { - return NormalizeEndings(content, runtime.GOOS) -} - // IsTextFile applies heuristics to determine // whether specified filename is a text file func IsTextFile(filename string) bool { From 101d80718963f3ca1f7fcf0617b4797ff84a6634 Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 6 Aug 2023 18:59:49 +0700 Subject: [PATCH 10/12] Normalize to LF in database --- models/endings_linux.go | 2 +- models/endings_windows.go | 2 +- models/files.go | 8 ++++---- server/admin/problem.go | 2 +- server/admin/test_groups.go | 8 ++++---- server/contests/problem.go | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/models/endings_linux.go b/models/endings_linux.go index 3e5b521e..f4305097 100644 --- a/models/endings_linux.go +++ b/models/endings_linux.go @@ -2,5 +2,5 @@ package models // NormalizeEndingsNative normalize file line endings to the current OS's endings func NormalizeEndingsNative(content []byte) ([]byte, error) { - return crlftoLF(content) + return NormalizeEndingsUnix(content) } diff --git a/models/endings_windows.go b/models/endings_windows.go index 9579cebd..5f6b041b 100644 --- a/models/endings_windows.go +++ b/models/endings_windows.go @@ -2,5 +2,5 @@ package models // NormalizeEndingsNative normalize file line endings to the current OS's endings func NormalizeEndingsNative(content []byte) ([]byte, error) { - return lftoCRLF(content) + return NormalizeEndingsWindows(content) } diff --git a/models/files.go b/models/files.go index 73bdc40c..65361f11 100644 --- a/models/files.go +++ b/models/files.go @@ -12,11 +12,11 @@ import ( "github.com/pkg/errors" ) -func crlftoLF(content []byte) ([]byte, error) { +func NormalizeEndingsUnix(content []byte) ([]byte, error) { return bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n")), nil } -func lftoCRLF(content []byte) ([]byte, error) { +func NormalizeEndingsWindows(content []byte) ([]byte, error) { lf := bytes.Count(content, []byte("\n")) crlf := bytes.Count(content, []byte("\r\n")) if crlf == lf { @@ -35,9 +35,9 @@ func lftoCRLF(content []byte) ([]byte, error) { func NormalizeEndings(content []byte, target string) ([]byte, error) { switch target { case "windows": - return lftoCRLF(content) + return NormalizeEndingsWindows(content) case "linux": - return crlftoLF(content) + return NormalizeEndingsUnix(content) default: return nil, errors.Errorf("%s not supported for line ending conversion", runtime.GOOS) } diff --git a/server/admin/problem.go b/server/admin/problem.go index 05b88622..ca21a391 100644 --- a/server/admin/problem.go +++ b/server/admin/problem.go @@ -205,7 +205,7 @@ func (g *Group) ProblemAddFile(c echo.Context) error { } for _, file := range files { if models.IsTextFile(file.Filename) { - file.Content, err = models.NormalizeEndingsNative(file.Content) + file.Content, err = models.NormalizeEndingsUnix(file.Content) if err != nil { return err } diff --git a/server/admin/test_groups.go b/server/admin/test_groups.go index 03dae226..edcb385e 100644 --- a/server/admin/test_groups.go +++ b/server/admin/test_groups.go @@ -105,7 +105,7 @@ func (g *Group) TestGroupUploadSingle(c echo.Context) error { if err != nil { return err } - input, err = models.NormalizeEndingsNative(input) + input, err = models.NormalizeEndingsUnix(input) if err != nil { return err } @@ -113,7 +113,7 @@ func (g *Group) TestGroupUploadSingle(c echo.Context) error { if err != nil { return err } - output, err = models.NormalizeEndingsNative(output) + output, err = models.NormalizeEndingsUnix(output) if err != nil { return err } @@ -269,7 +269,7 @@ func (r *TestGroupCtx) WriteTestsNormalized(db db.DBContext, tests []*tests.Lazy if err != nil { return errors.Wrapf(err, "test %v input", test.Name) } - input, err = models.NormalizeEndingsNative(input) + input, err = models.NormalizeEndingsUnix(input) if err != nil { return errors.Wrapf(err, "test %v input", test.Name) } @@ -277,7 +277,7 @@ func (r *TestGroupCtx) WriteTestsNormalized(db db.DBContext, tests []*tests.Lazy if err != nil { return errors.Wrapf(err, "test %v output", test.Name) } - output, err = models.NormalizeEndingsNative(output) + output, err = models.NormalizeEndingsUnix(output) if err != nil { return errors.Wrapf(err, "test %v output", test.Name) } diff --git a/server/contests/problem.go b/server/contests/problem.go index a4ac5512..477bd867 100644 --- a/server/contests/problem.go +++ b/server/contests/problem.go @@ -161,7 +161,7 @@ func (g *Group) SubmitPost(c echo.Context) error { // Submitted files can be executable if models.IsTextFile(file.Filename) { - source, err = models.NormalizeEndingsNative(source) + source, err = models.NormalizeEndingsUnix(source) if err != nil { return err } From 06f11c6743a69ca3b92a266e8b8f7bf64e9015dd Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Sun, 6 Aug 2023 19:03:23 +0700 Subject: [PATCH 11/12] Cmt for NormalizeEndings --- models/files.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/models/files.go b/models/files.go index 65361f11..3b30da7d 100644 --- a/models/files.go +++ b/models/files.go @@ -12,10 +12,13 @@ import ( "github.com/pkg/errors" ) +// NormalizeEndingsUnix normalize file line endings to LF func NormalizeEndingsUnix(content []byte) ([]byte, error) { return bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n")), nil } +// NormalizeEndingsWindows normalize file line endings to CRLF +// and throws if there is LF and CRLF mixed together func NormalizeEndingsWindows(content []byte) ([]byte, error) { lf := bytes.Count(content, []byte("\n")) crlf := bytes.Count(content, []byte("\r\n")) From 44ded42e3051293b7d487f13e965256c06132f4a Mon Sep 17 00:00:00 2001 From: minhnhatnoe <86871862+minhnhatnoe@users.noreply.github.com> Date: Mon, 7 Aug 2023 10:29:52 +0700 Subject: [PATCH 12/12] WIP. Made generate template accepts explicit private fields --- models/generate/main.go | 18 +++++++++++++++++- models/models.toml | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/models/generate/main.go b/models/generate/main.go index 81ff7ae8..33fd352b 100644 --- a/models/generate/main.go +++ b/models/generate/main.go @@ -26,6 +26,9 @@ type TomlTables map[string]TomlTable // SnakeToGocase translates snake case to go-case. // If export is true, the returned value has the first character in uppercase. func SnakeToGocase(s string, export bool) string { + s, explicit_private := RipPrivate(s) + export = export && !explicit_private + parts := strings.Split(s, "_") result := strings.Builder{} for i, part := range parts { @@ -42,6 +45,14 @@ func SnakeToGocase(s string, export bool) string { return result.String() } +// RipPrivate removes the "__" prefix from a string. +func RipPrivate(s string) (string, bool) { + if strings.HasPrefix(s, "__") { + return s[len("__"):], true + } + return s, false +} + var t = template.New("main") func init() { @@ -54,6 +65,7 @@ func init() { "args": JoinArguments, "marks": Marks, "fkey": ForeignKey, + "rppriv": func(s string) string { s, _ = RipPrivate(s); return s }, }) } @@ -80,6 +92,7 @@ func JoinCondition(keys map[string]string, sep string) string { s.WriteString(sep) } first = false + key, _ = RipPrivate(key) s.WriteString(key + " = ?") } return s.String() @@ -99,10 +112,13 @@ func sortKeys(k map[string]string) []string { func JoinArguments(keys map[string]string, structName string) string { var s []string for _, key := range sortKeys(keys) { + key, private := RipPrivate(key) if structName == "" { s = append(s, SnakeToGocase(key, false)) } else if structName == "-" { s = append(s, key) + } else if private { + s = append(s, structName+"."+SnakeToGocase(key, true)+"()") } else { s = append(s, structName+"."+SnakeToGocase(key, true)) } @@ -196,7 +212,7 @@ const TableTemplate = ` // {{$name}} is the struct generated from table "{{.Name}}". type {{$name}} struct { {{- range $field, $type := .Fields}} - {{$field | field}} {{$type}} {{$tick}}db:"{{$field}}"{{$tick}} + {{$field | field}} {{$type}} {{$tick}}db:"{{$field | rppriv}}"{{$tick}} {{- end}} } diff --git a/models/models.toml b/models/models.toml index 0b8a8aea..bed861c7 100644 --- a/models/models.toml +++ b/models/models.toml @@ -90,7 +90,7 @@ _order_by = "priority DESC, id ASC" id = "int" problem_id = "int" filename = "string" -content = "[]byte" +__content = "[]byte" public = "bool" [announcements]