Normalize file endings

models/files.go

@@ -1,15 +1,61 @@
 package models
 
 import (
+	"bytes"
 	"fmt"
 	"path/filepath"
+	"runtime"
 	"strings"
 
 	"github.com/natsukagami/kjudge/db"
 	"github.com/natsukagami/kjudge/models/verify"
 	"github.com/pkg/errors"
 )
 
+func crlftoLF(content []byte) ([]byte, error) {
+	return bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n")), nil
+}
+
+func lftoCRLF(content []byte) ([]byte, error) {
+	lf := bytes.Count(content, []byte("\n"))
+	crlf := bytes.Count(content, []byte("\r\n"))
+	if crlf == lf {
+		return content, nil
+	}
+	var err error = nil
+	if crlf != 0 {
+		err = errors.Errorf("number of crlf and lf (%v, %v) does not match", crlf, lf)
+	}
+	return bytes.ReplaceAll(content, []byte("\r\n"), []byte("\n")), err
+}
+
+// NormalizeEndings normalize file line endings to the target OS's endings
+// target accepts "windows" or "linux". Returns error if OS is not supported
+// or there is LF and CRLF mixed together
+func NormalizeEndings(content []byte, target string) ([]byte, error) {
+	switch target {
+	case "windows":
+		return lftoCRLF(content)
+	case "linux":
+		return crlftoLF(content)
+	default:
+		return nil, errors.Errorf("%s not supported for line ending conversion", runtime.GOOS)
+	}
+}
+
+// NormalizeEndings normalize file line endings to the current OS's endings
+// target accepts "windows" or "linux"
+func NormalizeEndingsNative(content []byte) ([]byte, error) {
+	return NormalizeEndings(content, runtime.GOOS)
+}
+
+// IsTextFile applies heuristics to determine
+// whether specified filename is a text file
+func IsTextFile(filename string) bool {
+	ext := filepath.Ext(filename)
+	return !(ext == "" || ext == "exe" || ext == "pdf" || ext == "zip")
+}
+
 // GetFileWithName returns a file with a given name.
 func GetFileWithName(db db.DBContext, problemID int, filename string) (*File, error) {
 	var f File

server/admin/problem.go

@@ -203,6 +203,14 @@ func (g *Group) ProblemAddFile(c echo.Context) error {
 	if rename != "" && len(files) == 1 {
 		files[0].Filename = rename
 	}
+	for _, file := range files {
+		if models.IsTextFile(file.Filename) {
+			file.Content, err = models.NormalizeEndingsNative(file.Content)
+			if err != nil {
+				return err
+			}
+		}
+	}
 	if err := ctx.Problem.WriteFiles(g.db, files); err != nil {
 		return httperr.BadRequestf("cannot write files: %v", err)
 	}

server/admin/test_groups.go

@@ -105,10 +105,18 @@ func (g *Group) TestGroupUploadSingle(c echo.Context) error {
 	if err != nil {
 		return err
 	}
+	input, err = models.NormalizeEndingsNative(input)
+	if err != nil {
+		return err
+	}
 	output, err := readFromForm("output", mp)
 	if err != nil {
 		return err
 	}
+	output, err = models.NormalizeEndingsNative(output)
+	if err != nil {
+		return err
+	}
 	// Make the test
 	test := &models.Test{
 		TestGroupID: tg.ID,
@@ -150,7 +158,7 @@ func (g *Group) TestGroupUploadMultiple(c echo.Context) error {
 	if err != nil {
 		return httperr.BadRequestf("cannot unpack tests: %v", err)
 	}
-	if err := tg.WriteTests(tx, tests, override); err != nil {
+	if err := tg.WriteTestsNormalized(tx, tests, override); err != nil {
 		return httperr.BadRequestf("Cannot write tests: %v", err)
 	}
 	if err := tx.Commit(); err != nil {
@@ -241,10 +249,10 @@ func (g *Group) TestGroupRejudgePost(c echo.Context) error {
 	return c.Redirect(http.StatusSeeOther, fmt.Sprintf("/admin/problems/%d/submissions", tg.ProblemID))
 }
 
-// WriteTests writes the given set of tests into the Database.
-// If override is set, all tests in the test group gets deleted first.
-// The LazyTests are STILL invalid models.Tests. DO NOT USE.
-func (r *TestGroupCtx) WriteTests(db db.DBContext, tests []*tests.LazyTest, override bool) error {
+// WriteTestsNormalized normalizes line endings and writes the given set of
+// tests into the Database. If override is set, all tests in the test group
+// gets deleted first. The LazyTests are STILL invalid models.Tests. DO NOT USE.
+func (r *TestGroupCtx) WriteTestsNormalized(db db.DBContext, tests []*tests.LazyTest, override bool) error {
 	for _, test := range tests {
 		test.TestGroupID = r.ID
 		if err := test.Verify(); err != nil {
@@ -261,10 +269,18 @@ func (r *TestGroupCtx) WriteTests(db db.DBContext, tests []*tests.LazyTest, over
 		if err != nil {
 			return errors.Wrapf(err, "test %v input", test.Name)
 		}
+		input, err = models.NormalizeEndingsNative(input)
+		if err != nil {
+			return errors.Wrapf(err, "test %v input", test.Name)
+		}
 		output, err := readZip(test.Output)
 		if err != nil {
 			return errors.Wrapf(err, "test %v output", test.Name)
 		}
+		output, err = models.NormalizeEndingsNative(output)
+		if err != nil {
+			return errors.Wrapf(err, "test %v output", test.Name)
+		}
 		if _, err := db.Exec(
 			"INSERT INTO tests(name, test_group_id, input, output) VALUES (?, ?, ?, ?)",
 			test.Name,

server/contests/problem.go

@@ -158,6 +158,14 @@ func (g *Group) SubmitPost(c echo.Context) error {
 	if err != nil {
 		return errors.WithStack(err)
 	}
+
+	// Submitted files can be executable
+	if models.IsTextFile(file.Filename) {
+		source, err = models.NormalizeEndingsNative(source)
+		if err != nil {
+			return err
+		}
+	}
 	sub := models.Submission{
 		ProblemID:   ctx.Problem.ID,
 		UserID:      ctx.Me.ID,