feat(server): add task to move submissions from local to S3

judgels-backends/judgels-commons/judgels-fs/src/main/java/judgels/fs/local/LocalFileSystem.java

@@ -7,8 +7,6 @@
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import com.google.common.io.ByteStreams;
-import com.google.common.io.MoreFiles;
-import com.google.common.io.RecursiveDeleteOption;
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileInputStream;
@@ -32,6 +30,7 @@
 import judgels.fs.FileInfo;
 import judgels.fs.FileSystem;
 import judgels.fs.NaturalFilenameComparator;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.FilenameUtils;
 
 public final class LocalFileSystem implements FileSystem {
@@ -82,7 +81,7 @@ public void createFile(Path filePath) {
     @Override
     public void removeFile(Path filePath) {
         try {
-            MoreFiles.deleteRecursively(baseDir.resolve(filePath), RecursiveDeleteOption.ALLOW_INSECURE);
+            FileUtils.deleteDirectory(baseDir.resolve(filePath).toFile());
         } catch (IOException e) {
             throw new RuntimeException(e);
         }

judgels-backends/judgels-server-app/src/main/java/judgels/JudgelsServerApplication.java

@@ -270,5 +270,6 @@ private void runJerahmeel(JudgelsServerApplicationConfiguration config, Environm
 
         env.admin().addTask(component.problemSetStatsTask());
         env.admin().addTask(component.contestStatsTask());
+        env.admin().addTask(component.submissionsDuplexToAwsTask());
     }
 }

judgels-backends/judgels-server-app/src/main/java/judgels/jerahmeel/JerahmeelComponent.java

@@ -24,6 +24,7 @@
 import judgels.jerahmeel.submission.programming.StatsModule;
 import judgels.jerahmeel.submission.programming.SubmissionModule;
 import judgels.jerahmeel.submission.programming.SubmissionResource;
+import judgels.jerahmeel.submission.programming.SubmissionsDuplexToAwsTask;
 import judgels.jophiel.hibernate.JophielHibernateDaoModule;
 import judgels.messaging.rabbitmq.RabbitMQModule;
 import judgels.sandalphon.SandalphonClientModule;
@@ -83,4 +84,5 @@ public interface JerahmeelComponent {
     GradingResponsePoller gradingResponsePoller();
     ProblemSetStatsTask problemSetStatsTask();
     ContestStatsTask contestStatsTask();
+    SubmissionsDuplexToAwsTask submissionsDuplexToAwsTask();
 }

judgels-backends/judgels-server-app/src/main/java/judgels/jerahmeel/submission/programming/SubmissionModule.java

@@ -106,6 +106,23 @@ static SubmissionRegrader submissionRegrader(
         return new SubmissionRegrader(submissionStore, executorService, processor);
     }
 
+    @Provides
+    @Singleton
+    static SubmissionsDuplexToAwsTask submissionsDuplexToAwsTask(
+            UnitOfWorkAwareProxyFactory unitOfWorkAwareProxyFactory,
+            @SubmissionFs FileSystem submissionFs,
+            @JerahmeelSubmissionStore SubmissionStore submissionStore) {
+
+        return unitOfWorkAwareProxyFactory.create(
+                SubmissionsDuplexToAwsTask.class,
+                new Class<?>[] {
+                        FileSystem.class,
+                        SubmissionStore.class},
+                new Object[] {
+                        submissionFs,
+                        submissionStore});
+    }
+
     @Provides
     @Singleton
     static GradingResponsePoller gradingResponsePoller(

judgels-backends/judgels-server-app/src/main/java/judgels/jerahmeel/submission/programming/SubmissionsDuplexToAwsTask.java

@@ -0,0 +1,75 @@
+package judgels.jerahmeel.submission.programming;
+
+import static java.util.Optional.empty;
+import static java.util.Optional.of;
+
+import io.dropwizard.hibernate.UnitOfWork;
+import io.dropwizard.servlets.tasks.Task;
+import java.io.PrintWriter;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import judgels.fs.FileSystem;
+import judgels.fs.duplex.DuplexFileSystem;
+import judgels.jerahmeel.submission.JerahmeelSubmissionStore;
+import judgels.persistence.api.Page;
+import judgels.sandalphon.api.submission.programming.Submission;
+import judgels.sandalphon.submission.programming.SubmissionSourceBuilder;
+import judgels.sandalphon.submission.programming.SubmissionStore;
+
+/**
+ * Moves submission files from local to AWS S3.
+ */
+public class SubmissionsDuplexToAwsTask extends Task {
+    private final FileSystem submissionFs;
+    private final SubmissionStore submissionStore;
+    private final SubmissionSourceBuilder submissionSourceBuilder;
+
+    public SubmissionsDuplexToAwsTask(
+            @SubmissionFs FileSystem submissionFs,
+            @JerahmeelSubmissionStore SubmissionStore submissionStore) {
+
+        super("jerahmeel-submissions-duplex-to-aws");
+
+        this.submissionFs = submissionFs;
+        this.submissionStore = submissionStore;
+        this.submissionSourceBuilder = new SubmissionSourceBuilder(submissionFs);
+    }
+
+    @Override
+    @UnitOfWork
+    public void execute(Map<String, List<String>> parameters, PrintWriter output) {
+        if (!(submissionFs instanceof DuplexFileSystem)) {
+            return;
+        }
+
+        List<String> lastSubmissionIds = parameters.get("lastSubmissionId");
+        Optional<Long> lastSubmissionId = lastSubmissionIds == null || lastSubmissionIds.isEmpty()
+                ? empty()
+                : of(Long.parseLong(lastSubmissionIds.get(0)));
+
+        List<String> limits = parameters.get("limit");
+        Optional<Integer> limit = limits == null || limits.isEmpty() ? empty() : of(Integer.parseInt(limits.get(0)));
+
+        Page<Submission> submissions =
+                submissionStore.getSubmissionsForStats(empty(), lastSubmissionId, limit.orElse(1000));
+
+        Submission lastSubmission = null;
+        for (Submission s : submissions.getPage()) {
+            var source = submissionSourceBuilder.fromPastSubmission(s.getJid(), false);
+
+            // Store to AWS
+            submissionSourceBuilder.storeSubmissionSource(s.getJid(), source);
+
+            // Remove from local
+            submissionFs.removeFile(Paths.get(s.getJid()));
+
+            lastSubmission = s;
+        }
+
+        if (lastSubmission != null) {
+            output.write(lastSubmission.getId() + "\n");
+        }
+    }
+}