Implement generation of split-source Manifest entries

build.gradle

@@ -153,6 +153,11 @@ idea {
                     programParameters = "--help"
                     moduleRef(project, sourceSets.main)
                 }
+                "Run Neoforge 1.21.4 (joined) + Parchment"(Application) {
+                    mainClass = mainClassName
+                    programParameters = "run --dist joined --neoforge net.neoforged:neoforge:21.4.1-beta:userdev --add-repository=https://maven.parchmentmc.org --parchment-data=org.parchmentmc.data:parchment-1.21:2024.06.23@zip --parchment-conflict-prefix=p_ --write-result=compiled:build/minecraft-1.21.4.jar --write-result=clientResources:build/client-extra-1.21.4.jar --write-result=sources:build/minecraft-sources-1.21.4.jar"
+                    moduleRef(project, sourceSets.main)
+                }
                 "Run Neoforge 1.21 (joined) + Parchment"(Application) {
                     mainClass = mainClassName
                     programParameters = "run --dist joined --neoforge net.neoforged:neoforge:21.0.0-beta:userdev --add-repository=https://maven.parchmentmc.org --parchment-data=org.parchmentmc.data:parchment-1.21:2024.06.23@zip --parchment-conflict-prefix=p_ --write-result=compiled:build/minecraft-1.21.jar --write-result=clientResources:build/client-extra-1.21.jar --write-result=sources:build/minecraft-sources-1.21.jar"

src/main/java/net/neoforged/neoform/runtime/actions/SplitResourcesFromClassesAction.java

@@ -2,32 +2,66 @@
 
 import net.neoforged.neoform.runtime.cache.CacheKeyBuilder;
 import net.neoforged.neoform.runtime.engine.ProcessingEnvironment;
+import net.neoforged.srgutils.IMappingFile;
+import org.jetbrains.annotations.Nullable;
 
-import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.LocalDateTime;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Objects;
+import java.util.Set;
 import java.util.function.Predicate;
-import java.util.jar.JarEntry;
-import java.util.jar.JarInputStream;
+import java.util.jar.Attributes;
+import java.util.jar.JarFile;
 import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
 
 /**
  * Copies a Jar file while applying a filename filter.
  */
 public final class SplitResourcesFromClassesAction extends BuiltInAction {
+
+    public static final String INPUT_OTHER_DIST_JAR = "otherDistJar";
+    public static final String INPUT_MAPPINGS = "mappings";
+
+    /**
+     * Use a fixed timestamp for the manifest entry.
+     */
+    private static final LocalDateTime MANIFEST_TIME = LocalDateTime.of(2000, 1, 1, 0, 0, 0, 0);
+
     /**
      * Patterns for filenames that should not be written to either output jar.
      */
     private final List<Pattern> denyListPatterns = new ArrayList<>();
 
+    /**
+     * When non-null, the action expects additional inputs ({@link #INPUT_OTHER_DIST_JAR} and {@link #INPUT_MAPPINGS})
+     * pointing to the Jar file of the *other* distribution (i.e. this action processes the client resources,
+     * then the other distribution jar is the server jar).
+     * The mapping file is required to produce a Manifest using named file names instead of obfuscated names.
+     */
+    @Nullable
+    private GenerateDistManifestSettings generateDistManifestSettings;
+
     @Override
     public void run(ProcessingEnvironment environment) throws IOException, InterruptedException {
         var inputJar = environment.getRequiredInputPath("input");
+        Path otherDistJarPath = null;
+        Path mappingsPath = null;
+        if (generateDistManifestSettings != null) {
+            otherDistJarPath = environment.getRequiredInputPath(INPUT_OTHER_DIST_JAR);
+            mappingsPath = environment.getRequiredInputPath(INPUT_MAPPINGS);
+        }
+
         var classesJar = environment.getOutputPath("output");
         var resourcesJar = environment.getOutputPath("resourcesOutput");
 
@@ -39,19 +73,35 @@ public void run(ProcessingEnvironment environment) throws IOException, Interrupt
                     .asMatchPredicate();
         }
 
-        try (var is = new JarInputStream(new BufferedInputStream(Files.newInputStream(inputJar)));
+        try (var jar = new ZipFile(inputJar.toFile());
              var classesFileOut = new BufferedOutputStream(Files.newOutputStream(classesJar));
              var resourcesFileOut = new BufferedOutputStream(Files.newOutputStream(resourcesJar));
              var classesJarOut = new JarOutputStream(classesFileOut);
              var resourcesJarOut = new JarOutputStream(resourcesFileOut);
         ) {
-            // Ignore any entry that's not allowed
-            JarEntry entry;
-            while ((entry = is.getNextJarEntry()) != null) {
+            if (generateDistManifestSettings != null) {
+                generateDistSourceManifest(
+                        generateDistManifestSettings.distId(),
+                        jar,
+                        generateDistManifestSettings.otherDistId(),
+                        otherDistJarPath,
+                        mappingsPath,
+                        resourcesJarOut
+                );
+            }
+
+            var entries = jar.entries();
+            while (entries.hasMoreElements()) {
+                var entry = entries.nextElement();
                 if (entry.isDirectory()) {
                     continue; // For simplicity, we ignore directories completely
                 }
 
+                // If we injected a manifest earlier, ignore any subsequent manifests
+                if (generateDistManifestSettings != null && entry.getName().equals(JarFile.MANIFEST_NAME)) {
+                    continue;
+                }
+
                 var filename = entry.getName();
 
                 // Skip anything that looks like a signature file
@@ -62,12 +112,77 @@ public void run(ProcessingEnvironment environment) throws IOException, Interrupt
                 var destinationStream = filename.endsWith(".class") ? classesJarOut : resourcesJarOut;
 
                 destinationStream.putNextEntry(entry);
-                is.transferTo(destinationStream);
+                try (var is = jar.getInputStream(entry)) {
+                    is.transferTo(destinationStream);
+                }
                 destinationStream.closeEntry();
             }
         }
     }
 
+    private static void generateDistSourceManifest(String distId,
+                                                   ZipFile jar,
+                                                   String otherDistId,
+                                                   Path otherDistJarPath,
+                                                   Path mappingsPath,
+                                                   JarOutputStream resourcesJarOut) throws IOException {
+        var mappings = mappingsPath != null ? IMappingFile.load(mappingsPath.toFile()) : null;
+
+        // Use the time-stamp of either of the two input files (whichever is newer)
+        var ourFiles = getFileIndex(jar);
+        ourFiles.remove(JarFile.MANIFEST_NAME);
+        Set<String> theirFiles;
+        try (var otherDistJar = new ZipFile(otherDistJarPath.toFile())) {
+            theirFiles = getFileIndex(otherDistJar);
+        }
+        theirFiles.remove(JarFile.MANIFEST_NAME);
+
+        var manifest = new Manifest();
+        manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+        manifest.getMainAttributes().putValue("Minecraft-Dists", distId + " " + otherDistId);
+
+        addSourceDistEntries(ourFiles, theirFiles, distId, mappings, manifest);
+        addSourceDistEntries(theirFiles, ourFiles, otherDistId, mappings, manifest);
+
+        var manifestEntry = new ZipEntry(JarFile.MANIFEST_NAME);
+        manifestEntry.setTimeLocal(MANIFEST_TIME);
+        resourcesJarOut.putNextEntry(manifestEntry);
+        manifest.write(resourcesJarOut);
+        resourcesJarOut.closeEntry();
+    }
+
+    private static void addSourceDistEntries(Set<String> distFiles,
+                                             Set<String> otherDistFiles,
+                                             String dist,
+                                             IMappingFile mappings,
+                                             Manifest manifest) {
+        for (var file : distFiles) {
+            if (!otherDistFiles.contains(file)) {
+                var fileAttr = new Attributes(1);
+                fileAttr.putValue("Minecraft-Dist", dist);
+
+                if (mappings != null && file.endsWith(".class")) {
+                    file = mappings.remapClass(file.substring(0, file.length() - ".class".length())) + ".class";
+                }
+                manifest.getEntries().put(file, fileAttr);
+            }
+        }
+    }
+
+    private static Set<String> getFileIndex(ZipFile zipFile) {
+        var result = new HashSet<String>(zipFile.size());
+
+        var entries = zipFile.entries();
+        while (entries.hasMoreElements()) {
+            ZipEntry entry = entries.nextElement();
+            if (!entry.isDirectory()) {
+                result.add(entry.getName());
+            }
+        }
+
+        return result;
+    }
+
     /**
      * Adds a regular expression for filenames that should be filtered out completely.
      */
@@ -77,9 +192,31 @@ public void addDenyPatterns(String... patterns) {
         }
     }
 
+    /**
+     * Enable generation of a Jar manifest in the output resources jar which contains
+     * entries detailing which distribution each file came from.
+     * This adds new required inputs.
+     */
+    public void generateSplitManifest(String distId, String otherDistId) {
+        generateDistManifestSettings = new GenerateDistManifestSettings(
+                Objects.requireNonNull(distId, "distId"),
+                Objects.requireNonNull(otherDistId, "otherDistId")
+        );
+    }
+
     @Override
     public void computeCacheKey(CacheKeyBuilder ck) {
         super.computeCacheKey(ck);
         ck.addStrings("deny patterns", denyListPatterns.stream().map(Pattern::pattern).toList());
+        if (generateDistManifestSettings != null) {
+            ck.add("generate dist manifest - our dist", generateDistManifestSettings.distId);
+            ck.add("generate dist manifest - other dist", generateDistManifestSettings.otherDistId);
+        }
+    }
+
+    private record GenerateDistManifestSettings(
+            String distId,
+            String otherDistId
+    ) {
     }
 }

src/main/java/net/neoforged/neoform/runtime/cli/RunNeoFormCommand.java

@@ -69,7 +69,7 @@ public class RunNeoFormCommand extends NeoFormEngineCommand {
     String parchmentConflictPrefix;
 
     static class SourceArtifacts {
-        @CommandLine.ArgGroup(multiplicity = "1")
+        @CommandLine.ArgGroup
         NeoFormArtifact neoform;
         @CommandLine.Option(names = "--neoforge")
         String neoforge;
@@ -93,10 +93,10 @@ protected void runWithNeoFormEngine(NeoFormEngine engine, List<AutoCloseable> cl
 
             // Allow it to be overridden with local or remote data
             Path neoformArtifact;
-            if (sourceArtifacts.neoform.file != null) {
+            if (sourceArtifacts.neoform != null && sourceArtifacts.neoform.file != null) {
                 LOG.println("Overriding NeoForm version " + neoforgeConfig.neoformArtifact() + " with NeoForm file " + sourceArtifacts.neoform.file);
                 neoformArtifact = sourceArtifacts.neoform.file;
-            } else if (sourceArtifacts.neoform.artifact != null) {
+            } else if (sourceArtifacts.neoform != null && sourceArtifacts.neoform.artifact != null) {
                 LOG.println("Overriding NeoForm version " + neoforgeConfig.neoformArtifact() + " with CLI argument " + sourceArtifacts.neoform.artifact);
                 neoformArtifact = artifactManager.get(MavenCoordinate.parse(sourceArtifacts.neoform.artifact)).path();
             } else {

src/main/java/net/neoforged/neoform/runtime/config/neoform/NeoFormDistConfig.java

@@ -21,6 +21,10 @@ public NeoFormDistConfig(NeoFormConfig config, String dist) {
         this.dist = dist;
     }
 
+    public String dist() {
+        return dist;
+    }
+
     public int javaVersion() {
         return config.javaVersion();
     }

src/main/java/net/neoforged/neoform/runtime/engine/NeoFormEngine.java

@@ -318,6 +318,23 @@ private void addNodeForStep(ExecutionGraph graph, NeoFormDistConfig config, NeoF
                 var action = new SplitResourcesFromClassesAction();
                 // The Minecraft jar contains nothing of interest in META-INF, and the signature files are useless.
                 action.addDenyPatterns("META-INF/.*");
+                if (processGeneration.generateDistSourceManifest() && config.dist().equals("joined")) {
+                    if ("stripClient".equals(step.getId())) {
+                        // Prefer the already extracted server
+                        var serverJarInput = graph.hasOutput("extractServer", "output") ?
+                                graph.getRequiredOutput("extractServer", "output").asInput()
+                                : graph.getRequiredOutput("downloadServer", "output").asInput();
+
+                        action.generateSplitManifest("client", "server");
+                        builder.input(SplitResourcesFromClassesAction.INPUT_OTHER_DIST_JAR, serverJarInput);
+                        builder.input(SplitResourcesFromClassesAction.INPUT_MAPPINGS, graph.getRequiredOutput("mergeMappings", "output").asInput());
+                    } else if ("stripServer".equals(step.getId())) {
+                        action.generateSplitManifest("server", "client");
+                        builder.input(SplitResourcesFromClassesAction.INPUT_OTHER_DIST_JAR, graph.getRequiredOutput("downloadClient", "output").asInput());
+                        builder.input(SplitResourcesFromClassesAction.INPUT_MAPPINGS, graph.getRequiredOutput("mergeMappings", "output").asInput());
+                    }
+                }
+
                 processGeneration.getAdditionalDenyListForMinecraftJars().forEach(action::addDenyPatterns);
                 builder.action(action);
             }
@@ -395,6 +412,7 @@ private void applyFunctionToNode(NeoFormStep step, NeoFormFunction function, Exe
                 if ("output".equals(variable)) {
                     var type = switch (step.type()) {
                         case "mergeMappings" -> NodeOutputType.TSRG;
+                        case "generateSplitManifest" -> NodeOutputType.JAR_MANIFEST;
                         default -> NodeOutputType.JAR;
                     };
                     if (!builder.hasOutput(variable)) {

src/main/java/net/neoforged/neoform/runtime/engine/ProcessGeneration.java

@@ -43,6 +43,7 @@ public int compareTo(@NotNull ProcessGeneration.MinecraftReleaseVersion o) {
 
     private static final MinecraftReleaseVersion MC_1_17_1 = new MinecraftReleaseVersion(1, 17, 1);
     private static final MinecraftReleaseVersion MC_1_20_1 = new MinecraftReleaseVersion(1, 20, 1);
+    private static final MinecraftReleaseVersion MC_1_21_4 = new MinecraftReleaseVersion(1, 21, 4);
 
     /**
      * Indicates whether the Minecraft server jar file contains third party
@@ -58,6 +59,12 @@ public int compareTo(@NotNull ProcessGeneration.MinecraftReleaseVersion o) {
      */
     private boolean sourcesUseIntermediaryNames;
 
+    /**
+     * Enables generation of the MANIFEST.MF in the client and server resource files that
+     * indicates which distribution each file came from. Only applies to joined distributions.
+     */
+    private boolean generateDistSourceManifest;
+
     /**
      * For (Neo)Forge 1.20.1 and below, we have to remap method and field names from
      * SRG to official names for development.
@@ -88,6 +95,9 @@ static ProcessGeneration fromMinecraftVersion(String minecraftVersion) {
         // In 1.20.2 and later, NeoForge switched to Mojmap at runtime and sources defined in Mojmap
         result.sourcesUseIntermediaryNames = isLessThanOrEqualTo(releaseVersion, MC_1_20_1);
 
+        // Technically 1.21.4 does not directly support this, but it does not harm it either
+        result.generateDistSourceManifest = isGreaterThanOrEqualTo(releaseVersion, MC_1_21_4);
+
         return result;
     }
 
@@ -98,13 +108,28 @@ private static boolean isLessThanOrEqualTo(@Nullable MinecraftReleaseVersion rel
         return releaseVersion.compareTo(version) <= 0;
     }
 
+    private static boolean isGreaterThanOrEqualTo(@Nullable MinecraftReleaseVersion releaseVersion, MinecraftReleaseVersion version) {
+        if (releaseVersion == null) {
+            return true; // We're working with a snapshot version, which we always use the latest processes for
+        }
+        return releaseVersion.compareTo(version) >= 0;
+    }
+
     /**
      * Does the Minecraft source code that MCP/NeoForm creates use SRG names?
      */
     public boolean sourcesUseIntermediaryNames() {
         return sourcesUseIntermediaryNames;
     }
 
+    /**
+     * Does the FML version on that MC generation support use of MANIFEST.MF entries
+     * for filtering out dist-specific classes in dev? (When using the joined distribution)
+     */
+    public boolean generateDistSourceManifest() {
+        return generateDistSourceManifest;
+    }
+
     /**
      * Allows additional resources to be completely removed from Minecraft jars before processing them.
      */