Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow multiple namespaces and input files for mergeTinyV2 #3

Closed
wants to merge 5 commits into from
Closed

Allow multiple namespaces and input files for mergeTinyV2 #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2f50208
Allow multiple input files and more than two namespaces
IotaBread Sep 22, 2021
6b6e38f
Update MergeTest
IotaBread Sep 22, 2021
4be763c
Fix merging names for more than two namespaces in a single file
IotaBread Sep 22, 2021
f339ef7
Update javadoc
IotaBread Sep 22, 2021
22394c5
Remove a TODO
IotaBread Sep 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
216 changes: 111 additions & 105 deletions src/main/java/net/fabricmc/stitch/commands/tinyv2/CommandMergeTinyV2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import java.util.regex.Pattern;
Expand All @@ -39,11 +40,11 @@
import net.fabricmc.stitch.util.Pair;

/**
* Merges a tiny file with 2 columns (namespaces) of mappings, with another tiny file that has
* the same namespace as the first column and a different namespace as the second column.
* The first column of the output will contain the shared namespace,
* the second column of the output would be the second namespace of input a,
* and the third column of the output would be the second namespace of input b
* Merges a tiny file with at least 2 columns (namespaces) of mappings, with another tiny
* file that has the same namespace as the first column and different namespaces as the
* other columns. The first column of the output will contain the shared namespace, the
* next columns of the output would be the namespaces of all inputs, excluding the first
* column namespace
* <p>
* Descriptors will remain as-is (using the namespace of the first column)
* <p>
Expand All @@ -65,12 +66,6 @@
* intermediary named official
* c net/minecraft/class_123 net/minecraft/somePackage/someClass a
* m (Lnet/minecraft/class_124;)V method_1234 someMethod a
* <p>
* <p>
* After intermediary-named mappings are obtained,
* and official-intermediary mappings are obtained and swapped using CommandReorderTinyV2, Loom merges them using this command,
* and then reorders it to official-intermediary-named using CommandReorderTinyV2 again.
* This is a convenient way of storing all the mappings in Loom.
*/
public class CommandMergeTinyV2 extends Command {
public CommandMergeTinyV2() {
Expand All @@ -82,62 +77,67 @@ public CommandMergeTinyV2() {
*/
@Override
public String getHelpString() {
return "<input-a> <input-b> <output>";
return "<input-a> <input-b> [<input-c>...] <output>";
}

@Override
public boolean isArgumentCountValid(int count) {
return count == 3;
return count >= 3;
}

@Override
public void run(String[] args) throws IOException {
Path inputA = Paths.get(args[0]);
Path inputB = Paths.get(args[1]);
System.out.println("Reading " + inputA);
TinyFile tinyFileA = TinyV2Reader.read(inputA);
System.out.println("Reading " + inputB);
TinyFile tinyFileB = TinyV2Reader.read(inputB);
TinyHeader headerA = tinyFileA.getHeader();
TinyHeader headerB = tinyFileB.getHeader();
if (headerA.getNamespaces().size() != 2) {
throw new IllegalArgumentException(inputA + " must have exactly 2 namespaces.");
Path[] inputs = new Path[args.length - 1];
for (int i = 0; i < args.length - 1; ++i) {
inputs[i] = Paths.get(args[i]);
}
if (headerB.getNamespaces().size() != 2) {
throw new IllegalArgumentException(inputB + " must have exactly 2 namespaces.");
Path output = Paths.get(args[args.length - 1]);

List<TinyFile> tinyFiles = new ArrayList<>();
TinyFile tinyFileA = TinyV2Reader.read(inputs[0]);
tinyFiles.add(tinyFileA);

TinyHeader headerA = tinyFileA.getHeader();
if (headerA.getNamespaces().size() < 2) {
throw new IllegalArgumentException(inputs[0] + " must have at least 2 namespaces.");
}

if (!headerA.getNamespaces().get(0).equals(headerB.getNamespaces().get(0))) {
throw new IllegalArgumentException(
String.format("The input tiny files must have the same namespaces as the first column. " +
"(%s has %s while %s has %s)",
inputA, headerA.getNamespaces().get(0), inputB, headerB.getNamespaces().get(0))
);
String baseNamespace = headerA.getNamespaces().get(0);
for (int i = 1; i < inputs.length; ++i) {
Path input = inputs[i];
TinyFile tinyFile = TinyV2Reader.read(input);
tinyFiles.add(tinyFile);
TinyHeader header = tinyFile.getHeader();
List<String> namespaces = header.getNamespaces();

if (header.getNamespaces().size() < 2) {
throw new IllegalArgumentException(inputs[i] + " must have at least 2 namespaces.");
}

if (!namespaces.get(0).equals(baseNamespace)) {
throw new IllegalArgumentException(String.format("The input tiny files must have the same namespaces as the first column. " +
"(%s has %s instead of %s)", input, namespaces.get(0), baseNamespace));
}
}
System.out.println("Merging " + inputA + " with " + inputB);
TinyFile mergedFile = merge(tinyFileA, tinyFileB);

TinyV2Writer.write(mergedFile, Paths.get(args[2]));
System.out.println("Merged mappings written to " + Paths.get(args[2]));
}
System.out.println("Merging " + inputs[0] + " with " + Arrays.stream(inputs).skip(1).map(Path::toString).collect(Collectors.joining(", ")));
TinyFile mergedFile = merge(tinyFiles);

TinyV2Writer.write(mergedFile, output);
System.out.println("Merged mappings written to " + output);
}

private TinyFile merge(TinyFile inputA, TinyFile inputB) {
private TinyFile merge(List<TinyFile> inputs) {
//TODO: how to merge properties?

TinyHeader mergedHeader = mergeHeaders(inputA.getHeader(), inputB.getHeader());
TinyHeader mergedHeader = mergeHeaders(inputs.stream().map(TinyFile::getHeader).collect(Collectors.toList()));

List<String> keyUnion = keyUnion(inputA.getClassEntries(), inputB.getClassEntries());
List<String> keyUnion = keyUnion(inputs.stream().map(TinyFile::getClassEntries).collect(Collectors.toList()));

Map<String, TinyClass> inputAClasses = inputA.mapClassesByFirstNamespace();
Map<String, TinyClass> inputBClasses = inputB.mapClassesByFirstNamespace();
List<Map<String, TinyClass>> inputsClasses = inputs.stream().map(TinyFile::mapClassesByFirstNamespace).collect(Collectors.toList());
List<TinyClass> mergedClasses = map(keyUnion, key -> {
TinyClass classA = inputAClasses.get(key);
TinyClass classB = inputBClasses.get(key);

classA = matchEnclosingClassIfNeeded(key, classA, inputAClasses);
classB = matchEnclosingClassIfNeeded(key, classB, inputBClasses);
return mergeClasses(key, classA, classB);
List<TinyClass> classes = inputsClasses.stream().map(inputClasses -> matchEnclosingClassIfNeeded(key, inputClasses.get(key), inputClasses)).collect(Collectors.toList());
return mergeClasses(key, classes);
});

return new TinyFile(mergedHeader, mergedClasses);
Expand Down Expand Up @@ -175,106 +175,109 @@ private String matchEnclosingClass(String sharedName, Map<String, TinyClass> inp
return sharedName;
}

private TinyClass mergeClasses(String sharedClassName, List<TinyClass> classes) {
List<String> mergedNames = mergeNames(sharedClassName, classes);
List<String> mergedComments = mergeComments(classes.stream().map(TinyClass::getComments).collect(Collectors.toList()));

private TinyClass mergeClasses(String sharedClassName, @Nonnull TinyClass classA, @Nonnull TinyClass classB) {
List<String> mergedNames = mergeNames(sharedClassName, classA, classB);
List<String> mergedComments = mergeComments(classA.getComments(), classB.getComments());

List<Pair<String, String>> methodKeyUnion = union(mapToFirstNamespaceAndDescriptor(classA), mapToFirstNamespaceAndDescriptor(classB));
Map<Pair<String, String>, TinyMethod> methodsA = classA.mapMethodsByFirstNamespaceAndDescriptor();
Map<Pair<String, String>, TinyMethod> methodsB = classB.mapMethodsByFirstNamespaceAndDescriptor();
List<TinyMethod> mergedMethods = map(methodKeyUnion,
(Pair<String, String> k) -> mergeMethods(k.getLeft(), methodsA.get(k), methodsB.get(k)));
List<Pair<String, String>> methodKeyUnion = union(classes.stream().map(clazz -> mapToFirstNamespaceAndDescriptor(clazz).collect(Collectors.toList())).collect(Collectors.toList()));
List<Map<Pair<String, String>, TinyMethod>> methods = classes.stream().map(TinyClass::mapMethodsByFirstNamespaceAndDescriptor).collect(Collectors.toList());
List<TinyMethod> mergedMethods = map(methodKeyUnion, (Pair<String, String> k) ->
mergeMethods(k.getLeft(), methods.stream().map(method -> method.get(k)).collect(Collectors.toList())));

List<String> fieldKeyUnion = keyUnion(classA.getFields(), classB.getFields());
Map<String, TinyField> fieldsA = classA.mapFieldsByFirstNamespace();
Map<String, TinyField> fieldsB = classB.mapFieldsByFirstNamespace();
List<TinyField> mergedFields = map(fieldKeyUnion, k -> mergeFields(k, fieldsA.get(k), fieldsB.get(k)));
List<String> fieldKeyUnion = keyUnion(classes.stream().map(TinyClass::getFields).collect(Collectors.toList()));
List<Map<String, TinyField>> fields = classes.stream().map(TinyClass::mapFieldsByFirstNamespace).collect(Collectors.toList());
List<TinyField> mergedFields = map(fieldKeyUnion, k -> mergeFields(k, fields.stream().map(map -> map.get(k)).collect(Collectors.toList())));

return new TinyClass(mergedNames, mergedMethods, mergedFields, mergedComments);
}

private static final TinyMethod EMPTY_METHOD = new TinyMethod(null, Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Collections.emptyList());

private TinyMethod mergeMethods(String sharedMethodName, List<TinyMethod> methods) {
List<String> mergedNames = mergeNames(sharedMethodName, methods);
methods.replaceAll(method -> method == null ? EMPTY_METHOD : method);
List<String> mergedComments = mergeComments(methods.stream().map(TinyMethod::getComments).collect(Collectors.toList()));

private TinyMethod mergeMethods(String sharedMethodName, @Nullable TinyMethod methodA, @Nullable TinyMethod methodB) {
List<String> mergedNames = mergeNames(sharedMethodName, methodA, methodB);
if (methodA == null) methodA = EMPTY_METHOD;
if (methodB == null) methodB = EMPTY_METHOD;
List<String> mergedComments = mergeComments(methodA.getComments(), methodB.getComments());

String descriptor = methodA.getMethodDescriptorInFirstNamespace() != null ? methodA.getMethodDescriptorInFirstNamespace()
: methodB.getMethodDescriptorInFirstNamespace();
String descriptor = methods.get(0).getMethodDescriptorInFirstNamespace() != null ? methods.get(0).getMethodDescriptorInFirstNamespace()
: methods.get(1).getMethodDescriptorInFirstNamespace();
if (descriptor == null) throw new RuntimeException("no descriptor for key " + sharedMethodName);


//TODO: this won't work too well when the first namespace is named or there is more than one named namespace (hack)
// TODO: Fix parameters
List<TinyMethodParameter> mergedParameters = new ArrayList<>();
addParameters(methodA, mergedParameters, 2);
addParameters(methodB, mergedParameters, 1);
addParameters(methods, mergedParameters);

List<TinyLocalVariable> mergedLocalVariables = new ArrayList<>();
addLocalVariables(methodA, mergedLocalVariables, 2);
addLocalVariables(methodB, mergedLocalVariables, 1);
addLocalVariables(methods, mergedLocalVariables);

return new TinyMethod(descriptor, mergedNames, mergedParameters, mergedLocalVariables, mergedComments);
}

private void addParameters(TinyMethod method, List<TinyMethodParameter> addTo, int emptySpacePos) {
for (TinyMethodParameter localVariable : method.getParameters()) {
List<String> names = new ArrayList<>(localVariable.getParameterNames());
names.add(emptySpacePos, "");
addTo.add(new TinyMethodParameter(localVariable.getLvIndex(), names, localVariable.getComments()));
private void addParameters(List<TinyMethod> methods, List<TinyMethodParameter> addTo) {
for (TinyMethod method : methods) {
for (TinyMethodParameter localVariable : method.getParameters()) {
List<String> names = new ArrayList<>(localVariable.getParameterNames());
addTo.add(new TinyMethodParameter(localVariable.getLvIndex(), names, localVariable.getComments()));
}
}
}

private void addLocalVariables(TinyMethod method, List<TinyLocalVariable> addTo, int emptySpacePos) {
for (TinyLocalVariable localVariable : method.getLocalVariables()) {
List<String> names = new ArrayList<>(localVariable.getLocalVariableNames());
names.add(emptySpacePos, "");
addTo.add(new TinyLocalVariable(localVariable.getLvIndex(), localVariable.getLvStartOffset(),
localVariable.getLvTableIndex(), names, localVariable.getComments()));
private void addLocalVariables(List<TinyMethod> methods, List<TinyLocalVariable> addTo) {
for (TinyMethod method : methods) {
for (TinyLocalVariable localVariable : method.getLocalVariables()) {
List<String> names = new ArrayList<>(localVariable.getLocalVariableNames());
addTo.add(new TinyLocalVariable(localVariable.getLvIndex(), localVariable.getLvStartOffset(),
localVariable.getLvTableIndex(), names, localVariable.getComments()));
}
}
}

private TinyField mergeFields(String sharedFieldName, List<TinyField> fields) {
List<String> mergedNames = mergeNames(sharedFieldName, fields);
List<String> mergedComments = mergeComments(fields.stream().map(field -> field != null ? field.getComments() : Collections.<String>emptyList()).collect(Collectors.toList()));

private TinyField mergeFields(String sharedFieldName, @Nullable TinyField fieldA, @Nullable TinyField fieldB) {
List<String> mergedNames = mergeNames(sharedFieldName, fieldA, fieldB);
List<String> mergedComments = mergeComments(fieldA != null ? fieldA.getComments() : Collections.emptyList(),
fieldB != null ? fieldB.getComments() : Collections.emptyList());

String descriptor = fieldA != null ? fieldA.getFieldDescriptorInFirstNamespace()
: fieldB != null ? fieldB.getFieldDescriptorInFirstNamespace() : null;
String descriptor = fields.stream().filter(Objects::nonNull).findFirst().map(TinyField::getFieldDescriptorInFirstNamespace).orElse(null);
if (descriptor == null) throw new RuntimeException("no descriptor for key " + sharedFieldName);

return new TinyField(descriptor, mergedNames, mergedComments);
}

private TinyHeader mergeHeaders(TinyHeader headerA, TinyHeader headerB) {
private TinyHeader mergeHeaders(List<TinyHeader> headers) {
TinyHeader headerA = headers.get(0);
List<String> namespaces = new ArrayList<>(headerA.getNamespaces());
namespaces.add(headerB.getNamespaces().get(1));
for (int i = 1; i < headers.size(); ++i) {
for (String namespace : headers.get(i).getNamespaces()) {
if (!namespaces.contains(namespace)) {
namespaces.add(namespace);
}
}
}
// TODO: how should versions and properties be merged?
return new TinyHeader(namespaces, headerA.getMajorVersion(), headerA.getMinorVersion(), headerA.getProperties());
}

private List<String> mergeComments(Collection<String> commentsA, Collection<String> commentsB) {
return union(commentsA, commentsB);
private List<String> mergeComments(List<Collection<String>> comments) {
return union(comments);
}

private <T extends Mapping> List<String> keyUnion(Collection<T> mappingsA, Collection<T> mappingB) {
return union(mappingsA.stream().map(m -> m.getMapping().get(0)), mappingB.stream().map(m -> m.getMapping().get(0)));
private <T extends Mapping> List<String> keyUnion(List<Collection<T>> mappings) {
return union(mappings.stream().map(c -> c.stream().map(m -> m.getMapping().get(0)).collect(Collectors.toList())).collect(Collectors.toList()));
}

private Stream<Pair<String, String>> mapToFirstNamespaceAndDescriptor(TinyClass tinyClass) {
return tinyClass.getMethods().stream().map(m -> Pair.of(m.getMapping().get(0), m.getMethodDescriptorInFirstNamespace()));
}


private List<String> mergeNames(String key, @Nullable Mapping mappingA, @Nullable Mapping mappingB) {
private List<String> mergeNames(String key, List<? extends Mapping> mappings) {
List<String> merged = new ArrayList<>();
merged.add(key);
merged.add(mappingExists(mappingA) ? mappingA.getMapping().get(1) : key);
merged.add(mappingExists(mappingB) ? mappingB.getMapping().get(1) : key);
mappings.forEach(mapping -> {
if (mapping != null) {
for (int i = 1; i < mapping.getMapping().size(); ++i) {
String m = mapping.getMapping().get(i);
merged.add(!m.isEmpty() ? m : key);
}
}
});

return merged;
}
Expand All @@ -283,8 +286,12 @@ private boolean mappingExists(@Nullable Mapping mapping) {
return mapping != null && !mapping.getMapping().get(1).isEmpty();
}

private <T> List<T> union(Stream<T> list1, Stream<T> list2) {
return union(list1.collect(Collectors.toList()), list2.collect(Collectors.toList()));
private <T> List<T> union(List<Collection<T>> lists) {
Set<T> set = new HashSet<>();

lists.forEach(set::addAll);

return new ArrayList<>(set);
}

private <T> List<T> union(Collection<T> list1, Collection<T> list2) {
Expand All @@ -303,5 +310,4 @@ private static String escape(String str) {
private <S, E> List<E> map(List<S> from, Function<S, E> mapper) {
return from.stream().map(mapper).collect(Collectors.toList());
}

}
29 changes: 29 additions & 0 deletions src/test/java/net/fabricmc/stitch/tinyv2/MergeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package net.fabricmc.stitch.tinyv2;

import net.fabricmc.stitch.commands.tinyv2.CommandMergeTinyV2;
import org.junit.jupiter.api.Test;

import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;

import static org.junit.jupiter.api.Assertions.*;

public class MergeTest {
private static final Path DIR = new File(MergeTest.class.getClassLoader().getResource("merge").getPath()).toPath().toAbsolutePath();

@Test
public void test() throws Exception {
Path expectedOutput = DIR.resolve("expected.tiny");
Path output = Files.createTempFile("stitch-merge-result-", ".tiny");
Path inputA = DIR.resolve("input-a.tiny");
Path inputB = DIR.resolve("input-b.tiny");
Path inputC = DIR.resolve("input-c.tiny");
new CommandMergeTinyV2().run(new String[]{inputA.toString(), inputB.toString(), inputC.toString(), output.toString()});

String expectedOutputContent = new String(Files.readAllBytes(expectedOutput), StandardCharsets.UTF_8).replace("\r\n", "\n");
String outputContent = new String(Files.readAllBytes(output), StandardCharsets.UTF_8).replace("\r\n", "\n");
assertEquals(expectedOutputContent, outputContent);
}
}
16 changes: 16 additions & 0 deletions src/test/resources/merge/expected.tiny
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
tiny 2 0 official hashed mojmap intermediary named
c a net/minecraft/unmapped/C_lctoxfsg com/mojang/math/Constants net/minecraft/class_5973 net/minecraft/util/math/MathConstants
f F a f_ygxirswm PI field_29658 PI
f F b f_ncoyhkkx RAD_TO_DEG field_29659 DEGREES_PER_RADIAN
f F c f_vaqzaaae DEG_TO_RAD field_29660 RADIANS_PER_DEGREE
f F d f_jhuobxwz EPSILON field_29661 EPSILON
c c net/minecraft/unmapped/C_ffukturc com/mojang/math/Matrix3f net/minecraft/class_4581 net/minecraft/util/math/Matrix3f
m (Lc;)V <init> <init> <init> <init> <init>
p 1 source
m (Ld;)V <init> <init> <init> <init> <init>
p 1 matrix
m (Lg;)V <init> <init> <init> <init> <init>
p 1 quaternion
m ()V a m_jdnetyjq transpose method_22847 transpose
m (F)V a m_asnpwjdg mul method_23274 multiply
p 1 scalar
12 changes: 12 additions & 0 deletions src/test/resources/merge/input-a.tiny
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
tiny 2 0 official hashed mojmap
c a net/minecraft/unmapped/C_lctoxfsg com/mojang/math/Constants
f F a f_ygxirswm PI
f F b f_ncoyhkkx RAD_TO_DEG
f F c f_vaqzaaae DEG_TO_RAD
f F d f_jhuobxwz EPSILON
c c net/minecraft/unmapped/C_ffukturc com/mojang/math/Matrix3f
m (Lc;)V <init> <init> <init>
m (Ld;)V <init> <init> <init>
m (Lg;)V <init> <init> <init>
m ()V a m_jdnetyjq transpose
m (F)V a m_asnpwjdg mul
Loading