From 79375c6a8912b51abb45911814d49a03b7062e32 Mon Sep 17 00:00:00 2001 From: EpicPlayerA10 Date: Sat, 12 Oct 2024 19:14:12 +0200 Subject: [PATCH] refactor and simplify saving output logic --- CONTRIBUTING.md | 2 +- .../deobfuscator/api/asm/ClassWrapper.java | 13 +- .../api/context/DeobfuscatorOptions.java | 16 +- .../deobfuscator/api/helper/ClassHelper.java | 12 +- .../deobfuscator/api/helper/FileHelper.java | 26 +-- .../uwu/narumi/deobfuscator/Deobfuscator.java | 172 +++++++++--------- .../base/AssertingResultSaver.java | 1 - .../base/TestDeobfuscationBase.java | 3 +- .../impl/hp888/HP888PackerTransformer.java | 17 +- reverse-engineering/README.md | 1 - 10 files changed, 135 insertions(+), 128 deletions(-) delete mode 100644 reverse-engineering/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f8dc8080..df2477ac 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,7 +10,7 @@ The project is structured as follows: - [`deobfuscator-api`](./deobfuscator-api) - The API for the deobfuscator. - [`deobfuscator-impl`](./deobfuscator-impl) - The main deobfuscator runner. - [`deobfuscator-transformers`](./deobfuscator-transformers) - Transformers for the deobfuscator. -- [`reverse-engineering`](./deobfuscator-impl/src/test/java/reverseengineering) - A place where you can throw your reverse-engineered classes. More info [here](./deobfuscator-impl/src/test/java/reverseengineering/README.md) +- [`reverse-engineering`](./reverse-engineering) - A place where you can throw your reverse-engineered classes (for example: complex number pool, complex string pool). You can reference them inside your transformers to make them more readable. - [`testData`](./testData) - Tests for transformers - [`src/java`](./testData/src/java) - You can write your java code to test transformers - [`compiled/custom-classes`](./testData/compiled/custom-classes) - Compiled classes to test transformers. You can throw here classes from your obfuscated jars. diff --git a/deobfuscator-api/src/main/java/uwu/narumi/deobfuscator/api/asm/ClassWrapper.java b/deobfuscator-api/src/main/java/uwu/narumi/deobfuscator/api/asm/ClassWrapper.java index 83d3666e..328e9fca 100644 --- a/deobfuscator-api/src/main/java/uwu/narumi/deobfuscator/api/asm/ClassWrapper.java +++ b/deobfuscator-api/src/main/java/uwu/narumi/deobfuscator/api/asm/ClassWrapper.java @@ -25,24 +25,21 @@ public class ClassWrapper implements Cloneable { private final ClassNode classNode; private final FieldCache fieldCache; private final ConstantPool constantPool; - private final int classWriterFlags; - public ClassWrapper(String pathInJar, ClassReader classReader, int classReaderFlags, int classWriterFlags) { + public ClassWrapper(String pathInJar, ClassReader classReader, int classReaderFlags) { this.pathInJar = pathInJar; this.classNode = new ClassNode(); this.constantPool = new ConstantPool(classReader); this.fieldCache = new FieldCache(); - this.classWriterFlags = classWriterFlags; classReader.accept(this.classNode, classReaderFlags); } - private ClassWrapper(String pathInJar, ClassNode classNode, FieldCache fieldCache, ConstantPool constantPool, int classWriterFlags) { + private ClassWrapper(String pathInJar, ClassNode classNode, FieldCache fieldCache, ConstantPool constantPool) { this.pathInJar = pathInJar; this.classNode = classNode; this.fieldCache = fieldCache; this.constantPool = constantPool; - this.classWriterFlags = classWriterFlags; } public Optional findMethod(String name, String desc) { @@ -129,9 +126,9 @@ public String canonicalName() { /** * Compiles class to bytes. */ - public byte[] compileToBytes(InheritanceGraph inheritanceGraph) { + public byte[] compileToBytes(InheritanceGraph inheritanceGraph, int classWriterFlags) { try { - ClassWriter classWriter = new InheritanceClassWriter(this.classWriterFlags, inheritanceGraph); + ClassWriter classWriter = new InheritanceClassWriter(classWriterFlags, inheritanceGraph); this.classNode.accept(classWriter); return classWriter.toByteArray(); @@ -166,6 +163,6 @@ public ConstantPool getConstantPool() { @Override public ClassWrapper clone() { - return new ClassWrapper(this.pathInJar, ClassHelper.copy(classNode), fieldCache.clone(), constantPool.clone(), this.classWriterFlags); + return new ClassWrapper(this.pathInJar, ClassHelper.copy(classNode), fieldCache.clone(), constantPool.clone()); } } diff --git a/deobfuscator-api/src/main/java/uwu/narumi/deobfuscator/api/context/DeobfuscatorOptions.java b/deobfuscator-api/src/main/java/uwu/narumi/deobfuscator/api/context/DeobfuscatorOptions.java index f60ea8a3..3c930cde 100644 --- a/deobfuscator-api/src/main/java/uwu/narumi/deobfuscator/api/context/DeobfuscatorOptions.java +++ b/deobfuscator-api/src/main/java/uwu/narumi/deobfuscator/api/context/DeobfuscatorOptions.java @@ -35,7 +35,8 @@ public record DeobfuscatorOptions( boolean printStacktraces, boolean continueOnError, - boolean verifyBytecode + boolean verifyBytecode, + boolean skipFiles ) { public static DeobfuscatorOptions.Builder builder() { return new DeobfuscatorOptions.Builder(); @@ -74,6 +75,7 @@ public static class Builder { private boolean printStacktraces = true; private boolean continueOnError = false; private boolean verifyBytecode = false; + private boolean skipFiles = false; private Builder() { } @@ -222,6 +224,15 @@ public DeobfuscatorOptions.Builder verifyBytecode() { return this; } + /** + * Skips files during saving. + */ + @Contract(" -> this") + public DeobfuscatorOptions.Builder skipFiles() { + this.skipFiles = true; + return this; + } + /** * Build immutable {@link DeobfuscatorOptions} with options verification */ @@ -252,7 +263,8 @@ public DeobfuscatorOptions build() { // Other config printStacktraces, continueOnError, - verifyBytecode + verifyBytecode, + skipFiles ); } } diff --git a/deobfuscator-api/src/main/java/uwu/narumi/deobfuscator/api/helper/ClassHelper.java b/deobfuscator-api/src/main/java/uwu/narumi/deobfuscator/api/helper/ClassHelper.java index 5a8883ce..e9588c35 100644 --- a/deobfuscator-api/src/main/java/uwu/narumi/deobfuscator/api/helper/ClassHelper.java +++ b/deobfuscator-api/src/main/java/uwu/narumi/deobfuscator/api/helper/ClassHelper.java @@ -34,15 +34,13 @@ public static boolean isClass(byte[] bytes) { * @param pathInJar Relative path of a class in a jar * @param bytes Class bytes * @param classReaderFlags {@link ClassReader} flags - * @param classWriterFlags {@link ClassWriter} flags */ public static ClassWrapper loadClass( String pathInJar, byte[] bytes, - @MagicConstant(flagsFromClass = ClassReader.class) int classReaderFlags, - @MagicConstant(flagsFromClass = ClassWriter.class) int classWriterFlags + @MagicConstant(flagsFromClass = ClassReader.class) int classReaderFlags ) { - return new ClassWrapper(pathInJar, new ClassReader(bytes), classReaderFlags, classWriterFlags); + return new ClassWrapper(pathInJar, new ClassReader(bytes), classReaderFlags); } /** @@ -51,18 +49,16 @@ public static ClassWrapper loadClass( * @param pathInJar Relative path of a class in a jar * @param bytes Class bytes * @param classReaderFlags {@link ClassReader} flags - * @param classWriterFlags {@link ClassWriter} flags */ public static ClassWrapper loadUnknownClass( String pathInJar, byte[] bytes, - @MagicConstant(flagsFromClass = ClassReader.class) int classReaderFlags, - @MagicConstant(flagsFromClass = ClassWriter.class) int classWriterFlags + @MagicConstant(flagsFromClass = ClassReader.class) int classReaderFlags ) throws InvalidClassException { // Fix class bytes = fixClass(bytes); - return loadClass(pathInJar, bytes, classReaderFlags, classWriterFlags); + return loadClass(pathInJar, bytes, classReaderFlags); } /** diff --git a/deobfuscator-api/src/main/java/uwu/narumi/deobfuscator/api/helper/FileHelper.java b/deobfuscator-api/src/main/java/uwu/narumi/deobfuscator/api/helper/FileHelper.java index 57469716..4ca44398 100644 --- a/deobfuscator-api/src/main/java/uwu/narumi/deobfuscator/api/helper/FileHelper.java +++ b/deobfuscator-api/src/main/java/uwu/narumi/deobfuscator/api/helper/FileHelper.java @@ -8,6 +8,7 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.function.BiConsumer; import java.util.jar.JarFile; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -15,22 +16,23 @@ public final class FileHelper { private static final Logger LOGGER = LogManager.getLogger(FileHelper.class); - private FileHelper() {} + private FileHelper() { + } public static void loadFilesFromZip(Path path, BiConsumer consumer) { try (JarFile zipFile = new JarFile(path.toFile())) { - zipFile - .entries() + zipFile.entries() .asIterator() - .forEachRemaining( - zipEntry -> { - try { - consumer.accept(zipEntry.getName(), zipFile.getInputStream(zipEntry).readAllBytes()); - } catch (Exception e) { - LOGGER.error("Could not load ZipEntry: {}", zipEntry.getName()); - LOGGER.debug("Error", e); - } - }); + .forEachRemaining(zipEntry -> { + if (zipEntry.isDirectory()) return; + + try { + consumer.accept(zipEntry.getName(), zipFile.getInputStream(zipEntry).readAllBytes()); + } catch (Exception e) { + LOGGER.error("Could not load ZipEntry: {}", zipEntry.getName()); + LOGGER.debug("Error", e); + } + }); } catch (Exception e) { LOGGER.error("Could not load file: {}", path); throw new RuntimeException(e); diff --git a/deobfuscator-impl/src/main/java/uwu/narumi/deobfuscator/Deobfuscator.java b/deobfuscator-impl/src/main/java/uwu/narumi/deobfuscator/Deobfuscator.java index cd4420bd..ed3c60bb 100644 --- a/deobfuscator-impl/src/main/java/uwu/narumi/deobfuscator/Deobfuscator.java +++ b/deobfuscator-impl/src/main/java/uwu/narumi/deobfuscator/Deobfuscator.java @@ -6,6 +6,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.*; +import java.util.function.BiConsumer; import java.util.function.Supplier; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -92,7 +93,7 @@ private void loadInput() { if (this.options.inputJar() != null) { LOGGER.info("Loading jar file: {}", this.options.inputJar()); // Load jar - FileHelper.loadFilesFromZip(this.options.inputJar(), this::loadClass); + FileHelper.loadFilesFromZip(this.options.inputJar(), this::loadClassOrFile); LOGGER.info("Loaded jar file: {}", this.options.inputJar()); } @@ -101,7 +102,7 @@ private void loadInput() { try (InputStream inputStream = new FileInputStream(clazz.path().toFile())) { // Load class - this.loadClass(clazz.pathInJar(), inputStream.readAllBytes()); + this.loadClassOrFile(clazz.pathInJar(), inputStream.readAllBytes()); LOGGER.info("Loaded class: {}", clazz.pathInJar()); } catch (IOException e) { @@ -110,29 +111,27 @@ private void loadInput() { } } - private void loadClass(String pathInJar, byte[] bytes) { - try { - if (ClassHelper.isClass(pathInJar, bytes)) { - ClassWrapper classWrapper = ClassHelper.loadUnknownClass( - pathInJar, - bytes, - ClassReader.SKIP_FRAMES, - this.options.classWriterFlags() - ); + private void loadClassOrFile(String pathInJar, byte[] bytes) { + // Load class + if (ClassHelper.isClass(pathInJar, bytes)) { + try { + ClassWrapper classWrapper = ClassHelper.loadUnknownClass(pathInJar, bytes, ClassReader.SKIP_FRAMES); context.getClasses().putIfAbsent(classWrapper.name(), classWrapper); - } else if (!context.getFiles().containsKey(pathInJar)) { - context.getFiles().put(pathInJar, bytes); + return; + } catch (Exception e) { + LOGGER.error("Could not load class: {}, adding as file", pathInJar, e); + // Will add as a file } - } catch (Exception e) { - LOGGER.error("Could not load class: {}, adding as file", pathInJar); - if (this.options.printStacktraces()) LOGGER.throwing(e); + } - context.getFiles().putIfAbsent(pathInJar, bytes); + // Load file + if (!context.getFiles().containsKey(pathInJar)) { + context.getFiles().put(pathInJar, bytes); } } public void transform(List> transformers) { - if (transformers == null || transformers.isEmpty()) return; + if (transformers.isEmpty()) return; // Run all transformers! transformers.forEach(transformerSupplier -> Transformer.transform(transformerSupplier, null, this.context)); @@ -145,36 +144,28 @@ private void saveOutput() { if (this.options.outputJar() != null) { saveToJar(); } else if (this.options.outputDir() != null) { - saveClassesToDir(); + saveToDir(); } else { throw new IllegalStateException("No output file or directory provided"); } } - private void saveClassesToDir() { - LOGGER.info("Saving classes to output directory: {}", this.options.outputDir()); - - InheritanceGraph inheritanceGraph = new InheritanceGraph(this.context.getCombinedClasspath()); + private void saveToDir() { + LOGGER.info("Saving output to directory: {}", this.options.outputDir()); - context - .getClasses() - .forEach((ignored, classWrapper) -> { - try { - byte[] data = classWrapper.compileToBytes(inheritanceGraph); - - Path path = this.options.outputDir().resolve(classWrapper.getPathInJar()); - Files.createDirectories(path.getParent()); - Files.write(path, data); - } catch (Exception e) { - LOGGER.error("Could not save class: {}", classWrapper.name(), e); - } - - context.getClasses().remove(classWrapper.name()); - }); + save((path, data) -> { + Path realPath = this.options.outputDir().resolve(path); + try { + Files.createDirectories(realPath.getParent()); + Files.write(realPath, data); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } private void saveToJar() { - LOGGER.info("Saving output file: {}", this.options.outputJar()); + LOGGER.info("Saving output to jar: {}", this.options.outputJar()); // Create directories if not exists if (this.options.outputJar().getParent() != null) { @@ -185,58 +176,69 @@ private void saveToJar() { } } - InheritanceGraph inheritanceGraph = new InheritanceGraph(this.context.getCombinedClasspath()); - try (ZipOutputStream zipOutputStream = new ZipOutputStream(Files.newOutputStream(this.options.outputJar()))) { zipOutputStream.setLevel(9); - context - .getClasses() - .forEach( - (ignored, classWrapper) -> { - try { - byte[] data = classWrapper.compileToBytes(inheritanceGraph); - - zipOutputStream.putNextEntry(new ZipEntry(classWrapper.name() + ".class")); - zipOutputStream.write(data); - } catch (Exception e) { - LOGGER.error("Could not save class, saving original class instead of deobfuscated: {}", classWrapper.name()); - if (this.options.printStacktraces()) LOGGER.throwing(e); - - try { - // Save original class as a fallback - byte[] data = context.getPrimaryClasspath().rawClasses().get(classWrapper.name()); - - zipOutputStream.putNextEntry(new ZipEntry(classWrapper.name() + ".class")); - zipOutputStream.write(data); - } catch (Exception e2) { - LOGGER.error("Could not save original class: {}", classWrapper.name()); - if (this.options.printStacktraces()) LOGGER.throwing(e2); - } - } - - context.getClasses().remove(classWrapper.name()); - }); - - context - .getFiles() - .forEach( - (name, data) -> { - try { - zipOutputStream.putNextEntry(new ZipEntry(name)); - zipOutputStream.write(data); - } catch (Exception e) { - LOGGER.error("Could not save file: {}", name); - if (this.options.printStacktraces()) LOGGER.throwing(e); - } - - context.getFiles().remove(name); - }); + save((path, data) -> { + try { + zipOutputStream.putNextEntry(new ZipEntry(path)); + zipOutputStream.write(data); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } catch (IOException e) { - LOGGER.error("Could not save output file: {}", this.options.outputJar()); + LOGGER.error("Could not save output to jar: {}", this.options.outputJar()); throw new RuntimeException(e); } - LOGGER.info("Saved output file: {}\n", this.options.outputJar()); + LOGGER.info("Saved output to jar: {}\n", this.options.outputJar()); + } + + /** + * Saves classes and files using provided saver + * + * @param saver a consumer that accepts a path and data to save + */ + private void save(BiConsumer saver) { + InheritanceGraph inheritanceGraph = new InheritanceGraph(this.context.getCombinedClasspath()); + + // Save classes + context.getClasses().forEach((ignored, classWrapper) -> { + String path = classWrapper.getPathInJar(); + + try { + byte[] data = classWrapper.compileToBytes(inheritanceGraph, this.options.classWriterFlags()); + saver.accept(path, data); + } catch (Exception e) { + LOGGER.error("Could not save class, saving original class instead of deobfuscated: {}", classWrapper.name()); + if (this.options.printStacktraces()) LOGGER.throwing(e); + + try { + // Save original class as a fallback + byte[] data = context.getPrimaryClasspath().rawClasses().get(classWrapper.name()); + saver.accept(path, data); + } catch (Exception e2) { + LOGGER.error("Could not save original class: {}", classWrapper.name()); + if (this.options.printStacktraces()) LOGGER.throwing(e2); + } + } + + context.getClasses().remove(classWrapper.name()); + }); + + // Save files + if (!this.options.skipFiles()) { + context.getFiles().forEach((path, data) -> { + try { + saver.accept(path, data); + } catch (Exception e) { + LOGGER.error("Could not save file: {}", path); + if (this.options.printStacktraces()) LOGGER.throwing(e); + } + + context.getFiles().remove(path); + }); + } } } diff --git a/deobfuscator-impl/src/test/java/uwu/narumi/deobfuscator/base/AssertingResultSaver.java b/deobfuscator-impl/src/test/java/uwu/narumi/deobfuscator/base/AssertingResultSaver.java index 1dec045d..95ffd8a3 100644 --- a/deobfuscator-impl/src/test/java/uwu/narumi/deobfuscator/base/AssertingResultSaver.java +++ b/deobfuscator-impl/src/test/java/uwu/narumi/deobfuscator/base/AssertingResultSaver.java @@ -22,7 +22,6 @@ public AssertingResultSaver(TestDeobfuscationBase.InputType inputType, String ja this.root = inputType == TestDeobfuscationBase.InputType.CUSTOM_JAR ? TestDeobfuscationBase.RESULTS_CLASSES_PATH.resolve(inputType.directory()).resolve(jarRelativePath) : TestDeobfuscationBase.RESULTS_CLASSES_PATH.resolve(inputType.directory()); - } @Override diff --git a/deobfuscator-impl/src/test/java/uwu/narumi/deobfuscator/base/TestDeobfuscationBase.java b/deobfuscator-impl/src/test/java/uwu/narumi/deobfuscator/base/TestDeobfuscationBase.java index 4a335ffe..aba901c7 100644 --- a/deobfuscator-impl/src/test/java/uwu/narumi/deobfuscator/base/TestDeobfuscationBase.java +++ b/deobfuscator-impl/src/test/java/uwu/narumi/deobfuscator/base/TestDeobfuscationBase.java @@ -125,7 +125,8 @@ private void runTest() { // Last configurations optionsBuilder .outputJar(null) - .outputDir(decompilerInputDir); + .outputDir(decompilerInputDir) + .skipFiles(); // Build and run deobfuscator! Deobfuscator.from(optionsBuilder.build()).start(); diff --git a/deobfuscator-transformers/src/main/java/uwu/narumi/deobfuscator/core/other/impl/hp888/HP888PackerTransformer.java b/deobfuscator-transformers/src/main/java/uwu/narumi/deobfuscator/core/other/impl/hp888/HP888PackerTransformer.java index 02c1f776..dd1fc309 100644 --- a/deobfuscator-transformers/src/main/java/uwu/narumi/deobfuscator/core/other/impl/hp888/HP888PackerTransformer.java +++ b/deobfuscator-transformers/src/main/java/uwu/narumi/deobfuscator/core/other/impl/hp888/HP888PackerTransformer.java @@ -32,7 +32,7 @@ public HP888PackerTransformer(String encryptedClassFilesSuffix) { @Override protected void transform(ClassWrapper scope, Context context) throws Exception { - Set toRemove = new HashSet<>(); + Set filesToRemove = new HashSet<>(); HashMap newClasses = new HashMap<>(); AtomicReference key = new AtomicReference<>(); @@ -56,16 +56,13 @@ protected void transform(ClassWrapper scope, Context context) throws Exception { context.getFiles().forEach((file, bytes) -> { if (file.endsWith(encryptedClassFilesSuffix)) { String cleanFileName = file.replace(encryptedClassFilesSuffix, "").replace(".", "/"); - toRemove.add(file); + filesToRemove.add(file); try { // Decrypt! byte[] decrypted = cipher.doFinal(bytes); - newClasses.put(cleanFileName, ClassHelper.loadUnknownClass(cleanFileName, - decrypted, - ClassReader.SKIP_FRAMES, - ClassWriter.COMPUTE_MAXS - )); + // Load class + newClasses.put(cleanFileName, ClassHelper.loadUnknownClass(cleanFileName, decrypted, ClassReader.SKIP_FRAMES)); markChange(); } catch (Exception e) { LOGGER.error(e); @@ -73,8 +70,10 @@ protected void transform(ClassWrapper scope, Context context) throws Exception { } }); - // Cleanup - toRemove.forEach(context.getFiles()::remove); + // Put all new classes context.getClasses().putAll(newClasses); + + // Cleanup + filesToRemove.forEach(context.getFiles()::remove); } } diff --git a/reverse-engineering/README.md b/reverse-engineering/README.md deleted file mode 100644 index b317af66..00000000 --- a/reverse-engineering/README.md +++ /dev/null @@ -1 +0,0 @@ -You can place here your reverse-engineered classes from obfuscated jars (for example: complex number pool, complex string pool), and you can reference them inside your transformers. This will make complex transformers' code more readable.