From 5bf7804664606dd6280d5a0dc6e33f3a9ffbb5a6 Mon Sep 17 00:00:00 2001 From: Em3rs0n Date: Fri, 23 Nov 2018 19:45:35 -0500 Subject: [PATCH] =?UTF-8?q?=E5=B0=86APK=E8=A7=A3=E6=9E=90=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E4=BC=98=E5=8C=96=E5=88=B0=E5=8E=9F=E5=85=88=E7=9A=84?= =?UTF-8?q?=E4=B8=89=E5=88=86=E4=B9=8B=E4=B8=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 重写apk parser，移除了大量不必要的计算，将～70M的APK解析时间从1500～1700ms降低到大约500ms。 --- build.gradle | 6 - .../spellbook/MirrorUnitTest.kt | 36 +++- .../wechatmagician/spellbook/WechatGlobal.kt | 6 +- .../spellbook/parser/ApkFile.kt | 36 ++++ .../spellbook/parser/DexHeader.kt | 55 ++++++ .../spellbook/parser/DexParser.kt | 184 ++++++++++++++++++ .../spellbook/util/ReflectionUtil.kt | 2 +- 7 files changed, 306 insertions(+), 19 deletions(-) create mode 100644 src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/parser/ApkFile.kt create mode 100644 src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/parser/DexHeader.kt create mode 100644 src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/parser/DexParser.kt diff --git a/build.gradle b/build.gradle index 2450204..857bfa4 100644 --- a/build.gradle +++ b/build.gradle @@ -19,12 +19,6 @@ apply plugin: 'com.android.library' apply plugin: 'kotlin-android' apply plugin: 'org.jetbrains.dokka-android' -kotlin { - experimental { - coroutines "enable" - } -} - android { compileSdkVersion 28 defaultConfig { diff --git a/src/androidTest/kotlin/com/gh0u1l5/wechatmagician/spellbook/MirrorUnitTest.kt b/src/androidTest/kotlin/com/gh0u1l5/wechatmagician/spellbook/MirrorUnitTest.kt index 60279cf..8f2f249 100644 --- a/src/androidTest/kotlin/com/gh0u1l5/wechatmagician/spellbook/MirrorUnitTest.kt +++ b/src/androidTest/kotlin/com/gh0u1l5/wechatmagician/spellbook/MirrorUnitTest.kt @@ -8,18 +8,20 @@ import com.gh0u1l5.wechatmagician.spellbook.base.Version import com.gh0u1l5.wechatmagician.spellbook.mirror.MirrorClasses import com.gh0u1l5.wechatmagician.spellbook.mirror.MirrorFields import com.gh0u1l5.wechatmagician.spellbook.mirror.MirrorMethods +import com.gh0u1l5.wechatmagician.spellbook.parser.ApkFile import com.gh0u1l5.wechatmagician.spellbook.util.FileUtil import com.gh0u1l5.wechatmagician.spellbook.util.MirrorUtil import com.gh0u1l5.wechatmagician.spellbook.util.ReflectionUtil import dalvik.system.PathClassLoader -import net.dongliu.apk.parser.ApkFile import org.junit.Assert.* import org.junit.Before import org.junit.Test import org.junit.runner.RunWith import java.io.File import java.lang.ClassLoader.getSystemClassLoader +import kotlin.system.measureTimeMillis +@ExperimentalUnsignedTypes @RunWith(AndroidJUnit4::class) class MirrorUnitTest { companion object { @@ -34,8 +36,13 @@ class MirrorUnitTest { } private fun verifyPackage(apkPath: String) { - val cacheDir = context!!.cacheDir + // Parse the version of the apk + val regex = Regex("wechat-v(.*)\\.apk") + val match = regex.find(apkPath) ?: throw Exception("Unexpected path format") + val version = match.groupValues[1] + // Store APK file to cache directory. + val cacheDir = context!!.cacheDir val apkFile = File(cacheDir, apkPath) try { javaClass.classLoader!!.getResourceAsStream(apkPath).use { @@ -46,16 +53,21 @@ class MirrorUnitTest { return // ignore if the apk isn't accessible } + // Ensure the apk is presented, and start the test assertTrue(apkFile.exists()) ApkFile(apkFile).use { + // Benchmark the APK parser + val timeParseDex = measureTimeMillis { it.classTypes } + Log.d("MirrorUnitTest", "Benchmark: Parse DexClasses takes $timeParseDex ms.") + + // Initialize WechatGlobal WechatGlobal.wxUnitTestMode = true - WechatGlobal.wxVersion = Version(it.apkMeta.versionName) - WechatGlobal.wxPackageName = it.apkMeta.packageName + WechatGlobal.wxVersion = Version(version) + WechatGlobal.wxPackageName = "com.tencent.mm" WechatGlobal.wxLoader = PathClassLoader(apkFile.absolutePath, getSystemClassLoader()) - WechatGlobal.wxClasses = it.dexClasses.map { clazz -> - ReflectionUtil.ClassName(clazz.classType) - } + WechatGlobal.wxClasses = it.classTypes + // Clear cached lazy evaluations val objects = MirrorClasses + MirrorMethods + MirrorFields ReflectionUtil.clearClassCache() ReflectionUtil.clearMethodCache() @@ -63,8 +75,14 @@ class MirrorUnitTest { MirrorUtil.clearUnitTestLazyFields(instance) } - MirrorUtil.generateReportWithForceEval(objects).forEach { - Log.d("MirrorUnitTest", "Verified ${it.first} -> ${it.second}") + // Test each lazy evaluation and generate result. + var result: List>? = null + val timeSearch = measureTimeMillis { + result = MirrorUtil.generateReportWithForceEval(objects) + } + Log.d("MirrorUnitTest", "Benchmark: Search over classes takes $timeSearch ms.") + result?.forEach { entry -> + Log.d("MirrorUnitTest", "Verified: ${entry.first} -> ${entry.second}") } } diff --git a/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/WechatGlobal.kt b/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/WechatGlobal.kt index 7c039dd..1c204f5 100644 --- a/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/WechatGlobal.kt +++ b/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/WechatGlobal.kt @@ -5,9 +5,9 @@ import android.widget.BaseAdapter import com.gh0u1l5.wechatmagician.spellbook.SpellBook.getApplicationVersion import com.gh0u1l5.wechatmagician.spellbook.base.Version import com.gh0u1l5.wechatmagician.spellbook.base.WaitChannel +import com.gh0u1l5.wechatmagician.spellbook.parser.ApkFile import com.gh0u1l5.wechatmagician.spellbook.util.BasicUtil.tryAsynchronously import de.robv.android.xposed.callbacks.XC_LoadPackage -import net.dongliu.apk.parser.ApkFile import java.lang.ref.WeakReference /** @@ -38,7 +38,7 @@ object WechatGlobal { * * Example: "Ljava/lang/String;" */ - @Volatile var wxClasses: List? = null + @Volatile var wxClasses: Array? = null /** * A flag indicating whether the codes are running under unit test mode. @@ -114,7 +114,7 @@ object WechatGlobal { wxLoader = lpparam.classLoader ApkFile(lpparam.appInfo.sourceDir).use { - wxClasses = it.dexClasses.map { it.classType } + wxClasses = it.classTypes } } finally { initializeChannel.done() diff --git a/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/parser/ApkFile.kt b/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/parser/ApkFile.kt new file mode 100644 index 0000000..1b347a1 --- /dev/null +++ b/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/parser/ApkFile.kt @@ -0,0 +1,36 @@ +package com.gh0u1l5.wechatmagician.spellbook.parser + +import java.io.Closeable +import java.io.File +import java.nio.ByteBuffer +import java.util.zip.ZipEntry +import java.util.zip.ZipFile + +@ExperimentalUnsignedTypes +class ApkFile(apkFile: File) : Closeable { + companion object { + const val DEX_FILE = "classes.dex" + const val DEX_ADDITIONAL = "classes%d.dex" + } + + constructor(path: String) : this(File(path)) + + private val zipFile: ZipFile = ZipFile(apkFile) + + private fun readEntry(entry: ZipEntry): ByteArray = + zipFile.getInputStream(entry).use { it.readBytes() } + + override fun close() = + zipFile.close() + + val classTypes: Array by lazy { + var ret = emptyArray() + for (i in 1 until 1000) { + val path = if (i == 1) DEX_FILE else String.format(DEX_ADDITIONAL, i) + val entry = zipFile.getEntry(path) ?: break + val buffer = ByteBuffer.wrap(readEntry(entry)) + ret += DexParser(buffer).parseClassTypes() + } + return@lazy ret + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/parser/DexHeader.kt b/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/parser/DexHeader.kt new file mode 100644 index 0000000..199281a --- /dev/null +++ b/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/parser/DexHeader.kt @@ -0,0 +1,55 @@ +package com.gh0u1l5.wechatmagician.spellbook.parser + +@ExperimentalUnsignedTypes +class DexHeader { + var version: Int = 0 + + var checksum: UInt = 0u + + var signature: ByteArray = ByteArray(kSHA1DigestLen) + + var fileSize: UInt = 0u + + var headerSize: UInt = 0u + + var endianTag: UInt = 0u + + var linkSize: UInt = 0u + + var linkOff: UInt = 0u + + var mapOff: UInt = 0u + + var stringIdsSize: Int = 0 + + var stringIdsOff: UInt = 0u + + var typeIdsSize: Int = 0 + + var typeIdsOff: UInt = 0u + + var protoIdsSize: Int = 0 + + var protoIdsOff: UInt = 0u + + var fieldIdsSize: Int = 0 + + var fieldIdsOff: UInt = 0u + + var methodIdsSize: Int = 0 + + var methodIdsOff: UInt = 0u + + var classDefsSize: Int = 0 + + var classDefsOff: UInt = 0u + + var dataSize: Int = 0 + + var dataOff: UInt = 0u + + companion object { + const val kSHA1DigestLen = 20 + const val kSHA1DigestOutputLen = kSHA1DigestLen * 2 + 1 + } +} \ No newline at end of file diff --git a/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/parser/DexParser.kt b/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/parser/DexParser.kt new file mode 100644 index 0000000..7b0f104 --- /dev/null +++ b/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/parser/DexParser.kt @@ -0,0 +1,184 @@ +package com.gh0u1l5.wechatmagician.spellbook.parser + +import java.nio.Buffer +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** + * parse dex file. + * current we only get the class name. + * see: + * http://source.android.com/devices/tech/dalvik/dex-format.html + * http://dexandroid.googlecode.com/svn/trunk/dalvik/libdex/DexFile.h + * + * @author dongliu + */ +@ExperimentalUnsignedTypes +class DexParser(buffer: ByteBuffer) { + private val buffer: ByteBuffer = buffer.duplicate().apply { + order(ByteOrder.LITTLE_ENDIAN) + } + + private fun ByteBuffer.readBytes(size: Int) = ByteArray(size).also { get(it) } + + fun parseClassTypes(): Array { + // read magic + val magic = String(buffer.readBytes(8)) + if (!magic.startsWith("dex\n")) { + return arrayOf() + } + val version = Integer.parseInt(magic.substring(4, 7)) + // now the version is 035 + if (version < 35) { + // version 009 was used for the M3 releases of the Android platform (November–December 2007), + // and version 013 was used for the M5 releases of the Android platform (February–March 2008) + throw Exception("Dex file version: $version is not supported") + } + + // read header + val header = readDexHeader() + header.version = version + + // read string offsets + val stringOffsets = readStringOffsets(header.stringIdsOff, header.stringIdsSize) + + // read type ids + val typeIds = readTypeIds(header.typeIdsOff, header.typeIdsSize) + + // read class ids + val classIds = readClassIds(header.classDefsOff, header.classDefsSize) + + // read class types + return Array(classIds.size) { i -> + val classId = classIds[i] + val typeId = typeIds[classId] + val offset = stringOffsets[typeId] + readStringAtOffset(offset) + } + } + + private fun readDexHeader() = DexHeader().apply { + checksum = buffer.int.toUInt() + + buffer.get(signature) + + fileSize = buffer.int.toUInt() + headerSize = buffer.int.toUInt() + + endianTag = buffer.int.toUInt() + + linkSize = buffer.int.toUInt() + linkOff = buffer.int.toUInt() + + mapOff = buffer.int.toUInt() + + stringIdsSize = buffer.int + stringIdsOff = buffer.int.toUInt() + + typeIdsSize = buffer.int + typeIdsOff = buffer.int.toUInt() + + protoIdsSize = buffer.int + protoIdsOff = buffer.int.toUInt() + + fieldIdsSize = buffer.int + fieldIdsOff = buffer.int.toUInt() + + methodIdsSize = buffer.int + methodIdsOff = buffer.int.toUInt() + + classDefsSize = buffer.int + classDefsOff = buffer.int.toUInt() + + dataSize = buffer.int + dataOff = buffer.int.toUInt() + } + + private fun readStringOffsets(stringIdsOff: UInt, stringIdsSize: Int): IntArray { + (buffer as Buffer).position(stringIdsOff.toInt()) + return IntArray(stringIdsSize) { + buffer.int + } + } + + private fun readTypeIds(typeIdsOff: UInt, typeIdsSize: Int): IntArray { + (buffer as Buffer).position(typeIdsOff.toInt()) + return IntArray(typeIdsSize) { + buffer.int + } + } + + private fun readClassIds(classDefsOff: UInt, classDefsSize: Int): Array { + (buffer as Buffer).position(classDefsOff.toInt()) + return Array(classDefsSize) { + val classIdx = buffer.int + // access_flags, skip + buffer.int + // superclass_idx, skip + buffer.int + // interfaces_off, skip + buffer.int + // source_file_idx, skip + buffer.int + // annotations_off, skip + buffer.int + // class_data_off, skip + buffer.int + // static_values_off, skip + buffer.int + + classIdx + } + } + + private fun readStringAtOffset(offset: Int): String { + (buffer as Buffer).position(offset) + val len = readULEB128Int() + return readString(len) + } + + private fun readULEB128Int(): Int { + var ret = 0 + + var count = 0 + var byte: Int + do { + if (count > 4) { + throw Exception("read varints error.") + } + byte = buffer.get().toInt() + ret = ret or (byte and 0x7f shl count * 7) + count++ + } while (byte and 0x80 != 0) + + return ret + } + + private fun readString(len: Int): String { + val chars = CharArray(len) + + for (i in 0 until len) { + val a = buffer.get().toInt() + when { + a and 0x80 == 0 -> { // ascii char + chars[i] = a.toChar() + } + a and 0xe0 == 0xc0 -> { // read one more + val b = buffer.get().toInt() + chars[i] = (a and 0x1F shl 6 or (b and 0x3F)).toChar() + } + a and 0xf0 == 0xe0 -> { + val b = buffer.get().toInt() + val c = buffer.get().toInt() + chars[i] = (a and 0x0F shl 12 or (b and 0x3F shl 6) or (c and 0x3F)).toChar() + } + else -> { + // throw UTFDataFormatException() + } + } + } + + return String(chars) + } +} + diff --git a/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/util/ReflectionUtil.kt b/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/util/ReflectionUtil.kt index a26c8a8..8dc12ae 100644 --- a/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/util/ReflectionUtil.kt +++ b/src/main/kotlin/com/gh0u1l5/wechatmagician/spellbook/util/ReflectionUtil.kt @@ -121,7 +121,7 @@ object ReflectionUtil { } // findClassesFromPackage returns a list of all the classes contained in the given package. - @JvmStatic fun findClassesFromPackage(loader: ClassLoader, classes: List, packageName: String, depth: Int = 0): Classes { + @JvmStatic fun findClassesFromPackage(loader: ClassLoader, classes: Array, packageName: String, depth: Int = 0): Classes { if ((packageName to depth) in classCache) { return classCache[packageName to depth]!! }