diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..3270137 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tools/python-uncompyle6"] + path = scripts/python-uncompyle6 + url = https://github.com/xforce/python-uncompyle6.git diff --git a/Cargo.toml b/Cargo.toml index 4d592bd..a53ab55 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,3 +12,4 @@ simple_logger = "1.6" compress = "0.2" tree_magic = "0.2" bytesize = "1.0.0" +regex = "1" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..fa214f9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Alexander Guettler + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 574bfae..6368277 100644 --- a/README.md +++ b/README.md @@ -35,10 +35,17 @@ cargo install --path . Example: ``` -npktool x script.npk out +npktool x script.npk ``` -This will extract all the files in script.npk to the out `out/script` directory. +This will extract all the files in script.npk to the `out` directory. + +You can also supply a list of .npk files, and if those contain a filelist file, the tool +will automatically detect it and put the files in the original file structure: +> In Eve Echoes the filelist file usually resides in res0.npk +``` +npktool x res0.npk res1.npk res2.npk res3.npk res4.npk res5.npk res6.npk res7.npk res8.npk res9.npk res10.npk res11.npk res12.npk +``` More info on how to use it can be found in the help section. `npktool --help` diff --git a/scripts/decompile_pyc.py b/scripts/decompile_pyc.py new file mode 100755 index 0000000..ea02ae8 --- /dev/null +++ b/scripts/decompile_pyc.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python + +# NOTE(alexander): DO NOT REORDER THIS! +import sys +import os +dir_path = os.path.dirname(os.path.realpath(__file__)) + +sys.path.insert(0, os.path.join(dir_path, "python-uncompyle6")) + +from uncompyle6.bin.uncompile import main_bin + +main_bin() diff --git a/scripts/pyc_decryptor.py b/scripts/pyc_decryptor.py new file mode 100755 index 0000000..141d19f --- /dev/null +++ b/scripts/pyc_decryptor.py @@ -0,0 +1,175 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +import os +import zlib +import marshal +import binascii +import argparse +import pymarshal + + +class PYCEncryptor(object): + def __init__(self): + # LEFT (ORIGNAL) RIGHT (NEOX) + self.opcode_encrypt_map = { + 0: 0, # STOP CODE (TODO) + 1: 38, # POP TOP + 2: 46, # ROT TWO + 3: 37, # ROT THREE + 4: 66, # DUP TOP + 5: 12, # ROT FOUR + # 9: 13, # NOP (TODO) + 10: 35, # UNARY POSITIVE + 11: 67, # UNARY NEGATIVE + 12: 81, # UNARY_NOT + 13: 32, # UNARY_CONVERT + 15: 9, # UNARY_INVERT + 19: 63, # BINARY_POWER + 20: 70, # BINARY_MULTIPLY + 21: 44, # BINARY_DIVIDE + 22: 36, # BINARY_MODULO + 23: 39, # BINARY_ADD + 24: 57, # BINARY_SUBTRACT + 25: 10, # BINARY_SUBSCR + 26: 52, # BINARY_FLOOR_DIVIDE + 27: 13, # BINARY_TRUE_DIVIDE (TODO) + 28: 49, # INPLACE_FLOOR_DIVIDE + # 29: 29, # INPLACE_TRUE_DIVIDE (TODO) + 30: 86, # SLICE + 31: 87, # SLICE_1 + 32: 88, # SLICE_2 + 33: 89, # SLICE_3 + 40: 24, # STORE_SLICE + 41: 25, # STORE_SLICE_1 + 42: 26, # STORE_SLICE_2 + 43: 27, # STORE_SLICE_3 + 50: 14, # DELETE_SLICE + 51: 15, # DELETE_SLICE_1 + 52: 16, # DELETE_SLICE_2 + 53: 17, # DELETE_SLICE_3 + 54: 8, # STORE_MAP + 55: 21, # INPLACE_ADD + 56: 55, # INPLACE_SUBTRACT + 57: 82, # INPLACE_MULTIPLY + 58: 34, # INPLACE_DIVIDE + 59: 22, # INPLACE_MODULO + 60: 65, # STORE_SUBSCR + 61: 6, # DELETE_SUBSCR + 62: 58, # BINARY_LSHIFT + 63: 71, # BINARY_RSHIFT + 64: 43, # BINARY_AND + 65: 30, # BINARY_XOR + 66: 19, # BINARY_OR + 67: 5, # INPLACE_POWER + 68: 60, # GET_ITER + # 70: 75, # PRINT_EXPR (TODO, WIP) + 71: 53, # PRINT_ITEM + 72: 42, # PRINT_NEWLINE + 73: 3, # PRINT_ITEM_TO + 74: 48, # PRINT_NEWLINE_TO + 75: 84, # INPLACE_LSHIFT + 76: 77, # INPLACE_RSHIFT + 77: 78, # INPLACE_AND + 78: 85, # INPLACE_XOR + 79: 47, # INPLACE_OR + 80: 51, # BREAK_LOOP + 81: 54, # WITH_CLEANUP + 82: 50, # LOAD_LOCALS + 83: 83, # RETURN_VALUE + 84: 74, # IMPORT_STAR + 85: 64, # EXEC_STMT + 86: 31, # YIELD_VALUE + 87: 72, # POP_BLOCK + 88: 45, # END_FINALLY + 89: 33, # BUILD_CLASS + 90: 145, # HAVE_ARGUMENT/ STORE_NAME + 91: 159, # DELETE_NAME + 92: 125, # UNPACK_SEQUENCE + 93: 149, # FOR_ITER + 94: 157, # LIST_APPEND + 95: 132, # STORE_ATTR + 96: 95, # DELETE_ATTR + 97: 113, # STORE_GLOBAL + 98: 111, # DELETE_GLOBAL + 99: 138, # DUP_TOPX + 100: 153, # LOAD_CONST + 101: 101, # LOAD_NAME + 102: 135, # BUILD_TUPLE + 103: 90, # BUILD_LIST + 104: 99, # BUILD_SET + 105: 151, # BUILD_MAP + 106: 96, # LOAD_ATTR + 107: 114, # COMPARE_OP + 108: 134, # IMPORT_NAME + 109: 116, # IMPORT_FROM + 110: 156, # JUMP_FORWARD + 111: 105, # JUMP_IF_FALSE_OR_POP + 112: 130, # JUMP_IF_TRUE_OR_POP + 113: 137, # JUMP_ABSOLUTE + 114: 148, # POP_JUMP_IF_FALSE + 115: 172, # POP_JUMP_IF_TRUE + 116: 155, # LOAD_GLOBAL + 119: 103, # CONTINUE_LOOP + 120: 158, # SETUP_LOOP + 121: 128, # SETUP_EXCEPT + 122: 110, # SETUP_FINALLY + 124: 97, # LOAD_FAST + 125: 104, # STORE_FAST + 126: 118, # DELETE_FAST + 130: 93, # RAISE_VARARGS + 131: 131, # CALL_FUNCTION + 132: 136, # MAKE_FUNCTION + 133: 115, # BUILD_SLICE + 134: 100, # MAKE_CLOSURE + 135: 120, # LOAD_CLOSURE + 136: 129, # LOAD_DEREF + 137: 102, # STORE_DEREF + 140: 140, # CALL_FUNCTION_VAR + 141: 141, # CALL_FUNCTION_KW + 142: 142, # CALL_FUNCTION_VAR_KW + 143: 94, # SETUP_WITH + # SPECIAL NEOX THING, LOAD CONST + LOAD FAST, I think (TODO, GUESS) + 173: 173, + 146: 109, # SET_ADD + 147: 123 # MAP_ADD + } + self.opcode_decrypt_map = { + self.opcode_encrypt_map[key]: key for key in self.opcode_encrypt_map} + self.pyc27_header = "\x03\xf3\x0d\x0a\x00\x00\x00\x00" + + def _decrypt_file(self, filename): + os.path.splitext(filename) + content = open(filename).read() + try: + m = pymarshal.loads(content) + except: + try: + m = marshal.loads(content) + except Exception as e: + print("[!] error: %s" % str(e)) + return None + # pymarshal.dumps(m, self.opcode_decrypt_map) + return m.co_filename.replace('\\', '/'), pymarshal.dumps(m, self.opcode_decrypt_map) + + def decrypt_file(self, input_file, output_file=None): + result = self._decrypt_file(input_file) + if not result: + return + pyc_filename, pyc_content = result + if not output_file: + output_file = os.path.basename(pyc_filename) + '.pyc' + with open(output_file, 'wb') as fd: + fd.write(self.pyc27_header + pyc_content) + + +def main(): + parser = argparse.ArgumentParser(description='onmyoji py decrypt tool') + parser.add_argument("INPUT_NAME", help='input file') + parser.add_argument("OUTPUT_NAME", help='output file') + args = parser.parse_args() + encryptor = PYCEncryptor() + encryptor.decrypt_file(args.INPUT_NAME, args.OUTPUT_NAME) + + +if __name__ == '__main__': + main() diff --git a/scripts/pymarshal.py b/scripts/pymarshal.py new file mode 100755 index 0000000..599265d --- /dev/null +++ b/scripts/pymarshal.py @@ -0,0 +1,499 @@ +import types +import cStringIO + +TYPE_NULL = '0' +TYPE_NONE = 'N' +TYPE_FALSE = 'F' +TYPE_TRUE = 'T' +TYPE_STOPITER = 'S' +TYPE_ELLIPSIS = '.' +TYPE_INT = 'i' +TYPE_INT64 = 'I' +TYPE_FLOAT = 'f' +TYPE_COMPLEX = 'x' +TYPE_LONG = 'l' +TYPE_STRING = 's' +TYPE_INTERNED = 't' +TYPE_STRINGREF = 'R' +TYPE_TUPLE = '(' +TYPE_LIST = '[' +TYPE_DICT = '{' +TYPE_CODE = 'c' +TYPE_UNICODE = 'u' +TYPE_UNKNOWN = '?' +TYPE_SET = '<' +TYPE_FROZENSET = '>' + + +UNKNOWN_BYTECODE = 0 + + +class _NULL: + pass + + +class _Marshaller: + dispatch = {} + + def __init__(self, writefunc, opmap=None): + self._write = writefunc + self._opmap = opmap or {} + + def dump(self, x): + try: + self.dispatch[type(x)](self, x) + except KeyError: + for tp in type(x).mro(): + func = self.dispatch.get(tp) + if func: + break + else: + raise ValueError("unmarshallable object") + func(self, x) + + def w_long64(self, x): + self.w_long(x) + self.w_long(x >> 32) + + def w_long(self, x): + a = chr(x & 0xff) + x >>= 8 + b = chr(x & 0xff) + x >>= 8 + c = chr(x & 0xff) + x >>= 8 + d = chr(x & 0xff) + self._write(a + b + c + d) + + def w_short(self, x): + self._write(chr((x) & 0xff)) + self._write(chr((x >> 8) & 0xff)) + + def dump_none(self, x): + self._write(TYPE_NONE) + + dispatch[type(None)] = dump_none + + def dump_bool(self, x): + if x: + self._write(TYPE_TRUE) + else: + self._write(TYPE_FALSE) + + dispatch[bool] = dump_bool + + def dump_stopiter(self, x): + if x is not StopIteration: + raise ValueError("unmarshallable object") + self._write(TYPE_STOPITER) + + dispatch[type(StopIteration)] = dump_stopiter + + def dump_ellipsis(self, x): + self._write(TYPE_ELLIPSIS) + + try: + dispatch[type(Ellipsis)] = dump_ellipsis + except NameError: + pass + + # In Python3, this function is not used; see dump_long() below. + def dump_int(self, x): + y = x >> 31 + if y and y != -1: + self._write(TYPE_INT64) + self.w_long64(x) + else: + self._write(TYPE_INT) + self.w_long(x) + + dispatch[int] = dump_int + + def dump_long(self, x): + self._write(TYPE_LONG) + sign = 1 + if x < 0: + sign = -1 + x = -x + digits = [] + while x: + digits.append(x & 0x7FFF) + x = x >> 15 + self.w_long(len(digits) * sign) + for d in digits: + self.w_short(d) + + try: + long + except NameError: + dispatch[int] = dump_long + else: + dispatch[long] = dump_long + + def dump_float(self, x): + write = self._write + write(TYPE_FLOAT) + s = repr(x) + write(chr(len(s))) + write(s) + + dispatch[float] = dump_float + + def dump_complex(self, x): + write = self._write + write(TYPE_COMPLEX) + s = repr(x.real) + write(chr(len(s))) + write(s) + s = repr(x.imag) + write(chr(len(s))) + write(s) + + try: + dispatch[complex] = dump_complex + except NameError: + pass + + def dump_string(self, x): + # XXX we can't check for interned strings, yet, + # so we (for now) never create TYPE_INTERNED or TYPE_STRINGREF + self._write(TYPE_STRING) + self.w_long(len(x)) + self._write(x) + + dispatch[bytes] = dump_string + + def dump_unicode(self, x): + self._write(TYPE_UNICODE) + s = x.encode('utf8') + self.w_long(len(s)) + self._write(s) + + try: + unicode + except NameError: + dispatch[str] = dump_unicode + else: + dispatch[unicode] = dump_unicode + + def dump_tuple(self, x): + self._write(TYPE_TUPLE) + self.w_long(len(x)) + for item in x: + self.dump(item) + + dispatch[tuple] = dump_tuple + + def dump_list(self, x): + self._write(TYPE_LIST) + self.w_long(len(x)) + for item in x: + self.dump(item) + + dispatch[list] = dump_list + + def dump_dict(self, x): + self._write(TYPE_DICT) + for key, value in x.items(): + self.dump(key) + self.dump(value) + self._write(TYPE_NULL) + + dispatch[dict] = dump_dict + + def dump_code(self, x): + self._write(TYPE_CODE) + self.w_long(x.co_argcount) + self.w_long(x.co_nlocals) + self.w_long(x.co_stacksize) + self.w_long(x.co_flags) + # self.dump(x.co_code) + + self.dump(self._transform_opcode(x.co_code)) + + self.dump(x.co_consts) + self.dump(x.co_names) + self.dump(x.co_varnames) + self.dump(x.co_freevars) + self.dump(x.co_cellvars) + self.dump(x.co_filename) + self.dump(x.co_name) + self.w_long(x.co_firstlineno) + self.dump(x.co_lnotab) + + try: + dispatch[types.CodeType] = dump_code + except NameError: + pass + + def _transform_opcode(self, x): + if not self._opmap: + return x + + opcode = bytearray(x) + c = 0 + while c < len(opcode): + try: + n = self._opmap[opcode[c]] + except Exception as e: + print("unmapping %s" % opcode[c]) + print(e) + + opcode[c] = n + + if n < 90: + c += 1 + else: + c += 3 + + return str(opcode) + + def dump_set(self, x): + self._write(TYPE_SET) + self.w_long(len(x)) + for each in x: + self.dump(each) + + try: + dispatch[set] = dump_set + except NameError: + pass + + def dump_frozenset(self, x): + self._write(TYPE_FROZENSET) + self.w_long(len(x)) + for each in x: + self.dump(each) + + try: + dispatch[frozenset] = dump_frozenset + except NameError: + pass + + +class _Unmarshaller: + dispatch = {} + + def __init__(self, readfunc): + self._read = readfunc + self._stringtable = [] + + def load(self): + c = self._read(1) + if not c: + raise EOFError + try: + return self.dispatch[c](self) + except KeyError: + raise ValueError("bad marshal code: %c (%d)" % (c, ord(c))) + + def r_short(self): + lo = ord(self._read(1)) + hi = ord(self._read(1)) + x = lo | (hi << 8) + if x & 0x8000: + x = x - 0x10000 + return x + + def r_long(self): + s = self._read(4) + a = ord(s[0]) + b = ord(s[1]) + c = ord(s[2]) + d = ord(s[3]) + x = a | (b << 8) | (c << 16) | (d << 24) + if d & 0x80 and x > 0: + x = -((1 << 32) - x) + return int(x) + else: + return x + + def r_long64(self): + a = ord(self._read(1)) + b = ord(self._read(1)) + c = ord(self._read(1)) + d = ord(self._read(1)) + e = ord(self._read(1)) + f = ord(self._read(1)) + g = ord(self._read(1)) + h = ord(self._read(1)) + x = a | (b << 8) | (c << 16) | (d << 24) + x = x | (e << 32) | (f << 40) | (g << 48) | (h << 56) + if h & 0x80 and x > 0: + x = -((1 << 64) - x) + return x + + def load_null(self): + return _NULL + + dispatch[TYPE_NULL] = load_null + + def load_none(self): + return None + + dispatch[TYPE_NONE] = load_none + + def load_true(self): + return True + + dispatch[TYPE_TRUE] = load_true + + def load_false(self): + return False + + dispatch[TYPE_FALSE] = load_false + + def load_stopiter(self): + return StopIteration + + dispatch[TYPE_STOPITER] = load_stopiter + + def load_ellipsis(self): + return Ellipsis + + dispatch[TYPE_ELLIPSIS] = load_ellipsis + + dispatch[TYPE_INT] = r_long + + dispatch[TYPE_INT64] = r_long64 + + def load_long(self): + size = self.r_long() + sign = 1 + if size < 0: + sign = -1 + size = -size + x = 0 + for i in range(size): + d = self.r_short() + x = x | (d << (i * 15)) + return x * sign + + dispatch[TYPE_LONG] = load_long + + def load_float(self): + n = ord(self._read(1)) + s = self._read(n) + return float(s) + + dispatch[TYPE_FLOAT] = load_float + + def load_complex(self): + n = ord(self._read(1)) + s = self._read(n) + real = float(s) + n = ord(self._read(1)) + s = self._read(n) + imag = float(s) + return complex(real, imag) + + dispatch[TYPE_COMPLEX] = load_complex + + def load_string(self): + n = self.r_long() + return self._read(n) + + dispatch[TYPE_STRING] = load_string + + def load_interned(self): + n = self.r_long() + ret = intern(self._read(n)) + self._stringtable.append(ret) + return ret + + dispatch[TYPE_INTERNED] = load_interned + + def load_stringref(self): + n = self.r_long() + return self._stringtable[n] + + dispatch[TYPE_STRINGREF] = load_stringref + + def load_unicode(self): + n = self.r_long() + s = self._read(n) + ret = s.decode('utf8') + return ret + + dispatch[TYPE_UNICODE] = load_unicode + + def load_tuple(self): + return tuple(self.load_list()) + + dispatch[TYPE_TUPLE] = load_tuple + + def load_list(self): + n = self.r_long() + list = [self.load() for i in range(n)] + return list + + dispatch[TYPE_LIST] = load_list + + def load_dict(self): + d = {} + while 1: + key = self.load() + if key is _NULL: + break + value = self.load() + d[key] = value + return d + + dispatch[TYPE_DICT] = load_dict + + def load_code(self): + argcount = self.r_long() + nlocals = self.r_long() + stacksize = self.r_long() + flags = self.r_long() + code = self.load() + consts = self.load() + names = self.load() + varnames = self.load() + freevars = self.load() + cellvars = self.load() + filename = self.load() + name = self.load() + firstlineno = self.r_long() + lnotab = self.load() + return types.CodeType(argcount, nlocals, stacksize, flags, code, consts, + names, varnames, filename, name, firstlineno, + lnotab, freevars, cellvars) + + dispatch[TYPE_CODE] = load_code + + def load_set(self): + n = self.r_long() + args = [self.load() for i in range(n)] + return set(args) + + dispatch[TYPE_SET] = load_set + + def load_frozenset(self): + n = self.r_long() + args = [self.load() for i in range(n)] + return frozenset(args) + + dispatch[TYPE_FROZENSET] = load_frozenset + + +def dump(x, f, opmap=None): + m = _Marshaller(f.write, opmap) + m.dump(x) + + +def load(f): + um = _Unmarshaller(f.read) + return um.load() + + +def loads(content): + io = cStringIO.StringIO(content) + return load(io) + + +def dumps(x, opmap=None): + io = cStringIO.StringIO() + dump(x, io, opmap) + io.seek(0) + return io.read() diff --git a/scripts/python-uncompyle6 b/scripts/python-uncompyle6 new file mode 160000 index 0000000..7a6a510 --- /dev/null +++ b/scripts/python-uncompyle6 @@ -0,0 +1 @@ +Subproject commit 7a6a51058c3dc092e9ecb63f0b564856096a6ceb diff --git a/scripts/script_decomp.sh b/scripts/script_decomp.sh new file mode 100755 index 0000000..7bb45e5 --- /dev/null +++ b/scripts/script_decomp.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +mkdir -p script/temp +mkdir -p script/pyc +mkdir -p script/out +mkdir -p script/failed + +filecount=$(ls -1 script_nxs | wc -l) +counter=0 + +for file in script_nxs/* +do + file="$(basename "$file")" + echo "$((100*$counter/$filecount))% - $file" + + python2 scripts/script_redirect.py script_nxs/$file > script/temp/$file.out + python2 scripts/pyc_decryptor.py script/temp/$file.out script/pyc/$file.pyc + python3 scripts/decompile_pyc.py -o script/out/$file.py script/pyc/$file.pyc 2> /dev/null + if [ $? -ne 0 ]; then + # A lot of the time Python 2 works instead + python2 scripts/decompile_pyc.py -o script/out/$file.py script/pyc/$file.pyc 2> /dev/null + + fi + + if [ ! -f script/out/$file.py ]; then + echo "Failed...sad face. Copied to 'script/failed/'" + cp "script_nxs/$file" "script/failed" + counter=$(($counter+1)) + continue + fi + + file_name="$(head -n 5 script/out/$file.py | tail -n 1)" + file_name=${file_name//\\/\/} + regexp="# Embedded.*" + if [[ "$file_name" =~ $regexp ]]; then + file_name="$(echo "$file_name" | sed -e "s/# Embedded file name: //")" + file_dir="$(dirname "$file_name")" + echo $file_name + echo $file_dir + mkdir -p script/layout/$file_dir + cp script/out/$file.py script/layout/$file_name + fi + counter=$(($counter+1)) +done diff --git a/scripts/script_redirect.py b/scripts/script_redirect.py new file mode 100755 index 0000000..8c03ccc --- /dev/null +++ b/scripts/script_redirect.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python2 +import zlib +import argparse + +def _reverse_string(s): + l = list(s) + l = map(lambda x: chr(ord(x) ^ 154), l[0:128]) + l[128:] + l.reverse() + return ''.join(l) + + +def unpack_pyc(data): + asdf_dn = 'j2h56ogodh3se' + asdf_dt = '=dziaq.' + asdf_df = '|os=5v7!"-234' + asdf_tm = asdf_dn * 4 + (asdf_dt + asdf_dn + asdf_df) * 5 + '!' + '#' + asdf_dt * 7 + asdf_df * 2 + '*' + '&' + "'" + import rotor + rotor = rotor.newrotor(asdf_tm) + data = rotor.decrypt(data) + data = zlib.decompress(data) + data = _reverse_string(data) + return data + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("INPUT_NAME", help='input file') + args = parser.parse_args() + input_file = args.INPUT_NAME + data = unpack_pyc(open(input_file).read()) + print(data) + +if __name__ == '__main__': + main() diff --git a/src/main.rs b/src/main.rs index 849e862..fcf71d0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ use byteorder::{LittleEndian, ReadBytesExt}; use clap::{App, Arg}; use log::{debug, error, info, trace, warn}; -use std::io::{BufReader, Read, Seek}; +use std::io::{BufRead, BufReader, Read, Seek}; fn is_eof(reader: &mut std::io::BufReader) -> std::io::Result where @@ -237,12 +237,6 @@ impl Npk2Reader { let index_size = reader.read_u32::()?; let mut field_28 = reader.read_u32::()?; // DATA END MAYBE? - // info!("File Size:\t{}", size); - // info!("NPK Magic:\t{:X}", magic); - // info!("NPK File Count:\t{}", file_count); - // info!("NPK Is Large File:\t{}", large_file_index_offset > 0); - // info!("NPK Index Offset:\t{}", index_offset); - // TODO(alexander): Figure out what this actually does and why we need it // in Eve Echoes NPKs we usually run into the first case, where we change field 28 let v5 = ((field_28 as i64 - index_size as i64) >> 3) as u64 / 5; // No idea why divide by 5, but it is a thing :) @@ -282,12 +276,6 @@ impl Npk2Reader { pub fn open(&mut self) -> Result<(), Npk2Error> { let mut reader = BufReader::new(&self.file); - // info!("File Size:\t{}", size); - // info!("NPK Magic:\t{:X}", magic); - // info!("NPK File Count:\t{}", file_count); - // info!("NPK Is Large File:\t{}", large_file_index_offset > 0); - // info!("NPK Index Offset:\t{}", index_offset); - let pos = reader.seek(std::io::SeekFrom::Start(self.header.index_offset()))?; assert!(pos == self.header.index_offset()); @@ -307,10 +295,8 @@ impl Npk2Reader { } else { // Load all the indices from the NPK File let mut buffer_cursor = std::io::Cursor::new(buffer); - let mut sub_buffer = vec![0; 0x28 as usize]; - let mut index_buffer = buffer_cursor.read_exact(&mut sub_buffer); - while index_buffer.is_ok() { - index_buffer = buffer_cursor.read_exact(&mut sub_buffer); + let mut sub_buffer = vec![0; self.header.size_of_index_entry() as usize]; + while buffer_cursor.read_exact(&mut sub_buffer).is_ok() { let index = NeoXIndex2::from_slice(sub_buffer.as_mut_slice())?; self.indices.push(index); } @@ -329,113 +315,185 @@ impl Npk2Reader { } } +fn load_file_name_hash_mappings(reader: &mut T) -> std::collections::HashMap +where + T: std::io::BufRead, +{ + info!("Parsing filelist"); + let mut file_mappings = std::collections::HashMap::new(); + for line in reader.lines() { + if let Ok(line) = line { + // + // 0 + let r = regex::Regex::new( + r"(\S+)(?:\s+)(\S+)(?:\s+)(\S+)(?:\s+)(\S+)(?:\s+)(\S+)(?:\s+)(\S.*)", + ) + .unwrap(); + let caps = r.captures(&line); + if let Some(caps) = caps { + let name_hash = caps.get(2).unwrap().as_str().parse::().unwrap(); + let filename = caps.get(6).unwrap().as_str(); + file_mappings.insert(name_hash, filename.to_string()); + } + } + } + file_mappings +} + fn main() -> Result<(), Npk2Error> { simple_logger::init_with_level(log::Level::Info).unwrap(); - // let string = "redirect.nxs"; - // let v1 = Hash32::hash_with_seed(string, 0x9747B28C); - // let v2 = Hash32::hash_with_seed(string, 0xC82B7479); + // let string = "test"; + // let v1 = fasthash::murmur3::hash32_with_seed(string, 0x9747B28C); + // let v2 = fasthash::murmur3::hash32_with_seed(string, 0xC82B7479); - // println!("{:X}", (v1 as u64 | (v2 as u64) << 0x20) as u64); + // println!("{:X}", (v2 as u64 | (v1 as u64) << 0x20) as u64); let matches = App::new("NeoX NPK Tool") .version("1.0") .author("Alexander Guettler ") - .arg( - Arg::with_name("MODE") - .possible_values(&["x", "p"]) // TODO(alexander): Add some kind of info mode, print file count etc. - .about("Specifies whether to extract (x) or pack (p) the specified directory") - .required(true) - .index(1), - ) + .subcommand(App::new("x") + .about("Unpack one or more NPKS") .arg( Arg::with_name("INPUT") - .about("The NPK file to be operated on") + .about("The NPK file(s) to be operated on") .required(true) - .index(2), + .multiple(true) + .index(1), ) .arg( Arg::with_name("DIR") + .short('d') + .long("dir") .value_name("DIR") .about("The directory where this NPK file should be extracted to") .default_value("out") - .index(3) .takes_value(true), ) + .arg( + Arg::with_name("FILELIST") + .short('f') + .long("filelist") + .value_name( + "FILELIST" + ).about("Supplies a file list to the npk unpack which will be used to try and reconstruct the original file tree\nWhen INPUT is supplied with a list of all resX.npk files this may be determined and used automatically.") + )) .get_matches(); - let input_file = matches.value_of("INPUT").unwrap(); - let mut npk_file = Npk2Reader::new(input_file)?; - npk_file.open()?; + match matches.subcommand() { + ("x", Some(sub_m)) => { + let input_files: Vec<&str> = sub_m.values_of("INPUT").unwrap().collect(); - let mode = matches.value_of("MODE").unwrap(); + let mut npk_readers = Vec::new(); + for input_file in input_files { + let mut npk_file = Npk2Reader::new(input_file)?; + npk_file.open()?; + npk_readers.push(npk_file); + } - match mode { - "x" => { - let output_directory = std::path::Path::new(matches.value_of("DIR").unwrap()); - let output_sub_directory = match std::path::Path::new(&input_file).file_stem() { - Some(v) => v.to_str().unwrap_or(""), - None => "", + let file_list = match sub_m.value_of("FILELIST") { + Some(path) => { + let file = std::fs::File::open(path)?; + load_file_name_hash_mappings(&mut BufReader::new(file)) + } + None => { + match npk_readers + .iter() + .map(|x| x.indices().iter().map(|i| (x, i)).collect::>()) + .collect::>>() + .into_iter() + .flatten() + .find(|(_, x)| x.name_hash() == 0xD4A17339F75381FD) + { + Some((npk_file, index)) => { + let content = npk_file.read_content_for_index(index)?; + let mut decompressed = Vec::new(); + compress::zlib::Decoder::new(std::io::Cursor::new(&content)) + .read_to_end(&mut decompressed)?; + load_file_name_hash_mappings(&mut std::io::Cursor::new(&decompressed)) + } + None => std::collections::HashMap::new(), + } + } }; - let output_directory = output_directory.join(output_sub_directory); + + let output_directory = std::path::Path::new(sub_m.value_of("DIR").unwrap()); std::fs::create_dir_all(&output_directory)?; - for index in npk_file.indices() { - debug!("Reading Index {:?}", index); - // - let content = npk_file.read_content_for_index(index)?; - let result = tree_magic::from_u8(&content); - let extension = match result.as_str() { - "text/plain" => "txt", - "application/octet-stream" => { - let mut rdr = std::io::Cursor::new(&content); - let magic = rdr.read_u32::(); - match magic { - Ok(magic) => { - // Detect NXS and stuff, which is a NeoX Script File - if magic == 0x041D { - "nxs" - } else if magic & 0xFFFF == 0x041D { - "nxs" - } else { - "dat" - } + for npk_file in npk_readers { + for index in npk_file.indices() { + debug!("Reading Index {:?}", index); + + let content = npk_file.read_content_for_index(index)?; + let file_name = match file_list.get(&index.name_hash()) { + Some(file_name) => file_name.clone(), + None => { + // This is a massive hack, but oh well + // We know the hash, and we know what the file is + // zlib compressed data + if index.name_hash() == 0xD4A17339F75381FD { + "filelist.txt".to_string() + } else { + let result = tree_magic::from_u8(&content); + let extension = match result.as_str() { + "text/plain" => "txt", + "application/octet-stream" => { + let mut rdr = std::io::Cursor::new(&content); + let magic = rdr.read_u32::(); + match magic { + Ok(magic) => { + // Detect NXS and stuff, which is a NeoX Script File + if magic == 0x041D { + "nxs" + } else if magic & 0xFFFF == 0x041D { + "nxs" + } else { + "dat" + } + } + Err(_) => "dat", + } + } + "application/x-executable" => "exe", + "application/x-cpio" => "cpio", + "image/ktx" => "ktx", + "image/png" => "png", + "image/x-dds" => "dds", + "image/x-win-bitmap" => "bmp", + "application/xml" => "xml", + "text/x-matlab" => "mat", // Maybe m instead? + "application/x-apple-systemprofiler+xml" => "xml", + "text/x-modelica" => "mo", + "text/x-csrc" => "c", + "font/ttf" => "ttf", + "image/bmp" => "bmp", + "application/zip" => "zip", + "image/jpeg" => "jpg", + _ => { + error!("Unhandled mime type {}", result); + "dat" + } + }; + format!("unknown_file_name/{:X}.{}", index.name_hash(), extension) } - Err(_) => "dat", } + }; + // + + let out_file = output_directory.join(file_name); + if let Some(dir_path) = std::path::Path::new(&out_file).parent() { + std::fs::create_dir_all(dir_path)?; } - "application/x-executable" => "exe", - "application/x-cpio" => "cpio", - "image/ktx" => "ktx", - "image/png" => "png", - "image/x-dds" => "dds", - "image/x-win-bitmap" => "bmp", - "application/xml" => "xml", - "text/x-matlab" => "mat", // Maybe m instead? - "application/x-apple-systemprofiler+xml" => "xml", - "text/x-modelica" => "mo", - "text/x-csrc" => "c", - "font/ttf" => "ttf", - "image/bmp" => "bmp", - "application/zip" => "zip", - "image/jpeg" => "jpg", - _ => { - error!("Unhandled mime type {}", result); - "dat" - } - }; - - let out_file = - output_directory.join(format!("{:X}.{}", index.name_hash(), extension)); - info!( - "Writing {} bytes to {}", - bytesize::ByteSize(content.len() as u64), - out_file.as_path().to_str().unwrap() - ); - std::fs::write(out_file, &content)?; + info!( + "Writing {} bytes to {}", + bytesize::ByteSize(content.len() as u64), + out_file.as_path().to_str().unwrap() + ); + std::fs::write(out_file, &content)?; + } } } - "p" => unimplemented!("Packing is currently not supported"), + ("p", Some(sub_m)) => unimplemented!("Packing is currently not supported"), _ => {} }