diff --git a/.gitignore b/.gitignore index 68bc17f..e64089c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,160 +1,139 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ + +/.idea/ + +#Fin manuales + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ +.idea/misc.xml +*.pak +/outFilesBad +/outFiles +/binary_playground diff --git a/env.py b/env.py new file mode 100644 index 0000000..cfbf7ac --- /dev/null +++ b/env.py @@ -0,0 +1,5 @@ +RELEASE_BUILD = True +PYINSTALLER = False +EXE_NAME = 'unxip.exe' +_PY_NAME = 'python main.py' +COMMAND_NAME : str = EXE_NAME if PYINSTALLER else _PY_NAME \ No newline at end of file diff --git a/etcXorers.py b/etcXorers.py new file mode 100644 index 0000000..ea467a7 --- /dev/null +++ b/etcXorers.py @@ -0,0 +1,127 @@ +from functools import cache +from typing import List, Optional, Tuple +import xipFormatStrategy +import struct + + +class Crc32Table: # 256 32bit numbers + size : int = 256 + _table : Optional[Tuple] = None + _tableHex : Optional[bytes] = None + + def __init__(self) -> None: + if Crc32Table._table is None: + table = [0 for _ in range(Crc32Table.size)] + Crc32Table._tableHex = bytes() + for i in range(Crc32Table.size): + crc = i + for _ in range(8): + crc = (crc >> 1) ^ (0xEDB88320 if crc & 1 else 0) + table[i] = crc + Crc32Table._tableHex += crc.to_bytes(4, "little") # type: ignore + Crc32Table._table = tuple(table) + + def get(self, index:int) -> int: + if Crc32Table._table is None: + Crc32Table() + index &= 255 + assert Crc32Table._table is not None + return Crc32Table._table[index] + + def getTable(self) -> bytes: + if Crc32Table._tableHex is None: + Crc32Table() + assert Crc32Table._tableHex is not None + return Crc32Table._tableHex + + + def __index__(self, index: int) -> int: + return Crc32Table().get(index) + + def __getitem__(self, index: int) -> int: + return Crc32Table().get(index) + +class VcKey: + key : Optional[bytes] = None + + @staticmethod + def get() -> bytes: + if VcKey.key is None: + VcKey() + assert VcKey.key is not None + return VcKey.key + + def __init__(self) -> None: + if VcKey.key is None: + VcKey.key = bytes(Crc32Table().get(i)&0xff for i in range(256)) + +def deXorVisualClip(input_: bytes, vc:bool = False ) -> bytes: + key : bytes = VcKey().get() + output = bytearray(input_[:]) + offset: int = 0xED + if vc: + offset += 9 + for i in range(8, len(input_) ):# preserve the first 8 bytes. + offset &= 255 #len(key) + output[i] = key[offset % 256] ^ input_[i] + offset += 1 + return output + + +@cache +def japaneseTextXorEncoder() -> bytes: + japText : str = """……耕一さん……あなたを殺します +私はあなたを、愛してはいませんから… +生きて…ラカン… +百年…貴方を待っていたの…千年…貴方に恋していたわ +私…世界より貴方がほしい…… +夜空に星が輝くように溶けた心は離れない +たとえこの手が離れてもふたりがそれを忘れぬ限り""" + return japText.replace("\n","\r\n").encode("shift-jis")[1:(1+256)] + + +def japDeXor(input_: bytes, offset: int , strategy: xipFormatStrategy.XipFormatStrategy ) -> bytes: + assert offset >= 0 + + xorKey = japaneseTextXorEncoder() + + output = bytes() + for i in range( strategy.FILE_HEADER_LENGTH ): + offset &= 255 #len(xorKey) + byteA = (xorKey[offset] ^ input_[i]).to_bytes(1, "big") + output += bytearray(byteA) + offset += 1 + return output + +@cache +def txtKey() -> Tuple[int]: + + vckey = Crc32Table().getTable() + key = vckey[0x70:0xA0] + vckey[0x30:0x70] + vckey[0xA0:0x100] + vckey[0x00:0x30] + vckey[0x180:0x200] + vckey[0x100:0x180] + vckey[0x270:0x2B0] + vckey[0x210:0x270] + vckey[0x2B0:0x300] + vckey[0x200:0x210] + vckey[0x370:0x400] + vckey[0x300:0x370] + return struct.unpack_from("<256I", key ) + +def unmaskTxt(input_: bytes) -> bytes: + + key = txtKey() + output = bytes() + key_offset = len(input_) % 0x100 + i = 0 + for i in range(len(input_) // 4): + inputInt4 = struct.unpack_from(" 0: + output += input_[-i:] + + return output + + +def checksumA(a:int , b:int, c:int) -> bool: + if c == 0: + #if __debug__: print("Unimplemented border case") TODO? + b ^= 1 + var1 = (a * b) & 0xFF + return var1 == c \ No newline at end of file diff --git a/extractor.py b/extractor.py new file mode 100644 index 0000000..58002cf --- /dev/null +++ b/extractor.py @@ -0,0 +1,142 @@ +import pstats, logging, struct, sys, bcolors +from typing import Any, Dict, List, Optional +import etcXorers +import xipFormatStrategy +from xipFormatStrategy import XipFormatStrategy +import os, sys, pathlib +from rsaDecryptor import WrongKeysError +from fileBlock import FileBlock + +def extract(*args, **kwargs): + # there's no indication if a pak uses the chinese keys or not (which are less popular), so we do this simple wrapper temporarily. + try: + extract_(*args, **kwargs) + except WrongKeysError: + try: + extract_( *args, chineseVersion = True, **kwargs) + except : + print(f"{bcolors.ERR} file/key error with '{args[0]}'{bcolors.ENDC}") + + +def extract_(pakPath:str , enable_extractFiles: bool = True , enable_listFiles:bool = False, outSubfolder:str = "", chineseVersion:bool = False, **kwargs): + + pakFileName = pathlib.Path(pakPath).name + if kwargs.get("noIndividualFoldersParam") is not True: + outSubfolder = "./outFiles/" + pakFileName + "/" + else: + outSubfolder = "./outFiles/" + + with open(pakPath, "rb") as xipFile: + xipFile=bytes(xipFile.read()) + + if xipFile[0:3] != b"XIP": + raise Exception("Not a valid XIP file.") + + xipDecoder : XipFormatStrategy + + if xipFile[3:4] == b"2": + if chineseVersion: + xipDecoder = xipFormatStrategy.Xip2ChDecoder() + else: + xipDecoder = xipFormatStrategy.Xip2Decoder() + elif xipFile[3:4] == b"3": + xipDecoder = xipFormatStrategy.Xip3Decoder() + else: + raise Exception("Not a compatible XIP file.") + + if xipDecoder.environment_ok == False: + for key in xipDecoder.REQUIRED_KEYS: + if key not in os.listdir("./keyFiles"): + raise Exception(f"Can't execute extractor, missing key file: {key}") + xipDecoder.__class__.environment_ok = True + + # 00 13 fb 3f + + offsetSecretA: bytes = xipFile[4:5]+xipFile[6:7]+xipFile[8:9]+xipFile[11:12] + offsetSecretA = struct.unpack_from(' 0x3000 or numberOfFiles > 10000: + raise WrongKeysError() + + etcXorers.checksumA(fileOffset&0xff,numberOfFiles&0xff,checksumAData) + + files : List[FileBlock] = [] + for fileNumber in range(numberOfFiles): + if fileOffset == offsetSecretA: + fileOffset += secretASkipSize + fb = FileBlock(xipFile, fileOffset, (numberOfFiles - fileNumber), xipDecoder, fileNumber) + files.append(fb) + fileOffset += xipDecoder.FILE_HEADER_LENGTH + fb.fileBlockSize + if not fb.noErrors : + logging.debug(f"{bcolors.ERR}Parsing failed for some file{bcolors.ENDC}") + continue + #logging.debug(pf.fileName) + + if enable_listFiles: + for fb in files: + print(fb.fileName) + + if enable_extractFiles : + pathsToMake = set() + for fb in files: + # make folder structures before extracting + path = fb.fileName.split("\\") + if any( [ p in [".", ".."] for p in path ] ): + print(f"This filename/path is not supported for security reasons: {fb.fileName}") # this is not optimal + continue + if len(path) > 1: + folder = path[:-1] + folder = "/".join(folder) + pathsToMake.add(f"{outSubfolder}{folder}") + + for path in pathsToMake: + os.makedirs(path, exist_ok=True) + + i: int = 0 + for fb in files: + try: + if not fb.noErrors : + print(f"\n{bcolors.ERR}Exception when parsing '{fb.fileName}'{bcolors.ENDC}") + else: + result = fb.extractFile() + finalPath = (outSubfolder) + fb.fileName.replace('\\','/') + + #os.access( os.path.dirname(finalPath), os.W_OK) # this is to check if the path is valid + + # check if file already exists + if os.path.exists(finalPath): + ... + #print(f"\n{bcolors.WARN}File '{pf.fileName}' already exists.{bcolors.ENDC}") + with open( finalPath, "wb") as file: + file.write(result["finalData"]) + + if not result["crcOk"]: + print(f"{bcolors.ERR}CRC32 check failed for '{fb.fileName}'{bcolors.ENDC}") + i += 1 + print(f"\r{i}/{len(files)} files completed.", end="", flush=True) + + except Exception as e: + print(f"\n{bcolors.ERR}Exception when decompressing '{fb.fileName}'{bcolors.ENDC}") + print(e) + print("") + + + print(f"Done {pakFileName}.") + + return files diff --git a/fileBlock.py b/fileBlock.py new file mode 100644 index 0000000..ec83759 --- /dev/null +++ b/fileBlock.py @@ -0,0 +1,181 @@ +import struct, os +from typing import Any, Dict, Optional +import lzo, bcolors, etcXorers + + +from rsaDecryptor import RsaDecryptor +from xipFormatStrategy import XipFormatStrategy +class FileBlock: + """ + structure of a file block for xip2: + + 0x11c bytes : xored data with the japanese text + 4 bytes: fileBlockSize (size of the whole file block) + 4 bytes: size of the final file once decompressed and everything + 4 bytes: unknown1 # some may be a hash of the filename or something to easily find the file- + 0x104 bytes : path+filename , zero terminated, 0xCC padding, encoding could vary? + 4 bytes: crc32 of the final extracted file but before the additional xoring for .vce + 4 bytes: unknown2 + 1 byte: cryptKeyIndex (for the rsa-like decryption of the cryptChunk) + 3 bytes: unknown3 , could be just the rest of the above byte that may come from something like random_int32() ? + + 4 bytes: _size1 , cryptChunkSize but with bits shuffled + 4 bytes: _size2 , rsa-likeDecryptedSize but with bits shuffled + + (cryptChunkSize) bytes : cryptChunk , it's the beginning of the compressed data, encrypted. Some parts could be the same as the raw data because of a flaw in the encryption algorithm. + + x bytes : rest of the file (compressed) + + """ + #instance variables: + + fileDescriptor: bytes # all bytes of the header + + fileName: str + baseOffset: int # address in the xip file + uncompressedSize: int + crc32: int + cryptKeyIndex: int + fileBlockSize: int # how much to jump to the next file header + + + xorParam: int + format_: XipFormatStrategy + + fileDataCompressed: bytes # file content after decryption + + finalData: bytes + + #for debug: + decryptedData: bytes + noErrors: bool + fileIndex: Optional[int] + + def __init__(self, xipFile, baseOffset, xorKeyOffset, format_: XipFormatStrategy, fileIndex: Optional[int] = None): + self.finalData = bytes() + self.xipFile = xipFile + self.noErrors = True + self.fileIndex = fileIndex + self.xorParam = xorKeyOffset + self.baseOffset = baseOffset + self.format_ = format_ + + + self.fileDescriptor = etcXorers.japDeXor(xipFile[baseOffset: baseOffset + format_.FILE_HEADER_LENGTH], xorKeyOffset , format_) + + a = self.fileDescriptor[0x0c:(0x0c + format_.FILE_HEADER_LENGTH)] + a = a[ : a.find(b'\x00')] + + fileName: Optional[str] = None + for enc in ["ascii", "shift-jis", "EUC-KR", "utf-8" ]: + try: + fileName = a.decode( enc ) + os.access(fileName, os.W_OK) # this is to check if path+name is valid + break + except UnicodeDecodeError: # wrong encoding + fileName = None + except ValueError: # windows impossible filename + fileName = None + if fileName is None: + raise Exception("Could not decode the file name.") + self.fileName = fileName + + self.fileBlockSize , self.uncompressedSize = struct.unpack_from(' Dict[ str, Any ] : + + self._size1 , self._size2 = struct.unpack_from('> 8 & 0xff) << 8 | A1 & 0xff) << 8 | A1 >> 0x18) << 8 | A1 >> 8 & 0xff + rsaDecryptedSize = (((A1 >> 0x10 & 0xff) << 8 | A2 >> 0x18) << 8 | A2 & 0xff) << 8 | A2 >> 0x10 & 0xff + #offsetToStartOverwriting = (cryptChunkSize - rsaDecryptedSize) + 8 + 0x11c + self.baseOffset # writeZoneStartAddr + + try: + off1=8 + 0x11c + self.baseOffset + self.decryptedData = self.format_.xipRsaDecrypt( self.xipFile[off1:off1+rsaDecryptedSize*2] , self.cryptKeyIndex) + except Exception as e: + self.noErrors = False + raise e + + '''# TODO : there's this loop that could be executed (timesLoopX) times, for big files? + for (A2 = A2 >> 0x10 & 3; A2 != 0; A2 = A2 - 1) { + *(byte *)writeZoneAddr = *(byte *)RSADecryptedIncrPointer; + RSADecryptedIncrPointer = (dword *)((int)RSADecryptedIncrPointer + 1); + writeZoneAddr = (dword *)((int)writeZoneAddr + 1); + } + ''' + timesLoopX = (A2 >> 16) & 3 #(0,1,2,3) + if timesLoopX != 0: + raise NotImplementedError + + sizeOfEncryptedData: int = cryptChunkSize + + unCryptDataOff = self.baseOffset + 0x011C + 8 + sizeOfEncryptedData + + fileDataCompressed = self.decryptedData + self.xipFile [ unCryptDataOff : unCryptDataOff + self.fileBlockSize - 8 - sizeOfEncryptedData ] + + #if __debug__: + # self.fileDataCompressed = fileDataCompressed + + + # unfortunately, the line below could suddenly crash python.exe because it's a C dll which will crash if the output buffer allocated (for the expected decompressed size) gets overflowed (because of a badly formed data) , even in a try-except block. + dataout=lzo.decompress(fileDataCompressed , False, self.uncompressedSize, algorithm="LZO1X") + + return {"index": self.fileIndex, "error":False, "finalData": dataout} + + def extractFile( self ): + enableCrcCheck:bool = False + + fileData = self.extract() + if not self.noErrors: + raise Exception(f"{bcolors.ERR}Exception when decompressing '{self.fileName}'{bcolors.ENDC}") + + dataout = fileData["finalData"] + + # additonal decryption for some file types + if len(self.fileName.split("\\")[-1]) > 3: + extension: str = self.fileName.split("\\")[-1][-4:] + if extension in self.format_.MASKED_CONFIG_FILE_TYPES: + dataout = etcXorers.unmaskTxt(dataout) + + crcOk: bool + if enableCrcCheck: + import zlib + crcCalculated : int = zlib.crc32(dataout) + crcOk = self.crc32 == crcCalculated + else: + crcOk = True + + + if len(self.fileName.split("\\")[-1]) > 3: + extension: str = "." + self.fileName.split("\\")[-1].split(".")[-1] + if extension in self.format_.MASKED_VISUALCLIP_FILE_TYPES: + if extension == ".vc": + dataout = etcXorers.deXorVisualClip(dataout, True) + else: + dataout = etcXorers.deXorVisualClip(dataout) + + return_ = {"index": self.fileIndex, "crcOk": crcOk, "finalData": dataout } + + return return_ + + + @property + def unknown1(self): + return self.fileDescriptor[0x8 : 0x0c] # 4 bytes + + @property + def unknown2(self): + return self.fileDescriptor[0x114 : 0x118] # 4 bytes + + @property + def unknown3(self): + return self.fileDescriptor[0x119 : 0x11c ] # 3 bytes + @property + def unknown3_4BB(self): + return self.fileDescriptor[0x118 : 0x11c ] \ No newline at end of file diff --git a/functions.py b/functions.py deleted file mode 100644 index aac7923..0000000 --- a/functions.py +++ /dev/null @@ -1,54 +0,0 @@ -from typing import List - -import numba -import xipDecoder_strategy - -def checksumA(a:int , b:int, c:int) -> bool: - if c == 0: - #if __debug__: print("Unimplemented border case") TODO? - b ^= 1 - var1 = (a * b) & 0xFF - return var1 == c - -def japDeXor(input_: bytes, offset: int , strategy: xipDecoder_strategy.XipDecoderStrategy ) -> bytes: - assert offset >= 0 - - xorKey = xipDecoder_strategy.XipDecoderStrategy().japaneseTextXorEncoder() - - output = bytes() - for i in range( strategy.fileHeaderLength()): - offset &= 255 #len(xorKey) - byteA = (xorKey[offset] ^ input_[i]).to_bytes(1, "big") - output += bytearray(byteA) - offset += 1 - return output - -def deXorTxt(input_: bytes) -> bytes: - import struct - with open("./keyFiles/txtCrcMask.bin", "rb") as file: - key = bytes(file.read()) - key = struct.unpack_from("<256I", key) - output = bytes() - key_offset = len(input_) % 0x100 - i = 0 - for i in range(len(input_) // 4): - inputInt4 = struct.unpack_from(" 0: - output += input_[-i:] - - return output - -def deXorVisualClip(input_: bytes) -> bytes: - key : bytes = xipDecoder_strategy.VcKey().get() - offset = 0xED - output = bytearray(input_[:]) - for i in range(8, len(input_) ):# preserve the first 8 bytes. - offset &= 255 #len(key) - output[i] = key[offset % 256] ^ input_[i] - offset += 1 - return output diff --git a/inputFiles/dummyfile.txt b/inputFiles/dummyfile.txt new file mode 100644 index 0000000..e69de29 diff --git a/japText.py b/japText.py deleted file mode 100644 index dc132df..0000000 --- a/japText.py +++ /dev/null @@ -1,9 +0,0 @@ -# -*- coding: utf-8 -*- - -japText : str = """……耕一さん……あなたを殺します -私はあなたを、愛してはいませんから… -生きて…ラカン… -百年…貴方を待っていたの…千年…貴方に恋していたわ -私…世界より貴方がほしい…… -夜空に星が輝くように溶けた心は離れない -たとえこの手が離れてもふたりがそれを忘れぬ限り""" \ No newline at end of file diff --git a/keyFiles/disclaimer.txt b/keyFiles/disclaimer.txt deleted file mode 100644 index ce152aa..0000000 --- a/keyFiles/disclaimer.txt +++ /dev/null @@ -1 +0,0 @@ -txtKey is like a crc32 table \ No newline at end of file diff --git a/keyFiles/dummyfile.txt b/keyFiles/dummyfile.txt new file mode 100644 index 0000000..e69de29 diff --git a/keyFiles/txtCrcMask.bin b/keyFiles/txtCrcMask.bin deleted file mode 100644 index 81f7590..0000000 Binary files a/keyFiles/txtCrcMask.bin and /dev/null differ diff --git a/keyGrabber.py b/keyGrabber.py new file mode 100644 index 0000000..a7e1f5d --- /dev/null +++ b/keyGrabber.py @@ -0,0 +1,45 @@ +import os, hashlib, env +from typing import Dict, List + +def main(**kwargs): + # name, first bytes, length, md5, description + keySignatures : Dict[str, List] = { + "key1a.bin": [b"\x3D\x8E\x82\x3D", 0x800, "a6e4dbb2e181e653083c6d5ee623b044", "FDK key a"], + "key1b.bin": [b"\x01\xA9\xD6\xB2", 0x800, "b305cb2ee7986b016eb1b81b110cf4c4", "FDK key b"], + "key1c.bin": [b"\xC5\x00\x5B\x01\x00\x00", 0x800, "b2039ff0f38400038176616610e3a854", "FDK key c (used only in TR)"], + "USB_16128_10.dat" : [b"\x34\x12\x01\x00", 0x80, "e547ac65c94f74df882f917e27234703", "USB key (TR)"], + "key1a_ch.bin": [b"\xE7\xE3\x4F\xB1\x8D", 0x800, "6a4fba222fe8e192f4afe2c004eba6c6", "FDK key a (for Chinese client)"], + "key1b_ch.bin": [b"\xF9\xCB\x59\xD5\xD9", 0x800, "e55e431103937d161fb08d263445cc60", "FDK key b (for Chinese client)"],} + + files = os.listdir("./") + files = [f for f in files if f.count(".") != 0 and f.split(".")[-1] in ["exe", "dll", "ex_", "client"] and f != env.EXE_NAME ] + + if not os.path.exists("./keyFiles"): + os.makedirs("./keyFiles") + + if test := False: + existingKeys = [] + else: + existingKeys = os.listdir("./keyFiles") + + for f in files: + with open(f, "rb") as f: + data = f.read() + for key in (set(keySignatures.keys()) - set(existingKeys)): + pos = -1 + pos = data.find(keySignatures[key][0], pos+1) + while pos != -1: + content = data[pos:(pos+keySignatures[key][1])] + if hashlib.md5(content).hexdigest() == keySignatures[key][2]: + with open(f"./keyFiles/{key}", "wb") as keyFile: + keyFile.write( content ) + print(f"Got a new key, {key} from {f.name} . ({keySignatures[key][3]})") + break + else: + #print(f"a part of {key} is in {f.name} .") + pos = data.find(keySignatures[key][0], pos+1) + if len(files) == 0: + print("No game executables were found to get the keys from ") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/main.py b/main.py index 8775ebf..95a6054 100644 --- a/main.py +++ b/main.py @@ -1,377 +1,75 @@ -import pstats -import logging, struct, sys, bcolors -from typing import Any, Dict, List, Optional - -import functions -import xipDecoder_strategy -from xipDecoder_strategy import XipDecoderStrategy -import os -import numba - - -@numba.njit() -def xipRsa2(inputB: int, input_: int, key: int) -> int: - '''decoded = (encoded **e) %m - decoded = (inputB ** input) % key - https://en.wikipedia.org/wiki/Modular_exponentiation#:~:text=)-,Pseudocode,-%5Bedit%5D - https://en.wikipedia.org/wiki/RSA_(cryptosystem)#:~:text=calculations%20can%20be-,computed%20efficiently,-using%20the%20square - ''' - output = 0 - while input_ != 0: - if input_ % 2 == 1: - output = (inputB + output) % key - inputB *= 2 - inputB %= key - input_ //= 2 - return output - -@numba.njit() -def xipRsa1(input_: int, key2: int, key1: int) -> int: - # input: 8Btyes. output: 4Bytes - output = 1 - while key2 != 0: - if key2 % 2 == 1: - output = xipRsa2(output, input_, key1) - input_ = xipRsa2(input_, input_, key1) - key2 //= 2 - return output - -class RsaDecryptor: - key1: Optional[bytes] = None - key2: Optional[bytes] = None - - @staticmethod - def initializeStaticVars(): - if RsaDecryptor.key1 is None or RsaDecryptor.key2 is None: - with open("./keyFiles/key1", "rb") as file: - RsaDecryptor.key1=bytes(file.read()) - with open("./keyFiles/key2", "rb") as file: - RsaDecryptor.key2=bytes(file.read()) - - def __init__(self): - if RsaDecryptor.key1 is None: RsaDecryptor.initializeStaticVars() - - def xipRsa(self, input_: bytes, keyIndex: int) -> bytes: - output = bytes() - assert RsaDecryptor.key1 is not None and RsaDecryptor.key2 is not None - - size=len(input_) - assert size % 8 == 0 - size //= 8 - for i in range(size): # decrypt 8 bytes to 4 in each cycle. - input_a = struct.unpack_from(' Dict[ Any, Any ] : - ''' - This extracts the file content, it's compatible with async and parallel procesing. - ''' - - self._size1 , self._size2 = struct.unpack_from('> 8 & 0xff) << 8 | A1 & 0xff) << 8 | A1 >> 0x18) << 8 | A1 >> 8 & 0xff - rsaDecryptedSize = (((A1 >> 0x10 & 0xff) << 8 | A2 >> 0x18) << 8 | A2 & 0xff) << 8 | A2 >> 0x10 & 0xff - #offsetToStartOverwriting = (cryptChunkSize - rsaDecryptedSize) + 8 + 0x11c + self.baseOffset # writeZoneStartAddr - - try: - off1=8 + 0x11c + self.baseOffset - self.decryptedData = RsaDecryptor().xipRsa( self.xipFile[off1:off1+rsaDecryptedSize*2] , self.cryptKeyIndex) +import keyGrabber, os, sys, pathlib, env +from extractor import extract + +def extractMany(dir_:str = "./", **kwargs): + files = os.listdir( dir_ ) + if "inputFiles" in files: + files += os.listdir( dir_ + "inputFiles/") + extracted = 0 + for file_ in (f for f in files if len(f) > 4 and f[-4:] == ".pak") : + print(f"Extracting {file_.split('/')[-1]}") + try: + extract(dir_+file_ , **kwargs) except Exception as e: - self.noErrors = False - raise e - - '''# TODO : there's this loop that could be executed (timesLoopX) times, for big files? - for (A2 = A2 >> 0x10 & 3; A2 != 0; A2 = A2 - 1) { - *(byte *)writeZoneAddr = *(byte *)RSADecryptedIncrPointer; - RSADecryptedIncrPointer = (dword *)((int)RSADecryptedIncrPointer + 1); - writeZoneAddr = (dword *)((int)writeZoneAddr + 1); - } - ''' - timesLoopX = (A2 >> 16) & 3 #(0,1,2,3) - if timesLoopX != 0: - raise NotImplementedError - - sizeOfEncryptedData: int = cryptChunkSize + print(e) + extracted += 1 + if extracted == 0: + print("No .pak files found.") + - unCryptDataOff = self.baseOffset + 0x011C + 8 + sizeOfEncryptedData +def autoExtractAll(): + keyGrabber.main() + extractMany( ) - fileDataCompressed = self.decryptedData + self.xipFile [ unCryptDataOff : unCryptDataOff + self.fileBlockSize - 8 - sizeOfEncryptedData ] - #if __debug__: - # self.fileDataCompressed = fileDataCompressed - - import lzo - # unfortunately, the line below could suddenly crash python.exe because it's a C dll which will crash if the output buffer allocated (for the expected decompressed size) gets overflowed (because of a badly formed data) , even in a try-except block. - dataout=lzo.decompress(fileDataCompressed , False, self.uncompressedSize, algorithm="LZO1X") - - return {"index": self.fileIndex, "error":False, "finalData": dataout} - - def extractFileAsyncComplete( self ): - enableCrcCheck:bool = False - - fileData = self.extractAsync() - if not self.noErrors: - raise Exception(f"{bcolors.ERR}Exception when decompressing '{self.fileName}'{bcolors.ENDC}") - - dataout = fileData["finalData"] - - # additonal decryption for some file types - if len(self.fileName.split("\\")[-1]) > 3: - extension: str = self.fileName.split("\\")[-1][-4:] - if extension in self.format_.maskedFileTypes_ConfigFile(): - dataout = functions.deXorTxt(dataout) - - crcOk: bool - if enableCrcCheck: - import zlib - crcCalculated : int = zlib.crc32(dataout) - crcOk = self.crc32 == crcCalculated +if __name__ == "__main__": + kwargs_ = {} + if len(sys.argv) > 1: + if "-nofolders" in sys.argv: + kwargs_["noIndividualFoldersParam"] = True + if "-getkeys" in sys.argv: + keyGrabber.main( ) + if "-extractall" in sys.argv: + extractMany( **kwargs_ ) + if ".pak" in "".join(sys.argv): + for file_ in [a for a in sys.argv if len(a) >= 4 and a[-4:] == ".pak"]: + print(f"Extracting {pathlib.Path(file_).name}") + try: + extract( file_ , **kwargs_) + except Exception as e: + print(e) else: - crcOk = True - - if len(self.fileName.split("\\")[-1]) > 3: - extension: str = self.fileName.split("\\")[-1][-4:] - if extension in self.format_.maskedFileTypes_VisualClip(): - dataout = functions.deXorVisualClip(dataout) - - return_ = {"index": self.fileIndex, "crcOk": crcOk, "finalData": dataout } - - return return_ - - - @property - def unknown1(self): - return self.fileDescriptor[0x8 : 0x0c] # 4 bytes - - @property - def unknown2(self): - return self.fileDescriptor[0x114 : 0x118] # 4 bytes - - @property - def unknown3(self): - return self.fileDescriptor[0x119 : 0x11c ] # 3 bytes - @property - def unknown3b(self): - return self.fileDescriptor[0x118 : 0x11c ] - - -def openXip(pakFilename:str , enableParallel:bool = False ) : - with open("./inputFiles/"+pakFilename, "rb") as xipFile: - xipFile=bytes(xipFile.read()) - if xipFile[0:3] != b"XIP": - raise Exception("Not a valid XIP file.") - - xipDecoder : XipDecoderStrategy - - if xipFile[3:4] == b"2": - xipDecoder = xipDecoder_strategy.Xip2Decoder() - elif xipFile[3:4] == b"3": - xipDecoder = xipDecoder_strategy.Xip3Decoder() + extractMany( **kwargs_ ) else: - raise Exception("Not a compatible XIP file.") - - if xipDecoder.environment_ok == False: - for key in xipDecoder.required_keys(): - if key not in os.listdir("./keyFiles"): - raise Exception(f"Can't execute extractor, missing key file: {key}") - xipDecoder.environment_ok = True - if 'key' in locals(): del key - - - offsetSecretA: bytes = xipFile[4:5]+xipFile[6:7]+xipFile[8:9]+xipFile[11:12] - offsetSecretA = struct.unpack_from(' 1: - folder = path[:-1] - folder = "\\".join(folder) - pathsToMake.add(f"./outFiles/{folder}") - - for path in pathsToMake: - os.makedirs(path, exist_ok=True) - - for i, pf in enumerate(files): - try: - if not pf.noErrors : - print(f"{bcolors.ERR}Exception when parsing '{pf.fileName}'{bcolors.ENDC}") - else: - result = pf.extractFileAsyncComplete() - finalPath = f"./outFiles/" + pf.fileName.replace('\\','/') - - os.access( os.path.dirname(finalPath), os.W_OK) - - with open( finalPath, "wb") as file: - file.write(result["finalData"]) - - print(f"{i+1}/{len(files)} files completed.", end="\r", flush=True) - - if not result["crcOk"]: - print(f"{bcolors.ERR}CRC32 check failed for '{pf.fileName}'{bcolors.ENDC}") + if ( env.RELEASE_BUILD ) : + print(f'''No file specified. +Usages: + Easy mode: Automatically extract all paks in the folder (place a game .exe in the root folder also) : {env.COMMAND_NAME} + + Extract a file: {env.COMMAND_NAME} file1.pak + Extract all paks in the folder : {env.COMMAND_NAME} -extractall + + parameter "-nofolders" : do not make individual pak folders. + +Requirement: The keys, or an .exe file with them , must be placed in this folder. +Some known files containing the keys are: TR1.exe , O2Mania*.exe , djmax*.exe , Pak Extract.exe , yomax.exe +For trilogy (not implemented yet), you must also be able to get USB_16128_10.dat or get it with a tool that reads the usb stick . + +Executing easy mode... +''') + autoExtractAll() + else: + ... - except Exception as e: - print(f"{bcolors.ERR}Exception when decompressing '{pf.fileName}'{bcolors.ENDC}") - print(e) - print("") - print(f"Done {pakFilename}.") - return files -def main_profile(fileName : str = "./System.pak"): +'''def main_profile(fileName : str = "./System.pak"): import cProfile logging.basicConfig(level=logging.ERROR) cProfile.run(f"openXip('{fileName}')", "profile_output.prof", sort="cumulative" ) stats = pstats.Stats("profile_output.prof") stats.sort_stats('cumulative') - stats.print_stats(30) - -def main(fileName : str = "./System.pak"): - - logging.basicConfig(level=logging.ERROR) - openXip(fileName) - -def openMany(): - files = os.listdir("./inputFiles/") - for file_ in files: - if len(file_) > 4 and file_[-4:] == ".pak": - print(f"Extracting {file_.split('/')[-1]}") - openXip(file_) - -if __name__ == "__main__": - import sys - main(sys.argv[1] if len(sys.argv) > 1 else "./System.pak") - - #openMany() - + stats.print_stats(30)''' \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 885af49..9e5d3fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ +# py >= 3.8 + python-lzo attrs numba \ No newline at end of file diff --git a/rsaDecryptor.py b/rsaDecryptor.py new file mode 100644 index 0000000..2fb6f03 --- /dev/null +++ b/rsaDecryptor.py @@ -0,0 +1,62 @@ +import struct +from typing import Optional +import numba + + +class WrongKeysError(Exception): + pass + +@numba.njit() +def xipRsa2(inputB: int, input_: int, key: int) -> int: + '''decoded = (encoded **e) %m + decoded = (inputB ** input) % key + https://en.wikipedia.org/wiki/Modular_exponentiation#:~:text=)-,Pseudocode,-%5Bedit%5D + https://en.wikipedia.org/wiki/RSA_(cryptosystem)#:~:text=calculations%20can%20be-,computed%20efficiently,-using%20the%20square + ''' + output = 0 + while input_ != 0: + if input_ % 2 == 1: + output = (inputB + output) % key + inputB *= 2 + inputB %= key + input_ //= 2 + return output + +@numba.njit() +def xipRsa1(input_: int, key2: int, key1: int) -> int: + # input: 8Btyes. output: 4Bytes + output = 1 + while key2 != 0: + if key2 % 2 == 1: + output = xipRsa2(output, input_, key1) + input_ = xipRsa2(input_, input_, key1) + key2 //= 2 + return output + +class RsaDecryptor: # RSA-like decryption + key1: bytes + key2: bytes + + def __init__(self, chKeyMode: bool = False): + with open(f'./keyFiles/key1a{"_ch" if chKeyMode else ""}.bin', "rb") as file_: + self.key1=bytes(file_.read()) + with open(f'./keyFiles/key1b{"_ch" if chKeyMode else ""}.bin', "rb") as file_: + self.key2=bytes(file_.read()) + + def xipRsaDecrypt(self, input_: bytes, keyIndex: int) -> bytes: + output = bytes() + + size=len(input_) + assert size % 8 == 0 + size //= 8 + for i in range(size): # decrypt 8 bytes to 4 in each cycle. + input_a = struct.unpack_from(' bytes: - text: str = japText.japText - return text.replace("\n","\r\n").encode("shift-jis")[1:(1+256)] - - def maskedFileTypes_VisualClip(self) -> List[str]: - return [".vci", ".vce"] - - @abstractmethod - def fileHeaderLength(self) -> int: - ... - - @abstractmethod - def fileNameLength(self) -> int: - ... - - @abstractmethod - def maskedFileTypes_ConfigFile(self) -> List[str]: - ... - - @abstractmethod - def required_keys(self) -> List[str]: - ... - - -class Xip2Decoder(XipDecoderStrategy): - # class variables: - environment_ok: bool = False - - def fileHeaderLength(self) -> int: - return 0x11c - - def fileNameLength(self) -> int: - return 0x104 - - def maskedFileTypes_ConfigFile(self) -> List[str]: - # they made a typo, 'cvs' instead of 'csv' - return [".ini", ".crc", ".cvs", ".vgi", ".gsi", ".txt", ".gds"] - - def required_keys(self) -> List[str]: - return ["key1", "key2", "txtCrcMask.bin"] - - -class Xip3Decoder(XipDecoderStrategy): - # class variables: - environment_ok: bool = False - - def fileHeaderLength(self) ->int: - # 9c ? - raise NotImplementedError - - def fileNameLength(self) -> int: - return 0x80 - - def maskedFileTypes_ConfigFile(self) -> List[str]: - return [".ini", ".crc", ".cvs", ".vgi", ".gsi", ".txt", ".cgi", ".gdi"] - - def required_keys(self) -> List[str]: - return ["key1", "key2", "key3", "txtCrcMask.bin", "USB_16128_10.dat"] - - -class Crc32Table: # a.k.a. "vcKey", 256 32bit numbers - # class variables: - size : int = 256 - table : Optional[List[int]] = None - - def __init__(self) -> None: - if Crc32Table.table is None: - Crc32Table.table = [0 for _ in range(Crc32Table.size)] - for i in range(Crc32Table.size): - crc = i - for _ in range(8): - crc = (crc >> 1) ^ (0xEDB88320 if crc & 1 else 0) - Crc32Table.table[i] = crc - - @staticmethod - def get(index:int) -> int: - if Crc32Table.table is None: - Crc32Table() - index &= 255 - assert Crc32Table.table is not None - return Crc32Table.table[index] - - def __index__(self, index: int) -> int: - return Crc32Table().get(index) - - def __getitem__(self, index: int) -> int: - return Crc32Table().get(index) - -class VcKey: - key : Optional[bytes] = None - - @staticmethod - def get() -> bytes: - if VcKey.key is None: - VcKey() - assert VcKey.key is not None - return VcKey.key - - def __init__(self) -> None: - if VcKey.key is None: - VcKey.key = bytes(Crc32Table().get(i)&0xff for i in range(256)) \ No newline at end of file diff --git a/xipFormatStrategy.py b/xipFormatStrategy.py new file mode 100644 index 0000000..92e7cd7 --- /dev/null +++ b/xipFormatStrategy.py @@ -0,0 +1,48 @@ +from abc import abstractmethod +from typing import List, Optional, Set, Tuple +from rsaDecryptor import RsaDecryptor + + +class XipFormatStrategy: + + # class vars: + MASKED_VISUALCLIP_FILE_TYPES = {".vci", ".vce",".vc"} + rsaDecryptor : Optional[RsaDecryptor] = None + environment_ok: bool = False + FILE_HEADER_LENGTH = 0x11c + FILE_NAME_LENGTH : int + MASKED_CONFIG_FILE_TYPES : Set[str] + REQUIRED_KEYS : Set[str] + # end class vars + + def xipRsaDecrypt(self, *args, **kwargs) -> bytes: + if XipFormatStrategy.rsaDecryptor is None: + XipFormatStrategy.rsaDecryptor = RsaDecryptor() + return XipFormatStrategy.rsaDecryptor.xipRsaDecrypt(*args, **kwargs) + + +class Xip2Decoder(XipFormatStrategy): + + FILE_NAME_LENGTH = 0x104 + MASKED_CONFIG_FILE_TYPES = {".ini", ".crc", ".cvs", ".vgi", ".gsi", ".txt", ".gds"} + REQUIRED_KEYS = {"key1a.bin", "key1b.bin"} + +class Xip2ChDecoder(Xip2Decoder): + _rsaDecryptor : Optional[RsaDecryptor] = None + + REQUIRED_KEYS = {"key1a_ch.bin", "key1b_ch.bin"} + + def xipRsaDecrypt(self, *args, **kwargs) -> bytes: + if Xip2ChDecoder._rsaDecryptor is None: + Xip2ChDecoder._rsaDecryptor = RsaDecryptor(True) + return Xip2ChDecoder._rsaDecryptor.xipRsaDecrypt(*args, **kwargs) + + +class Xip3Decoder(XipFormatStrategy): + + FILENAME_LENGTH = 0x9C # check + MASKED_CONFIG_FILE_TYPES = {".ini", ".crc", ".cvs", ".vgi", ".gsi", ".txt", ".cgi", ".gdi"} + REQUIRED_KEYS = {"key1a.bin", "key1b.bin", "key1c.bin", "USB_16128_10.dat"} + + def __init__(self) -> None: + raise NotImplementedError() \ No newline at end of file