From 156f1084f1520416cede452464427b5e5a988b8c Mon Sep 17 00:00:00 2001 From: Yallxe <82591945+yallxe@users.noreply.github.com> Date: Sat, 29 Jan 2022 12:03:00 +0100 Subject: [PATCH 1/3] Add IDEA to gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 985254e..b70cfd2 100644 --- a/.gitignore +++ b/.gitignore @@ -131,3 +131,6 @@ dmypy.json # Temporal files *~ + +# IDEA Dev Environment +.idea \ No newline at end of file From 43b1612dfec5a687750403f02424c8f333edcd6d Mon Sep 17 00:00:00 2001 From: Yallxe <82591945+yallxe@users.noreply.github.com> Date: Sat, 29 Jan 2022 12:18:18 +0100 Subject: [PATCH 2/3] Set utf-8 as an encoding for open() Not every system uses 'utf-8' as a default encoding for opening files in Python. Before you say that there must be a way to set default encoding in one line, no, there is not. At least, I didn't found a way to do this. --- dev/mvt-android | 4 ++++ mvt/android/download_apks.py | 4 ++-- mvt/android/modules/adb/dumpsys_batterystats.py | 4 ++-- mvt/android/modules/adb/dumpsys_full.py | 2 +- mvt/android/modules/adb/dumpsys_packages.py | 2 +- mvt/android/modules/adb/dumpsys_procstats.py | 2 +- mvt/android/modules/adb/logcat.py | 4 ++-- mvt/common/indicators.py | 2 +- mvt/common/module.py | 2 +- mvt/ios/decrypt.py | 2 +- mvt/ios/modules/fs/shutdownlog.py | 2 +- mvt/ios/modules/fs/version_history.py | 2 +- tests/artifacts/generate_stix.py | 2 +- 13 files changed, 19 insertions(+), 15 deletions(-) diff --git a/dev/mvt-android b/dev/mvt-android index 53ae739..52d5868 100755 --- a/dev/mvt-android +++ b/dev/mvt-android @@ -6,6 +6,10 @@ import os import sys +import importlib + +importlib.reload(sys) +print(sys.getdefaultencoding()) sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/mvt/android/download_apks.py b/mvt/android/download_apks.py index 7a9b8b2..797e0a4 100644 --- a/mvt/android/download_apks.py +++ b/mvt/android/download_apks.py @@ -57,7 +57,7 @@ class DownloadAPKs(AndroidExtraction): :param json_path: Path to the apks.json file to parse. """ - with open(json_path, "r") as handle: + with open(json_path, "r", encoding="utf-8") as handle: packages = json.load(handle) return cls(packages=packages) @@ -173,7 +173,7 @@ class DownloadAPKs(AndroidExtraction): def save_json(self): """Save the results to the package.json file.""" json_path = os.path.join(self.output_folder, "apks.json") - with open(json_path, "w") as handle: + with open(json_path, "w", encoding="utf-8") as handle: json.dump(self.packages, handle, indent=4) def run(self): diff --git a/mvt/android/modules/adb/dumpsys_batterystats.py b/mvt/android/modules/adb/dumpsys_batterystats.py index ecec627..1160761 100644 --- a/mvt/android/modules/adb/dumpsys_batterystats.py +++ b/mvt/android/modules/adb/dumpsys_batterystats.py @@ -27,7 +27,7 @@ class DumpsysBatterystats(AndroidExtraction): if self.output_folder: stats_path = os.path.join(self.output_folder, "dumpsys_batterystats.txt") - with open(stats_path, "w") as handle: + with open(stats_path, "w", encoding="utf-8") as handle: handle.write(stats) log.info("Records from dumpsys batterystats stored at %s", @@ -37,7 +37,7 @@ class DumpsysBatterystats(AndroidExtraction): if self.output_folder: history_path = os.path.join(self.output_folder, "dumpsys_batterystats_history.txt") - with open(history_path, "w") as handle: + with open(history_path, "w", encoding="utf-8") as handle: handle.write(history) log.info("History records from dumpsys batterystats stored at %s", diff --git a/mvt/android/modules/adb/dumpsys_full.py b/mvt/android/modules/adb/dumpsys_full.py index ffed0dc..3f6408d 100644 --- a/mvt/android/modules/adb/dumpsys_full.py +++ b/mvt/android/modules/adb/dumpsys_full.py @@ -27,7 +27,7 @@ class DumpsysFull(AndroidExtraction): if self.output_folder: stats_path = os.path.join(self.output_folder, "dumpsys.txt") - with open(stats_path, "w") as handle: + with open(stats_path, "w", encoding="utf-8") as handle: handle.write(stats) log.info("Full dumpsys output stored at %s", diff --git a/mvt/android/modules/adb/dumpsys_packages.py b/mvt/android/modules/adb/dumpsys_packages.py index 8bd366a..510f4b4 100644 --- a/mvt/android/modules/adb/dumpsys_packages.py +++ b/mvt/android/modules/adb/dumpsys_packages.py @@ -28,7 +28,7 @@ class DumpsysPackages(AndroidExtraction): if self.output_folder: packages_path = os.path.join(self.output_folder, "dumpsys_packages.txt") - with open(packages_path, "w") as handle: + with open(packages_path, "w", encoding="utf-8") as handle: handle.write(output) log.info("Records from dumpsys package stored at %s", diff --git a/mvt/android/modules/adb/dumpsys_procstats.py b/mvt/android/modules/adb/dumpsys_procstats.py index 74a321d..f925d65 100644 --- a/mvt/android/modules/adb/dumpsys_procstats.py +++ b/mvt/android/modules/adb/dumpsys_procstats.py @@ -27,7 +27,7 @@ class DumpsysProcstats(AndroidExtraction): if self.output_folder: procstats_path = os.path.join(self.output_folder, "dumpsys_procstats.txt") - with open(procstats_path, "w") as handle: + with open(procstats_path, "w", encoding="utf-8") as handle: handle.write(output) log.info("Records from dumpsys procstats stored at %s", diff --git a/mvt/android/modules/adb/logcat.py b/mvt/android/modules/adb/logcat.py index 284f6b0..00d659b 100644 --- a/mvt/android/modules/adb/logcat.py +++ b/mvt/android/modules/adb/logcat.py @@ -31,7 +31,7 @@ class Logcat(AndroidExtraction): if self.output_folder: logcat_path = os.path.join(self.output_folder, "logcat.txt") - with open(logcat_path, "w") as handle: + with open(logcat_path, "w", encoding="utf-8") as handle: handle.write(output) log.info("Current logcat logs stored at %s", @@ -39,7 +39,7 @@ class Logcat(AndroidExtraction): logcat_last_path = os.path.join(self.output_folder, "logcat_last.txt") - with open(logcat_last_path, "w") as handle: + with open(logcat_last_path, "w", encoding="utf-8") as handle: handle.write(last_output) log.info("Logcat logs prior to last reboot stored at %s", diff --git a/mvt/common/indicators.py b/mvt/common/indicators.py index 68f8db6..e283707 100644 --- a/mvt/common/indicators.py +++ b/mvt/common/indicators.py @@ -82,7 +82,7 @@ class Indicators: ioc_file["file_path"] = file_path ioc_file["file_name"] = os.path.basename(file_path) - with open(file_path, "r") as handle: + with open(file_path, "r", encoding="utf-8") as handle: try: data = json.load(handle) except json.decoder.JSONDecodeError: diff --git a/mvt/common/module.py b/mvt/common/module.py index f8efad2..fc13852 100644 --- a/mvt/common/module.py +++ b/mvt/common/module.py @@ -58,7 +58,7 @@ class MVTModule(object): @classmethod def from_json(cls, json_path, log=None): - with open(json_path, "r") as handle: + with open(json_path, "r", encoding="utf-8") as handle: results = json.load(handle) if log: log.info("Loaded %d results from \"%s\"", diff --git a/mvt/ios/decrypt.py b/mvt/ios/decrypt.py index f827b56..1f90b98 100644 --- a/mvt/ios/decrypt.py +++ b/mvt/ios/decrypt.py @@ -185,7 +185,7 @@ class DecryptBackup: return try: - with open(key_path, 'w') as handle: + with open(key_path, 'w', encoding="utf-8") as handle: handle.write(self._decryption_key) except Exception as e: log.exception(e) diff --git a/mvt/ios/modules/fs/shutdownlog.py b/mvt/ios/modules/fs/shutdownlog.py index ee86c89..819d768 100644 --- a/mvt/ios/modules/fs/shutdownlog.py +++ b/mvt/ios/modules/fs/shutdownlog.py @@ -86,5 +86,5 @@ class ShutdownLog(IOSExtraction): def run(self): self._find_ios_database(root_paths=SHUTDOWN_LOG_PATH) self.log.info("Found shutdown log at path: %s", self.file_path) - with open(self.file_path, "r") as handle: + with open(self.file_path, "r", encoding="utf-8") as handle: self.process_shutdownlog(handle.read()) diff --git a/mvt/ios/modules/fs/version_history.py b/mvt/ios/modules/fs/version_history.py index b6379ea..49282c6 100644 --- a/mvt/ios/modules/fs/version_history.py +++ b/mvt/ios/modules/fs/version_history.py @@ -34,7 +34,7 @@ class IOSVersionHistory(IOSExtraction): def run(self): for found_path in self._get_fs_files_from_patterns(IOS_ANALYTICS_JOURNAL_PATHS): - with open(found_path, "r") as analytics_log: + with open(found_path, "r", encoding="utf-8") as analytics_log: log_line = json.loads(analytics_log.readline().strip()) timestamp = datetime.datetime.strptime(log_line["timestamp"], diff --git a/tests/artifacts/generate_stix.py b/tests/artifacts/generate_stix.py index 418a732..6fd6459 100644 --- a/tests/artifacts/generate_stix.py +++ b/tests/artifacts/generate_stix.py @@ -41,7 +41,7 @@ def generate_test_stix_file(file_path): res.append(Relationship(i, "indicates", malware)) bundle = Bundle(objects=res) - with open(file_path, "w+") as f: + with open(file_path, "w+", encoding="utf-8") as f: f.write(bundle.serialize(pretty=True)) From bc86d159b8d529fe8d5bc9119feeab7c44799e6c Mon Sep 17 00:00:00 2001 From: Yallxe <82591945+yallxe@users.noreply.github.com> Date: Sat, 29 Jan 2022 12:28:22 +0100 Subject: [PATCH 3/3] Clear 'debugging' things --- dev/mvt-android | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dev/mvt-android b/dev/mvt-android index 52d5868..53ae739 100755 --- a/dev/mvt-android +++ b/dev/mvt-android @@ -6,10 +6,6 @@ import os import sys -import importlib - -importlib.reload(sys) -print(sys.getdefaultencoding()) sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))