Automatically recover malformed sqlite3 databases (closes: #25 #37)

docs/install.md

@@ -7,7 +7,7 @@ Before proceeding, please note that mvt requires Python 3.6+ to run. While it sh
 First install some basic dependencies that will be necessary to build all required tools:
 
 ```bash
-sudo apt install python3 python3-pip libusb-1.0-0
+sudo apt install python3 python3-pip libusb-1.0-0 sqlite3
 ```
 
 *libusb-1.0-0* is not required if you intend to only use `mvt-ios` and not `mvt-android`.
@@ -19,7 +19,7 @@ Running MVT on Mac requires Xcode and [homebrew](https://brew.sh) to be installe
 In order to install dependencies use:
 
 ```bash
-brew install python3 libusb
+brew install python3 libusb sqlite3
 ```
 
 *libusb* is not required if you intend to only use `mvt-ios` and not `mvt-android`.

mvt/common/module.py

@@ -13,6 +13,12 @@ import simplejson as json
 
 from .indicators import Indicators
 
+class DatabaseNotFoundError(Exception):
+    pass
+
+class DatabaseCorruptedError(Exception):
+    pass
+
 class MVTModule(object):
     """This class provides a base for all extraction modules."""
 
@@ -136,9 +142,12 @@ def run_module(module):
     except NotImplementedError:
         module.log.exception("The run() procedure of module %s was not implemented yet!",
                              module.__class__.__name__)
-    except FileNotFoundError as e:
+    except DatabaseNotFoundError as e:
         module.log.info("There might be no data to extract by module %s: %s",
                         module.__class__.__name__, e)
+    except DatabaseCorruptedError as e:
+        module.log.error("The %s module database seems to be corrupted and recovery failed: %s",
+                         module.__class__.__name__, e)
     except Exception as e:
         module.log.exception("Error in running extraction from module %s: %s",
                              module.__class__.__name__, e)

mvt/ios/modules/fs/base.py

@@ -3,10 +3,15 @@
 # See the file 'LICENSE' for usage and copying permissions, or find a copy at
 #   https://github.com/mvt-project/mvt/blob/main/LICENSE
 
+import io
 import os
 import glob
+import shutil
+import sqlite3
+import subprocess
 
 from mvt.common.module import MVTModule
+from mvt.common.module import DatabaseNotFoundError, DatabaseCorruptedError
 
 class IOSExtraction(MVTModule):
     """This class provides a base for all iOS filesystem/backup extraction modules."""
@@ -15,6 +20,31 @@ class IOSExtraction(MVTModule):
     is_fs_dump = False
     is_sysdiagnose = False
 
+    def _recover_database(self, file_path):
+        """Tries to recover a malformed database by running a .clone command.
+        :param file_path: Path to the malformed database file.
+        """
+        # TODO: Find a better solution.
+
+        self.log.info("Database at path %s is malformed. Trying to recover...", file_path)
+
+        if not os.path.exists(file_path):
+            return
+
+        if not shutil.which("sqlite3"):
+            raise DatabaseCorruptedError("Unable to recover without sqlite3 binary. Please install sqlite3!")
+
+        bak_path = f"{file_path}.bak"
+        shutil.move(file_path, bak_path)
+
+        cmd = f"sqlite3 {bak_path} '.clone {file_path}'"
+        ret = subprocess.call(cmd, shell=True, stdout=subprocess.PIPE,
+                              stderr=subprocess.PIPE)
+        if ret != 0:
+            raise DatabaseCorruptedError("Recovery of database failed")
+
+        self.log.info("Database at path %s recovered successfully!", file_path)
+
     def _find_ios_database(self, backup_ids=None, root_paths=[]):
         """Try to locate the module's database file from either an iTunes
         backup or a full filesystem dump.
@@ -52,4 +82,20 @@ class IOSExtraction(MVTModule):
         if file_path:
             self.file_path = file_path
         else:
-            raise FileNotFoundError("Unable to find the module's database file")
+            raise DatabaseNotFoundError("Unable to find the module's database file")
+
+        # Check if the database is corrupted.
+        conn = sqlite3.connect(self.file_path)
+        cur = conn.cursor()
+
+        try:
+            recover = False
+            cur.execute("SELECT name FROM sqlite_master WHERE type='table';")
+        except sqlite3.DatabaseError as e:
+            if "database disk image is malformed" in str(e):
+                recover = True
+        finally:
+            conn.close()
+
+        if recover:
+            self._recover_database(self.file_path)