Added more date conversion wrappers

mvt/android/parsers/backup.py

@@ -3,7 +3,6 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-import datetime
 import io
 import json
 import tarfile

mvt/common/utils.py

@@ -9,36 +9,6 @@ import re
 from typing import Union
 
 
-def convert_mactime_to_datetime(timestamp: Union[int, float],
-                                from_2001: bool = True):
-    """Converts Mac Standard Time to a datetime.
-
-    :param timestamp: MacTime timestamp (either int or float).
-    :type timestamp: int
-    :param from_2001: bool: Whether to (Default value = True)
-    :param from_2001: Default value = True)
-    :returns: datetime.
-
-    """
-    if not timestamp:
-        return None
-
-    # This is to fix formats in case of, for example, SMS messages database
-    # timestamp format.
-    if isinstance(timestamp, int) and len(str(timestamp)) == 18:
-        timestamp = int(str(timestamp)[:9])
-
-    # MacTime counts from 2001-01-01.
-    if from_2001:
-        timestamp = timestamp + 978307200
-
-    # TODO: This is rather ugly. Happens sometimes with invalid timestamps.
-    try:
-        return datetime.datetime.utcfromtimestamp(timestamp)
-    except Exception:
-        return None
-
-
 def convert_chrometime_to_datetime(timestamp: int) -> int:
     """Converts Chrome timestamp to a datetime.
 
@@ -67,6 +37,17 @@ def convert_datetime_to_iso(datetime: datetime.datetime) -> str:
         return ""
 
 
+def convert_unix_to_utc_datetime(timestamp: int) -> datetime.datetime:
+    """Converts a unix epoch timestamp to UTC datetime.
+
+    :param timestamp: Epoc timestamp to convert.
+    :type timestamp: int
+    :returns: datetime.
+
+    """
+    return datetime.datetime.utcfromtimestamp(int(timestamp))
+
+
 def convert_unix_to_iso(timestamp: int) -> str:
     """Converts a unix epoch to ISO string.
 
@@ -77,11 +58,41 @@ def convert_unix_to_iso(timestamp: int) -> str:
 
     """
     try:
-        return convert_datetime_to_iso(datetime.datetime.utcfromtimestamp(int(timestamp)))
+        return convert_datetime_to_iso(convert_unix_to_utc_datetime(timestamp))
     except Exception:
         return ""
 
 
+def convert_mactime_to_datetime(timestamp: Union[int, float],
+                                from_2001: bool = True):
+    """Converts Mac Standard Time to a datetime.
+
+    :param timestamp: MacTime timestamp (either int or float).
+    :type timestamp: int
+    :param from_2001: bool: Whether to (Default value = True)
+    :param from_2001: Default value = True)
+    :returns: datetime.
+
+    """
+    if not timestamp:
+        return None
+
+    # This is to fix formats in case of, for example, SMS messages database
+    # timestamp format.
+    if isinstance(timestamp, int) and len(str(timestamp)) == 18:
+        timestamp = int(str(timestamp)[:9])
+
+    # MacTime counts from 2001-01-01.
+    if from_2001:
+        timestamp = timestamp + 978307200
+
+    # TODO: This is rather ugly. Happens sometimes with invalid timestamps.
+    try:
+        return convert_unix_to_utc_datetime(timestamp)
+    except Exception:
+        return None
+
+
 def convert_mactime_to_iso(timestamp: int, from_2001: bool = True):
     """Wraps two conversions from mactime to iso date.
 

mvt/ios/modules/fs/filesystem.py

@@ -3,7 +3,6 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-import datetime
 import logging
 import os
 from typing import Union

mvt/ios/modules/fs/webkit_base.py

@@ -3,10 +3,9 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-import datetime
 import os
 
-from mvt.common.utils import convert_datetime_to_iso
+from mvt.common.utils import convert_unix_to_iso
 
 from ..base import IOSExtraction
 
@@ -35,10 +34,9 @@ class WebkitBase(IOSExtraction):
                 name = name.replace("http_", "http://")
                 name = name.replace("https_", "https://")
                 url = name.split("_")[0]
-                utc_timestamp = datetime.datetime.utcfromtimestamp(os.stat(found_path).st_mtime)
 
                 self.results.append({
                     "folder": key,
                     "url": url,
-                    "isodate": convert_datetime_to_iso(utc_timestamp),
+                    "isodate": convert_unix_to_iso(os.stat(found_path).st_mtime),
                 })

mvt/ios/modules/mixed/firefox_favicon.py

@@ -5,7 +5,6 @@
 
 import logging
 import sqlite3
-from datetime import datetime
 from typing import Union
 
 from mvt.common.utils import convert_unix_to_iso

mvt/ios/modules/mixed/webkit_resource_load_statistics.py

@@ -3,12 +3,11 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-import datetime
 import logging
 import os
 import sqlite3
 
-from mvt.common.utils import convert_datetime_to_iso
+from mvt.common.utils import convert_unix_to_iso
 
 from ..base import IOSExtraction
 
@@ -72,11 +71,12 @@ class WebkitResourceLoadStatistics(IOSExtraction):
                 "registrable_domain": row[1],
                 "last_seen": row[2],
                 "had_user_interaction": bool(row[3]),
-                "last_seen_isodate": convert_datetime_to_iso(datetime.datetime.utcfromtimestamp(int(row[2]))),
+                "last_seen_isodate": convert_unix_to_iso(row[2]),
             })
 
         if len(self.results[key]) > 0:
-            self.log.info("Extracted a total of %d records from %s", len(self.results[key]), db_path)
+            self.log.info("Extracted a total of %d records from %s",
+                          len(self.results[key]), db_path)
 
     def run(self) -> None:
         if self.is_backup:
@@ -87,9 +87,8 @@ class WebkitResourceLoadStatistics(IOSExtraction):
                     if db_path:
                         self._process_observations_db(db_path=db_path, key=key)
             except Exception as exc:
-                self.log.info("Unable to search for WebKit observations.db: %s",
-                              exc)
+                self.log.info("Unable to find WebKit observations.db: %s", exc)
         elif self.is_fs_dump:
             for db_path in self._get_fs_files_from_patterns(WEBKIT_RESOURCELOADSTATICS_ROOT_PATHS):
-                self._process_observations_db(db_path=db_path,
-                                              key=os.path.relpath(db_path, self.target_path))
+                db_rel_path = os.path.relpath(db_path, self.target_path)
+                self._process_observations_db(db_path=db_path, key=db_rel_path)

setup.cfg

@@ -61,7 +61,7 @@ output-format = colorized
 max-locals = 25
 max-args = 10
 
-good-names = i,e,m
+good-names = i,m
 
 min-similarity-lines = 10
 ignore-comments = yes

tests/common/test_date_conversions.py

@@ -3,7 +3,9 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-from mvt.common.utils import convert_mactime_to_iso, convert_unix_to_iso
+from mvt.common.utils import (convert_datetime_to_iso, convert_mactime_to_iso,
+                              convert_unix_to_iso,
+                              convert_unix_to_utc_datetime)
 
 TEST_DATE_EPOCH = 1626566400
 TEST_DATE_ISO = "2021-07-18 00:00:00.000000"
@@ -17,3 +19,13 @@ class TestDateConversions:
 
     def test_convert_mactime_to_iso(self):
         assert convert_mactime_to_iso(TEST_DATE_MAC) == TEST_DATE_ISO
+
+    def test_convert_unix_to_utc_datetime(self):
+        converted = convert_unix_to_utc_datetime(TEST_DATE_EPOCH)
+        assert converted.year == 2021
+        assert converted.month == 7
+        assert converted.day == 18
+
+    def test_convert_datetime_to_iso(self):
+        converted = convert_unix_to_utc_datetime(TEST_DATE_EPOCH)
+        assert convert_datetime_to_iso(converted) == TEST_DATE_ISO

tests/common/test_indicators.py

@@ -10,6 +10,7 @@ from mvt.common.indicators import Indicators
 
 
 class TestIndicators:
+
     def test_parse_stix2(self, indicator_file):
         ind = Indicators(log=logging)
         ind.load_indicators_files([indicator_file], load_default=False)