mirror of https://github.com/mvt-project/mvt.git
Merge branch 'main' into feature/ios-check-usb
This commit is contained in:
commit
4a568835d2
|
@ -13,8 +13,8 @@ from mvt.common.cmd_check_iocs import CmdCheckIOCS
|
||||||
from mvt.common.help import (HELP_MSG_FAST, HELP_MSG_IOC,
|
from mvt.common.help import (HELP_MSG_FAST, HELP_MSG_IOC,
|
||||||
HELP_MSG_LIST_MODULES, HELP_MSG_MODULE,
|
HELP_MSG_LIST_MODULES, HELP_MSG_MODULE,
|
||||||
HELP_MSG_OUTPUT, HELP_MSG_SERIAL)
|
HELP_MSG_OUTPUT, HELP_MSG_SERIAL)
|
||||||
from mvt.common.indicators import download_indicators_files
|
|
||||||
from mvt.common.logo import logo
|
from mvt.common.logo import logo
|
||||||
|
from mvt.common.updates import IndicatorsUpdates
|
||||||
|
|
||||||
from .cmd_check_adb import CmdAndroidCheckADB
|
from .cmd_check_adb import CmdAndroidCheckADB
|
||||||
from .cmd_check_backup import CmdAndroidCheckBackup
|
from .cmd_check_backup import CmdAndroidCheckBackup
|
||||||
|
@ -215,4 +215,5 @@ def check_iocs(ctx, iocs, list_modules, module, folder):
|
||||||
#==============================================================================
|
#==============================================================================
|
||||||
@cli.command("download-iocs", help="Download public STIX2 indicators")
|
@cli.command("download-iocs", help="Download public STIX2 indicators")
|
||||||
def download_indicators():
|
def download_indicators():
|
||||||
download_indicators_files(log)
|
ioc_updates = IndicatorsUpdates()
|
||||||
|
ioc_updates.update()
|
||||||
|
|
|
@ -7,11 +7,13 @@ import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
|
||||||
import requests
|
|
||||||
from appdirs import user_data_dir
|
from appdirs import user_data_dir
|
||||||
|
|
||||||
from .url import URL
|
from .url import URL
|
||||||
|
|
||||||
|
MVT_DATA_FOLDER = user_data_dir("mvt")
|
||||||
|
MVT_INDICATORS_FOLDER = os.path.join(MVT_DATA_FOLDER, "indicators")
|
||||||
|
|
||||||
|
|
||||||
class Indicators:
|
class Indicators:
|
||||||
"""This class is used to parse indicators from a STIX2 file and provide
|
"""This class is used to parse indicators from a STIX2 file and provide
|
||||||
|
@ -19,18 +21,17 @@ class Indicators:
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, log=logging.Logger) -> None:
|
def __init__(self, log=logging.Logger) -> None:
|
||||||
self.data_dir = user_data_dir("mvt")
|
|
||||||
self.log = log
|
self.log = log
|
||||||
self.ioc_collections = []
|
self.ioc_collections = []
|
||||||
self.total_ioc_count = 0
|
self.total_ioc_count = 0
|
||||||
|
|
||||||
def _load_downloaded_indicators(self) -> None:
|
def _load_downloaded_indicators(self) -> None:
|
||||||
if not os.path.isdir(self.data_dir):
|
if not os.path.isdir(MVT_INDICATORS_FOLDER):
|
||||||
return
|
return
|
||||||
|
|
||||||
for f in os.listdir(self.data_dir):
|
for ioc_file_name in os.listdir(MVT_INDICATORS_FOLDER):
|
||||||
if f.lower().endswith(".stix2"):
|
if ioc_file_name.lower().endswith(".stix2"):
|
||||||
self.parse_stix2(os.path.join(self.data_dir, f))
|
self.parse_stix2(os.path.join(MVT_INDICATORS_FOLDER, ioc_file_name))
|
||||||
|
|
||||||
def _check_stix2_env_variable(self) -> None:
|
def _check_stix2_env_variable(self) -> None:
|
||||||
"""
|
"""
|
||||||
|
@ -446,36 +447,3 @@ class Indicators:
|
||||||
self.log.warning("Found a known suspicious app with ID \"%s\" matching indicators from \"%s\"",
|
self.log.warning("Found a known suspicious app with ID \"%s\" matching indicators from \"%s\"",
|
||||||
app_id, ioc["name"])
|
app_id, ioc["name"])
|
||||||
return ioc
|
return ioc
|
||||||
|
|
||||||
|
|
||||||
def download_indicators_files(log: logging.Logger) -> None:
|
|
||||||
"""
|
|
||||||
Download indicators from repo into MVT app data directory.
|
|
||||||
"""
|
|
||||||
data_dir = user_data_dir("mvt")
|
|
||||||
if not os.path.isdir(data_dir):
|
|
||||||
os.makedirs(data_dir, exist_ok=True)
|
|
||||||
|
|
||||||
# Download latest list of indicators from the MVT repo.
|
|
||||||
res = requests.get("https://raw.githubusercontent.com/mvt-project/mvt-indicators/main/indicators.json")
|
|
||||||
if res.status_code != 200:
|
|
||||||
log.warning("Unable to find retrieve list of indicators from the MVT repository.")
|
|
||||||
return
|
|
||||||
|
|
||||||
for ioc_entry in res.json():
|
|
||||||
ioc_url = ioc_entry["stix2_url"]
|
|
||||||
log.info("Downloading indicator file %s from %s", ioc_entry["name"], ioc_url)
|
|
||||||
|
|
||||||
res = requests.get(ioc_url)
|
|
||||||
if res.status_code != 200:
|
|
||||||
log.warning("Could not find indicator file %s", ioc_url)
|
|
||||||
continue
|
|
||||||
|
|
||||||
clean_file_name = ioc_url.lstrip("https://").replace("/", "_")
|
|
||||||
ioc_path = os.path.join(data_dir, clean_file_name)
|
|
||||||
|
|
||||||
# Write file to disk. This will overwrite any older version of the STIX2 file.
|
|
||||||
with open(ioc_path, "w", encoding="utf-8") as handle:
|
|
||||||
handle.write(res.text)
|
|
||||||
|
|
||||||
log.info("Saved indicator file to %s", os.path.basename(ioc_path))
|
|
||||||
|
|
|
@ -5,22 +5,55 @@
|
||||||
|
|
||||||
from rich import print
|
from rich import print
|
||||||
|
|
||||||
from .updates import check_for_updates
|
from .updates import IndicatorsUpdates, MVTUpdates
|
||||||
from .version import MVT_VERSION
|
from .version import MVT_VERSION
|
||||||
|
|
||||||
|
|
||||||
|
def check_updates() -> None:
|
||||||
|
# First we check for MVT version udpates.
|
||||||
|
mvt_updates = MVTUpdates()
|
||||||
|
try:
|
||||||
|
latest_version = mvt_updates.check()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
if latest_version:
|
||||||
|
print(f"\t\t[bold]Version {latest_version} is available! Upgrade mvt![/bold]")
|
||||||
|
|
||||||
|
# Then we check for indicators files updates.
|
||||||
|
ioc_updates = IndicatorsUpdates()
|
||||||
|
|
||||||
|
# Before proceeding, we check if we have downloaded an indicators index.
|
||||||
|
# If not, there's no point in proceeding with the updates check.
|
||||||
|
if ioc_updates.get_latest_update() == 0:
|
||||||
|
print("\t\t[bold]You have not yet downloaded any indicators, check the `download-iocs` command![/bold]")
|
||||||
|
return
|
||||||
|
|
||||||
|
# We only perform this check at a fixed frequency, in order to not
|
||||||
|
# overburden the user with too many lookups if the command is being run
|
||||||
|
# multiple times.
|
||||||
|
should_check, hours = ioc_updates.should_check()
|
||||||
|
if not should_check:
|
||||||
|
print(f"\t\tIndicators updates checked recently, next automatic check in {int(hours)} hours")
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
ioc_to_update = ioc_updates.check()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
if ioc_to_update:
|
||||||
|
print("\t\t[bold]There are updates to your indicators files! Run the `download-iocs` command to update![/bold]")
|
||||||
|
else:
|
||||||
|
print("\t\tYour indicators files seem to be up to date.")
|
||||||
|
|
||||||
|
|
||||||
def logo() -> None:
|
def logo() -> None:
|
||||||
print("\n")
|
print("\n")
|
||||||
print("\t[bold]MVT[/bold] - Mobile Verification Toolkit")
|
print("\t[bold]MVT[/bold] - Mobile Verification Toolkit")
|
||||||
print("\t\thttps://mvt.re")
|
print("\t\thttps://mvt.re")
|
||||||
print(f"\t\tVersion: {MVT_VERSION}")
|
print(f"\t\tVersion: {MVT_VERSION}")
|
||||||
|
|
||||||
try:
|
check_updates()
|
||||||
latest_version = check_for_updates()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
if latest_version:
|
|
||||||
print(f"\t\t[bold]Version {latest_version} is available! Upgrade mvt![/bold]")
|
|
||||||
|
|
||||||
print("\n")
|
print("\n")
|
||||||
|
|
|
@ -3,18 +3,209 @@
|
||||||
# Use of this software is governed by the MVT License 1.1 that can be found at
|
# Use of this software is governed by the MVT License 1.1 that can be found at
|
||||||
# https://license.mvt.re/1.1/
|
# https://license.mvt.re/1.1/
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
import yaml
|
||||||
from packaging import version
|
from packaging import version
|
||||||
|
|
||||||
|
from .indicators import MVT_DATA_FOLDER, MVT_INDICATORS_FOLDER
|
||||||
from .version import MVT_VERSION
|
from .version import MVT_VERSION
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
def check_for_updates() -> str:
|
# In hours.
|
||||||
res = requests.get("https://pypi.org/pypi/mvt/json")
|
INDICATORS_CHECK_FREQUENCY = 12
|
||||||
data = res.json()
|
|
||||||
latest_version = data.get("info", {}).get("version", "")
|
|
||||||
|
|
||||||
if version.parse(latest_version) > version.parse(MVT_VERSION):
|
|
||||||
return latest_version
|
|
||||||
|
|
||||||
return ""
|
class MVTUpdates:
|
||||||
|
|
||||||
|
def check(self) -> str:
|
||||||
|
res = requests.get("https://pypi.org/pypi/mvt/json")
|
||||||
|
data = res.json()
|
||||||
|
latest_version = data.get("info", {}).get("version", "")
|
||||||
|
|
||||||
|
if version.parse(latest_version) > version.parse(MVT_VERSION):
|
||||||
|
return latest_version
|
||||||
|
|
||||||
|
return ""
|
||||||
|
|
||||||
|
|
||||||
|
class IndicatorsUpdates:
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.github_raw_url = "https://raw.githubusercontent.com/{}/{}/{}/{}"
|
||||||
|
|
||||||
|
self.index_owner = "mvt-project"
|
||||||
|
self.index_repo = "mvt-indicators"
|
||||||
|
self.index_branch = "main"
|
||||||
|
self.index_path = "indicators.yaml"
|
||||||
|
|
||||||
|
self.latest_update_path = os.path.join(MVT_DATA_FOLDER,
|
||||||
|
"latest_indicators_update")
|
||||||
|
self.latest_check_path = os.path.join(MVT_DATA_FOLDER,
|
||||||
|
"latest_indicators_check")
|
||||||
|
|
||||||
|
def get_latest_check(self) -> int:
|
||||||
|
if not os.path.exists(self.latest_check_path):
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with open(self.latest_check_path, "r") as handle:
|
||||||
|
data = handle.read().strip()
|
||||||
|
if data:
|
||||||
|
return int(data)
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def set_latest_check(self) -> None:
|
||||||
|
timestamp = int(datetime.utcnow().timestamp())
|
||||||
|
with open(self.latest_check_path, "w") as handle:
|
||||||
|
handle.write(str(timestamp))
|
||||||
|
|
||||||
|
def get_latest_update(self) -> int:
|
||||||
|
if not os.path.exists(self.latest_update_path):
|
||||||
|
return 0
|
||||||
|
|
||||||
|
with open(self.latest_update_path, "r") as handle:
|
||||||
|
data = handle.read().strip()
|
||||||
|
if data:
|
||||||
|
return int(data)
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def set_latest_update(self) -> None:
|
||||||
|
timestamp = int(datetime.utcnow().timestamp())
|
||||||
|
with open(self.latest_update_path, "w") as handle:
|
||||||
|
handle.write(str(timestamp))
|
||||||
|
|
||||||
|
def get_remote_index(self) -> dict:
|
||||||
|
url = self.github_raw_url.format(self.index_owner, self.index_repo,
|
||||||
|
self.index_branch, self.index_path)
|
||||||
|
res = requests.get(url)
|
||||||
|
if res.status_code != 200:
|
||||||
|
log.error("Failed to retrieve indicators index located at %s (error %d)",
|
||||||
|
url, res.status_code)
|
||||||
|
return None
|
||||||
|
|
||||||
|
return yaml.safe_load(res.content)
|
||||||
|
|
||||||
|
def download_remote_ioc(self, ioc_url: str) -> str:
|
||||||
|
res = requests.get(ioc_url)
|
||||||
|
if res.status_code != 200:
|
||||||
|
log.error("Failed to download indicators file from %s (error %d)",
|
||||||
|
ioc_url, res.status_code)
|
||||||
|
return None
|
||||||
|
|
||||||
|
clean_file_name = ioc_url.lstrip("https://").replace("/", "_")
|
||||||
|
ioc_path = os.path.join(MVT_INDICATORS_FOLDER, clean_file_name)
|
||||||
|
|
||||||
|
with open(ioc_path, "w", encoding="utf-8") as handle:
|
||||||
|
handle.write(res.text)
|
||||||
|
|
||||||
|
return ioc_path
|
||||||
|
|
||||||
|
def update(self) -> None:
|
||||||
|
self.set_latest_check()
|
||||||
|
|
||||||
|
if not os.path.exists(MVT_INDICATORS_FOLDER):
|
||||||
|
os.makedirs(MVT_INDICATORS_FOLDER)
|
||||||
|
|
||||||
|
index = self.get_remote_index()
|
||||||
|
for ioc in index.get("indicators", []):
|
||||||
|
ioc_type = ioc.get("type", "")
|
||||||
|
|
||||||
|
if ioc_type == "github":
|
||||||
|
github = ioc.get("github", {})
|
||||||
|
owner = github.get("owner", "")
|
||||||
|
repo = github.get("repo", "")
|
||||||
|
branch = github.get("branch", "main")
|
||||||
|
path = github.get("path", "")
|
||||||
|
|
||||||
|
ioc_url = self.github_raw_url.format(owner, repo, branch, path)
|
||||||
|
else:
|
||||||
|
ioc_url = ioc.get("download_url", "")
|
||||||
|
|
||||||
|
if not ioc_url:
|
||||||
|
log.error("Could not find a way to download indicator file for %s",
|
||||||
|
ioc.get("name"))
|
||||||
|
continue
|
||||||
|
|
||||||
|
ioc_local_path = self.download_remote_ioc(ioc_url)
|
||||||
|
if not ioc_local_path:
|
||||||
|
continue
|
||||||
|
|
||||||
|
log.info("Downloaded indicators \"%s\" to %s",
|
||||||
|
ioc.get("name"), ioc_local_path)
|
||||||
|
|
||||||
|
self.set_latest_update()
|
||||||
|
|
||||||
|
def _get_remote_file_latest_commit(self, owner: str, repo: str,
|
||||||
|
branch: str, path: str) -> bool:
|
||||||
|
file_commit_url = f"https://api.github.com/repos/{self.index_owner}/{self.index_repo}/commits?path={self.index_path}"
|
||||||
|
res = requests.get(file_commit_url)
|
||||||
|
if res.status_code != 200:
|
||||||
|
log.error("Failed to get details about file %s (error %d)",
|
||||||
|
file_commit_url, res.status_code)
|
||||||
|
return False
|
||||||
|
|
||||||
|
details = res.json()
|
||||||
|
if len(details) == 0:
|
||||||
|
return False
|
||||||
|
|
||||||
|
latest_commit = details[0]
|
||||||
|
latest_commit_date = latest_commit.get("commit", {}).get("author", {}).get("date", None)
|
||||||
|
if not latest_commit_date:
|
||||||
|
log.error("Failed to retrieve date of latest update to indicators index file")
|
||||||
|
return False
|
||||||
|
|
||||||
|
latest_commit_dt = datetime.strptime(latest_commit_date, '%Y-%m-%dT%H:%M:%SZ')
|
||||||
|
latest_commit_ts = int(latest_commit_dt.timestamp())
|
||||||
|
|
||||||
|
return latest_commit_ts
|
||||||
|
|
||||||
|
def should_check(self) -> (bool, int):
|
||||||
|
now = datetime.utcnow()
|
||||||
|
latest_check_ts = self.get_latest_check()
|
||||||
|
latest_check_dt = datetime.fromtimestamp(latest_check_ts)
|
||||||
|
|
||||||
|
diff = now - latest_check_dt
|
||||||
|
diff_hours = divmod(diff.total_seconds(), 3600)[0]
|
||||||
|
|
||||||
|
if diff_hours >= INDICATORS_CHECK_FREQUENCY:
|
||||||
|
return True, 0
|
||||||
|
|
||||||
|
return False, INDICATORS_CHECK_FREQUENCY - diff_hours
|
||||||
|
|
||||||
|
def check(self) -> bool:
|
||||||
|
self.set_latest_check()
|
||||||
|
|
||||||
|
latest_update = self.get_latest_update()
|
||||||
|
latest_commit_ts = self._get_remote_file_latest_commit(self.index_owner,
|
||||||
|
self.index_repo,
|
||||||
|
self.index_branch,
|
||||||
|
self.index_path)
|
||||||
|
|
||||||
|
if latest_update < latest_commit_ts:
|
||||||
|
return True
|
||||||
|
|
||||||
|
index = self.get_remote_index()
|
||||||
|
for ioc in index.get("indicators", []):
|
||||||
|
if ioc.get("type", "") != "github":
|
||||||
|
continue
|
||||||
|
|
||||||
|
github = ioc.get("github", {})
|
||||||
|
owner = github.get("owner", "")
|
||||||
|
repo = github.get("repo", "")
|
||||||
|
branch = github.get("branch", "main")
|
||||||
|
path = github.get("path", "")
|
||||||
|
|
||||||
|
file_latest_commit_ts = self._get_remote_file_latest_commit(owner,
|
||||||
|
repo,
|
||||||
|
branch,
|
||||||
|
path)
|
||||||
|
if latest_update < file_latest_commit_ts:
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
|
@ -14,9 +14,9 @@ from mvt.common.cmd_check_iocs import CmdCheckIOCS
|
||||||
from mvt.common.help import (HELP_MSG_FAST, HELP_MSG_IOC,
|
from mvt.common.help import (HELP_MSG_FAST, HELP_MSG_IOC,
|
||||||
HELP_MSG_LIST_MODULES, HELP_MSG_MODULE,
|
HELP_MSG_LIST_MODULES, HELP_MSG_MODULE,
|
||||||
HELP_MSG_OUTPUT, HELP_MSG_SERIAL)
|
HELP_MSG_OUTPUT, HELP_MSG_SERIAL)
|
||||||
from mvt.common.indicators import download_indicators_files
|
|
||||||
from mvt.common.logo import logo
|
from mvt.common.logo import logo
|
||||||
from mvt.common.options import MutuallyExclusiveOption
|
from mvt.common.options import MutuallyExclusiveOption
|
||||||
|
from mvt.common.updates import IndicatorsUpdates
|
||||||
|
|
||||||
from .cmd_check_backup import CmdIOSCheckBackup
|
from .cmd_check_backup import CmdIOSCheckBackup
|
||||||
from .cmd_check_fs import CmdIOSCheckFS
|
from .cmd_check_fs import CmdIOSCheckFS
|
||||||
|
@ -214,7 +214,8 @@ def check_iocs(ctx, iocs, list_modules, module, folder):
|
||||||
#==============================================================================
|
#==============================================================================
|
||||||
@cli.command("download-iocs", help="Download public STIX2 indicators")
|
@cli.command("download-iocs", help="Download public STIX2 indicators")
|
||||||
def download_iocs():
|
def download_iocs():
|
||||||
download_indicators_files(log)
|
ioc_updates = IndicatorsUpdates()
|
||||||
|
ioc_updates.update()
|
||||||
|
|
||||||
|
|
||||||
#==============================================================================
|
#==============================================================================
|
||||||
|
|
Loading…
Reference in New Issue