Merge ac157a4421 into 7a4946e2c6

Signed-off-by: Donncha Ó Cearbhaill <donncha.ocearbhaill@amnesty.org>

mvt/common/alerting.py

@@ -0,0 +1,138 @@
+# Mobile Verification Toolkit (MVT)
+# Copyright (c) 2021-2023 The MVT Authors.
+# Use of this software is governed by the MVT License 1.1 that can be found at
+#   https://license.mvt.re/1.1/
+from enum import Enum
+
+
+class AlertLevel(Enum):
+    """
+    informational: Rule is intended for enrichment of events, e.g. by tagging them. No case or alerting should be triggered by such rules because it is expected that a huge amount of events will match these rules.
+    low: Notable event but rarely an incident. Low rated events can be relevant in high numbers or combination with others. Immediate reaction shouldn’t be necessary, but a regular review is recommended.
+    medium: Relevant event that should be reviewed manually on a more frequent basis.
+    high: Relevant event that should trigger an internal alert and requires a prompt review.
+    critical: Highly relevant event that indicates an incident. Critical events should be reviewed immediately.
+    """
+
+    INFORMATIONAL = 0
+    LOW = 10
+    MEDIUM = 20
+    HIGH = 30
+    CRITICAL = 40
+
+
+class AlertStore(object):
+    """
+    Track all of the alerts and detections generated during an analysis.
+
+    Results can be logged as log messages or in JSON format for processing by other tools.
+    """
+
+    def __init__(self) -> None:
+        self.alerts = []
+
+    def add_alert(
+        self, level, message=None, event_time=None, event=None, ioc=None, detected=True
+    ):
+        """
+        Add an alert to the alert store.
+        """
+        self.alerts.append(
+            Alert(
+                level=level,
+                message=message,
+                event_time=event_time,
+                event=event,
+                ioc=ioc,
+                detected=detected,
+            )
+        )
+
+    def informational(
+        self, message=None, event_time=None, event=None, ioc=None, detected=False
+    ):
+        self.add_alert(
+            AlertLevel.INFORMATIONAL,
+            message=message,
+            event_time=event_time,
+            event=event,
+            ioc=ioc,
+            detected=detected,
+        )
+
+    def low(self, message=None, event_time=None, event=None, ioc=None, detected=False):
+        self.add_alert(
+            AlertLevel.LOW,
+            message=message,
+            event_time=event_time,
+            event=event,
+            ioc=ioc,
+            detected=detected,
+        )
+
+    def medium(
+        self, message=None, event_time=None, event=None, ioc=None, detected=False
+    ):
+        self.add_alert(
+            AlertLevel.MEDIUM,
+            message=message,
+            event_time=event_time,
+            event=event,
+            ioc=ioc,
+            detected=detected,
+        )
+
+    def high(self, message=None, event_time=None, event=None, ioc=None, detected=False):
+        self.add_alert(
+            AlertLevel.HIGH,
+            message=message,
+            event_time=event_time,
+            event=event,
+            ioc=ioc,
+            detected=detected,
+        )
+
+    def critical(
+        self, message=None, event_time=None, event=None, ioc=None, detected=False
+    ):
+        self.add_alert(
+            AlertLevel.CRITICAL,
+            message=message,
+            event_time=event_time,
+            event=event,
+            ioc=ioc,
+            detected=detected,
+        )
+
+
+class Alert(object):
+    """
+    An alert generated by an MVT module.
+    """
+
+    def __init__(self, level, message, event_time, event, ioc, detected):
+        self.level = level
+        self.message = message
+        self.event_time = event_time
+        self.event = event
+        self.ioc = ioc
+        self.detected = detected
+
+    def __repr__(self):
+        return f"<Alert level={self.level} message={self.message} event_time={self.event_time} event={self.event}>"
+
+    def __str__(self):
+        return f"{self.level} {self.message} {self.event_time} {self.event}"
+
+    def to_log(self):
+        return f"{self.level} {self.message} {self.event_time} {self.event}"
+
+    def to_json(self):
+        return {
+            "level": self.level,
+            "message": self.message,
+            "event_time": self.event_time,
+            "event": self.event,
+            "ioc": self.ioc,
+            "detected": self.detected,
+        }

mvt/common/url.py

@@ -9,47 +9,65 @@ import requests
 from tld import get_tld
 
 SHORTENER_DOMAINS = [
+    "0rz.tw",
     "1drv.ms",
     "1link.in",
     "1url.com",
     "2big.at",
+    "2.gp",
     "2pl.us",
     "2tu.us",
     "2ya.com",
+    "3.ly",
+    "4sq.com",
     "4url.cc",
     "6url.com",
-    "a.gg",
-    "a.nf",
+    "7.ly",
     "a2a.me",
     "abbrr.com",
     "adf.ly",
     "adjix.com",
+    "a.gg",
     "alturl.com",
+    "a.nf",
+    "anon.to",
+    "apple.news",
     "atu.ca",
     "b23.ru",
     "bacn.me",
+    "bc.vc",
+    "bfy.tw",
+    "binged.it",
     "bit.do",
     "bit.ly",
+    "bizj.us",
     "bkite.com",
     "bloat.me",
     "budurl.com",
     "buff.ly",
     "buk.me",
     "burnurl.com",
-    "c-o.in",
     "chilp.it",
+    "chn.ge",
     "clck.ru",
-    "cli.gs",
     "clickmeter.com",
+    "cli.gs",
+    "c-o.in",
     "cort.as",
     "cut.ly",
+    "cutt.ly",
     "cuturl.com",
-    "decenturl.com",
+    "dai.ly",
+    "dailym.ai",
+    "db.tt",
     "decenturl.com",
     "dfl8.me",
     "digbig.com",
     "digg.com",
+    "disq.us",
+    "dlvr.it",
     "doiop.com",
+    "do.my",
     "dwarfurl.com",
     "dy.fi",
     "easyuri.com",
@@ -58,27 +76,35 @@ SHORTENER_DOMAINS = [
     "esyurl.com",
     "ewerl.com",
     "fa.b",
-    "ff.im",
+    "fa.by",
+    "fb.me",
     "fff.to",
+    "ff.im",
     "fhurl.com",
     "fire.to",
     "firsturl.de",
+    "firsturl.net",
     "flic.kr",
+    "flq.us",
     "fly2.ws",
     "fon.gs",
     "forms.gle",
     "fwd4.me",
+    "gdurl.com",
+    "gg.gg",
     "gl.am",
-    "go.9nl.com",
-    "go2.me",
     "go2cut.com",
+    "go2.me",
+    "go.9nl.com",
     "goo.gl",
     "goshrink.com",
+    "got.by",
     "gowat.ch",
     "gri.ms",
     "gurl.es",
     "hellotxt.com",
     "hex.io",
+    "hongkiat.shorturl.com",
     "hover.com",
     "href.in",
     "ht.ly",
@@ -87,13 +113,15 @@ SHORTENER_DOMAINS = [
     "hurl.it",
     "hurl.me",
     "hurl.ws",
+    "ibb.co",
     "icanhaz.com",
     "idek.net",
     "inreply.to",
-    "is.gd",
     "iscool.net",
+    "is.gd",
     "iterasi.net",
     "jijr.com",
+    "j.mp",
     "jmp2.net",
     "just.as",
     "kissa.be",
@@ -101,21 +129,23 @@ SHORTENER_DOMAINS = [
     "klck.me",
     "korta.nu",
     "krunchd.com",
+    "lat.ms",
     "liip.to",
     "liltext.com",
     "lin.cr",
     "linkbee.com",
     "linkbun.ch",
     "liurl.cn",
-    "ln-s.net",
-    "ln-s.ru",
+    "lnkd.in",
     "lnk.gd",
     "lnk.in",
-    "lnkd.in",
+    "ln-s.net",
+    "ln-s.ru",
     "loopt.us",
     "lru.jp",
     "lt.tl",
     "lurl.no",
+    "lyhyt.eu",
     "metamark.net",
     "migre.me",
     "minilien.com",
@@ -123,52 +153,71 @@ SHORTENER_DOMAINS = [
     "minurl.fr",
     "moourl.com",
     "myurl.in",
+    "nbcnews.to",
     "ne1.net",
     "njx.me",
     "nn.nf",
     "notlong.com",
+    "n.pr",
     "nsfw.in",
-    "o-x.fr",
+    "nyti.ms",
     "om.ly",
+    "onforb.es",
+    "on.mktw.net",
     "ow.ly",
+    "o-x.fr",
+    "pca.st",
     "pd.am",
     "pic.gd",
     "ping.fm",
     "piurl.com",
     "pnt.me",
+    "politi.co",
     "poprl.com",
-    "post.ly",
     "posted.at",
+    "post.ly",
     "profile.to",
+    "q.gs",
     "qicute.com",
     "qlnk.net",
+    "qr.ae",
+    "qte.me",
     "quip-art.com",
     "rb6.me",
+    "rb.gy",
+    "read.bi",
+    "redir.ec",
     "redirx.com",
-    "ri.ms",
+    "redr.me",
+    "reut.rs",
     "rickroll.it",
+    "r.im",
+    "ri.ms",
     "riz.gd",
     "rsmonkey.com",
-    "ru.ly",
     "rubyurl.com",
+    "ru.ly",
     "s7y.us",
     "safe.mn",
     "sharein.com",
     "sharetabs.com",
     "shorl.com",
     "short.ie",
-    "short.to",
     "shortlinks.co.uk",
     "shortna.me",
+    "short.to",
+    "shorturl.at",
     "shorturl.com",
     "shoturl.us",
+    "shout.to",
     "shrinkify.com",
     "shrinkster.com",
-    "shrt.st",
     "shrten.com",
+    "shrt.st",
     "shrunkin.com",
     "shw.me",
     "simurl.com",
+    "smsh.me",
     "sn.im",
     "snipr.com",
     "snipurl.com",
@@ -179,24 +228,30 @@ SHORTENER_DOMAINS = [
     "starturl.com",
     "sturly.com",
     "su.pr",
+    "t.cn",
     "t.co",
     "tcrn.ch",
+    "tgr.ph",
     "thrdl.es",
     "tighturl.com",
-    "tiny.cc",
-    "tiny.pl",
     "tiny123.com",
     "tinyarro.ws",
+    "tiny.cc",
+    "tinylink.in",
+    "tiny.pl",
+    "tiny.tw",
     "tinytw.it",
     "tinyuri.ca",
     "tinyurl.com",
     "tinyvid.io",
+    "t.me",
     "tnij.org",
-    "to.ly",
+    "tnw.to",
     "togoto.us",
+    "to.ly",
+    "traceurl.com",
     "tr.im",
     "tr.my",
-    "traceurl.com",
     "turo.us",
     "tweetburner.com",
     "twirl.at",
@@ -206,49 +261,62 @@ SHORTENER_DOMAINS = [
     "twiturl.de",
     "twurl.cc",
     "twurl.nl",
-    "u.mavrev.com",
-    "u.nu",
     "u6e.de",
     "ub0.cc",
+    "ukl.me.uk",
+    "u.mavrev.com",
+    "u.nu",
     "updating.me",
     "ur1.ca",
-    "url.co.uk",
-    "url.ie",
     "url4.eu",
     "urlao.com",
     "urlbrief.com",
+    "url.co.uk",
     "urlcover.com",
     "urlcut.com",
     "urlenco.de",
     "urlhawk.com",
+    "url.ie",
     "urlkiss.com",
     "urlot.com",
     "urlpire.com",
     "urlx.ie",
     "urlx.org",
     "urlzen.com",
+    "use.my",
+    "u.to",
+    "v.gd",
     "virl.com",
     "vl.am",
+    "vurl.com",
+    "vzturl.com",
     "w3t.org",
+    "wapo.st",
     "wapurl.co.uk",
     "wipi.es",
     "wp.me",
-    "x.co",
-    "x.se",
     "xaddr.com",
+    "x.co",
     "xeeurl.com",
     "xr.com",
     "xrl.in",
     "xrl.us",
+    "x.se",
+    "xurl.es",
     "xurl.jp",
     "xzb.cc",
+    "ye.pe",
     "yep.it",
     "yfrog.com",
+    "yhoo.it",
     "ymlp.com",
+    "yuarel.com",
     "yweb.com",
     "zi.ma",
     "zi.pe",
     "zipmyurl.com",
+    "zurl.to",
+    "zurl.ws",
     "zz.gd",
 ]
 

mvt/common/version.py

@@ -3,4 +3,4 @@
 # Use of this software is governed by the MVT License 1.1 that can be found at
 #   https://license.mvt.re/1.1/
 
-MVT_VERSION = "2.5.0"
+MVT_VERSION = "2.5.1"

mvt/ios/modules/mixed/sms.py

@@ -68,9 +68,10 @@ class SMS(IOSExtraction):
         for message in self.results:
             alert = "ALERT: State-sponsored attackers may be targeting your iPhone"
             if message.get("text", "").startswith(alert):
-                self.log.warning(
-                    "Apple warning about state-sponsored attack received on the %s",
-                    message["isodate"],
+                self.alerts.medium(
+                    f"Apple warning about state-sponsored attack received on the {message['isodate']}",
+                    event_time=message["isodate"],
+                    event=message,
                 )
 
         if not self.indicators: