Merge pull request #247 from mvt-project/android-split-parsers

Android split parsers
This commit is contained in:
Nex 2022-02-03 00:06:53 +01:00 committed by GitHub
commit 998d87900d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 352 additions and 306 deletions

View File

@ -5,6 +5,8 @@
import logging
from mvt.android.parsers import parse_dumpsys_accessibility
from .base import AndroidExtraction
log = logging.getLogger(__name__)
@ -30,36 +32,11 @@ class DumpsysAccessibility(AndroidExtraction):
self.detected.append(result)
continue
@staticmethod
def parse_accessibility(output):
results = []
in_services = False
for line in output.splitlines():
if line.strip().startswith("installed services:"):
in_services = True
continue
if not in_services:
continue
if line.strip() == "}":
break
service = line.split(":")[1].strip()
results.append({
"package_name": service.split("/")[0],
"service": service,
})
return results
def run(self):
self._adb_connect()
output = self._adb_command("dumpsys accessibility")
self.results = self.parse_accessibility(output)
self.results = parse_dumpsys_accessibility(output)
for result in self.results:
log.info("Found installed accessibility service \"%s\"", result.get("service"))

View File

@ -5,6 +5,8 @@
import logging
from mvt.android.parsers import parse_dumpsys_activity_resolver_table
from .base import AndroidExtraction
log = logging.getLogger(__name__)
@ -33,66 +35,10 @@ class DumpsysActivities(AndroidExtraction):
self.detected.append({intent: activity})
continue
@staticmethod
def parse_activity_resolver_table(output):
results = {}
in_activity_resolver_table = False
in_non_data_actions = False
intent = None
for line in output.splitlines():
if line.startswith("Activity Resolver Table:"):
in_activity_resolver_table = True
continue
if not in_activity_resolver_table:
continue
if line.startswith(" Non-Data Actions:"):
in_non_data_actions = True
continue
if not in_non_data_actions:
continue
# If we hit an empty line, the Non-Data Actions section should be
# finished.
if line.strip() == "":
break
# We detect the action name.
if line.startswith(" " * 6) and not line.startswith(" " * 8) and ":" in line:
intent = line.strip().replace(":", "")
results[intent] = []
continue
# If we are not in an intent block yet, skip.
if not intent:
continue
# If we are in a block but the line does not start with 8 spaces
# it means the block ended a new one started, so we reset and
# continue.
if not line.startswith(" " * 8):
intent = None
continue
# If we got this far, we are processing receivers for the
# activities we are interested in.
activity = line.strip().split(" ")[1]
package_name = activity.split("/")[0]
results[intent].append({
"package_name": package_name,
"activity": activity,
})
return results
def run(self):
self._adb_connect()
output = self._adb_command("dumpsys package")
self.results = self.parse_activity_resolver_table(output)
self.results = parse_dumpsys_activity_resolver_table(output)
self._adb_disconnect()

View File

@ -5,6 +5,8 @@
import logging
from mvt.android.parsers import parse_dumpsys_battery_daily
from .base import AndroidExtraction
log = logging.getLogger(__name__)
@ -38,54 +40,11 @@ class DumpsysBatteryDaily(AndroidExtraction):
self.detected.append(result)
continue
@staticmethod
def parse_battery_daily(output):
results = []
daily = None
daily_updates = []
for line in output.splitlines()[1:]:
if line.startswith(" Daily from "):
if len(daily_updates) > 0:
results.extend(daily_updates)
daily_updates = []
timeframe = line[13:].strip()
date_from, date_to = timeframe.strip(":").split(" to ", 1)
daily = {"from": date_from[0:10], "to": date_to[0:10]}
continue
if not daily:
continue
if not line.strip().startswith("Update "):
continue
line = line.strip().replace("Update ", "")
package_name, vers = line.split(" ", 1)
vers_nr = vers.split("=", 1)[1]
already_seen = False
for update in daily_updates:
if package_name == update["package_name"] and vers_nr == update["vers"]:
already_seen = True
break
if not already_seen:
daily_updates.append({
"action": "update",
"from": daily["from"],
"to": daily["to"],
"package_name": package_name,
"vers": vers_nr,
})
return results
def run(self):
self._adb_connect()
output = self._adb_command("dumpsys batterystats --daily")
self.results = self.parse_battery_daily(output)
self.results = parse_dumpsys_battery_daily(output)
self.log.info("Extracted %d records from battery daily stats", len(self.results))

View File

@ -5,6 +5,8 @@
import logging
from mvt.android.parsers import parse_dumpsys_battery_history
from .base import AndroidExtraction
log = logging.getLogger(__name__)
@ -30,61 +32,11 @@ class DumpsysBatteryHistory(AndroidExtraction):
self.detected.append(result)
continue
@staticmethod
def parse_battery_history(output):
results = []
for line in output.splitlines()[1:]:
if line.strip() == "":
break
time_elapsed, rest = line.strip().split(" ", 1)
start = line.find(" 100 ")
if start == -1:
continue
line = line[start+5:]
event = ""
if line.startswith("+job"):
event = "start_job"
elif line.startswith("-job"):
event = "end_job"
elif line.startswith("+running +wake_lock="):
event = "wake"
else:
continue
if event in ["start_job", "end_job"]:
uid = line[line.find("=")+1:line.find(":")]
service = line[line.find(":")+1:].strip('"')
package_name = service.split("/")[0]
elif event == "wake":
uid = line[line.find("=")+1:line.find(":")]
service = line[line.find("*walarm*:")+9:].split(" ")[0].strip('"').strip()
if service == "" or "/" not in service:
continue
package_name = service.split("/")[0]
else:
continue
results.append({
"time_elapsed": time_elapsed,
"event": event,
"uid": uid,
"package_name": package_name,
"service": service,
})
return results
def run(self):
self._adb_connect()
output = self._adb_command("dumpsys batterystats --history")
self.results = self.parse_battery_history(output)
self.results = parse_dumpsys_battery_history(output)
self.log.info("Extracted %d records from battery history", len(self.results))

View File

@ -6,6 +6,8 @@
import logging
import re
from mvt.android.parsers import parse_dumpsys_dbinfo
from .base import AndroidExtraction
log = logging.getLogger(__name__)
@ -35,45 +37,11 @@ class DumpsysDBInfo(AndroidExtraction):
self.detected.append(result)
continue
@staticmethod
def parse_dbinfo(output):
results = []
rxp = re.compile(r'.*\[([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3})\].*\[Pid:\((\d+)\)\](\w+).*sql\=\"(.+?)\".*path\=(.*?$)')
in_operations = False
for line in output.splitlines():
if line.strip() == "Most recently executed operations:":
in_operations = True
continue
if not in_operations:
continue
if not line.startswith(" "):
in_operations = False
continue
matches = rxp.findall(line)
if not matches:
continue
match = matches[0]
results.append({
"isodate": match[0],
"pid": match[1],
"action": match[2],
"sql": match[3],
"path": match[4],
})
return results
def run(self):
self._adb_connect()
output = self._adb_command("dumpsys dbinfo")
self.results = self.parse_dbinfo(output)
self.results = parse_dumpsys_dbinfo(output)
self.log.info("Extracted a total of %d records from database information",
len(self.results))

View File

@ -5,6 +5,8 @@
import logging
from mvt.android.parsers import parse_dumpsys_receiver_resolver_table
from .base import AndroidExtraction
log = logging.getLogger(__name__)
@ -55,66 +57,10 @@ class DumpsysReceivers(AndroidExtraction):
self.detected.append({intent: receiver})
continue
@staticmethod
def parse_receiver_resolver_table(output):
results = {}
in_receiver_resolver_table = False
in_non_data_actions = False
intent = None
for line in output.splitlines():
if line.startswith("Receiver Resolver Table:"):
in_receiver_resolver_table = True
continue
if not in_receiver_resolver_table:
continue
if line.startswith(" Non-Data Actions:"):
in_non_data_actions = True
continue
if not in_non_data_actions:
continue
# If we hit an empty line, the Non-Data Actions section should be
# finished.
if line.strip() == "":
break
# We detect the action name.
if line.startswith(" " * 6) and not line.startswith(" " * 8) and ":" in line:
intent = line.strip().replace(":", "")
results[intent] = []
continue
# If we are not in an intent block yet, skip.
if not intent:
continue
# If we are in a block but the line does not start with 8 spaces
# it means the block ended a new one started, so we reset and
# continue.
if not line.startswith(" " * 8):
intent = None
continue
# If we got this far, we are processing receivers for the
# activities we are interested in.
receiver = line.strip().split(" ")[1]
package_name = receiver.split("/")[0]
results[intent].append({
"package_name": package_name,
"receiver": receiver,
})
return results
def run(self):
self._adb_connect()
output = self._adb_command("dumpsys package")
self.results = self.parse_receiver_resolver_table(output)
self.results = parse_dumpsys_receiver_resolver_table(output)
self._adb_disconnect()

View File

@ -6,6 +6,8 @@
import logging
import re
from mvt.android.parsers import parse_getprop
from .base import AndroidExtraction
log = logging.getLogger(__name__)
@ -22,31 +24,11 @@ class Getprop(AndroidExtraction):
self.results = {} if not results else results
@staticmethod
def parse_getprop(output):
results = {}
rxp = re.compile(r"\[(.+?)\]: \[(.+?)\]")
for line in output.splitlines():
line = line.strip()
if line == "":
continue
matches = re.findall(rxp, line)
if not matches or len(matches[0]) != 2:
continue
key = matches[0][0]
value = matches[0][1]
results[key] = value
return results
def run(self):
self._adb_connect()
output = self._adb_command("getprop")
self._adb_disconnect()
self.results = self.parse_getprop(output)
self.results = parse_getprop(output)
self.log.info("Extracted %d Android system properties", len(self.results))

View File

@ -19,6 +19,16 @@ class Processes(AndroidExtraction):
output_folder=output_folder, fast_mode=fast_mode,
log=log, results=results)
def check_indicators(self):
if not self.indicators:
return
for result in self.results:
ioc = self.indicators.check_app_id(result.get("name", ""))
if ioc:
result["matched_indicator"] = ioc
self.detected.append(result)
def run(self):
self._adb_connect()

View File

@ -5,7 +5,7 @@
import logging
from mvt.android.modules.adb.dumpsys_accessibility import DumpsysAccessibility
from mvt.android.parsers import parse_dumpsys_accessibility
from .base import BugReportModule
@ -56,7 +56,7 @@ class Accessibility(BugReportModule):
lines.append(line)
self.results = DumpsysAccessibility.parse_accessibility("\n".join(lines))
self.results = parse_dumpsys_accessibility("\n".join(lines))
for result in self.results:
log.info("Found installed accessibility service \"%s\"", result.get("service"))

View File

@ -5,7 +5,7 @@
import logging
from mvt.android.modules.adb.dumpsys_activities import DumpsysActivities
from mvt.android.parsers import parse_dumpsys_activity_resolver_table
from .base import BugReportModule
@ -59,4 +59,4 @@ class Activities(BugReportModule):
lines.append(line)
self.results = DumpsysActivities.parse_activity_resolver_table("\n".join(lines))
self.results = parse_dumpsys_activity_resolver_table("\n".join(lines))

View File

@ -5,7 +5,7 @@
import logging
from mvt.android.modules.adb.dumpsys_battery_daily import DumpsysBatteryDaily
from mvt.android.parsers import parse_dumpsys_battery_daily
from .base import BugReportModule
@ -73,4 +73,4 @@ class BatteryDaily(BugReportModule):
lines.append(line)
self.results = DumpsysBatteryDaily.parse_battery_daily("\n".join(lines))
self.results = parse_dumpsys_battery_daily("\n".join(lines))

View File

@ -5,8 +5,7 @@
import logging
from mvt.android.modules.adb.dumpsys_battery_history import \
DumpsysBatteryHistory
from mvt.android.parsers import parse_dumpsys_battery_history
from .base import BugReportModule
@ -66,4 +65,4 @@ class BatteryHistory(BugReportModule):
lines.append(line)
self.results = DumpsysBatteryHistory.parse_battery_history("\n".join(lines))
self.results = parse_dumpsys_battery_history("\n".join(lines))

View File

@ -5,7 +5,7 @@
import logging
from mvt.android.modules.adb.dumpsys_dbinfo import DumpsysDBInfo
from mvt.android.parsers import parse_dumpsys_dbinfo
from .base import BugReportModule
@ -60,4 +60,4 @@ class DBInfo(BugReportModule):
lines.append(line)
self.results = DumpsysDBInfo.parse_dbinfo("\n".join(lines))
self.results = parse_dumpsys_dbinfo("\n".join(lines))

View File

@ -6,7 +6,7 @@
import logging
import re
from mvt.android.modules.adb.getprop import Getprop as GP
from mvt.android.parsers import parse_getprop
from .base import BugReportModule
@ -48,6 +48,6 @@ class Getprop(BugReportModule):
lines.append(line)
self.results = GP.parse_getprop("\n".join(lines))
self.results = parse_getprop("\n".join(lines))
self.log.info("Extracted %d Android system properties", len(self.results))

View File

@ -5,7 +5,7 @@
import logging
from mvt.android.modules.adb.dumpsys_receivers import DumpsysReceivers
from mvt.android.parsers import parse_dumpsys_receiver_resolver_table
from .base import BugReportModule
@ -81,4 +81,4 @@ class Receivers(BugReportModule):
lines.append(line)
self.results = DumpsysReceivers.parse_receiver_resolver_table("\n".join(lines))
self.results = parse_dumpsys_receiver_resolver_table("\n".join(lines))

View File

@ -0,0 +1,11 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021-2022 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
from .dumpsys import (parse_dumpsys_accessibility,
parse_dumpsys_activity_resolver_table,
parse_dumpsys_battery_daily,
parse_dumpsys_battery_history, parse_dumpsys_dbinfo,
parse_dumpsys_receiver_resolver_table)
from .getprop import parse_getprop

View File

@ -0,0 +1,270 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021-2022 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import re
def parse_dumpsys_accessibility(output):
results = []
in_services = False
for line in output.splitlines():
if line.strip().startswith("installed services:"):
in_services = True
continue
if not in_services:
continue
if line.strip() == "}":
break
service = line.split(":")[1].strip()
results.append({
"package_name": service.split("/")[0],
"service": service,
})
return results
def parse_dumpsys_activity_resolver_table(output):
results = {}
in_activity_resolver_table = False
in_non_data_actions = False
intent = None
for line in output.splitlines():
if line.startswith("Activity Resolver Table:"):
in_activity_resolver_table = True
continue
if not in_activity_resolver_table:
continue
if line.startswith(" Non-Data Actions:"):
in_non_data_actions = True
continue
if not in_non_data_actions:
continue
# If we hit an empty line, the Non-Data Actions section should be
# finished.
if line.strip() == "":
break
# We detect the action name.
if line.startswith(" " * 6) and not line.startswith(" " * 8) and ":" in line:
intent = line.strip().replace(":", "")
results[intent] = []
continue
# If we are not in an intent block yet, skip.
if not intent:
continue
# If we are in a block but the line does not start with 8 spaces
# it means the block ended a new one started, so we reset and
# continue.
if not line.startswith(" " * 8):
intent = None
continue
# If we got this far, we are processing receivers for the
# activities we are interested in.
activity = line.strip().split(" ")[1]
package_name = activity.split("/")[0]
results[intent].append({
"package_name": package_name,
"activity": activity,
})
return results
def parse_dumpsys_battery_daily(output):
results = []
daily = None
daily_updates = []
for line in output.splitlines()[1:]:
if line.startswith(" Daily from "):
if len(daily_updates) > 0:
results.extend(daily_updates)
daily_updates = []
timeframe = line[13:].strip()
date_from, date_to = timeframe.strip(":").split(" to ", 1)
daily = {"from": date_from[0:10], "to": date_to[0:10]}
continue
if not daily:
continue
if not line.strip().startswith("Update "):
continue
line = line.strip().replace("Update ", "")
package_name, vers = line.split(" ", 1)
vers_nr = vers.split("=", 1)[1]
already_seen = False
for update in daily_updates:
if package_name == update["package_name"] and vers_nr == update["vers"]:
already_seen = True
break
if not already_seen:
daily_updates.append({
"action": "update",
"from": daily["from"],
"to": daily["to"],
"package_name": package_name,
"vers": vers_nr,
})
return results
def parse_dumpsys_battery_history(output):
results = []
for line in output.splitlines()[1:]:
if line.strip() == "":
break
time_elapsed, rest = line.strip().split(" ", 1)
start = line.find(" 100 ")
if start == -1:
continue
line = line[start+5:]
event = ""
if line.startswith("+job"):
event = "start_job"
elif line.startswith("-job"):
event = "end_job"
elif line.startswith("+running +wake_lock="):
event = "wake"
else:
continue
if event in ["start_job", "end_job"]:
uid = line[line.find("=")+1:line.find(":")]
service = line[line.find(":")+1:].strip('"')
package_name = service.split("/")[0]
elif event == "wake":
uid = line[line.find("=")+1:line.find(":")]
service = line[line.find("*walarm*:")+9:].split(" ")[0].strip('"').strip()
if service == "" or "/" not in service:
continue
package_name = service.split("/")[0]
else:
continue
results.append({
"time_elapsed": time_elapsed,
"event": event,
"uid": uid,
"package_name": package_name,
"service": service,
})
return results
def parse_dumpsys_dbinfo(output):
results = []
rxp = re.compile(r'.*\[([0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3})\].*\[Pid:\((\d+)\)\](\w+).*sql\=\"(.+?)\".*path\=(.*?$)')
in_operations = False
for line in output.splitlines():
if line.strip() == "Most recently executed operations:":
in_operations = True
continue
if not in_operations:
continue
if not line.startswith(" "):
in_operations = False
continue
matches = rxp.findall(line)
if not matches:
continue
match = matches[0]
results.append({
"isodate": match[0],
"pid": match[1],
"action": match[2],
"sql": match[3],
"path": match[4],
})
return results
def parse_dumpsys_receiver_resolver_table(output):
results = {}
in_receiver_resolver_table = False
in_non_data_actions = False
intent = None
for line in output.splitlines():
if line.startswith("Receiver Resolver Table:"):
in_receiver_resolver_table = True
continue
if not in_receiver_resolver_table:
continue
if line.startswith(" Non-Data Actions:"):
in_non_data_actions = True
continue
if not in_non_data_actions:
continue
# If we hit an empty line, the Non-Data Actions section should be
# finished.
if line.strip() == "":
break
# We detect the action name.
if line.startswith(" " * 6) and not line.startswith(" " * 8) and ":" in line:
intent = line.strip().replace(":", "")
results[intent] = []
continue
# If we are not in an intent block yet, skip.
if not intent:
continue
# If we are in a block but the line does not start with 8 spaces
# it means the block ended a new one started, so we reset and
# continue.
if not line.startswith(" " * 8):
intent = None
continue
# If we got this far, we are processing receivers for the
# activities we are interested in.
receiver = line.strip().split(" ")[1]
package_name = receiver.split("/")[0]
results[intent].append({
"package_name": package_name,
"receiver": receiver,
})
return results

View File

@ -0,0 +1,26 @@
# Mobile Verification Toolkit (MVT)
# Copyright (c) 2021-2022 The MVT Project Authors.
# Use of this software is governed by the MVT License 1.1 that can be found at
# https://license.mvt.re/1.1/
import re
def parse_getprop(output):
results = {}
rxp = re.compile(r"\[(.+?)\]: \[(.+?)\]")
for line in output.splitlines():
line = line.strip()
if line == "":
continue
matches = re.findall(rxp, line)
if not matches or len(matches[0]) != 2:
continue
key = matches[0][0]
value = matches[0][1]
results[key] = value
return results