mirror of
https://github.com/servo/servo.git
synced 2025-06-06 16:45:39 +00:00
Devtools parser: reassemble fragmented messages (#36033)
* Devtools parser: reassemble fragmented messages Co-authored-by: Aria Edmonds <8436007+ar1a@users.noreply.github.com> Signed-off-by: Delan Azabani <dazabani@igalia.com> * Enable devtools parser tests on Linux only for now Signed-off-by: Delan Azabani <dazabani@igalia.com> --------- Signed-off-by: Delan Azabani <dazabani@igalia.com> Co-authored-by: Aria Edmonds <8436007+ar1a@users.noreply.github.com>
This commit is contained in:
Delan Azabani
GitHub
parent
5d1c64dba9
commit
2c94110952
4 changed files with 147 additions and 25 deletions
|
|
@ -38,8 +38,8 @@
|
|||
# ./devtools_parser.py --use cap.pcap --port 1234 --filter watcher --range 10:30
|
||||
|
||||
import json
|
||||
import re
|
||||
import signal
|
||||
import sys
|
||||
from argparse import ArgumentParser
|
||||
from subprocess import Popen, PIPE
|
||||
try:
|
||||
|
|
@ -49,7 +49,6 @@ except ImportError:
|
|||
return text
|
||||
|
||||
fields = ["frame.time", "tcp.srcport", "tcp.payload"]
|
||||
pattern = r"(\d+:){"
|
||||
|
||||
|
||||
# Use tshark to capture network traffic and save the result in a
|
||||
|
|
@ -98,40 +97,84 @@ def read_data(file):
|
|||
|
||||
|
||||
# Transform the raw output of wireshark into a more manageable one
|
||||
def process_data(out, port):
|
||||
out = [line.split("\t") for line in out.split("\n")]
|
||||
def process_data(input, port):
|
||||
# Split the input into lines.
|
||||
# `input` = newline-terminated lines of tab-delimited tshark(1) output
|
||||
lines = [line.split("\t") for line in input.split("\n")]
|
||||
|
||||
# Process fields
|
||||
data = []
|
||||
for line in out:
|
||||
# Remove empty lines and empty sends, and decode hex to bytes.
|
||||
# `lines` = [[date, port, hex-encoded data]], e.g.
|
||||
# `["Mar 18, 2025 21:09:51.879661797 AWST", "6080", "3133"]`
|
||||
# `["Mar 18, 2025 21:09:51.879661797 AWST", "6080", "393a"]`
|
||||
# `["Mar 18, 2025 21:09:51.879661797 AWST", "6080", "7b..."]`
|
||||
sends = []
|
||||
for line in lines:
|
||||
if len(line) != 3:
|
||||
continue
|
||||
|
||||
curr_time, curr_port, curr_data = line
|
||||
if len(curr_data) == 0:
|
||||
continue
|
||||
elif len(curr_data) % 2 == 1:
|
||||
print(f"[WARNING] Extra byte in hex-encoded data: {curr_data[-1]}", file=sys.stderr)
|
||||
curr_data = curr_data[:-1]
|
||||
if len(sends) > 0 and sends[-1][1] == curr_port:
|
||||
sends[-1][2] += bytearray.fromhex(curr_data)
|
||||
else:
|
||||
sends.append([curr_time, curr_port, bytearray.fromhex(curr_data)])
|
||||
|
||||
# Split and merge consecutive sends with the same port, to yield exactly one record per message.
|
||||
# Message records are of the form `length:{...}`, where `length` is an integer in ASCII decimal.
|
||||
# Incomplete messages are deferred until they are complete.
|
||||
# `sends` = [[date, port, record data]], e.g.
|
||||
# `["Mar 18, 2025 21:09:51.879661797 AWST", "6080", b"13"]`
|
||||
# `["Mar 18, 2025 21:09:51.879661797 AWST", "6080", b"9:"]`
|
||||
# `["Mar 18, 2025 21:09:51.879661797 AWST", "6080", b"{..."]`
|
||||
# `["Mar 18, 2025 21:09:51.879661797 AWST", "6080", b"...}"]`
|
||||
records = []
|
||||
scunge = {} # Map from port to incomplete message data
|
||||
for curr_time, curr_port, rest in sends:
|
||||
rest = scunge.pop(curr_port, b"") + rest
|
||||
while rest != b"":
|
||||
try:
|
||||
length, new_rest = rest.split(b":", 1) # Can raise ValueError
|
||||
length = int(length)
|
||||
if len(new_rest) < length:
|
||||
raise ValueError("Incomplete message (for now)")
|
||||
# If we found a `length:` prefix and we have enough data to satisfy it,
|
||||
# cut off the prefix so `rest` is just `{...}length:{...}length:{...}`.
|
||||
rest = new_rest
|
||||
except ValueError:
|
||||
print(f"[WARNING] Incomplete message detected (will try to reassemble): {repr(rest)}", file=sys.stderr)
|
||||
scunge[curr_port] = rest
|
||||
# Wait for more data from later sends, potentially after sends with the other port.
|
||||
break
|
||||
# Cut off the message so `rest` is just `length:{...}length:{...}`.
|
||||
message = rest[:length]
|
||||
rest = rest[length:]
|
||||
try:
|
||||
records.append([curr_time, curr_port, message.decode()])
|
||||
except UnicodeError as e:
|
||||
print(f"[WARNING] Failed to decode message as UTF-8: {e}")
|
||||
continue
|
||||
|
||||
# Process message records.
|
||||
# `records` = [[date, port, message text]], e.g.
|
||||
# `["Mar 18, 2025 21:09:51.879661797 AWST", "6080", "{...}"]`
|
||||
result = []
|
||||
for line in records:
|
||||
if len(line) != 3:
|
||||
continue
|
||||
curr_time, curr_port, text = line
|
||||
# Time
|
||||
curr_time = curr_time.split(" ")[-2].split(".")[0]
|
||||
# Port
|
||||
curr_port = "Servo" if curr_port == port else "Firefox"
|
||||
# Data
|
||||
if not curr_data:
|
||||
continue
|
||||
try:
|
||||
dec = bytearray.fromhex(curr_data).decode()
|
||||
except UnicodeError:
|
||||
continue
|
||||
result.append([curr_time, curr_port, len(result), text])
|
||||
|
||||
indices = [m.span() for m in re.finditer(pattern, dec)]
|
||||
indices.append((len(dec), -1))
|
||||
prev = 0
|
||||
|
||||
for min, max in indices:
|
||||
if min > 0:
|
||||
text = dec[prev - 1:min]
|
||||
data.append([curr_time, curr_port, len(data), text])
|
||||
prev = max
|
||||
|
||||
return data
|
||||
# `result` = [[date, endpoint, index, message text]], e.g.
|
||||
# `["Mar 18, 2025 21:09:51.879661797 AWST", "Servo", 0, "{...}"]`
|
||||
return result
|
||||
|
||||
|
||||
# Pretty prints the json message
|
||||
|
|
|
|||
59
etc/devtools_parser_test.json
Normal file
59
etc/devtools_parser_test.json
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
{"_from": "root", "message": {"applicationType": "browser", "from": "root", "traits": {"customHighlighters": true, "highlightable": true, "networkMonitor": false, "sources": true}}}
|
||||
{"_to": "root", "message": {"frontendVersion": "135.0.1", "to": "root", "type": "connect"}}
|
||||
{"_from": "root", "message": {"from": "root"}}
|
||||
{"_to": "root", "message": {"to": "root", "type": "getRoot"}}
|
||||
{"_from": "root", "message": {"deviceActor": "server1.conn0.device1", "from": "root", "performanceActor": "server1.conn0.performance0", "preferenceActor": "server1.conn0.preference2", "selected": 0}}
|
||||
{"_to": "server1.conn0.device1", "message": {"to": "server1.conn0.device1", "type": "getDescription"}}
|
||||
{"_from": "server1.conn0.device1", "message": {"from": "server1.conn0.device1", "value": {"appbuildid": "20250318210028", "apptype": "servo", "brandName": "Servo", "platformversion": "133.0", "version": "0.0.1"}}}
|
||||
{"_to": "server1.conn0.device1", "message": {"to": "server1.conn0.device1", "type": "getDescription"}}
|
||||
{"_from": "server1.conn0.device1", "message": {"from": "server1.conn0.device1", "value": {"appbuildid": "20250318210028", "apptype": "servo", "brandName": "Servo", "platformversion": "133.0", "version": "0.0.1"}}}
|
||||
{"_to": "server1.conn0.preference2", "message": {"to": "server1.conn0.preference2", "type": "getBoolPref", "value": "devtools.debugger.prompt-connection"}}
|
||||
{"_from": "server1.conn0.preference2", "message": {"from": "server1.conn0.preference2", "value": false}}
|
||||
{"_to": "server1.conn0.preference2", "message": {"to": "server1.conn0.preference2", "type": "getBoolPref", "value": "browser.privatebrowsing.autostart"}}
|
||||
{"_from": "server1.conn0.preference2", "message": {"from": "server1.conn0.preference2", "value": false}}
|
||||
{"_to": "server1.conn0.preference2", "message": {"to": "server1.conn0.preference2", "type": "getBoolPref", "value": "dom.serviceWorkers.enabled"}}
|
||||
{"_from": "server1.conn0.preference2", "message": {"from": "server1.conn0.preference2", "value": false}}
|
||||
{"_to": "root", "message": {"iconDataURL": true, "to": "root", "type": "listAddons"}}
|
||||
{"_to": "root", "message": {"to": "root", "type": "listTabs"}}
|
||||
{"_from": "root", "message": {"addons": [], "from": "root"}}
|
||||
{"_from": "root", "message": {"from": "root", "tabs": [{"actor": "server1.conn0.tab-description11", "browserId": 1, "browsingContextID": 1, "isZombieTab": false, "outerWindowID": 1, "selected": false, "title": "", "traits": {"supportsReloadDescriptor": true, "watcher": true}, "url": "file:///cuffs/code/servo/attic/iframe4.html"}]}}
|
||||
{"_to": "server1.conn0.tab-description11", "message": {"to": "server1.conn0.tab-description11", "type": "getFavicon"}}
|
||||
{"_from": "server1.conn0.tab-description11", "message": {"favicon": "", "from": "server1.conn0.tab-description11"}}
|
||||
{"_to": "root", "message": {"to": "root", "type": "listWorkers"}}
|
||||
{"_to": "root", "message": {"to": "root", "type": "listProcesses"}}
|
||||
{"_to": "root", "message": {"id": 0, "to": "root", "type": "getProcess"}}
|
||||
{"_from": "root", "message": {"from": "root", "workers": []}}
|
||||
{"_from": "root", "message": {"from": "root", "processes": [{"actor": "server1.conn0.process3", "id": 0, "isParent": true, "isWindowlessParent": false, "traits": {"supportsReloadDescriptor": false, "watcher": false}}]}}
|
||||
{"_from": "root", "message": {"from": "root", "processDescriptor": {"actor": "server1.conn0.process3", "id": 0, "isParent": true, "isWindowlessParent": false, "traits": {"supportsReloadDescriptor": false, "watcher": false}}}}
|
||||
{"_to": "root", "message": {"to": "root", "type": "listServiceWorkerRegistrations"}}
|
||||
{"_from": "root", "message": {"from": "root", "registrations": []}}
|
||||
{"_to": "root", "message": {"browserId": 1, "to": "root", "type": "getTab"}}
|
||||
{"_from": "root", "message": {"from": "root", "tab": {"actor": "server1.conn0.tab-description11", "browserId": 1, "browsingContextID": 1, "isZombieTab": false, "outerWindowID": 1, "selected": true, "title": "", "traits": {"supportsReloadDescriptor": true, "watcher": true}, "url": "file:///cuffs/code/servo/attic/iframe4.html"}}}
|
||||
{"_to": "server1.conn0.tab-description11", "message": {"isPopupDebuggingEnabled": false, "isServerTargetSwitchingEnabled": true, "to": "server1.conn0.tab-description11", "type": "getWatcher"}}
|
||||
{"_from": "server1.conn0.tab-description11", "message": {"actor": "server1.conn0.watcher16", "from": "server1.conn0.tab-description11", "traits": {"frame": true, "process": false, "resources": {"Cache": false, "console-message": true, "cookies": false, "css-change": true, "css-message": false, "css-registered-properties": false, "document-event": false, "error-message": true, "extension-storage": false, "indexed-db": false, "jstracer-state": false, "jstracer-trace": false, "last-private-context-exit": false, "local-storage": false, "network-event": false, "network-event-stacktrace": false, "platform-message": false, "reflow": false, "server-sent-event": false, "session-storage": false, "source": true, "stylesheet": false, "thread-state": false, "websocket": false}, "service_worker": false, "shared_worker": false, "worker": false}}}
|
||||
{"_to": "server1.conn0.watcher16", "message": {"targetType": "frame", "to": "server1.conn0.watcher16", "type": "watchTargets"}}
|
||||
{"_from": "server1.conn0.watcher16", "message": {"from": "server1.conn0.watcher16", "target": {"accessibilityActor": "server1.conn0.accessibility6", "actor": "server1.conn0.target5", "browserId": 1, "browsingContextID": 1, "consoleActor": "server1.conn0.console4", "cssPropertiesActor": "server1.conn0.css-properties7", "inspectorActor": "server1.conn0.inspector8", "isTopLevelTarget": true, "outerWindowID": 1, "reflowActor": "server1.conn0.reflow9", "styleSheetsActor": "server1.conn0.stylesheets10", "threadActor": "server1.conn0.thread12", "title": "", "traits": {"frames": true, "isBrowsingContext": true, "logInPage": false, "navigation": true, "supportsTopLevelTargetFlag": true, "watchpoints": true}, "url": "file:///cuffs/code/servo/attic/iframe4.html"}, "type": "target-available-form"}}
|
||||
{"_from": "server1.conn0.target5", "message": {"frames": [{"id": 1, "isTopLevel": true, "title": "", "url": "file:///cuffs/code/servo/attic/iframe4.html"}], "from": "server1.conn0.target5", "type": "frameUpdate"}}
|
||||
{"_from": "server1.conn0.watcher16", "message": {"from": "server1.conn0.watcher16"}}
|
||||
{"_to": "server1.conn0.console4", "message": {"listeners": ["DocumentEvents"], "to": "server1.conn0.console4", "type": "startListeners"}}
|
||||
{"_from": "server1.conn0.console4", "message": {"from": "server1.conn0.console4", "nativeConsoleApi": true, "startedListeners": ["DocumentEvents"], "traits": null}}
|
||||
{"_to": "server1.conn0.watcher16", "message": {"to": "server1.conn0.watcher16", "type": "getTargetConfigurationActor"}}
|
||||
{"_from": "server1.conn0.watcher16", "message": {"configuration": {"actor": "server1.conn0.target-configuration14", "configuration": {}, "traits": {"supportedOptions": {"cacheDisabled": false, "colorSchemeSimulation": false, "customFormatters": false, "customUserAgent": false, "javascriptEnabled": false, "overrideDPPX": false, "printSimulationEnabled": false, "rdmPaneMaxTouchPoints": false, "rdmPaneOrientation": false, "recordAllocations": false, "reloadOnTouchSimulationToggle": false, "restoreFocus": false, "serviceWorkersTestingEnabled": false, "setTabOffline": false, "touchEventsOverride": false, "tracerOptions": false, "useSimpleHighlightersForReducedMotion": false}}}, "from": "server1.conn0.watcher16"}}
|
||||
{"_to": "server1.conn0.target-configuration14", "message": {"configuration": {"cacheDisabled": true, "customFormatters": false, "isTracerFeatureEnabled": false, "serviceWorkersTestingEnabled": false, "useSimpleHighlightersForReducedMotion": false}, "to": "server1.conn0.target-configuration14", "type": "updateConfiguration"}}
|
||||
{"_from": "server1.conn0.target-configuration14", "message": {"from": "server1.conn0.target-configuration14"}}
|
||||
{"_to": "server1.conn0.watcher16", "message": {"to": "server1.conn0.watcher16", "type": "getThreadConfigurationActor"}}
|
||||
{"_from": "server1.conn0.watcher16", "message": {"configuration": {"actor": "server1.conn0.thread-configuration15"}, "from": "server1.conn0.watcher16"}}
|
||||
{"_to": "server1.conn0.thread-configuration15", "message": {"configuration": {"ignoreCaughtExceptions": true, "logEventBreakpoints": false, "observeAsmJS": true, "pauseOnExceptions": false, "pauseOverlay": true, "shouldIncludeAsyncLiveFrames": false, "shouldIncludeSavedFrames": true, "shouldPauseOnDebuggerStatement": true, "skipBreakpoints": false}, "to": "server1.conn0.thread-configuration15", "type": "updateConfiguration"}}
|
||||
{"_from": "server1.conn0.thread-configuration15", "message": {"from": "server1.conn0.thread-configuration15"}}
|
||||
{"_to": "server1.conn0.target5", "message": {"to": "server1.conn0.target5", "type": "listFrames"}}
|
||||
{"_to": "server1.conn0.watcher16", "message": {"resourceTypes": ["console-message"], "to": "server1.conn0.watcher16", "type": "watchResources"}}
|
||||
{"_from": "server1.conn0.target5", "message": {"from": "server1.conn0.target5"}}
|
||||
{"_to": "server1.conn0.watcher16", "message": {"resourceTypes": ["error-message"], "to": "server1.conn0.watcher16", "type": "watchResources"}}
|
||||
{"_from": "server1.conn0.target5", "message": {"array": [["console-message", []]], "from": "server1.conn0.target5", "type": "resources-available-array"}}
|
||||
{"_from": "server1.conn0.watcher16", "message": {"from": "server1.conn0.watcher16"}}
|
||||
{"_from": "server1.conn0.target5", "message": {"array": [["error-message", []]], "from": "server1.conn0.target5", "type": "resources-available-array"}}
|
||||
{"_from": "server1.conn0.watcher16", "message": {"from": "server1.conn0.watcher16"}}
|
||||
{"_to": "server1.conn0.target5", "message": {"to": "server1.conn0.target5", "type": "listWorkers"}}
|
||||
{"_to": "server1.conn0.thread-configuration15", "message": {"configuration": {"shouldPauseOnDebuggerStatement": true}, "to": "server1.conn0.thread-configuration15", "type": "updateConfiguration"}}
|
||||
{"_from": "server1.conn0.thread-configuration15", "message": {"from": "server1.conn0.thread-configuration15"}}
|
||||
{"_to": "server1.conn0.thread-configuration15", "message": {"configuration": {"ignoreCaughtExceptions": false, "pauseOnExceptions": false}, "to": "server1.conn0.thread-configuration15", "type": "updateConfiguration"}}
|
||||
BIN
etc/devtools_parser_test.pcap
Normal file
BIN
etc/devtools_parser_test.pcap
Normal file
Binary file not shown.
|
|
@ -13,6 +13,7 @@ import re
|
|||
import sys
|
||||
import os
|
||||
import os.path as path
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import textwrap
|
||||
|
|
@ -291,6 +292,25 @@ class MachCommands(CommandBase):
|
|||
print("Running WPT tests...")
|
||||
passed = wpt.run_tests() and passed
|
||||
|
||||
print("Running devtools parser tests...")
|
||||
# TODO: Enable these tests on other platforms once mach bootstrap installs tshark(1) for them
|
||||
if platform.system() == "Linux":
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["etc/devtools_parser.py", "--json", "--use", "etc/devtools_parser_test.pcap"],
|
||||
check=True, capture_output=True)
|
||||
expected = open("etc/devtools_parser_test.json", "rb").read()
|
||||
actual = result.stdout
|
||||
assert actual == expected, f"Incorrect output!\nExpected: {repr(expected)}\nActual: {repr(actual)}"
|
||||
print("OK")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"Process failed with exit status {e.returncode}: {e.cmd}", file=sys.stderr)
|
||||
print(f"stdout: {repr(e.stdout)}", file=sys.stderr)
|
||||
print(f"stderr: {repr(e.stderr)}", file=sys.stderr)
|
||||
raise e
|
||||
else:
|
||||
print("SKIP")
|
||||
|
||||
if all or tests:
|
||||
print("Running WebIDL tests...")
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue