Devtools parser: reassemble fragmented messages (#36033)

* Devtools parser: reassemble fragmented messages

Co-authored-by: Aria Edmonds <8436007+ar1a@users.noreply.github.com>
Signed-off-by: Delan Azabani <dazabani@igalia.com>

* Enable devtools parser tests on Linux only for now

Signed-off-by: Delan Azabani <dazabani@igalia.com>

---------

Signed-off-by: Delan Azabani <dazabani@igalia.com>
Co-authored-by: Aria Edmonds <8436007+ar1a@users.noreply.github.com>

etc/devtools_parser.py

@@ -38,8 +38,8 @@
 # ./devtools_parser.py --use cap.pcap --port 1234 --filter watcher --range 10:30
 
 import json
-import re
 import signal
+import sys
 from argparse import ArgumentParser
 from subprocess import Popen, PIPE
 try:
@@ -49,7 +49,6 @@ except ImportError:
         return text
 
 fields = ["frame.time", "tcp.srcport", "tcp.payload"]
-pattern = r"(\d+:){"
 
 
 # Use tshark to capture network traffic and save the result in a
@@ -98,40 +97,84 @@ def read_data(file):
 
 
 # Transform the raw output of wireshark into a more manageable one
-def process_data(out, port):
+def process_data(input, port):
-    out = [line.split("\t") for line in out.split("\n")]
+    # Split the input into lines.
+    # `input` = newline-terminated lines of tab-delimited tshark(1) output
+    lines = [line.split("\t") for line in input.split("\n")]
 
-    # Process fields
+    # Remove empty lines and empty sends, and decode hex to bytes.
-    data = []
+    # `lines` = [[date, port, hex-encoded data]], e.g.
-    for line in out:
+    # `["Mar 18, 2025 21:09:51.879661797 AWST", "6080", "3133"]`
+    # `["Mar 18, 2025 21:09:51.879661797 AWST", "6080", "393a"]`
+    # `["Mar 18, 2025 21:09:51.879661797 AWST", "6080", "7b..."]`
+    sends = []
+    for line in lines:
         if len(line) != 3:
             continue
-
         curr_time, curr_port, curr_data = line
+        if len(curr_data) == 0:
+            continue
+        elif len(curr_data) % 2 == 1:
+            print(f"[WARNING] Extra byte in hex-encoded data: {curr_data[-1]}", file=sys.stderr)
+            curr_data = curr_data[:-1]
+        if len(sends) > 0 and sends[-1][1] == curr_port:
+            sends[-1][2] += bytearray.fromhex(curr_data)
+        else:
+            sends.append([curr_time, curr_port, bytearray.fromhex(curr_data)])
 
+    # Split and merge consecutive sends with the same port, to yield exactly one record per message.
+    # Message records are of the form `length:{...}`, where `length` is an integer in ASCII decimal.
+    # Incomplete messages are deferred until they are complete.
+    # `sends` = [[date, port, record data]], e.g.
+    # `["Mar 18, 2025 21:09:51.879661797 AWST", "6080", b"13"]`
+    # `["Mar 18, 2025 21:09:51.879661797 AWST", "6080", b"9:"]`
+    # `["Mar 18, 2025 21:09:51.879661797 AWST", "6080", b"{..."]`
+    # `["Mar 18, 2025 21:09:51.879661797 AWST", "6080", b"...}"]`
+    records = []
+    scunge = {}  # Map from port to incomplete message data
+    for curr_time, curr_port, rest in sends:
+        rest = scunge.pop(curr_port, b"") + rest
+        while rest != b"":
+            try:
+                length, new_rest = rest.split(b":", 1)  # Can raise ValueError
+                length = int(length)
+                if len(new_rest) < length:
+                    raise ValueError("Incomplete message (for now)")
+                # If we found a `length:` prefix and we have enough data to satisfy it,
+                # cut off the prefix so `rest` is just `{...}length:{...}length:{...}`.
+                rest = new_rest
+            except ValueError:
+                print(f"[WARNING] Incomplete message detected (will try to reassemble): {repr(rest)}", file=sys.stderr)
+                scunge[curr_port] = rest
+                # Wait for more data from later sends, potentially after sends with the other port.
+                break
+            # Cut off the message so `rest` is just `length:{...}length:{...}`.
+            message = rest[:length]
+            rest = rest[length:]
+            try:
+                records.append([curr_time, curr_port, message.decode()])
+            except UnicodeError as e:
+                print(f"[WARNING] Failed to decode message as UTF-8: {e}")
+                continue
+
+    # Process message records.
+    # `records` = [[date, port, message text]], e.g.
+    # `["Mar 18, 2025 21:09:51.879661797 AWST", "6080", "{...}"]`
+    result = []
+    for line in records:
+        if len(line) != 3:
+            continue
+        curr_time, curr_port, text = line
         # Time
         curr_time = curr_time.split(" ")[-2].split(".")[0]
         # Port
         curr_port = "Servo" if curr_port == port else "Firefox"
         # Data
-        if not curr_data:
+        result.append([curr_time, curr_port, len(result), text])
-            continue
-        try:
-            dec = bytearray.fromhex(curr_data).decode()
-        except UnicodeError:
-            continue
 
-        indices = [m.span() for m in re.finditer(pattern, dec)]
+    # `result` = [[date, endpoint, index, message text]], e.g.
-        indices.append((len(dec), -1))
+    # `["Mar 18, 2025 21:09:51.879661797 AWST", "Servo", 0, "{...}"]`
-        prev = 0
+    return result
-
-        for min, max in indices:
-            if min > 0:
-                text = dec[prev - 1:min]
-                data.append([curr_time, curr_port, len(data), text])
-            prev = max
-
-    return data
 
 
 # Pretty prints the json message

etc/devtools_parser_test.json

@@ -0,0 +1,59 @@
+{"_from": "root", "message": {"applicationType": "browser", "from": "root", "traits": {"customHighlighters": true, "highlightable": true, "networkMonitor": false, "sources": true}}}
+{"_to": "root", "message": {"frontendVersion": "135.0.1", "to": "root", "type": "connect"}}
+{"_from": "root", "message": {"from": "root"}}
+{"_to": "root", "message": {"to": "root", "type": "getRoot"}}
+{"_from": "root", "message": {"deviceActor": "server1.conn0.device1", "from": "root", "performanceActor": "server1.conn0.performance0", "preferenceActor": "server1.conn0.preference2", "selected": 0}}
+{"_to": "server1.conn0.device1", "message": {"to": "server1.conn0.device1", "type": "getDescription"}}
+{"_from": "server1.conn0.device1", "message": {"from": "server1.conn0.device1", "value": {"appbuildid": "20250318210028", "apptype": "servo", "brandName": "Servo", "platformversion": "133.0", "version": "0.0.1"}}}
+{"_to": "server1.conn0.device1", "message": {"to": "server1.conn0.device1", "type": "getDescription"}}
+{"_from": "server1.conn0.device1", "message": {"from": "server1.conn0.device1", "value": {"appbuildid": "20250318210028", "apptype": "servo", "brandName": "Servo", "platformversion": "133.0", "version": "0.0.1"}}}
+{"_to": "server1.conn0.preference2", "message": {"to": "server1.conn0.preference2", "type": "getBoolPref", "value": "devtools.debugger.prompt-connection"}}
+{"_from": "server1.conn0.preference2", "message": {"from": "server1.conn0.preference2", "value": false}}
+{"_to": "server1.conn0.preference2", "message": {"to": "server1.conn0.preference2", "type": "getBoolPref", "value": "browser.privatebrowsing.autostart"}}
+{"_from": "server1.conn0.preference2", "message": {"from": "server1.conn0.preference2", "value": false}}
+{"_to": "server1.conn0.preference2", "message": {"to": "server1.conn0.preference2", "type": "getBoolPref", "value": "dom.serviceWorkers.enabled"}}
+{"_from": "server1.conn0.preference2", "message": {"from": "server1.conn0.preference2", "value": false}}
+{"_to": "root", "message": {"iconDataURL": true, "to": "root", "type": "listAddons"}}
+{"_to": "root", "message": {"to": "root", "type": "listTabs"}}
+{"_from": "root", "message": {"addons": [], "from": "root"}}
+{"_from": "root", "message": {"from": "root", "tabs": [{"actor": "server1.conn0.tab-description11", "browserId": 1, "browsingContextID": 1, "isZombieTab": false, "outerWindowID": 1, "selected": false, "title": "", "traits": {"supportsReloadDescriptor": true, "watcher": true}, "url": "file:///cuffs/code/servo/attic/iframe4.html"}]}}
+{"_to": "server1.conn0.tab-description11", "message": {"to": "server1.conn0.tab-description11", "type": "getFavicon"}}
+{"_from": "server1.conn0.tab-description11", "message": {"favicon": "", "from": "server1.conn0.tab-description11"}}
+{"_to": "root", "message": {"to": "root", "type": "listWorkers"}}
+{"_to": "root", "message": {"to": "root", "type": "listProcesses"}}
+{"_to": "root", "message": {"id": 0, "to": "root", "type": "getProcess"}}
+{"_from": "root", "message": {"from": "root", "workers": []}}
+{"_from": "root", "message": {"from": "root", "processes": [{"actor": "server1.conn0.process3", "id": 0, "isParent": true, "isWindowlessParent": false, "traits": {"supportsReloadDescriptor": false, "watcher": false}}]}}
+{"_from": "root", "message": {"from": "root", "processDescriptor": {"actor": "server1.conn0.process3", "id": 0, "isParent": true, "isWindowlessParent": false, "traits": {"supportsReloadDescriptor": false, "watcher": false}}}}
+{"_to": "root", "message": {"to": "root", "type": "listServiceWorkerRegistrations"}}
+{"_from": "root", "message": {"from": "root", "registrations": []}}
+{"_to": "root", "message": {"browserId": 1, "to": "root", "type": "getTab"}}
+{"_from": "root", "message": {"from": "root", "tab": {"actor": "server1.conn0.tab-description11", "browserId": 1, "browsingContextID": 1, "isZombieTab": false, "outerWindowID": 1, "selected": true, "title": "", "traits": {"supportsReloadDescriptor": true, "watcher": true}, "url": "file:///cuffs/code/servo/attic/iframe4.html"}}}
+{"_to": "server1.conn0.tab-description11", "message": {"isPopupDebuggingEnabled": false, "isServerTargetSwitchingEnabled": true, "to": "server1.conn0.tab-description11", "type": "getWatcher"}}
+{"_from": "server1.conn0.tab-description11", "message": {"actor": "server1.conn0.watcher16", "from": "server1.conn0.tab-description11", "traits": {"frame": true, "process": false, "resources": {"Cache": false, "console-message": true, "cookies": false, "css-change": true, "css-message": false, "css-registered-properties": false, "document-event": false, "error-message": true, "extension-storage": false, "indexed-db": false, "jstracer-state": false, "jstracer-trace": false, "last-private-context-exit": false, "local-storage": false, "network-event": false, "network-event-stacktrace": false, "platform-message": false, "reflow": false, "server-sent-event": false, "session-storage": false, "source": true, "stylesheet": false, "thread-state": false, "websocket": false}, "service_worker": false, "shared_worker": false, "worker": false}}}
+{"_to": "server1.conn0.watcher16", "message": {"targetType": "frame", "to": "server1.conn0.watcher16", "type": "watchTargets"}}
+{"_from": "server1.conn0.watcher16", "message": {"from": "server1.conn0.watcher16", "target": {"accessibilityActor": "server1.conn0.accessibility6", "actor": "server1.conn0.target5", "browserId": 1, "browsingContextID": 1, "consoleActor": "server1.conn0.console4", "cssPropertiesActor": "server1.conn0.css-properties7", "inspectorActor": "server1.conn0.inspector8", "isTopLevelTarget": true, "outerWindowID": 1, "reflowActor": "server1.conn0.reflow9", "styleSheetsActor": "server1.conn0.stylesheets10", "threadActor": "server1.conn0.thread12", "title": "", "traits": {"frames": true, "isBrowsingContext": true, "logInPage": false, "navigation": true, "supportsTopLevelTargetFlag": true, "watchpoints": true}, "url": "file:///cuffs/code/servo/attic/iframe4.html"}, "type": "target-available-form"}}
+{"_from": "server1.conn0.target5", "message": {"frames": [{"id": 1, "isTopLevel": true, "title": "", "url": "file:///cuffs/code/servo/attic/iframe4.html"}], "from": "server1.conn0.target5", "type": "frameUpdate"}}
+{"_from": "server1.conn0.watcher16", "message": {"from": "server1.conn0.watcher16"}}
+{"_to": "server1.conn0.console4", "message": {"listeners": ["DocumentEvents"], "to": "server1.conn0.console4", "type": "startListeners"}}
+{"_from": "server1.conn0.console4", "message": {"from": "server1.conn0.console4", "nativeConsoleApi": true, "startedListeners": ["DocumentEvents"], "traits": null}}
+{"_to": "server1.conn0.watcher16", "message": {"to": "server1.conn0.watcher16", "type": "getTargetConfigurationActor"}}
+{"_from": "server1.conn0.watcher16", "message": {"configuration": {"actor": "server1.conn0.target-configuration14", "configuration": {}, "traits": {"supportedOptions": {"cacheDisabled": false, "colorSchemeSimulation": false, "customFormatters": false, "customUserAgent": false, "javascriptEnabled": false, "overrideDPPX": false, "printSimulationEnabled": false, "rdmPaneMaxTouchPoints": false, "rdmPaneOrientation": false, "recordAllocations": false, "reloadOnTouchSimulationToggle": false, "restoreFocus": false, "serviceWorkersTestingEnabled": false, "setTabOffline": false, "touchEventsOverride": false, "tracerOptions": false, "useSimpleHighlightersForReducedMotion": false}}}, "from": "server1.conn0.watcher16"}}
+{"_to": "server1.conn0.target-configuration14", "message": {"configuration": {"cacheDisabled": true, "customFormatters": false, "isTracerFeatureEnabled": false, "serviceWorkersTestingEnabled": false, "useSimpleHighlightersForReducedMotion": false}, "to": "server1.conn0.target-configuration14", "type": "updateConfiguration"}}
+{"_from": "server1.conn0.target-configuration14", "message": {"from": "server1.conn0.target-configuration14"}}
+{"_to": "server1.conn0.watcher16", "message": {"to": "server1.conn0.watcher16", "type": "getThreadConfigurationActor"}}
+{"_from": "server1.conn0.watcher16", "message": {"configuration": {"actor": "server1.conn0.thread-configuration15"}, "from": "server1.conn0.watcher16"}}
+{"_to": "server1.conn0.thread-configuration15", "message": {"configuration": {"ignoreCaughtExceptions": true, "logEventBreakpoints": false, "observeAsmJS": true, "pauseOnExceptions": false, "pauseOverlay": true, "shouldIncludeAsyncLiveFrames": false, "shouldIncludeSavedFrames": true, "shouldPauseOnDebuggerStatement": true, "skipBreakpoints": false}, "to": "server1.conn0.thread-configuration15", "type": "updateConfiguration"}}
+{"_from": "server1.conn0.thread-configuration15", "message": {"from": "server1.conn0.thread-configuration15"}}
+{"_to": "server1.conn0.target5", "message": {"to": "server1.conn0.target5", "type": "listFrames"}}
+{"_to": "server1.conn0.watcher16", "message": {"resourceTypes": ["console-message"], "to": "server1.conn0.watcher16", "type": "watchResources"}}
+{"_from": "server1.conn0.target5", "message": {"from": "server1.conn0.target5"}}
+{"_to": "server1.conn0.watcher16", "message": {"resourceTypes": ["error-message"], "to": "server1.conn0.watcher16", "type": "watchResources"}}
+{"_from": "server1.conn0.target5", "message": {"array": [["console-message", []]], "from": "server1.conn0.target5", "type": "resources-available-array"}}
+{"_from": "server1.conn0.watcher16", "message": {"from": "server1.conn0.watcher16"}}
+{"_from": "server1.conn0.target5", "message": {"array": [["error-message", []]], "from": "server1.conn0.target5", "type": "resources-available-array"}}
+{"_from": "server1.conn0.watcher16", "message": {"from": "server1.conn0.watcher16"}}
+{"_to": "server1.conn0.target5", "message": {"to": "server1.conn0.target5", "type": "listWorkers"}}
+{"_to": "server1.conn0.thread-configuration15", "message": {"configuration": {"shouldPauseOnDebuggerStatement": true}, "to": "server1.conn0.thread-configuration15", "type": "updateConfiguration"}}
+{"_from": "server1.conn0.thread-configuration15", "message": {"from": "server1.conn0.thread-configuration15"}}
+{"_to": "server1.conn0.thread-configuration15", "message": {"configuration": {"ignoreCaughtExceptions": false, "pauseOnExceptions": false}, "to": "server1.conn0.thread-configuration15", "type": "updateConfiguration"}}

python/servo/testing_commands.py

@@ -13,6 +13,7 @@ import re
 import sys
 import os
 import os.path as path
+import platform
 import shutil
 import subprocess
 import textwrap
@@ -291,6 +292,25 @@ class MachCommands(CommandBase):
         print("Running WPT tests...")
         passed = wpt.run_tests() and passed
 
+        print("Running devtools parser tests...")
+        # TODO: Enable these tests on other platforms once mach bootstrap installs tshark(1) for them
+        if platform.system() == "Linux":
+            try:
+                result = subprocess.run(
+                    ["etc/devtools_parser.py", "--json", "--use", "etc/devtools_parser_test.pcap"],
+                    check=True, capture_output=True)
+                expected = open("etc/devtools_parser_test.json", "rb").read()
+                actual = result.stdout
+                assert actual == expected, f"Incorrect output!\nExpected: {repr(expected)}\nActual:   {repr(actual)}"
+                print("OK")
+            except subprocess.CalledProcessError as e:
+                print(f"Process failed with exit status {e.returncode}: {e.cmd}", file=sys.stderr)
+                print(f"stdout: {repr(e.stdout)}", file=sys.stderr)
+                print(f"stderr: {repr(e.stderr)}", file=sys.stderr)
+                raise e
+        else:
+            print("SKIP")
+
         if all or tests:
             print("Running WebIDL tests...")