diff --git a/.github/scripts/README.md b/.github/scripts/README.md new file mode 100644 index 000000000..729d0a046 --- /dev/null +++ b/.github/scripts/README.md @@ -0,0 +1,121 @@ +# Content & Feedback widget UI test runner + +Automated UI test for the demo app's content overlay (sticky / modal / +half-modal / fullscreen) and feedback widgets (NPS / rating / survey). +Records a video per variant, drives the app via `adb` + UIAutomator, and +writes a per-variant verdict + summary. + +## What it does + +For each content variant in `content_test_config.CONTENT_VARIANTS`: + +1. Starts `adb screenrecord` in the background. +2. Force-stops the demo and launches `ActivityExampleContentZone`. +3. Sets the device ID to `__` (e.g. `modal_a3f9_03`) so + the server returns the matching content type. Adjust prefixes in the + config file if your server routing changes. +4. Taps **Change Device ID**, then **Enter Content Zone**. +5. Waits for `[ContentOverlayView] page loaded successfully` in logcat + (max `TIMEOUTS["content_load"]` seconds). +6. Rotates landscape → presses back → rotates portrait. +7. Navigates through `POKE_ACTIVITIES` (CustomEvents, ViewTracking, + UserDetails by default), tapping clickable buttons matched by + `button_text_hints` to record events. +8. Tap-passthrough probes (skipped on `fullscreen`): + - taps a coordinate outside content bounds, asserts background activity + received an event; + - taps inside, asserts no background event was emitted. +9. Best-effort WebView interactions: looks for "Go" / "X" nodes in the + accessibility dump, taps them, asserts Chrome opened / overlay closed. +10. Counts FATAL EXCEPTIONS and `IncorrectContextUseViolation` lines in the + full test logcat. +11. Stops `screenrecord`, pulls the MP4, writes `verdict.json`. + +Feedback widgets follow a similar but shorter flow — operations inside +the WebView are intentionally minimal until you specify them. + +## Requirements + +- `adb` on `PATH`. +- A connected emulator or device with the demo app installable + (`./gradlew :app:installDebug` first). +- Python 3.9+ (stdlib only, no `pip install`). + +## Run + +```sh +# All variants + all feedback widgets +python3 .github/scripts/content_test_runner.py + +# A subset +python3 .github/scripts/content_test_runner.py --only modal,sticky_up,nps + +# Skip a category +python3 .github/scripts/content_test_runner.py --no-feedback + +# Target a specific device +python3 .github/scripts/content_test_runner.py --device emulator-5556 +``` + +## Output + +Artifacts land under `.github/scripts/test_output/_/`: + +``` +2026-05-02_18-30-00_a3f9/ +├── summary.md +├── content_modal/ +│ ├── recording.mp4 +│ ├── logcat.txt +│ └── verdict.json +├── content_sticky_up/ +│ └── ... +├── feedback_nps/ +│ └── ... +└── ... +``` + +Open `summary.md` for a per-test PASS/FAIL/SKIP table. For a failing +variant, watch the corresponding `recording.mp4` and inspect +`logcat.txt`. + +## Tuning + +Edit `.github/scripts/content_test_config.py` to: + +- **Add new variants**: append to `CONTENT_VARIANTS`. Mark fullscreen-like + variants in `FULLSCREEN_VARIANTS` to skip passthrough probes. +- **Change log assertions**: every PASS/FAIL hinges on a regex in + `LOG_PATTERNS`. If the SDK renames a log message, update that one entry. +- **Adjust timeouts**: `TIMEOUTS` controls per-step waits. Increase if + running on a slow device or VM. +- **Change "poke" inventory**: `POKE_ACTIVITIES` lists which demo + activities the runner navigates to during a content session and which + button text hints it taps. Add or remove entries as the demo grows. + +## Limitations and known gaps + +- **WebView interactions are best-effort.** UIAutomator can see WebView + accessibility nodes on most modern Android versions, but some widgets + may not expose their X / Go elements. Those tests fall back to SKIP + rather than FAIL. +- **`adb screenrecord` is capped at 180 seconds per file.** Each test is + designed to fit comfortably under that. If you add long pauses, expect + the recording to truncate. +- **Tap-passthrough is heuristic.** "Background activity registered tap" + is inferred from any new event-record log line during the probe window — + it can produce WARN if the host activity at that screen position + doesn't have a tappable element. Place a known button at the probed + coordinates if you need a stronger assertion. +- **Tests are sequential, not parallel.** Running on multiple devices at + once would need a refactor of `_DEVICE_SERIAL` from a module-global to + per-test argument. + +## Adding new feedback operations + +When you want the runner to do more inside a feedback widget (fill +fields, submit answers, etc.), edit `run_feedback_test` in +`content_test_runner.py`. The accessibility dump (`dump_ui()`) and +`find_nodes_by_text_contains` should let you locate most form +controls. Open a per-widget code branch keyed on `feedback_type` if the +operations differ between NPS / rating / survey. diff --git a/.github/scripts/cdp_client.py b/.github/scripts/cdp_client.py new file mode 100644 index 000000000..7a955d251 --- /dev/null +++ b/.github/scripts/cdp_client.py @@ -0,0 +1,511 @@ +"""Minimal Chrome DevTools Protocol (CDP) client for the Countly content/feedback +widget WebView. + +Why this exists +--------------- +UIAutomator only sees the WebView's accessibility-tree projection of the DOM, +which strips HTML class names, IDs, `data-*` attributes, and most CSS state. +That makes locating buttons like `
` (no text, icon +font glyph) impossible through accessibility alone. + +The Chrome WebView ships with the DevTools Protocol enabled per process. When +the demo app calls `WebView.setWebContentsDebuggingEnabled(true)`, each +WebView's process exposes a debuggable Unix domain socket +`webview_devtools_remote_` in the abstract namespace (the leading `@`). +We use `adb forward tcp: localabstract:` to bridge that to +localhost, then speak HTTP+WebSocket to it the same way Chrome's +`chrome://inspect` page does. + +Stdlib only — no `websockets` / `websocket-client` dependency. The WebSocket +framing is a hand-rolled RFC 6455 client that supports the subset we need: +text frames, masking from client, single-shot send/recv (no fragmentation), +Sec-WebSocket-Key handshake. ~150 lines. + +What you can do with it +----------------------- +- Find any DOM element by CSS selector, including class/ID/aria selectors that + UIAutomator can't see. +- Trigger clicks via `.click()` instead of synthesising taps at coordinates — + fires the actual DOM click event, no pixel math, no DPI translation. +- Read live DOM state: button text, href values, computed bounds. + +Example +------- + cdp = CDP.connect_to_demo() + state = cdp.run_js(\"\"\" + JSON.stringify({ + close: !!document.querySelector('.close-button'), + buttons: Array.from(document.querySelectorAll('button')).map(b => ({ + text: b.innerText.trim(), href: b.getAttribute('data-href') || null, + })), + }) + \"\"\") + cdp.click('.close-button') +""" + +import base64 +import hashlib +import json +import secrets +import socket +import ssl +import struct +import subprocess +import time +import urllib.request +from typing import Optional +from urllib.parse import urlparse + + +CDP_LOCAL_PORT = 9222 + + +# --------------------------------------------------------------------------- +# adb plumbing — find the WebView socket and bridge it to localhost +# --------------------------------------------------------------------------- + +def _adb(args: list[str], device: Optional[str] = None, + timeout: float = 10) -> subprocess.CompletedProcess: + cmd = ["adb"] + if device: + cmd += ["-s", device] + cmd += args + return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + + +def find_webview_socket(package: str, device: Optional[str] = None) -> Optional[str]: + """Returns the abstract-namespace socket name (without the leading '@') for + the WebView devtools server in the named package's process, or None if + no debuggable WebView is currently running. + + Looks at /proc/net/unix for entries like + 000... 00010000 0001 01 12345 @webview_devtools_remote_12345 + + The PID at the end of the name corresponds to the process hosting the + WebView (usually the renderer subprocess on modern Android). We just want + a socket whose name starts with `webview_devtools_remote_` — the PID is + auto-routed to whichever WebView is currently alive. + """ + proc = _adb(["shell", "cat /proc/net/unix"], device=device) + if proc.returncode != 0: + return None + for line in proc.stdout.splitlines(): + # Last whitespace-separated token is the path. Abstract sockets begin + # with '@'; we want the literal name without it. + parts = line.split() + if not parts: + continue + path = parts[-1] + if path.startswith("@webview_devtools_remote"): + return path[1:] # strip leading @ + return None + + +def setup_forward(socket_name: str, local_port: int = CDP_LOCAL_PORT, + device: Optional[str] = None) -> bool: + """Sets up `adb forward tcp: localabstract:`. + Returns True on success.""" + proc = _adb( + ["forward", f"tcp:{local_port}", f"localabstract:{socket_name}"], + device=device, + ) + return proc.returncode == 0 + + +def remove_forward(local_port: int = CDP_LOCAL_PORT, + device: Optional[str] = None) -> None: + _adb(["forward", "--remove", f"tcp:{local_port}"], device=device) + + +# --------------------------------------------------------------------------- +# CDP page enumeration over HTTP +# --------------------------------------------------------------------------- + +def list_pages(local_port: int = CDP_LOCAL_PORT) -> list[dict]: + """GET http://localhost:/json — returns the array of debuggable + pages. Each entry has at least `id`, `title`, `url`, `webSocketDebuggerUrl`. + """ + url = f"http://localhost:{local_port}/json" + with urllib.request.urlopen(url, timeout=5) as r: + return json.loads(r.read()) + + +# --------------------------------------------------------------------------- +# Minimal WebSocket client (RFC 6455 text frames, client-side masking) +# --------------------------------------------------------------------------- + +# Per RFC 6455 §1.3, the server's Sec-WebSocket-Accept must equal +# base64(sha1(client_key + GUID)) +_WS_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + + +class _WSError(RuntimeError): + pass + + +class WSClient: + """Synchronous WebSocket client implementing the subset CDP needs: + text frames, no fragmentation, masked-from-client. Roughly 80 lines. + + Usage: + ws = WSClient.connect("ws://localhost:9222/devtools/page/") + ws.send('{"id":1,"method":"Runtime.evaluate","params":{...}}') + reply = ws.recv() + ws.close() + """ + + def __init__(self, sock: socket.socket): + self._sock = sock + self._recv_buf = b"" + + @classmethod + def connect(cls, ws_url: str, timeout: float = 5.0) -> "WSClient": + u = urlparse(ws_url) + if u.scheme not in ("ws", "wss"): + raise _WSError(f"not a ws:// URL: {ws_url}") + host = u.hostname or "localhost" + port = u.port or (443 if u.scheme == "wss" else 80) + path = u.path or "/" + if u.query: + path = f"{path}?{u.query}" + + sock = socket.create_connection((host, port), timeout=timeout) + if u.scheme == "wss": + sock = ssl.create_default_context().wrap_socket(sock, server_hostname=host) + + # Random 16-byte key, base64-encoded, sent as Sec-WebSocket-Key. + client_key = base64.b64encode(secrets.token_bytes(16)).decode("ascii") + request = ( + f"GET {path} HTTP/1.1\r\n" + f"Host: {host}:{port}\r\n" + f"Upgrade: websocket\r\n" + f"Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {client_key}\r\n" + f"Sec-WebSocket-Version: 13\r\n" + f"\r\n" + ) + sock.sendall(request.encode("ascii")) + + # Read until end-of-headers. + buf = b"" + while b"\r\n\r\n" not in buf: + chunk = sock.recv(4096) + if not chunk: + raise _WSError("server closed before handshake completed") + buf += chunk + head, _, leftover = buf.partition(b"\r\n\r\n") + head_text = head.decode("latin-1") + if " 101 " not in head_text.split("\r\n", 1)[0]: + raise _WSError(f"non-101 status: {head_text.splitlines()[0]}") + + # Verify Sec-WebSocket-Accept. + expected = base64.b64encode( + hashlib.sha1((client_key + _WS_GUID).encode("ascii")).digest() + ).decode("ascii") + for line in head_text.split("\r\n"): + if line.lower().startswith("sec-websocket-accept:"): + got = line.split(":", 1)[1].strip() + if got != expected: + raise _WSError(f"bad Sec-WebSocket-Accept: {got!r} != {expected!r}") + break + else: + raise _WSError("no Sec-WebSocket-Accept header in response") + + client = cls(sock) + client._recv_buf = leftover # any post-header bytes belong to the WS stream + return client + + def send(self, text: str) -> None: + """Send a single text frame, fin=1, masked (clients MUST mask).""" + payload = text.encode("utf-8") + header = bytearray() + header.append(0x81) # fin=1, opcode=1 (text) + ln = len(payload) + if ln < 126: + header.append(0x80 | ln) + elif ln < 65536: + header.append(0x80 | 126) + header += struct.pack(">H", ln) + else: + header.append(0x80 | 127) + header += struct.pack(">Q", ln) + mask = secrets.token_bytes(4) + header += mask + masked = bytearray(payload) + for i in range(len(masked)): + masked[i] ^= mask[i % 4] + self._sock.sendall(bytes(header) + bytes(masked)) + + def _read_exact(self, n: int) -> bytes: + data = bytearray(self._recv_buf[:n]) + self._recv_buf = self._recv_buf[n:] + while len(data) < n: + chunk = self._sock.recv(n - len(data)) + if not chunk: + raise _WSError("connection closed mid-frame") + data += chunk + return bytes(data) + + def recv(self, timeout: float = 10.0) -> str: + """Read one frame (assumes fin=1, no fragmentation, server frames are + unmasked per RFC 6455 §5.3). Returns the decoded text payload.""" + self._sock.settimeout(timeout) + b1, b2 = self._read_exact(2) + opcode = b1 & 0x0F + if opcode == 0x8: # close + raise _WSError("server sent close frame") + if opcode == 0x9: # ping → pong; not expected from CDP, but tolerate + payload_len = b2 & 0x7F + payload = self._read_exact(payload_len) if payload_len else b"" + self._sock.sendall(b"\x8a" + bytes([0x80 | payload_len]) + + secrets.token_bytes(4) + payload) + return self.recv(timeout) + if opcode != 0x1: + raise _WSError(f"unexpected opcode 0x{opcode:x}") + masked = bool(b2 & 0x80) + payload_len = b2 & 0x7F + if payload_len == 126: + payload_len = struct.unpack(">H", self._read_exact(2))[0] + elif payload_len == 127: + payload_len = struct.unpack(">Q", self._read_exact(8))[0] + if masked: + mask = self._read_exact(4) + payload = self._read_exact(payload_len) + if masked: + payload = bytes(b ^ mask[i % 4] for i, b in enumerate(payload)) + return payload.decode("utf-8") + + def close(self) -> None: + try: + # Send close frame (opcode 8, empty payload, masked). + self._sock.sendall(b"\x88\x80" + secrets.token_bytes(4)) + except Exception: + pass + try: + self._sock.close() + except Exception: + pass + + +# --------------------------------------------------------------------------- +# CDP wrapper — request/response with monotonically-increasing message IDs +# --------------------------------------------------------------------------- + +class CDPError(RuntimeError): + pass + + +class CDP: + """Thin wrapper around a CDP WebSocket: send-and-await-matching-id. + + CDP is bidirectional (server can send unsolicited events for things like + `Page.frameNavigated`), so we have to handle the case where `recv` returns + an event message before our reply. We just discard everything that doesn't + match our outgoing id. + """ + + def __init__(self, ws: WSClient): + self._ws = ws + self._next_id = 1 + + @classmethod + def connect_to_demo(cls, package: str = "ly.count.android.demo", + device: Optional[str] = None, + title_substring: Optional[str] = None, + local_port: int = CDP_LOCAL_PORT, + retries: int = 3) -> Optional["CDP"]: + """Locate the demo's WebView, set up adb forward, list pages, pick the + relevant one, and return a connected CDP. Returns None if no debuggable + WebView is found (e.g., the widget hasn't loaded yet, or + `setWebContentsDebuggingEnabled(true)` isn't on). + """ + for _ in range(retries): + sock_name = find_webview_socket(package, device=device) + if sock_name: + break + time.sleep(0.4) + else: + return None + if not setup_forward(sock_name, local_port=local_port, device=device): + return None + try: + pages = list_pages(local_port=local_port) + except Exception: + return None + # If a title hint is provided, prefer it; otherwise take the first + # page that's not an extension/devtools page. + chosen = None + for p in pages: + if p.get("type") not in ("page", None): + continue + if title_substring and title_substring.lower() not in p.get("title", "").lower(): + continue + chosen = p + break + if chosen is None and pages: + chosen = pages[0] + if not chosen: + return None + ws_url = chosen.get("webSocketDebuggerUrl") + if not ws_url: + return None + ws = WSClient.connect(ws_url) + return cls(ws) + + def _send(self, method: str, params: Optional[dict] = None) -> dict: + """Send one CDP command, return the matching response object.""" + msg_id = self._next_id + self._next_id += 1 + msg = {"id": msg_id, "method": method} + if params: + msg["params"] = params + self._ws.send(json.dumps(msg)) + # Drain events until we get our response. + while True: + reply = json.loads(self._ws.recv()) + if reply.get("id") == msg_id: + if "error" in reply: + err = reply["error"] + raise CDPError(f"{method}: {err.get('message')} ({err.get('code')})") + return reply.get("result", {}) + # else: an event/other-id reply; ignore. + + def run_js(self, expression: str) -> object: + """Run a JS expression in the page's main frame. Returns the JS value + marshaled by Runtime.evaluate. Use `JSON.stringify(...)` in the + expression and json.loads on the result for structured data. + + Despite the name, this only runs JS in a *remote* browser — the + Python interpreter never sees the expression as code, only sends it + as a string to the WebView's V8. + """ + result = self._send("Runtime.evaluate", { + "expression": expression, + "returnByValue": True, + "awaitPromise": False, + # Treat the eval as initiated by a user gesture. Some browser + # behaviors (popup blockers, target="_blank" navigation, certain + # event handlers) only fire when the engine believes the user + # initiated the action — without this, programmatic `.click()` + # on an `` after the first call can be + # silently suppressed. + "userGesture": True, + }) + if "exceptionDetails" in result: + raise CDPError(f"JS exception: {result['exceptionDetails'].get('text')}") + ret = result.get("result", {}) + return ret.get("value") + + def click(self, css_selector: str) -> bool: + """Trigger a click on the first matching element. Returns True if an + element was found and clicked, False otherwise.""" + js = ( + "(() => {" + f" const el = document.querySelector({json.dumps(css_selector)});" + " if (!el) return false;" + " el.click();" + " return true;" + "})()" + ) + return bool(self.run_js(js)) + + def set_value(self, css_selector: str, value: str) -> bool: + """Set the `value` of an `` or `