ci: drive gate exercises mouse/keyboard/iframe/canvas, not just navigate+eval
An adversarial audit of the pipeline found the drive gate only did goto+evaluate, so several historically-shipped breakages would still pass it green: - firefox-2 (jugglerSendMouseEvent missing) — no mouse/keyboard was tested - issue #20 (cross-origin iframe content_frame() None) — no iframe was tested - canvas non-determinism (stealth seed) and headless navigator tells ci_drive_gate.py now clicks a button, moves the mouse, types into an input, reaches into an iframe, checks an identical canvas draw is byte-stable, and checks navigator.languages/plugins — all offline (data: URLs), GPU-free, no proxy. Validated against the real build. Pipeline hardening from the same audit: - Windows: stop swallowing `mach package` failure and never fall back to the dev tree dist/bin (that masked the firefox-7/8 packaging bugs) - macOS: plutil -lint Info.plist + required-key checks (a malformed plist ships fine through a headless drive but Finder calls the .app "damaged") - publish: assert all 5 archives present + fail_on_unmatched_files (no silent partial release if a build leg drops out)
This commit is contained in:
@@ -181,19 +181,19 @@ jobs:
|
|||||||
if: matrix.family == 'win'
|
if: matrix.family == 'win'
|
||||||
run: |
|
run: |
|
||||||
set -e
|
set -e
|
||||||
./mach package || echo "mach package rc=$? (continuing to locate the app tree)"
|
# Do NOT swallow a mach failure: `./mach package || echo` lets set -e pass
|
||||||
# Prefer the clean packaged tree; fall back to dist/bin if cross didn't produce dist/firefox
|
# and would fall through to a stale tree. A release MUST come from the clean
|
||||||
if [ -f obj-rel/dist/firefox/firefox.exe ]; then WIN_APP=obj-rel/dist/firefox
|
# dist/firefox; dist/bin is the dev tree (cruft + loose juggler that masked
|
||||||
elif [ -f obj-rel/dist/bin/firefox.exe ]; then WIN_APP=obj-rel/dist/bin
|
# the firefox-7/8 packaging bugs), never acceptable for a release.
|
||||||
else echo "ERROR: firefox.exe not found in dist/firefox nor dist/bin"; exit 1; fi
|
./mach package
|
||||||
|
[ -f obj-rel/dist/firefox/firefox.exe ] \
|
||||||
|
|| { echo "ERROR: mach package did not produce a clean dist/firefox tree"; exit 1; }
|
||||||
|
WIN_APP=obj-rel/dist/firefox
|
||||||
echo "packaging from: $WIN_APP"
|
echo "packaging from: $WIN_APP"
|
||||||
# JUGGLER GATE: omni.ja must carry juggler (dist/firefox) or loose chrome/ (dist/bin fallback)
|
# JUGGLER GATE: omni.ja must carry juggler (else Playwright can't drive it)
|
||||||
if [ -f "$WIN_APP/omni.ja" ]; then
|
[ -f "$WIN_APP/omni.ja" ] || { echo "ERROR: no omni.ja in $WIN_APP"; exit 1; }
|
||||||
python3 -c "import zipfile,sys; sys.exit(0 if any('juggler' in n.lower() for n in zipfile.ZipFile('$WIN_APP/omni.ja').namelist()) else 1)" \
|
python3 -c "import zipfile,sys; sys.exit(0 if any('juggler' in n.lower() for n in zipfile.ZipFile('$WIN_APP/omni.ja').namelist()) else 1)" \
|
||||||
|| { echo "ERROR: juggler missing from $WIN_APP/omni.ja — Playwright can't drive it"; exit 1; }
|
|| { echo "ERROR: juggler missing from $WIN_APP/omni.ja — Playwright can't drive it"; exit 1; }
|
||||||
elif [ ! -d "$WIN_APP/chrome/juggler" ]; then
|
|
||||||
echo "ERROR: juggler missing from $WIN_APP (no omni.ja juggler, no loose chrome/juggler)"; exit 1
|
|
||||||
fi
|
|
||||||
echo "juggler GATE OK (win)"
|
echo "juggler GATE OK (win)"
|
||||||
mkdir -p out
|
mkdir -p out
|
||||||
( cd "$WIN_APP" && zip -qr "$GITHUB_WORKSPACE/out/${{ matrix.asset }}" . ) # firefox.exe at zip ROOT
|
( cd "$WIN_APP" && zip -qr "$GITHUB_WORKSPACE/out/${{ matrix.asset }}" . ) # firefox.exe at zip ROOT
|
||||||
@@ -220,6 +220,15 @@ jobs:
|
|||||||
for need in "Contents/MacOS/firefox" "Contents/Info.plist"; do
|
for need in "Contents/MacOS/firefox" "Contents/Info.plist"; do
|
||||||
[ -e "$APP/$need" ] || { echo "ERROR: missing $need"; exit 1; }
|
[ -e "$APP/$need" ] || { echo "ERROR: missing $need"; exit 1; }
|
||||||
done
|
done
|
||||||
|
echo "=== Info.plist well-formed + required keys (a malformed plist → Finder 'damaged') ==="
|
||||||
|
plutil -lint "$APP/Contents/Info.plist"
|
||||||
|
for key in CFBundleExecutable CFBundleIdentifier CFBundleShortVersionString; do
|
||||||
|
plutil -extract "$key" raw -o - "$APP/Contents/Info.plist" >/dev/null \
|
||||||
|
|| { echo "ERROR: Info.plist missing $key"; exit 1; }
|
||||||
|
done
|
||||||
|
EXEC="$(plutil -extract CFBundleExecutable raw -o - "$APP/Contents/Info.plist")"
|
||||||
|
[ -e "$APP/Contents/MacOS/$EXEC" ] \
|
||||||
|
|| { echo "ERROR: CFBundleExecutable '$EXEC' has no matching binary in Contents/MacOS"; exit 1; }
|
||||||
echo "=== verify NO absolute symlinks in the .app (relative-internal ones are fine) ==="
|
echo "=== verify NO absolute symlinks in the .app (relative-internal ones are fine) ==="
|
||||||
BAD="$(find "$APP" -type l -print0 | xargs -0 -I{} sh -c 't=$(readlink "{}"); case "$t" in /*) echo "{} -> $t";; esac')"
|
BAD="$(find "$APP" -type l -print0 | xargs -0 -I{} sh -c 't=$(readlink "{}"); case "$t" in /*) echo "{} -> $t";; esac')"
|
||||||
[ -z "$BAD" ] || { echo "ERROR: absolute symlinks in .app (break on user machines):"; echo "$BAD" | head -5; exit 1; }
|
[ -z "$BAD" ] || { echo "ERROR: absolute symlinks in .app (break on user machines):"; echo "$BAD" | head -5; exit 1; }
|
||||||
@@ -324,6 +333,20 @@ jobs:
|
|||||||
- name: Download all build assets
|
- name: Download all build assets
|
||||||
uses: actions/download-artifact@v4
|
uses: actions/download-artifact@v4
|
||||||
with: { pattern: asset-*, path: dl, merge-multiple: true }
|
with: { pattern: asset-*, path: dl, merge-multiple: true }
|
||||||
|
- name: Assert all 5 target archives present (no silent partial release)
|
||||||
|
run: |
|
||||||
|
cd dl
|
||||||
|
EXPECTED="
|
||||||
|
firefox-150.0.1-stealth-linux-x86_64.tar.gz
|
||||||
|
firefox-150.0.1-stealth-linux-arm64.tar.gz
|
||||||
|
firefox-150.0.1-stealth-win-x86_64.zip
|
||||||
|
firefox-150.0.1-stealth-macos-arm64.tar.gz
|
||||||
|
firefox-150.0.1-stealth-macos-x86_64.tar.gz
|
||||||
|
"
|
||||||
|
for a in $EXPECTED; do
|
||||||
|
[ -s "$a" ] || { echo "ERROR: missing/empty release asset: $a (a build leg silently dropped out?)"; exit 1; }
|
||||||
|
done
|
||||||
|
echo "all 5 target archives present"
|
||||||
- name: Generate checksums.txt
|
- name: Generate checksums.txt
|
||||||
run: |
|
run: |
|
||||||
cd dl; ls -la
|
cd dl; ls -la
|
||||||
@@ -344,6 +367,7 @@ jobs:
|
|||||||
name: invisible_firefox (150.0.1) rev ${{ steps.tag.outputs.tag }}
|
name: invisible_firefox (150.0.1) rev ${{ steps.tag.outputs.tag }}
|
||||||
draft: true
|
draft: true
|
||||||
prerelease: false
|
prerelease: false
|
||||||
|
fail_on_unmatched_files: true
|
||||||
files: |
|
files: |
|
||||||
dl/*.tar.gz
|
dl/*.tar.gz
|
||||||
dl/*.zip
|
dl/*.zip
|
||||||
|
|||||||
+79
-11
@@ -1,15 +1,26 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""CI drive gate — the firefox-8 catcher.
|
"""CI drive gate — the firefox-N catcher.
|
||||||
|
|
||||||
A raw `firefox --screenshot` proves nothing about automation: a juggler-less
|
A raw `firefox --screenshot` proves nothing about automation: a juggler-less
|
||||||
binary renders a screenshot just fine and ships broken (firefox-8 did exactly
|
binary renders a screenshot just fine and ships broken (firefox-8 did exactly
|
||||||
that). This DRIVES the binary the way users will — Playwright launches it over
|
that). This DRIVES the binary the way users will — Playwright launches it over
|
||||||
the juggler pipe, loads a real page, and round-trips JS. A binary with a
|
the juggler pipe and exercises the input/DOM paths real callers depend on.
|
||||||
missing/broken juggler throws TargetClosedError here and the gate fails.
|
|
||||||
|
|
||||||
Headless, NO screenshot → GPU-free, so it can't false-fail on GPU-less hosted
|
It deliberately covers the failure modes that HISTORICALLY shipped green:
|
||||||
runners. Zero proxy / zero secrets → safe in public CI. (The proxy realness
|
- juggler missing entirely → TargetClosedError on launch (firefox-8)
|
||||||
gate — fppro/webrtc — stays local, it needs secrets.)
|
- mouse/keyboard input broken → click/move/type assertions (firefox-2 #9:
|
||||||
|
jugglerSendMouseEvent / synthesizeMouseEvent)
|
||||||
|
- cross-origin iframe broken → content_frame() reachable (issue #20)
|
||||||
|
- canvas non-deterministic → identical draw → identical dataURL (stealth
|
||||||
|
seed must be per-session, not per-readback)
|
||||||
|
- headless navigator tells → navigator.webdriver falsy, languages
|
||||||
|
non-empty, plugins is a real PluginArray
|
||||||
|
|
||||||
|
All of this is headless, NO screenshot → GPU-free (can't false-fail on the
|
||||||
|
GPU-less hosted runners), and fully offline — data: URLs only, NO network, NO
|
||||||
|
proxy, NO secrets → safe in public CI. WebGL determinism is intentionally NOT
|
||||||
|
checked here (it needs SWGL and can false-fail headless); it lives in the local
|
||||||
|
proxy realness gate, alongside the fingerprint/WebRTC-vs-vanilla checks.
|
||||||
|
|
||||||
Usage: python ci_drive_gate.py /path/to/firefox[.exe | .app/Contents/MacOS/firefox]
|
Usage: python ci_drive_gate.py /path/to/firefox[.exe | .app/Contents/MacOS/firefox]
|
||||||
Exit 0 + "DRIVE GATE OK ..." on success; non-zero with a reason on failure.
|
Exit 0 + "DRIVE GATE OK ..." on success; non-zero with a reason on failure.
|
||||||
@@ -20,25 +31,82 @@ import sys
|
|||||||
|
|
||||||
from playwright.sync_api import sync_playwright
|
from playwright.sync_api import sync_playwright
|
||||||
|
|
||||||
|
# Single offline page that wires up every probe: a clickable button, a text
|
||||||
|
# input, a same-document iframe, and a mousemove counter. data: URLs execute
|
||||||
|
# inline scripts and are same-origin, so this needs no server.
|
||||||
|
PAGE = (
|
||||||
|
"data:text/html,"
|
||||||
|
"<title>dt</title>"
|
||||||
|
"<h1 id=x>hello-drive</h1>"
|
||||||
|
"<button id=b onclick=\"window.__clicked=1\">go</button>"
|
||||||
|
"<input id=inp>"
|
||||||
|
"<iframe id=f src=\"data:text/html,<b id=ok>ok</b>\"></iframe>"
|
||||||
|
"<script>window.__moves=0;"
|
||||||
|
"addEventListener('mousemove',function(){window.__moves++})</script>"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Identical 2D draw, evaluated twice in one session. The stealth canvas spoof is
|
||||||
|
# seeded per-session (see fingerprint-consistency rule), so two identical draws
|
||||||
|
# MUST produce byte-identical output. Per-readback noise → instant bot flag.
|
||||||
|
CANVAS_DRAW = (
|
||||||
|
"() => {const c=document.createElement('canvas');c.width=c.height=16;"
|
||||||
|
"const g=c.getContext('2d');g.fillStyle='#08f';g.fillRect(0,0,16,16);"
|
||||||
|
"g.fillStyle='#f40';g.fillText('s',2,12);return c.toDataURL();}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def main(exe: str) -> int:
|
def main(exe: str) -> int:
|
||||||
with sync_playwright() as p:
|
with sync_playwright() as p:
|
||||||
browser = p.firefox.launch(executable_path=exe, headless=True)
|
browser = p.firefox.launch(executable_path=exe, headless=True)
|
||||||
page = browser.new_page()
|
page = browser.new_page()
|
||||||
# data: URL → real HTML parse + DOM + JS, fully offline (no network/proxy).
|
page.goto(PAGE)
|
||||||
page.goto("data:text/html,<title>dt</title><h1 id=x>hello-drive</h1>")
|
|
||||||
ua = page.evaluate("navigator.userAgent")
|
ua = page.evaluate("navigator.userAgent")
|
||||||
webdriver = page.evaluate("navigator.webdriver")
|
webdriver = page.evaluate("navigator.webdriver")
|
||||||
text = page.evaluate("() => document.getElementById('x').textContent")
|
text = page.evaluate("() => document.getElementById('x').textContent")
|
||||||
|
|
||||||
|
# firefox-2 / issue-#9 catcher: real mouse + keyboard over juggler.
|
||||||
|
page.mouse.move(20, 20)
|
||||||
|
page.mouse.move(120, 90) # exercises synthesizeMouseEvent path
|
||||||
|
page.click("#b") # mousedown/up/click → onclick fires
|
||||||
|
page.click("#inp")
|
||||||
|
page.keyboard.type("ok")
|
||||||
|
clicked = page.evaluate("window.__clicked")
|
||||||
|
moves = page.evaluate("window.__moves")
|
||||||
|
typed = page.evaluate("() => document.getElementById('inp').value")
|
||||||
|
|
||||||
|
# issue-#20 catcher: the iframe must be reachable and runnable.
|
||||||
|
frame_el = page.query_selector("#f")
|
||||||
|
frame = frame_el.content_frame() if frame_el else None
|
||||||
|
iframe_ok = bool(frame) and frame.evaluate(
|
||||||
|
"() => document.getElementById('ok') && document.getElementById('ok').textContent"
|
||||||
|
) == "ok"
|
||||||
|
|
||||||
|
# stealth-determinism catcher: identical draw → identical dataURL.
|
||||||
|
canvas_a = page.evaluate(CANVAS_DRAW)
|
||||||
|
canvas_b = page.evaluate(CANVAS_DRAW)
|
||||||
|
|
||||||
|
# BotD navigator-surface tells (proxy-free subset).
|
||||||
|
langs = page.evaluate("navigator.languages.length")
|
||||||
|
plugins = page.evaluate("navigator.plugins instanceof PluginArray")
|
||||||
|
|
||||||
browser.close()
|
browser.close()
|
||||||
|
|
||||||
assert "Firefox" in ua, f"unexpected UA (binary not driving correctly): {ua!r}"
|
assert "Firefox" in ua, f"unexpected UA (binary not driving correctly): {ua!r}"
|
||||||
assert text == "hello-drive", f"DOM/JS roundtrip failed: {text!r}"
|
assert text == "hello-drive", f"DOM/JS roundtrip failed: {text!r}"
|
||||||
# Free stealth smoke: the patched build hides navigator.webdriver even when
|
|
||||||
# driven by bare Playwright. A True here is a stealth regression, not a crash.
|
|
||||||
assert not webdriver, f"navigator.webdriver leaked True (stealth regression): {webdriver!r}"
|
assert not webdriver, f"navigator.webdriver leaked True (stealth regression): {webdriver!r}"
|
||||||
|
assert clicked == 1, "page.click() did not fire onclick — mouse-event synthesis broken (firefox-2 class)"
|
||||||
|
assert moves >= 1, "page.mouse.move() produced no mousemove — jugglerSendMouseEvent regression"
|
||||||
|
assert typed == "ok", f"page.keyboard.type() failed: {typed!r}"
|
||||||
|
assert iframe_ok, "iframe content_frame() unreachable — fission.webContentIsolationStrategy regression (issue #20)"
|
||||||
|
assert canvas_a == canvas_b, "canvas non-deterministic across identical draws (stealth seed broken → bot tell)"
|
||||||
|
assert langs and langs > 0, "navigator.languages empty (headless tell)"
|
||||||
|
assert plugins, "navigator.plugins is not a PluginArray (headless tell)"
|
||||||
|
|
||||||
print(f"DRIVE GATE OK | UA={ua} | webdriver={webdriver} | dom-roundtrip=ok")
|
print(
|
||||||
|
f"DRIVE GATE OK | UA={ua} | webdriver={webdriver} | "
|
||||||
|
f"click+mousemove+keyboard+iframe+canvas-determinism+navsurface=ok"
|
||||||
|
)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user