diff --git a/docs/accessibility.md b/docs/accessibility.md
index 0432d5d..43d74dd 100644
--- a/docs/accessibility.md
+++ b/docs/accessibility.md
@@ -48,6 +48,8 @@ Options:
--bypass-csp Bypass Content-Security-Policy
--auth-password TEXT Password for HTTP Basic authentication
--auth-username TEXT Username for HTTP Basic authentication
+ --init-script TEXT JavaScript to run as a page init script, before the
+ page's own scripts run. Can be passed more than once.
--help Show this message and exit.
```
diff --git a/docs/har.md b/docs/har.md
index 3c8693f..9e8b1d6 100644
--- a/docs/har.md
+++ b/docs/har.md
@@ -131,6 +131,8 @@ Options:
--bypass-csp Bypass Content-Security-Policy
--auth-password TEXT Password for HTTP Basic authentication
--auth-username TEXT Username for HTTP Basic authentication
+ --init-script TEXT JavaScript to run as a page init script, before the
+ page's own scripts run. Can be passed more than once.
--help Show this message and exit.
```
diff --git a/docs/html.md b/docs/html.md
index b2676ee..df1428e 100644
--- a/docs/html.md
+++ b/docs/html.md
@@ -80,6 +80,9 @@ Options:
--silent Do not output any messages
--auth-password TEXT Password for HTTP Basic authentication
--auth-username TEXT Username for HTTP Basic authentication
+ --init-script TEXT JavaScript to run as a page init script,
+ before the page's own scripts run. Can be
+ passed more than once.
--help Show this message and exit.
```
diff --git a/docs/javascript.md b/docs/javascript.md
index 519b477..d5a89d3 100644
--- a/docs/javascript.md
+++ b/docs/javascript.md
@@ -137,6 +137,20 @@ This example [uses GitHub Actions](https://docs.github.com/en/actions/quickstart
}"
```
+## Running an init script
+
+The `-j/--javascript` option runs code *after* the page has loaded. Sometimes you need to run code *before* the page's own scripts execute instead - for example to seed `Math.random()`, freeze `Date`, or hide the `navigator.webdriver` flag that some sites use to detect headless browsers.
+
+The `--init-script` option registers JavaScript as a [Playwright init script](https://playwright.dev/python/docs/api/class-page#page-add-init-script), which runs on every navigation before any of the page's own scripts:
+
+```bash
+shot-scraper javascript https://example.com/ \
+ --init-script "delete Object.getPrototypeOf(navigator).webdriver" \
+ "navigator.webdriver"
+```
+
+You can pass `--init-script` more than once to register several scripts, and it works with the `shot`, `multi`, `javascript`, `accessibility`, `pdf`, `html` and `har` commands.
+
## Running JavaScript from a file
You can also save JavaScript to a file and execute it like this:
@@ -260,6 +274,9 @@ Options:
--bypass-csp Bypass Content-Security-Policy
--auth-password TEXT Password for HTTP Basic authentication
--auth-username TEXT Username for HTTP Basic authentication
+ --init-script TEXT JavaScript to run as a page init script,
+ before the page's own scripts run. Can be
+ passed more than once.
--help Show this message and exit.
```
diff --git a/docs/multi.md b/docs/multi.md
index a0ef14d..25bc086 100644
--- a/docs/multi.md
+++ b/docs/multi.md
@@ -269,6 +269,9 @@ Options:
--har Save all requests to trace.har file
--har-zip Save all requests to trace.har.zip file
--har-file FILE Path to HAR file to save all requests
+ --init-script TEXT JavaScript to run as a page init script,
+ before the page's own scripts run. Can be
+ passed more than once.
--help Show this message and exit.
```
diff --git a/docs/pdf.md b/docs/pdf.md
index c19b50d..f4a67bb 100644
--- a/docs/pdf.md
+++ b/docs/pdf.md
@@ -75,6 +75,9 @@ Options:
--silent Do not output any messages
--auth-password TEXT Password for HTTP Basic authentication
--auth-username TEXT Username for HTTP Basic authentication
+ --init-script TEXT JavaScript to run as a page init script,
+ before the page's own scripts run. Can be
+ passed more than once.
--help Show this message and exit.
```
diff --git a/docs/screenshots.md b/docs/screenshots.md
index 3cdf3d6..9e7277c 100644
--- a/docs/screenshots.md
+++ b/docs/screenshots.md
@@ -387,6 +387,9 @@ Options:
--silent Do not output any messages
--auth-password TEXT Password for HTTP Basic authentication
--auth-username TEXT Username for HTTP Basic authentication
+ --init-script TEXT JavaScript to run as a page init script,
+ before the page's own scripts run. Can be
+ passed more than once.
--help Show this message and exit.
```
diff --git a/shot_scraper/cli.py b/shot_scraper/cli.py
index 9015e24..6c233c9 100644
--- a/shot_scraper/cli.py
+++ b/shot_scraper/cli.py
@@ -171,6 +171,19 @@ def js_file_option(fn):
return fn
+def init_script_option(fn):
+ click.option(
+ "init_scripts",
+ "--init-script",
+ multiple=True,
+ help=(
+ "JavaScript to run as a page init script, before the page's own "
+ "scripts run. Can be passed more than once."
+ ),
+ )(fn)
+ return fn
+
+
def _load_javascript_source(source):
"Load JavaScript from a file path, '-' for stdin or gh:username/script"
if source.startswith("gh:"):
@@ -308,6 +321,7 @@ def cli():
@bypass_csp_option
@silent_option
@http_auth_options
+@init_script_option
def shot(
url,
auth,
@@ -342,6 +356,7 @@ def shot(
silent,
auth_username,
auth_password,
+ init_scripts,
):
"""
Take a single screenshot of a page or portion of a page.
@@ -410,6 +425,7 @@ def shot(
bypass_csp=bypass_csp,
auth_username=auth_username,
auth_password=auth_password,
+ init_scripts=init_scripts,
)
if interactive or devtools:
use_existing_page = True
@@ -469,6 +485,7 @@ def _browser_context(
record_video_dir=None,
record_video_size=None,
viewport=None,
+ init_scripts=None,
):
# Playwright 1.58 removed the `devtools` launch option. Emulate the
# previous behavior for Chromium by passing the corresponding flag.
@@ -512,6 +529,8 @@ def _browser_context(
if viewport:
context_args["viewport"] = viewport
context = browser_obj.new_context(**context_args)
+ for init_script in init_scripts or []:
+ context.add_init_script(script=init_script)
if timeout:
context.set_default_timeout(timeout)
return context, browser_obj
@@ -803,6 +822,7 @@ def _convert_video_to_mp4(output, silent=False):
type=click.Path(file_okay=True, writable=True, dir_okay=False),
help="Path to HAR file to save all requests",
)
+@init_script_option
def multi(
config,
auth,
@@ -826,6 +846,7 @@ def multi(
har,
har_zip,
har_file,
+ init_scripts,
):
"""
Take multiple screenshots, defined by a YAML file
@@ -877,6 +898,7 @@ def multi(
auth_username=auth_username,
auth_password=auth_password,
record_har_path=har_file or None,
+ init_scripts=init_scripts,
)
try:
for shot in shots:
@@ -954,6 +976,7 @@ def multi(
@skip_fail_options
@bypass_csp_option
@http_auth_options
+@init_script_option
def accessibility(
url,
auth,
@@ -967,6 +990,7 @@ def accessibility(
bypass_csp,
auth_username,
auth_password,
+ init_scripts,
):
"""
Dump the Chromium accessibility tree for the specifed page
@@ -985,6 +1009,7 @@ def accessibility(
bypass_csp=bypass_csp,
auth_username=auth_username,
auth_password=auth_password,
+ init_scripts=init_scripts,
)
page = context.new_page()
if log_console:
@@ -1037,6 +1062,7 @@ def accessibility(
@skip_fail_options
@bypass_csp_option
@http_auth_options
+@init_script_option
def har(
url,
zip_,
@@ -1054,6 +1080,7 @@ def har(
bypass_csp,
auth_username,
auth_password,
+ init_scripts,
):
"""
Record a HAR file for the specified page
@@ -1094,6 +1121,7 @@ def har(
auth_username=auth_username,
auth_password=auth_password,
record_har_path=str(output),
+ init_scripts=init_scripts,
)
page = context.new_page()
if log_console:
@@ -1272,6 +1300,7 @@ def _extract_har_entry(entry, extract_dir, existing_files, file_exists_fn, zip_f
@skip_fail_options
@bypass_csp_option
@http_auth_options
+@init_script_option
def javascript(
url,
javascript,
@@ -1292,6 +1321,7 @@ def javascript(
bypass_csp,
auth_username,
auth_password,
+ init_scripts,
):
"""
Execute JavaScript against the page and return the result as JSON
@@ -1334,6 +1364,7 @@ def javascript(
bypass_csp=bypass_csp,
auth_username=auth_username,
auth_password=auth_password,
+ init_scripts=init_scripts,
)
page = context.new_page()
if log_console:
@@ -1417,6 +1448,7 @@ def javascript(
@bypass_csp_option
@silent_option
@http_auth_options
+@init_script_option
def pdf(
url,
auth,
@@ -1440,6 +1472,7 @@ def pdf(
silent,
auth_username,
auth_password,
+ init_scripts,
):
"""
Create a PDF of the specified page
@@ -1468,6 +1501,7 @@ def pdf(
auth_username=auth_username,
auth_password=auth_password,
timeout=timeout,
+ init_scripts=init_scripts,
)
page = context.new_page()
if log_console:
@@ -1542,6 +1576,7 @@ def pdf(
@bypass_csp_option
@silent_option
@http_auth_options
+@init_script_option
def html(
url,
auth,
@@ -1561,6 +1596,7 @@ def html(
silent,
auth_username,
auth_password,
+ init_scripts,
):
"""
Output the final HTML of the specified page
@@ -1588,6 +1624,7 @@ def html(
bypass_csp=bypass_csp,
auth_username=auth_username,
auth_password=auth_password,
+ init_scripts=init_scripts,
)
page = context.new_page()
if log_console:
diff --git a/tests/test_shot_scraper.py b/tests/test_shot_scraper.py
index 7d80404..ead8113 100644
--- a/tests/test_shot_scraper.py
+++ b/tests/test_shot_scraper.py
@@ -386,6 +386,80 @@ def test_javascript(args, expected):
assert result.output == expected
+# HTML that records, at parse time, whether the init script already ran
+INIT_SCRIPT_HTML = """
+
+
+Init title
+
+
+
+
+"""
+
+
+def test_javascript_init_script():
+ runner = CliRunner()
+ with runner.isolated_filesystem():
+ open("index.html", "w").write(TEST_HTML)
+ result = runner.invoke(
+ cli,
+ [
+ "javascript",
+ "index.html",
+ "--init-script",
+ "window.injectedValue = 'hello';",
+ "window.injectedValue",
+ ],
+ )
+ assert result.exit_code == 0, str(result.exception)
+ assert result.output == '"hello"\n'
+
+
+def test_javascript_init_script_runs_before_page_scripts():
+ # An init script runs before the page's own scripts, so the inline
+ #