Add --init-script option for running scripts before page load

docs/accessibility.md

@@ -48,6 +48,8 @@ Options:
   --bypass-csp           Bypass Content-Security-Policy
   --auth-password TEXT   Password for HTTP Basic authentication
   --auth-username TEXT   Username for HTTP Basic authentication
+  --init-script TEXT     JavaScript to run as a page init script, before the
+                         page's own scripts run. Can be passed more than once.
   --help                 Show this message and exit.
 ```
 <!-- [[[end]]] -->

docs/har.md

@@ -131,6 +131,8 @@ Options:
   --bypass-csp           Bypass Content-Security-Policy
   --auth-password TEXT   Password for HTTP Basic authentication
   --auth-username TEXT   Username for HTTP Basic authentication
+  --init-script TEXT     JavaScript to run as a page init script, before the
+                         page's own scripts run. Can be passed more than once.
   --help                 Show this message and exit.
 ```
 <!-- [[[end]]] -->

docs/html.md

@@ -80,6 +80,9 @@ Options:
   --silent                        Do not output any messages
   --auth-password TEXT            Password for HTTP Basic authentication
   --auth-username TEXT            Username for HTTP Basic authentication
+  --init-script TEXT              JavaScript to run as a page init script,
+                                  before the page's own scripts run. Can be
+                                  passed more than once.
   --help                          Show this message and exit.
 ```
 <!-- [[[end]]] -->

docs/javascript.md

@@ -137,6 +137,20 @@ This example [uses GitHub Actions](https://docs.github.com/en/actions/quickstart
       }"
 ```
 
+## Running an init script
+
+The `-j/--javascript` option runs code *after* the page has loaded. Sometimes you need to run code *before* the page's own scripts execute instead - for example to seed `Math.random()`, freeze `Date`, or hide the `navigator.webdriver` flag that some sites use to detect headless browsers.
+
+The `--init-script` option registers JavaScript as a [Playwright init script](https://playwright.dev/python/docs/api/class-page#page-add-init-script), which runs on every navigation before any of the page's own scripts:
+
+```bash
+shot-scraper javascript https://example.com/ \
+  --init-script "delete Object.getPrototypeOf(navigator).webdriver" \
+  "navigator.webdriver"
+```
+
+You can pass `--init-script` more than once to register several scripts, and it works with the `shot`, `multi`, `javascript`, `accessibility`, `pdf`, `html` and `har` commands.
+
 ## Running JavaScript from a file
 
 You can also save JavaScript to a file and execute it like this:
@@ -260,6 +274,9 @@ Options:
   --bypass-csp                    Bypass Content-Security-Policy
   --auth-password TEXT            Password for HTTP Basic authentication
   --auth-username TEXT            Username for HTTP Basic authentication
+  --init-script TEXT              JavaScript to run as a page init script,
+                                  before the page's own scripts run. Can be
+                                  passed more than once.
   --help                          Show this message and exit.
 ```
 <!-- [[[end]]] -->

docs/multi.md

@@ -269,6 +269,9 @@ Options:
   --har                           Save all requests to trace.har file
   --har-zip                       Save all requests to trace.har.zip file
   --har-file FILE                 Path to HAR file to save all requests
+  --init-script TEXT              JavaScript to run as a page init script,
+                                  before the page's own scripts run. Can be
+                                  passed more than once.
   --help                          Show this message and exit.
 ```
 <!-- [[[end]]] -->

docs/pdf.md

@@ -75,6 +75,9 @@ Options:
   --silent                        Do not output any messages
   --auth-password TEXT            Password for HTTP Basic authentication
   --auth-username TEXT            Username for HTTP Basic authentication
+  --init-script TEXT              JavaScript to run as a page init script,
+                                  before the page's own scripts run. Can be
+                                  passed more than once.
   --help                          Show this message and exit.
 ```
 <!-- [[[end]]] -->

docs/screenshots.md

@@ -387,6 +387,9 @@ Options:
   --silent                        Do not output any messages
   --auth-password TEXT            Password for HTTP Basic authentication
   --auth-username TEXT            Username for HTTP Basic authentication
+  --init-script TEXT              JavaScript to run as a page init script,
+                                  before the page's own scripts run. Can be
+                                  passed more than once.
   --help                          Show this message and exit.
 ```
 <!-- [[[end]]] -->

shot_scraper/cli.py

@@ -171,6 +171,19 @@ def js_file_option(fn):
     return fn
 
 
+def init_script_option(fn):
+    click.option(
+        "init_scripts",
+        "--init-script",
+        multiple=True,
+        help=(
+            "JavaScript to run as a page init script, before the page's own "
+            "scripts run. Can be passed more than once."
+        ),
+    )(fn)
+    return fn
+
+
 def _load_javascript_source(source):
     "Load JavaScript from a file path, '-' for stdin or gh:username/script"
     if source.startswith("gh:"):
@@ -308,6 +321,7 @@ def cli():
 @bypass_csp_option
 @silent_option
 @http_auth_options
+@init_script_option
 def shot(
     url,
     auth,
@@ -342,6 +356,7 @@ def shot(
     silent,
     auth_username,
     auth_password,
+    init_scripts,
 ):
     """
     Take a single screenshot of a page or portion of a page.
@@ -410,6 +425,7 @@ def shot(
             bypass_csp=bypass_csp,
             auth_username=auth_username,
             auth_password=auth_password,
+            init_scripts=init_scripts,
         )
         if interactive or devtools:
             use_existing_page = True
@@ -469,6 +485,7 @@ def _browser_context(
     record_video_dir=None,
     record_video_size=None,
     viewport=None,
+    init_scripts=None,
 ):
     # Playwright 1.58 removed the `devtools` launch option. Emulate the
     # previous behavior for Chromium by passing the corresponding flag.
@@ -512,6 +529,8 @@ def _browser_context(
     if viewport:
         context_args["viewport"] = viewport
     context = browser_obj.new_context(**context_args)
+    for init_script in init_scripts or []:
+        context.add_init_script(script=init_script)
     if timeout:
         context.set_default_timeout(timeout)
     return context, browser_obj
@@ -803,6 +822,7 @@ def _convert_video_to_mp4(output, silent=False):
     type=click.Path(file_okay=True, writable=True, dir_okay=False),
     help="Path to HAR file to save all requests",
 )
+@init_script_option
 def multi(
     config,
     auth,
@@ -826,6 +846,7 @@ def multi(
     har,
     har_zip,
     har_file,
+    init_scripts,
 ):
     """
     Take multiple screenshots, defined by a YAML file
@@ -877,6 +898,7 @@ def multi(
             auth_username=auth_username,
             auth_password=auth_password,
             record_har_path=har_file or None,
+            init_scripts=init_scripts,
         )
         try:
             for shot in shots:
@@ -954,6 +976,7 @@ def multi(
 @skip_fail_options
 @bypass_csp_option
 @http_auth_options
+@init_script_option
 def accessibility(
     url,
     auth,
@@ -967,6 +990,7 @@ def accessibility(
     bypass_csp,
     auth_username,
     auth_password,
+    init_scripts,
 ):
     """
     Dump the Chromium accessibility tree for the specifed page
@@ -985,6 +1009,7 @@ def accessibility(
             bypass_csp=bypass_csp,
             auth_username=auth_username,
             auth_password=auth_password,
+            init_scripts=init_scripts,
         )
         page = context.new_page()
         if log_console:
@@ -1037,6 +1062,7 @@ def accessibility(
 @skip_fail_options
 @bypass_csp_option
 @http_auth_options
+@init_script_option
 def har(
     url,
     zip_,
@@ -1054,6 +1080,7 @@ def har(
     bypass_csp,
     auth_username,
     auth_password,
+    init_scripts,
 ):
     """
     Record a HAR file for the specified page
@@ -1094,6 +1121,7 @@ def har(
             auth_username=auth_username,
             auth_password=auth_password,
             record_har_path=str(output),
+            init_scripts=init_scripts,
         )
         page = context.new_page()
         if log_console:
@@ -1272,6 +1300,7 @@ def _extract_har_entry(entry, extract_dir, existing_files, file_exists_fn, zip_f
 @skip_fail_options
 @bypass_csp_option
 @http_auth_options
+@init_script_option
 def javascript(
     url,
     javascript,
@@ -1292,6 +1321,7 @@ def javascript(
     bypass_csp,
     auth_username,
     auth_password,
+    init_scripts,
 ):
     """
     Execute JavaScript against the page and return the result as JSON
@@ -1334,6 +1364,7 @@ def javascript(
             bypass_csp=bypass_csp,
             auth_username=auth_username,
             auth_password=auth_password,
+            init_scripts=init_scripts,
         )
         page = context.new_page()
         if log_console:
@@ -1417,6 +1448,7 @@ def javascript(
 @bypass_csp_option
 @silent_option
 @http_auth_options
+@init_script_option
 def pdf(
     url,
     auth,
@@ -1440,6 +1472,7 @@ def pdf(
     silent,
     auth_username,
     auth_password,
+    init_scripts,
 ):
     """
     Create a PDF of the specified page
@@ -1468,6 +1501,7 @@ def pdf(
             auth_username=auth_username,
             auth_password=auth_password,
             timeout=timeout,
+            init_scripts=init_scripts,
         )
         page = context.new_page()
         if log_console:
@@ -1542,6 +1576,7 @@ def pdf(
 @bypass_csp_option
 @silent_option
 @http_auth_options
+@init_script_option
 def html(
     url,
     auth,
@@ -1561,6 +1596,7 @@ def html(
     silent,
     auth_username,
     auth_password,
+    init_scripts,
 ):
     """
     Output the final HTML of the specified page
@@ -1588,6 +1624,7 @@ def html(
             bypass_csp=bypass_csp,
             auth_username=auth_username,
             auth_password=auth_password,
+            init_scripts=init_scripts,
         )
         page = context.new_page()
         if log_console:

tests/test_shot_scraper.py

@@ -386,6 +386,80 @@ def test_javascript(args, expected):
         assert result.output == expected
 
 
+# HTML that records, at parse time, whether the init script already ran
+INIT_SCRIPT_HTML = """
+<!DOCTYPE html>
+<html>
+<head><title>Init title</title></head>
+<body>
+<script>
+window.seenAtParse = (typeof window.injectedValue !== "undefined")
+  ? window.injectedValue
+  : "missing";
+</script>
+</body>
+</html>
+"""
+
+
+def test_javascript_init_script():
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        open("index.html", "w").write(TEST_HTML)
+        result = runner.invoke(
+            cli,
+            [
+                "javascript",
+                "index.html",
+                "--init-script",
+                "window.injectedValue = 'hello';",
+                "window.injectedValue",
+            ],
+        )
+        assert result.exit_code == 0, str(result.exception)
+        assert result.output == '"hello"\n'
+
+
+def test_javascript_init_script_runs_before_page_scripts():
+    # An init script runs before the page's own scripts, so the inline
+    # <script> in INIT_SCRIPT_HTML should observe the injected value.
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        open("index.html", "w").write(INIT_SCRIPT_HTML)
+        result = runner.invoke(
+            cli,
+            [
+                "javascript",
+                "index.html",
+                "--init-script",
+                "window.injectedValue = 'early';",
+                "window.seenAtParse",
+            ],
+        )
+        assert result.exit_code == 0, str(result.exception)
+        assert result.output == '"early"\n'
+
+
+def test_javascript_multiple_init_scripts():
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        open("index.html", "w").write(TEST_HTML)
+        result = runner.invoke(
+            cli,
+            [
+                "javascript",
+                "index.html",
+                "--init-script",
+                "window.parts = ['a'];",
+                "--init-script",
+                "window.parts.push('b');",
+                "window.parts.join('-')",
+            ],
+        )
+        assert result.exit_code == 0, str(result.exception)
+        assert result.output == '"a-b"\n'
+
+
 def test_javascript_width_height():
     runner = CliRunner()
     with runner.isolated_filesystem():