diff --git a/docs/multi.md b/docs/multi.md index a0ef14d..091663d 100644 --- a/docs/multi.md +++ b/docs/multi.md @@ -121,6 +121,29 @@ You can include desired `height`, `width`, `quality`, `wait` and `wait_for` opti wait_for: document.querySelector('#bighead') ``` +To save a WebP image, use an `output` filename ending in `.webp`. WebP is lossless by default; add `quality` from 0 to 99 for lossy compression: + +```yaml +- output: simon.webp + url: https://simonwillison.net/ + height: 800 +- output: bighead.webp + url: https://simonwillison.net/ + selector: "#bighead" + quality: 80 +``` + +You can also specify `format: png`, `format: jpeg` or `format: webp` on each item. This takes precedence over the output filename extension: + +```yaml +- output: screenshot + url: https://simonwillison.net/ + format: webp + quality: 80 +``` + +Without `format` or a `.webp` output filename, `quality` selects JPEG. Combining `format: png` with `quality` is an error. See {ref}`screenshots` for more about image formats and quality. + (multi-har)= ## Recording to an HTTP Archive diff --git a/docs/screenshots.md b/docs/screenshots.md index 3cdf3d6..c319348 100644 --- a/docs/screenshots.md +++ b/docs/screenshots.md @@ -125,7 +125,7 @@ The `--js-file` option is also available for the `pdf`, `html`, `accessibility` ## Using JPEGs instead of PNGs -Screenshots default to PNG. You can save as a JPEG by specifying a `-o` filename that ends with `.jpg`. +Screenshots default to PNG. Use a `-o` filename ending in `.jpg` or `.jpeg` to save as JPEG, or `.webp` to save as WebP. You can also use `--quality X` to save as a JPEG with the specified quality, in order to reduce the filesize. 80 is a good value to use here: ```bash @@ -136,6 +136,47 @@ shot-scraper https://simonwillison.net/ \ % ls -lah simonwillison.jpg -rw-r--r--@ 1 simon staff 168K Mar 9 13:53 simonwillison.jpg ``` +## Using WebP + +Save a screenshot as WebP by specifying a filename ending in `.webp`: + +```bash +shot-scraper https://simonwillison.net/ -h 800 -o simonwillison.webp +``` + +WebP screenshots are lossless by default, equivalent to `--quality 100`. Use a lower quality to apply lossy compression: + +```bash +shot-scraper https://simonwillison.net/ \ + -h 800 -o simonwillison.webp --quality 80 +``` + +WebP also works with selectors: + +```bash +shot-scraper https://simonwillison.net/ -s '#bighead' -o bighead.webp +``` + +WebP images are limited to [16,383 pixels in either dimension](https://developers.google.com/speed/webp/faq#what_is_the_maximum_size_a_webp_image_can_be), so use `--height` or a selector when capturing a page that exceeds that limit. + +## Specifying the image format + +Use `--format png`, `--format jpeg` or `--format webp` to select the format explicitly. This takes precedence over the output filename extension. + +Without `-o`, the automatic filename uses the selected format. This writes to `datasette-io.webp`: + +```bash +shot-scraper https://datasette.io/ --format webp +``` + +Use `--format` with `-o -` to write WebP or JPEG to standard output: + +```bash +shot-scraper https://datasette.io/ --format webp -o - > datasette.webp +``` + +The `--quality` option accepts values from 0 to 100 for JPEG and WebP. Combining `--format png` with `--quality` is an error because PNG does not use a quality setting. + ## Device scale factor The `--scale-factor` option sets a specific device scale factor, which effectively simulates different device pixel ratios. This setting is useful for testing high-definition displays or emulating screens with various pixel densities. @@ -164,11 +205,17 @@ Note: The `--retina` option should not be used in conjunction with the `--scale- ## Transparent background -The `--omit-background` option instructs the browser to ignore the default background, allowing for the capture of a page with a transparent background. Does not work with JPG images or when `quality` is set. +The `--omit-background` option instructs the browser to ignore the default background, allowing for the capture of a page with a transparent background. The default Chromium browser supports this for PNG and WebP images. JPEG does not support transparency. ```bash shot-scraper https://simonwillison.net/ -o simon.png \ --width 400 --height 600 --omit-background ``` +WebP can preserve transparency when using `--quality`: + +```bash +shot-scraper https://simonwillison.net/ -o simon.webp \ + --width 400 --height 600 --omit-background --quality 80 +``` ## Interacting with the page Sometimes it's useful to be able to manually interact with a page before the screenshot is captured. @@ -363,9 +410,12 @@ Options: together with '--scale-factor'. --omit-background Omit the default browser background from the shot, making it possible take advantage of - transparency. Does not work with JPEGs or when - using --quality. - --quality INTEGER Save as JPEG with this quality, e.g. 80 + transparency. Does not work with JPEGs. + --format [png|jpeg|webp] Image format. Overrides the output extension; + use with -o - for stdout. + --quality INTEGER RANGE JPEG or WebP quality, e.g. 80. Defaults to + JPEG unless WebP is selected. WebP defaults to + 100 (lossless). [0<=x<=100] --wait INTEGER Wait this many milliseconds before taking the screenshot --wait-for TEXT Wait until this JS expression returns true diff --git a/pyproject.toml b/pyproject.toml index f3da138..7950079 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "click", "PyYAML", "pydantic", - "playwright>=1.61.0", + "playwright>=1.62.0", "click-default-group", ] diff --git a/shot_scraper/cli.py b/shot_scraper/cli.py index 9015e24..9e5a15c 100644 --- a/shot_scraper/cli.py +++ b/shot_scraper/cli.py @@ -42,6 +42,7 @@ ) BROWSERS = ("chromium", "firefox", "webkit", "chrome", "chrome-beta") +SCREENSHOT_FORMATS = ("png", "jpeg", "webp") def console_log(msg): @@ -271,9 +272,19 @@ def cli(): @click.option( "--omit-background", is_flag=True, - help="Omit the default browser background from the shot, making it possible take advantage of transparency. Does not work with JPEGs or when using --quality.", + help="Omit the default browser background from the shot, making it possible take advantage of transparency. Does not work with JPEGs.", +) +@click.option( + "--format", + "format_", + type=click.Choice(SCREENSHOT_FORMATS, case_sensitive=False), + help="Image format. Overrides the output extension; use with -o - for stdout.", +) +@click.option( + "--quality", + type=click.IntRange(0, 100), + help="JPEG or WebP quality, e.g. 80. Defaults to JPEG unless WebP is selected. WebP defaults to 100 (lossless).", ) -@click.option("--quality", type=int, help="Save as JPEG with this quality, e.g. 80") @click.option( "--wait", type=int, help="Wait this many milliseconds before taking the screenshot" ) @@ -324,6 +335,7 @@ def shot( retina, scale_factor, omit_background, + format_, quality, wait, wait_for, @@ -370,8 +382,9 @@ def shot( shot-scraper https://simonwillison.net -s '#bighead' """ javascript = _resolve_javascript(javascript, js_file) + format_ = _screenshot_format(output, format_, quality) if output is None: - ext = "jpg" if quality else None + ext = "jpg" if format_ == "jpeg" else format_ output = filename_for_url(url, ext=ext, file_exists=os.path.exists) scale_factor = normalize_scale_factor(retina, scale_factor) @@ -385,6 +398,7 @@ def shot( "javascript": javascript, "width": width, "height": height, + "format": format_, "quality": quality, "wait": wait, "wait_for": wait_for, @@ -2180,6 +2194,28 @@ def _get_viewport(width, height): return {} +def _screenshot_format(output, format_, quality): + "Resolve explicit formats and quality, or let Playwright infer from the path." + if quality is not None and ( + not isinstance(quality, int) + or isinstance(quality, bool) + or not 0 <= quality <= 100 + ): + raise click.ClickException("quality must be an integer between 0 and 100") + if format_ is not None: + if not isinstance(format_, str) or format_.lower() not in SCREENSHOT_FORMATS: + raise click.ClickException("format must be one of: png, jpeg, webp") + format_ = format_.lower() + elif pathlib.Path((output or "").strip()).suffix.lower() == ".webp": + format_ = "webp" + elif quality is not None: + # Preserve the existing behavior of --quality selecting JPEG. + format_ = "jpeg" + if format_ == "png" and quality is not None: + raise click.ClickException("quality is not supported for PNG images") + return format_ + + def take_shot( context_or_page, shot, @@ -2201,9 +2237,14 @@ def take_shot( url = url_or_file_path(url, file_exists=_check_and_absolutize) output = (shot.get("output") or "").strip() - if not output and not return_bytes: - output = filename_for_url(url, ext="png", file_exists=os.path.exists) quality = shot.get("quality") + format_ = _screenshot_format(output, shot.get("format"), quality) + if not output and not return_bytes: + output = filename_for_url( + url, + ext="jpg" if format_ == "jpeg" else format_, + file_exists=os.path.exists, + ) omit_background = shot.get("omit_background") wait = shot.get("wait") wait_for = shot.get("wait_for") @@ -2281,8 +2322,10 @@ def on_response(response): page.wait_for_function(wait_for) screenshot_args = {} - if quality: - screenshot_args.update({"quality": quality, "type": "jpeg"}) + if format_ is not None: + screenshot_args["type"] = format_ + if quality is not None: + screenshot_args["quality"] = quality if omit_background: screenshot_args.update({"omit_background": True}) if not return_bytes: diff --git a/tests/test_image_formats.py b/tests/test_image_formats.py new file mode 100644 index 0000000..9f20c88 --- /dev/null +++ b/tests/test_image_formats.py @@ -0,0 +1,195 @@ +from pathlib import Path + +import pytest +import yaml +from click.testing import CliRunner + +from shot_scraper.cli import cli + + +@pytest.fixture +def image_runner(): + runner = CliRunner() + with runner.isolated_filesystem(): + Path("page.html").write_text( + '' + '
Screenshot
' + "" + ) + yield runner + + +def assert_image_format(content, image_format, lossless=None): + if image_format == "png": + assert content.startswith(b"\x89PNG\r\n\x1a\n") + elif image_format == "jpeg": + assert content.startswith(b"\xff\xd8\xff") + assert content.endswith(b"\xff\xd9") + else: + assert content[:4] == b"RIFF" + assert content[8:12] == b"WEBP" + # Read RIFF chunks instead of searching the compressed image payload. + chunks = [] + offset = 12 + while offset + 8 <= len(content): + chunks.append(content[offset : offset + 4]) + size = int.from_bytes(content[offset + 4 : offset + 8], "little") + offset += 8 + size + (size % 2) + if lossless is not None: + assert (b"VP8L" if lossless else b"VP8 ") in chunks + + +@pytest.mark.parametrize( + "output,options,image_format,lossless", + [ + ("image.webp", [], "webp", True), + ("image.WEBP", ["--quality", "80"], "webp", False), + ("image.png", ["--format", "WeBp"], "webp", True), + ("image.webp", ["--format", "png"], "png", None), + ("image.webp", ["--format", "jpeg"], "jpeg", None), + ("image.png", [], "png", None), + ("image.jpg", [], "jpeg", None), + ("image.jpeg", [], "jpeg", None), + ("image.png", ["--quality", "80"], "jpeg", None), + ], +) +def test_shot_file_format(image_runner, output, options, image_format, lossless): + result = image_runner.invoke( + cli, ["page.html", "-o", output, "--width", "120", *options] + ) + assert result.exit_code == 0, result.output + assert_image_format(Path(output).read_bytes(), image_format, lossless) + + +@pytest.mark.parametrize( + "options,output,image_format,lossless", + [ + ([], "page-html.png", "png", None), + (["--format", "webp"], "page-html.webp", "webp", True), + (["--format", "jpeg"], "page-html.jpg", "jpeg", None), + (["--quality", "0"], "page-html.jpg", "jpeg", None), + ( + ["--format", "webp", "--quality", "0"], + "page-html.webp", + "webp", + False, + ), + ], +) +def test_shot_automatic_filename(image_runner, options, output, image_format, lossless): + result = image_runner.invoke(cli, ["page.html", "--width", "120", *options]) + assert result.exit_code == 0, result.output + assert_image_format(Path(output).read_bytes(), image_format, lossless) + + +@pytest.mark.parametrize( + "quality,selector,lossless", [(None, None, True), (80, "#target", False)] +) +def test_webp_stdout(image_runner, quality, selector, lossless): + options = [] if quality is None else ["--quality", str(quality)] + if selector: + options.extend(["--selector", selector]) + result = image_runner.invoke( + cli, + [ + "page.html", + "--format", + "webp", + "-o", + "-", + "--width", + "120", + *options, + ], + ) + assert result.exit_code == 0, result.output + assert_image_format(result.stdout_bytes, "webp", lossless) + assert sorted(path.name for path in Path().iterdir()) == ["page.html"] + + +def test_multi_image_formats(image_runner): + shots = [ + {"output": "page.webp"}, + {"output": "lossless.webp", "quality": 100}, + {"output": "lossy.webp", "quality": 80}, + {"output": "minimum.webp", "quality": 0}, + {"output": "selector.webp", "selector": "#target", "quality": 80}, + {"output": "explicit.png", "format": "WEBP", "selector": "#target"}, + {"output": "png.webp", "format": "png"}, + {"output": "legacy.png", "quality": 80}, + {"format": "webp"}, + {"format": "jpeg"}, + {}, + ] + Path("shots.yml").write_text( + yaml.safe_dump([{"url": "page.html", "width": 120, **shot} for shot in shots]) + ) + result = image_runner.invoke(cli, ["multi", "shots.yml"]) + assert result.exit_code == 0, result.output + + for output, lossless in ( + ("page.webp", True), + ("lossless.webp", True), + ("lossy.webp", False), + ("minimum.webp", False), + ("selector.webp", False), + ("explicit.png", True), + ): + assert_image_format(Path(output).read_bytes(), "webp", lossless) + assert_image_format(Path("png.webp").read_bytes(), "png") + assert_image_format(Path("legacy.png").read_bytes(), "jpeg") + + # Local URLs may include the absolute file path in multi's generated name. + for suffix, image_format in (("webp", "webp"), ("jpg", "jpeg"), ("png", "png")): + generated = list(Path().glob(f"*page-html.{suffix}")) + assert len(generated) == 1 + assert_image_format(generated[0].read_bytes(), image_format) + + +@pytest.mark.parametrize( + "options,error", + [ + (["--quality", "-1"], "0<=x<=100"), + (["--quality", "101"], "0<=x<=100"), + (["--quality", "80.5"], "integer"), + (["--format", "png", "--quality", "0"], "quality"), + (["--format", "gif"], "Invalid value for '--format'"), + ], +) +def test_invalid_cli_format_options(image_runner, mocker, options, error): + playwright = mocker.patch("shot_scraper.cli.sync_playwright") + result = image_runner.invoke(cli, ["page.html", *options]) + assert result.exit_code != 0 + assert error in result.output + playwright.assert_not_called() + + +@pytest.mark.parametrize( + "options,error", + [ + ({"quality": -1}, "quality"), + ({"quality": 101}, "quality"), + ({"quality": 80.5}, "quality"), + ({"quality": "80"}, "quality"), + ({"quality": True}, "quality"), + ({"format": "png", "quality": 0}, "quality"), + ({"format": "gif"}, "format"), + ({"format": ["webp"]}, "format"), + ], +) +def test_invalid_yaml_format_options(image_runner, mocker, options, error): + mocker.patch("shot_scraper.cli.sync_playwright") + context = mocker.MagicMock() + mocker.patch( + "shot_scraper.cli._browser_context", + return_value=(context, mocker.MagicMock()), + ) + Path("shots.yml").write_text( + yaml.safe_dump([{"url": "page.html", "output": "image.webp", **options}]) + ) + result = image_runner.invoke(cli, ["multi", "shots.yml"]) + assert result.exit_code != 0 + assert error in result.output + context.new_page.assert_not_called() + assert not Path("image.webp").exists()