tests: Implement snapshot test suites for formatting errors

Cargo.toml

@@ -63,6 +63,7 @@ semver = "1.0.21"
 
 [dev-dependencies]
 tempfile = "3.23.0"
+insta = { version = "1.48.0", features = ["filters"] }
 
 # Rustc dependencies are loaded from the sysroot, Cargo doesn't know about them.
 

Contributing.md

@@ -63,6 +63,11 @@ would need a configuration file named `test-indent.toml` in that directory. As a
 example, the `issue-1111.rs` test file is configured by the file
 `./tests/config/issue-1111.toml`.
 
+### Updating snapshots
+
+Some tests that test rustfmt-specific output (e.g. `--help` output and formatting-specific errors) use [insta](https://insta.rs/) to snapshot their output.
+To update these tests, install [`cargo-insta`](https://insta.rs/docs/cli/) and run `cargo insta test --review`.
+
 ## Debugging
 
 Some `rewrite_*` methods use the `debug!` macro for printing useful information.

src/test/mod.rs

@@ -249,6 +249,46 @@ fn system_tests() {
     });
 }
 
+// Check formatting-specific warning/error emissions against snapshots.
+#[test]
+fn warning_tests() {
+    init_log();
+    let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
+    let manifest_dir_filter = regex::escape(manifest_dir.to_string_lossy().as_ref());
+    let files = get_test_files(Path::new("tests/warning/source"), true);
+    let mut config = Config::default();
+    config.set().error_on_line_overflow(true);
+    config.set().error_on_unformatted(true);
+
+    for file in &files {
+        let snapshot_name = file.file_stem().unwrap().to_str().unwrap();
+        let (parsing_errors, _, report) = format_file(file, config.clone());
+        assert!(!parsing_errors, "{} failed to parse", file.display());
+        assert!(
+            report.has_warnings(),
+            "{} did not emit a warning or error",
+            file.display()
+        );
+        let warning = FormatReportFormatterBuilder::new(&report)
+            .build()
+            .to_string();
+
+        insta::with_settings!({
+            snapshot_path => manifest_dir.join("tests/warning/snapshots"),
+            prepend_module_to_snapshot => false,
+            omit_expression => true,
+            filters => vec![
+                (manifest_dir_filter.as_str(), "$$DIR"),
+                (r"\r\n", "\n"),
+                (r"\\", "/"),
+            ],
+            strip_ansi_escape_codes => true,
+        }, {
+            insta::assert_snapshot!(snapshot_name, warning);
+        });
+    }
+}
+
 // Do the same for tests/coverage-source directory.
 // The only difference is the coverage mode.
 #[test]

tests/rustfmt/main.rs

@@ -5,7 +5,7 @@ use std::fs::{File, remove_file};
 use std::path::Path;
 use std::process::Command;
 
-use rustfmt_config_proc_macro::{nightly_only_test, rustfmt_only_ci_test};
+use rustfmt_config_proc_macro::{nightly_only_test, rustfmt_only_ci_test, stable_only_test};
 
 /// Run the rustfmt executable with environment vars set and return its output.
 fn rustfmt_with_extra(
@@ -117,11 +117,116 @@ fn inline_config() {
     );
 }
 
+#[stable_only_test]
 #[test]
 fn rustfmt_usage_text() {
     let args = ["--help"];
     let (stdout, _) = rustfmt(&args);
-    assert!(stdout.contains("Format Rust code\n\nusage: rustfmt [options] <file>..."));
+    insta::assert_snapshot!(stdout, @"
+    Format Rust code
+
+    usage: rustfmt [options] <file>...
+
+    Options:
+            --check         Run in 'check' mode. Exits with 0 if input is
+                            formatted correctly. Exits with 1 and prints a diff if
+                            formatting is required.
+            --emit [files|stdout]
+                            What data to emit and how
+            --backup        Backup any modified files.
+            --config-path [Path for the configuration file]
+                            Recursively searches the given path for the
+                            rustfmt.toml config file. If not found reverts to the
+                            input file path
+            --edition [2015|2018|2021|2024]
+                            Rust edition to use
+            --style-edition [2015|2018|2021|2024]
+                            The edition of the Style Guide (unstable).
+            --color [always|never|auto]
+                            Use colored output (if supported)
+            --print-config [default|minimal|current] PATH
+                            Dumps a default or minimal config to PATH. A minimal
+                            config is the subset of the current config file used
+                            for formatting the current program. `current` writes
+                            to stdout current config as if formatting the file at
+                            PATH.
+        -l, --files-with-diff 
+                            Prints the names of mismatched files that were
+                            formatted. Prints the names of files that would be
+                            formatted when used with `--check` mode.
+            --config [key1=val1,key2=val2...]
+                            Set options from command line. These settings take
+                            priority over .rustfmt.toml
+            --style-edition [2015|2018|2021|2024]
+                            The edition of the Style Guide.
+        -v, --verbose       Print verbose output
+        -q, --quiet         Print less output
+        -V, --version       Show version information
+        -h, --help [=TOPIC] Show this message or help about a specific topic:
+                            `config`
+    ");
+}
+
+#[nightly_only_test]
+#[test]
+fn rustfmt_nightly_usage_text() {
+    let args = ["--help"];
+    let (stdout, _) = rustfmt(&args);
+    insta::assert_snapshot!(stdout, @"
+    Format Rust code
+
+    usage: rustfmt [options] <file>...
+
+    Options:
+            --check         Run in 'check' mode. Exits with 0 if input is
+                            formatted correctly. Exits with 1 and prints a diff if
+                            formatting is required.
+            --emit [files|stdout|coverage|checkstyle|json]
+                            What data to emit and how
+            --backup        Backup any modified files.
+            --config-path [Path for the configuration file]
+                            Recursively searches the given path for the
+                            rustfmt.toml config file. If not found reverts to the
+                            input file path
+            --edition [2015|2018|2021|2024]
+                            Rust edition to use
+            --style-edition [2015|2018|2021|2024]
+                            The edition of the Style Guide (unstable).
+            --color [always|never|auto]
+                            Use colored output (if supported)
+            --print-config [default|minimal|current] PATH
+                            Dumps a default or minimal config to PATH. A minimal
+                            config is the subset of the current config file used
+                            for formatting the current program. `current` writes
+                            to stdout current config as if formatting the file at
+                            PATH.
+        -l, --files-with-diff 
+                            Prints the names of mismatched files that were
+                            formatted. Prints the names of files that would be
+                            formatted when used with `--check` mode.
+            --config [key1=val1,key2=val2...]
+                            Set options from command line. These settings take
+                            priority over .rustfmt.toml
+            --style-edition [2015|2018|2021|2024]
+                            The edition of the Style Guide.
+            --unstable-features 
+                            Enables unstable features. Only available on nightly
+                            channel.
+            --file-lines JSON
+                            Format specified line ranges. Run with
+                            `--help=file-lines` for more detail (unstable).
+            --error-on-unformatted 
+                            Error if unable to get comments or string literals
+                            within max_width, or they are left with trailing
+                            whitespaces (unstable).
+            --skip-children 
+                            Don't reformat child modules (unstable).
+        -v, --verbose       Print verbose output
+        -q, --quiet         Print less output
+        -V, --version       Show version information
+        -h, --help [=TOPIC] Show this message or help about a specific topic:
+                            `config` or `file-lines`
+    ");
 }
 
 #[test]

tests/warning/snapshots/deprecated_skip.snap

@@ -0,0 +1,10 @@
+---
+source: src/test/mod.rs
+---
+warning: `rustfmt_skip` is deprecated; use `rustfmt::skip`
+ --> tests/warning/source/deprecated_skip.rs:1
+  |
+1 | #[rustfmt_skip]
+  |
+
+warning: rustfmt has failed to format. See previous 1 errors.

tests/warning/snapshots/invalid_attribute.snap

@@ -0,0 +1,10 @@
+---
+source: src/test/mod.rs
+---
+error: invalid attribute
+ --> tests/warning/source/invalid_attribute.rs:1
+  |
+1 | #[rustfmt::invalid]
+  |
+
+warning: rustfmt has failed to format. See previous 1 errors.

tests/warning/snapshots/line_overflow.snap

@@ -0,0 +1,11 @@
+---
+source: src/test/mod.rs
+---
+error[internal]: line formatted, but exceeded maximum width (maximum: 100 (see `max_width` option), found: 109)
+ --> tests/warning/source/line_overflow.rs:1:1:101
+  |
+1 | fn this_function_name_is_intentionally_long_enough_to_exceed_the_default_one_hundred_character_maximum_width(
+  |                                                                                                     ^^^^^^^^^
+  |
+
+warning: rustfmt has failed to format. See previous 1 errors.

tests/warning/snapshots/lost_comment.snap

@@ -0,0 +1,11 @@
+---
+source: src/test/mod.rs
+---
+error[internal]: not formatted because a comment would be lost
+ --> tests/warning/source/lost_comment.rs:1
+  |
+1 | fn main() { let _ = 1 /* This comment cannot be retained by the expression formatter. */ + 2; }
+  |
+  = note: set `error_on_unformatted = false` to suppress the warning against comments or string literals
+
+warning: rustfmt has failed to format. See previous 1 errors.

tests/warning/snapshots/multiple_errors.snap

@@ -0,0 +1,38 @@
+---
+source: src/test/mod.rs
+---
+warning: `rustfmt_skip` is deprecated; use `rustfmt::skip`
+ --> tests/warning/source/multiple_errors.rs:1
+  |
+1 | #[rustfmt_skip]
+  |
+
+error: invalid attribute
+ --> tests/warning/source/multiple_errors.rs:4
+  |
+4 | #[rustfmt::invalid]
+  |
+
+error[internal]: not formatted because a comment would be lost
+ --> tests/warning/source/multiple_errors.rs:9
+  |
+9 | fn lost_comment() { let _ = 1 /* This comment cannot be retained by the expression formatter. */ + 2; }
+  |
+  = note: set `error_on_unformatted = false` to suppress the warning against comments or string literals
+
+error[internal]: line formatted, but exceeded maximum width (maximum: 100 (see `max_width` option), found: 109)
+ --> tests/warning/source/multiple_errors.rs:7:7:101
+  |
+7 | fn this_function_name_is_intentionally_long_enough_to_exceed_the_default_one_hundred_character_maximum_width(
+  |                                                                                                     ^^^^^^^^^
+  |
+
+error[internal]: left behind trailing whitespace
+  --> tests/warning/source/multiple_errors.rs:15:15:46
+   |
+15 | /// This doc comment has trailing whitespace.   
+   |                                              ^^^
+   |
+   = note: set `error_on_unformatted = false` to suppress the warning against comments or string literals
+
+warning: rustfmt has failed to format. See previous 5 errors.