diff --git a/claude.md b/claude.md index 94f004b..ec99576 100644 --- a/claude.md +++ b/claude.md @@ -127,5 +127,8 @@ Run it from the repository root. It pairs each received file with the verified f - `src/DeterministicIoPackaging/` - Main library - `Patching/` - XML patchers for different file types - `DeterministicPackage.cs` - Entry point with patcher registration +- `src/DeterministicIoPackaging.Tool/` - `detpackage`, the CliFx dotnet tool wrapping `DeterministicPackage.ConvertAsync` + - `ConvertCommand.cs` - the single (default) command + - `FileResolver.cs` - expands the path parameter into source/target file pairs - `src/Tests/` - Tests using Verify for snapshot testing - `tools/` - Utility projects (e.g., CreateDocx for generating test files) diff --git a/readme.md b/readme.md index b566be8..f7b22a5 100644 --- a/readme.md +++ b/readme.md @@ -80,6 +80,56 @@ var target = await DeterministicPackage.ConvertAsync(sourceStream); +## CLI tool + +A [dotnet tool](https://learn.microsoft.com/en-us/dotnet/core/tools/global-tools) that applies the same conversion to files on disk. + + * https://nuget.org/packages/DeterministicIoPackaging.Tool + +``` +dotnet tool install -g DeterministicIoPackaging.Tool +``` + + +### Usage + +``` +detpackage [options] +``` + +`path` is a package file, or a directory containing packages. It is converted in place unless `--target` is used. + + * `-t|--target` Write results here instead of modifying the input in place. An output file path when the input is a file, otherwise a directory mirroring the input tree. + * `-p|--pattern` Search patterns applied when the input is a directory. Defaults to every known package extension: `*.nupkg`, `*.snupkg`, `*.vsix`, `*.docx`, `*.docm`, `*.dotx`, `*.xlsx`, `*.xlsm`, `*.xltx`, `*.pptx`, `*.pptm`, `*.potx`. Repeat the option for multiple patterns. + * `-r|--recursive` Recurse into subdirectories when the input is a directory. + * `--check` Report which packages are not already deterministic without writing anything. Exits with code 1 if any are found. + * `--continue-on-error` Keep processing the remaining files after a failure, then exit with code 1. + * `-q|--quiet` Suppress per file and summary output. Errors are still written. + +A package that is already deterministic is left untouched, so an in place run does not disturb its timestamp. + + +### Examples + +Convert one package in place: + +``` +detpackage MyPackage.1.0.0.nupkg +``` + +Convert a tree into a separate output directory: + +``` +detpackage ./input -r --target ./output +``` + +Fail a build when any package is not deterministic: + +``` +detpackage ./artifacts -r --check +``` + + ## Icon [Pi](https://thenounproject.com/icon/pi-2131020/) designed by [Zaidan](https://thenounproject.com/creator/mzaidanfiros/) from [The Noun Project](https://thenounproject.com). diff --git a/src/DeterministicIoPackaging.Tool/ConvertCommand.cs b/src/DeterministicIoPackaging.Tool/ConvertCommand.cs new file mode 100644 index 0000000..89ed1bd --- /dev/null +++ b/src/DeterministicIoPackaging.Tool/ConvertCommand.cs @@ -0,0 +1,195 @@ +[Command( + Description = "Rewrites a System.IO.Packaging file so the same source package always produces byte-identical output.")] +public partial class ConvertCommand : ICommand +{ + // Every System.IO.Packaging format the library is known to handle: NuGet packages, the Office + // Open XML documents, and the VSIX container. + static string[] defaultPatterns = + [ + "*.nupkg", + "*.snupkg", + "*.vsix", + "*.docx", + "*.docm", + "*.dotx", + "*.xlsx", + "*.xlsm", + "*.xltx", + "*.pptx", + "*.pptm", + "*.potx" + ]; + + [CommandParameter( + 0, + Name = "path", + Description = "Package file, or directory containing packages, to convert. Converted in place unless --target is used.")] + public required string Input { get; set; } + + [CommandOption( + "target", + 't', + Description = "Write results here instead of modifying the input in place. An output file path when the input is a file, otherwise a directory mirroring the input tree.")] + public string? Target { get; set; } + + [CommandOption( + "pattern", + 'p', + Description = "Search patterns applied when the input is a directory. Defaults to every known package extension.")] + public string[] Patterns { get; set; } = defaultPatterns; + + [CommandOption( + "recursive", + 'r', + Description = "Recurse into subdirectories when the input is a directory.")] + public bool Recursive { get; set; } + + [CommandOption( + "check", + Description = "Report which packages are not already deterministic without writing anything. Exits with code 1 if any are found.")] + public bool Check { get; set; } + + [CommandOption( + "continue-on-error", + Description = "Keep processing the remaining files after a failure, then exit with code 1.")] + public bool ContinueOnError { get; set; } + + [CommandOption( + "quiet", + 'q', + Description = "Suppress per file and summary output. Errors are still written.")] + public bool Quiet { get; set; } + + public async ValueTask ExecuteAsync(IConsole console) + { + if (Check && + Target != null) + { + throw new CommandException("--check does not write anything, so it cannot be combined with --target."); + } + + if (Patterns.Length == 0) + { + throw new CommandException("--pattern requires at least one value."); + } + + var jobs = FileResolver.Resolve(Input, Target, Patterns, Recursive); + if (jobs.Count == 0) + { + throw new CommandException($"No files matching {string.Join(", ", Patterns)} found in: {Input}"); + } + + var cancel = console.RegisterCancellationHandler(); + var changed = 0; + var failed = 0; + + foreach (var job in jobs) + { + try + { + if (await Handle(console, job, cancel)) + { + changed++; + } + } + catch (Exception exception) + when (exception is not OperationCanceledException) + { + if (!ContinueOnError) + { + throw new CommandException($"{Relative(job.Source)}: {exception.Message}", innerException: exception); + } + + failed++; + await console.Error.WriteLineAsync($"failed: {Relative(job.Source)}: {exception.Message}"); + } + } + + await WriteSummary(console, jobs.Count, changed, failed); + } + + // Returns whether converting altered the package. + async Task Handle(IConsole console, FileJob job, Cancel cancel) + { + var source = await File.ReadAllBytesAsync(job.Source, cancel); + + // Read fully into memory first: an in place run overwrites the file the conversion read from. + using var sourceStream = new MemoryStream(source, writable: false); + using var targetStream = await DeterministicPackage.ConvertAsync(sourceStream, cancel); + + var converted = targetStream.ToArray(); + var isChanged = !converted.AsSpan().SequenceEqual(source); + + if (Check) + { + if (isChanged) + { + await Write(console, $"not deterministic: {Relative(job.Source)}"); + } + + return isChanged; + } + + // An unchanged package is left alone on an in place run rather than rewritten with the same + // bytes, so its timestamp is not disturbed. A separate target always has to be written. + if (isChanged || + !job.IsInPlace) + { + var directory = Path.GetDirectoryName(job.Target); + if (directory != null) + { + Directory.CreateDirectory(directory); + } + + await File.WriteAllBytesAsync(job.Target, converted, cancel); + } + + var status = isChanged ? "converted" : "unchanged"; + if (job.IsInPlace) + { + await Write(console, $"{status}: {Relative(job.Source)}"); + } + else + { + await Write(console, $"{status}: {Relative(job.Source)} -> {Relative(job.Target)}"); + } + + return isChanged; + } + + async Task WriteSummary(IConsole console, int total, int changed, int failed) + { + if (Check) + { + if (changed > 0 || + failed > 0) + { + throw new CommandException($"{Count(total)} checked, {changed} not deterministic{(failed > 0 ? $", {failed} failed" : null)}."); + } + + await Write(console, $"{Count(total)} checked, all deterministic."); + return; + } + + await Write(console, $"{Count(total)} processed, {changed} converted."); + + if (failed > 0) + { + throw new CommandException($"{Count(failed)} failed."); + } + } + + Task Write(IConsole console, string message) + { + if (Quiet) + { + return Task.CompletedTask; + } + + return console.Output.WriteLineAsync(message); + } + + static string Count(int value) => value == 1 ? "1 file" : $"{value} files"; + + static string Relative(string path) => Path.GetRelativePath(Directory.GetCurrentDirectory(), path); +} diff --git a/src/DeterministicIoPackaging.Tool/DeterministicIoPackaging.Tool.csproj b/src/DeterministicIoPackaging.Tool/DeterministicIoPackaging.Tool.csproj new file mode 100644 index 0000000..27462fa --- /dev/null +++ b/src/DeterministicIoPackaging.Tool/DeterministicIoPackaging.Tool.csproj @@ -0,0 +1,18 @@ + + + net10.0 + Exe + true + detpackage + false + LatestMajor + packaging, opc, nupkg, xlsx, docx, pptx, deterministic, reproducible, cli, dotnet-tool + Command line tool that modifies System.IO.Packaging files (nupkg, xlsx, docx, pptx) to ensure they are deterministic. Helpful for testing, build reproducibility, security verification, and ensuring package integrity across different build environments. + + + + + + + + diff --git a/src/DeterministicIoPackaging.Tool/FileJob.cs b/src/DeterministicIoPackaging.Tool/FileJob.cs new file mode 100644 index 0000000..0e70d66 --- /dev/null +++ b/src/DeterministicIoPackaging.Tool/FileJob.cs @@ -0,0 +1,5 @@ +// A single file to process, and where its result is written. Target equals Source for an in place run. +record FileJob(string Source, string Target) +{ + public bool IsInPlace => string.Equals(Source, Target, PathComparison.Value); +} diff --git a/src/DeterministicIoPackaging.Tool/FileResolver.cs b/src/DeterministicIoPackaging.Tool/FileResolver.cs new file mode 100644 index 0000000..ef10ef0 --- /dev/null +++ b/src/DeterministicIoPackaging.Tool/FileResolver.cs @@ -0,0 +1,104 @@ +// Expands the input path parameter into the set of files to process, pairing each with its output +// path. Both the file and the directory forms are supported, and an omitted target means in place. +static class FileResolver +{ + public static IReadOnlyList Resolve(string input, string? target, IReadOnlyList patterns, bool recursive) + { + var fullInput = Path.GetFullPath(input); + + if (File.Exists(fullInput)) + { + return [new(fullInput, ResolveFileTarget(fullInput, target))]; + } + + if (Directory.Exists(fullInput)) + { + return ResolveDirectory(fullInput, target, patterns, recursive); + } + + throw new CommandException($"Path not found: {input}"); + } + + // A target that names an existing directory, or is written with a trailing separator, keeps the + // source file name. Anything else is the output file path itself. + static string ResolveFileTarget(string source, string? target) + { + if (target == null) + { + return source; + } + + if (Directory.Exists(target) || + EndsWithSeparator(target)) + { + return Path.Combine(Path.GetFullPath(target), Path.GetFileName(source)); + } + + return Path.GetFullPath(target); + } + + static IReadOnlyList ResolveDirectory(string directory, string? target, IReadOnlyList patterns, bool recursive) + { + var fullTarget = target == null ? null : Path.GetFullPath(target); + var searchOption = recursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly; + + // Sorted so the order of the run does not depend on the order of the patterns or on the + // order the file system happens to enumerate in, and to drop duplicates when patterns overlap. + var sources = new SortedSet(PathComparison.Comparer); + foreach (var pattern in patterns) + { + foreach (var file in Directory.EnumerateFiles(directory, pattern, searchOption)) + { + if (!MatchesExtension(pattern, file)) + { + continue; + } + + // A target nested inside the input directory would otherwise feed its own output + // back in on a recursive run. + if (fullTarget != null && + IsUnder(fullTarget, file)) + { + continue; + } + + sources.Add(file); + } + } + + var jobs = new List(sources.Count); + foreach (var source in sources) + { + if (fullTarget == null) + { + jobs.Add(new(source, source)); + continue; + } + + jobs.Add(new(source, Path.Combine(fullTarget, Path.GetRelativePath(directory, source)))); + } + + return jobs; + } + + // Windows keeps legacy 8.3 name matching, so a "*.doc" search pattern also matches "report.docx". + // Re-check the extension for the plain "*.extension" pattern shape. + static bool MatchesExtension(string pattern, string file) + { + if (!pattern.StartsWith("*.") || + pattern.IndexOf('*', 2) != -1 || + pattern.Contains('?')) + { + return true; + } + + return string.Equals(Path.GetExtension(file), pattern[1..], StringComparison.OrdinalIgnoreCase); + } + + static bool IsUnder(string directory, string path) => + path.StartsWith(directory.TrimEnd(Path.DirectorySeparatorChar) + Path.DirectorySeparatorChar, PathComparison.Value); + + static bool EndsWithSeparator(string path) => + path.EndsWith(Path.DirectorySeparatorChar) || + path.EndsWith(Path.AltDirectorySeparatorChar); +} diff --git a/src/DeterministicIoPackaging.Tool/GlobalUsings.cs b/src/DeterministicIoPackaging.Tool/GlobalUsings.cs new file mode 100644 index 0000000..2061dba --- /dev/null +++ b/src/DeterministicIoPackaging.Tool/GlobalUsings.cs @@ -0,0 +1,4 @@ +global using CliFx; +global using CliFx.Binding; +global using CliFx.Infrastructure; +global using DeterministicIoPackaging; diff --git a/src/DeterministicIoPackaging.Tool/PathComparison.cs b/src/DeterministicIoPackaging.Tool/PathComparison.cs new file mode 100644 index 0000000..b692abb --- /dev/null +++ b/src/DeterministicIoPackaging.Tool/PathComparison.cs @@ -0,0 +1,9 @@ +// File system case sensitivity: Windows and macOS treat paths case insensitively, Linux does not. +static class PathComparison +{ + public static StringComparison Value { get; } = + OperatingSystem.IsLinux() ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; + + public static StringComparer Comparer { get; } = + OperatingSystem.IsLinux() ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase; +} diff --git a/src/DeterministicIoPackaging.Tool/Program.cs b/src/DeterministicIoPackaging.Tool/Program.cs new file mode 100644 index 0000000..422e57d --- /dev/null +++ b/src/DeterministicIoPackaging.Tool/Program.cs @@ -0,0 +1,7 @@ +return await new CommandLineApplicationBuilder() + .AddCommandsFromThisAssembly() + .SetExecutableName("detpackage") + .SetTitle("DeterministicIoPackaging CLI") + .SetDescription("Rewrites a System.IO.Packaging file (nupkg, xlsx, docx, pptx) so the same source package always produces byte-identical output.") + .Build() + .RunAsync(); diff --git a/src/DeterministicIoPackaging.slnx b/src/DeterministicIoPackaging.slnx index 2e925b8..03d5a7a 100644 --- a/src/DeterministicIoPackaging.slnx +++ b/src/DeterministicIoPackaging.slnx @@ -12,6 +12,7 @@ + diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index d164170..0ed70f9 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -4,6 +4,7 @@ true +