diff --git a/README.md b/README.md index bc65fb1..08370fa 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,17 @@ fmt.Println(expr.String()) // "MIT OR (GPL-2.0-only AND Apache-2.0)" // ParseStrict requires valid SPDX IDs (no fuzzy normalization) expr, err := spdx.ParseStrict("MIT OR Apache-2.0") // succeeds expr, err := spdx.ParseStrict("Apache 2 OR MIT") // fails + +// ParseSyntax validates expression grammar without requiring identifiers +// to exist in the SPDX list bundled by this module +expr, err := spdx.ParseSyntax("Future-License-1.0 OR MIT") // succeeds ``` +`ParseSyntax` is useful when the caller validates identifiers against another +pinned data source. Known identifiers are returned in their canonical form, +while well-formed unknown license and exception identifiers are preserved. +`ParseStrict` continues to require identifiers from the bundled SPDX list. + ### Rewrite expression identifiers Parse an expression once, then replace its identifiers while keeping its diff --git a/parse.go b/parse.go index c1b2301..fa5dfd7 100644 --- a/parse.go +++ b/parse.go @@ -255,13 +255,17 @@ const ( // parser parses SPDX expressions. type parser struct { - lexer *lexer - current token - depth int + lexer *lexer + current token + depth int + allowUnknownIdentifiers bool } -func newParser(input string) (*parser, error) { - p := &parser{lexer: newLexer(input)} +func newParser(input string, allowUnknownIdentifiers bool) (*parser, error) { + p := &parser{ + lexer: newLexer(input), + allowUnknownIdentifiers: allowUnknownIdentifiers, + } tok, err := p.lexer.next() if err != nil { return nil, err @@ -306,21 +310,12 @@ func Parse(expression string) (Expression, error) { return nil, err } - p, err := newParser(normalized) - if err != nil { - return nil, err - } - - expr, err := p.parseExpression() + p, err := newParser(normalized, false) if err != nil { return nil, err } - if p.current.typ != tokenEOF { - return nil, fmt.Errorf("%w: %s", ErrUnexpectedToken, p.current.value) - } - - return expr, nil + return p.parse() } // ParseStrict parses an SPDX expression requiring strict SPDX identifiers. @@ -333,6 +328,30 @@ func Parse(expression string) (Expression, error) { // ParseStrict("MIT OR Apache-2.0") // succeeds // ParseStrict("mit OR apache 2") // fails - "apache 2" is not a valid SPDX ID func ParseStrict(expression string) (Expression, error) { + return parseWithoutNormalization(expression, false) +} + +// ParseSyntax parses an SPDX expression without requiring bare license and +// exception identifiers to exist in the bundled SPDX identifier list. It +// validates identifier syntax, operators, modifiers, and grouping. Known +// identifiers are returned in their canonical form; unknown identifiers are +// preserved as written. +// +// Use ParseStrict when identifiers must also be present in the bundled SPDX +// list. +// +// Example: +// +// ParseSyntax("Future-License-1.0 OR MIT") // succeeds +// ParseStrict("Future-License-1.0 OR MIT") // fails +func ParseSyntax(expression string) (Expression, error) { + return parseWithoutNormalization(expression, true) +} + +func parseWithoutNormalization( + expression string, + allowUnknownIdentifiers bool, +) (Expression, error) { expression = strings.TrimSpace(expression) if expression == "" { return nil, ErrEmptyExpression @@ -341,11 +360,15 @@ func ParseStrict(expression string) (Expression, error) { return nil, ErrExpressionTooLarge } - p, err := newParser(expression) + p, err := newParser(expression, allowUnknownIdentifiers) if err != nil { return nil, err } + return p.parse() +} + +func (p *parser) parse() (Expression, error) { expr, err := p.parseExpression() if err != nil { return nil, err @@ -426,9 +449,9 @@ func (p *parser) parseWith() (Expression, error) { return nil, fmt.Errorf("%w: expected exception after WITH", ErrMissingOperand) } - exception := lookupException(p.current.value) - if exception == "" { - return nil, fmt.Errorf("%w: %s", ErrInvalidException, p.current.value) + exception, err := p.resolveExceptionIdentifier(p.current.value) + if err != nil { + return nil, err } license.Exception = exception @@ -471,59 +494,116 @@ func (p *parser) parseAtom() (Expression, error) { return expr, nil case tokenLicense: - value := p.current.value - upper := strings.ToUpper(value) - - // Handle special values - if upper == "NONE" || upper == "NOASSERTION" { - if err := p.advance(); err != nil { - return nil, err - } - return &SpecialValue{Value: upper}, nil - } + return p.parseLicenseAtom() - // Look up the canonical license ID - id := lookupLicense(value) - if id == "" { - return nil, fmt.Errorf("%w: %s", ErrInvalidLicenseID, value) - } - - license := &License{ID: id} + case tokenLicenseRef: + return p.parseLicenseReference(false) - if err := p.advance(); err != nil { - return nil, err - } + case tokenDocumentRef: + return p.parseLicenseReference(true) - // Check for + - if p.current.typ == tokenPlus { - license.Plus = true - if err := p.advance(); err != nil { - return nil, err - } - } + case tokenEOF: + return nil, ErrMissingOperand - return license, nil + default: + return nil, fmt.Errorf("%w: %s", ErrUnexpectedToken, p.current.value) + } +} - case tokenLicenseRef: - ref := parseLicenseRef(p.current.value) +func (p *parser) parseLicenseAtom() (Expression, error) { + value := p.current.value + upper := strings.ToUpper(value) + if upper == "NONE" || upper == "NOASSERTION" { if err := p.advance(); err != nil { return nil, err } - return ref, nil + return &SpecialValue{Value: upper}, nil + } - case tokenDocumentRef: - ref := parseDocumentRef(p.current.value) + id, err := p.resolveLicenseIdentifier(value) + if err != nil { + return nil, err + } + license := &License{ID: id} + if err := p.advance(); err != nil { + return nil, err + } + if p.current.typ == tokenPlus { + license.Plus = true if err := p.advance(); err != nil { return nil, err } - return ref, nil + } + return license, nil +} - case tokenEOF: - return nil, ErrMissingOperand +func (p *parser) resolveLicenseIdentifier(identifier string) (string, error) { + if id := lookupLicense(identifier); id != "" { + return id, nil + } + if p.allowUnknownIdentifiers && validIdentifier(identifier) { + return identifier, nil + } + return "", fmt.Errorf("%w: %s", ErrInvalidLicenseID, identifier) +} - default: - return nil, fmt.Errorf("%w: %s", ErrUnexpectedToken, p.current.value) +func (p *parser) resolveExceptionIdentifier(identifier string) (string, error) { + if exception := lookupException(identifier); exception != "" { + return exception, nil } + if p.allowUnknownIdentifiers && validIdentifier(identifier) { + return identifier, nil + } + return "", fmt.Errorf("%w: %s", ErrInvalidException, identifier) +} + +func (p *parser) parseLicenseReference(document bool) (Expression, error) { + value := p.current.value + var reference *LicenseRef + if document { + reference = parseDocumentRef(value) + } else { + reference = parseLicenseRef(value) + } + if !validLicenseReference(reference, document) { + return nil, fmt.Errorf("%w: %s", ErrInvalidLicenseID, value) + } + if err := p.advance(); err != nil { + return nil, err + } + return reference, nil +} + +func validLicenseReference(reference *LicenseRef, document bool) bool { + if reference == nil || !validIdentifier(reference.LicenseRef) { + return false + } + if document { + return validIdentifier(reference.DocumentRef) + } + return reference.DocumentRef == "" +} + +func validIdentifier(identifier string) bool { + if identifier == "" || !isIdentifierAlphanumeric(rune(identifier[0])) || + !isIdentifierAlphanumeric(rune(identifier[len(identifier)-1])) { + return false + } + for _, character := range identifier { + switch { + case isIdentifierAlphanumeric(character): + case character == '-', character == '.': + default: + return false + } + } + return true +} + +func isIdentifierAlphanumeric(character rune) bool { + return character >= 'a' && character <= 'z' || + character >= 'A' && character <= 'Z' || + character >= '0' && character <= '9' } // parseLicenseRef parses "LicenseRef-xxx" into a LicenseRef. diff --git a/parse_syntax_test.go b/parse_syntax_test.go new file mode 100644 index 0000000..ab394c0 --- /dev/null +++ b/parse_syntax_test.go @@ -0,0 +1,162 @@ +package spdx + +import ( + "errors" + "reflect" + "strings" + "testing" +) + +func TestParseSyntaxAllowsUnknownIdentifiers(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + want string + err error + }{ + { + name: "license", + input: "Future-License-9.9 OR MIT", + want: "Future-License-9.9 OR MIT", + err: ErrInvalidLicenseID, + }, + { + name: "license with interior punctuation", + input: "Future.License-9.9 OR MIT", + want: "Future.License-9.9 OR MIT", + err: ErrInvalidLicenseID, + }, + { + name: "exception", + input: "GPL-2.0-only WITH Future-exception-9.9", + want: "GPL-2.0-only WITH Future-exception-9.9", + err: ErrInvalidException, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + expression, err := ParseSyntax(test.input) + if err != nil { + t.Fatalf("ParseSyntax(%q): %v", test.input, err) + } + if got := expression.String(); got != test.want { + t.Errorf("ParseSyntax(%q) = %q, want %q", test.input, got, test.want) + } + + _, err = ParseStrict(test.input) + if !errors.Is(err, test.err) { + t.Errorf("ParseStrict(%q) error = %v, want %v", test.input, err, test.err) + } + }) + } +} + +func TestParseSyntaxMatchesStrictAST(t *testing.T) { + t.Parallel() + + inputs := []string{ + "mit", + "MIT OR Apache-2.0 AND BSD-3-Clause", + "(MIT OR Apache-2.0) AND GPL-2.0-only", + "GPL-2.0-only WITH Classpath-exception-2.0", + "GPL-2.0+", + "LicenseRef-custom AND DocumentRef-vendor:LicenseRef-proprietary", + "LicenseRef-custom.1-2 AND DocumentRef-vendor.1:LicenseRef-proprietary-2", + "NONE", + "NOASSERTION", + } + for _, input := range inputs { + t.Run(input, func(t *testing.T) { + t.Parallel() + + syntaxExpression, err := ParseSyntax(input) + if err != nil { + t.Fatalf("ParseSyntax(%q): %v", input, err) + } + strictExpression, err := ParseStrict(input) + if err != nil { + t.Fatalf("ParseStrict(%q): %v", input, err) + } + if !reflect.DeepEqual(syntaxExpression, strictExpression) { + t.Errorf( + "ASTs differ:\nParseSyntax: %#v\nParseStrict: %#v", + syntaxExpression, + strictExpression, + ) + } + }) + } +} + +func TestParseSyntaxRejectsMalformedExpressions(t *testing.T) { + t.Parallel() + + inputs := []string{ + "", + "MIT OR", + "AND MIT", + "(MIT OR Apache-2.0", + "MIT OR Apache-2.0)", + "MIT WITH", + "MIT/Apache-2.0", + "Future_License-1.0", + "-MIT", + "MIT-", + ".MIT", + "MIT.", + ".", + "-", + "MIT WITH Future/exception", + "MIT WITH -Future-exception", + "MIT WITH Future-exception-", + "MIT WITH .", + "LicenseRef-", + "LicenseRef--", + "LicenseRef-.", + "LicenseRef--custom", + "LicenseRef-custom-", + "LicenseRef-.custom", + "LicenseRef-custom.", + "LicenseRef-invalid/value", + "DocumentRef-vendor", + "DocumentRef--:LicenseRef-custom", + "DocumentRef-.:LicenseRef-custom", + "DocumentRef--vendor:LicenseRef-custom", + "DocumentRef-vendor-:LicenseRef-custom", + "DocumentRef-:LicenseRef-custom", + "DocumentRef-vendor:LicenseRef-", + "DocumentRef-vendor:LicenseRef-invalid/value", + } + for _, input := range inputs { + t.Run(input, func(t *testing.T) { + t.Parallel() + + if _, err := ParseSyntax(input); err == nil { + t.Errorf("ParseSyntax(%q) succeeded", input) + } + if _, err := ParseStrict(input); err == nil { + t.Errorf("ParseStrict(%q) succeeded", input) + } + }) + } +} + +func TestParseSyntaxEnforcesLimits(t *testing.T) { + t.Parallel() + + oversized := strings.Repeat("X", maxParseLength+1) + if _, err := ParseSyntax(oversized); !errors.Is(err, ErrExpressionTooLarge) { + t.Errorf("oversized expression error = %v, want %v", err, ErrExpressionTooLarge) + } + + deeplyNested := strings.Repeat("(", maxParseDepth+1) + + "Future-License-9.9" + + strings.Repeat(")", maxParseDepth+1) + if _, err := ParseSyntax(deeplyNested); !errors.Is(err, ErrExpressionTooLarge) { + t.Errorf("deeply nested expression error = %v, want %v", err, ErrExpressionTooLarge) + } +}