Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions pkg/guidednavigation/converter/converter.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
"golang.org/x/net/html/atom"
)

func Do(ctx context.Context, resource fetcher.Resource, locator manifest.Locator) (*guidednavigation.GuidedNavigationDocument, error) {
func Do(ctx context.Context, resource fetcher.Resource, locator manifest.Locator, opts ...Option) (*guidednavigation.GuidedNavigationDocument, error) {
raw, rerr := fetcher.ReadResourceAsString(ctx, resource)
if rerr != nil {
return nil, errors.Wrap(rerr, "failed reading HTML string of "+resource.Link().Href.String())
Expand Down Expand Up @@ -45,7 +45,7 @@ func Do(ctx context.Context, resource fetcher.Resource, locator manifest.Locator
return nil, errors.New("HTML of " + resource.Link().Href.String() + " doesn't have a <body>")
}

contentConverter := NewHTMLConverter(locator)
contentConverter := NewHTMLConverter(locator, opts...)
contentConverter.xmlParsed = xmlParsed

// Traverse the document's HTML
Expand Down
65 changes: 63 additions & 2 deletions pkg/guidednavigation/converter/converter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
"github.com/stretchr/testify/require"
)

func convertDoc(t *testing.T, doc string, mt *mediatype.MediaType) string {
func convertDoc(t *testing.T, doc string, mt *mediatype.MediaType, opts ...Option) string {
t.Helper()
f := fetcher.NewBytesResource(manifest.Link{
Href: manifest.MustNewHREFFromString("hello.xhtml", false),
Expand All @@ -23,7 +23,7 @@ func convertDoc(t *testing.T, doc string, mt *mediatype.MediaType) string {

nav, err := Do(context.Background(), f, manifest.Locator{
Href: f.Link().Href.Resolve(nil, nil),
})
}, opts...)
require.NoError(t, err)
bin, err := json.Marshal(nav)
require.NoError(t, err)
Expand All @@ -45,6 +45,12 @@ func convertBody(t *testing.T, body string) string {
return convertDoc(t, wrapXHTML("", body), &mediatype.XHTML)
}

// Like convertBody, with textref locators enabled.
func convertBodyWithLocators(t *testing.T, body string) string {
t.Helper()
return convertDoc(t, wrapXHTML("", body), &mediatype.XHTML, WithTextRefLocators())
}

// The expected guided navigation document JSON for the given children of <body>.
func guidedJSON(children string) string {
res := `{"guided":[{"role":["body"],"textref":"hello.xhtml"`
Expand Down Expand Up @@ -736,3 +742,58 @@ func TestDoErrorsWithoutBody(t *testing.T) {
})
require.Error(t, err)
}

// With WithTextRefLocators, every object carrying roles points back at its source
// element: by fragment id when the element has one, by css() selector otherwise.
// Selectors are anchored to the closest ancestor with an id.
func TestConvertTextRefLocators(t *testing.T) {
res := convertBodyWithLocators(t, `
<section><p>Hello <img src="img.png" alt="An image"/> world</p></section>
<section id="s2"><p>Two</p></section>`)
assert.JSONEq(t, guidedJSON(`{
"role": ["section"],
"textref": "hello.xhtml#css(body%20%3E%20section:nth-child(1))",
"children": [{
"role": ["paragraph"],
"textref": "hello.xhtml#css(body%20%3E%20section:nth-child(1)%20%3E%20p)",
"text": {
"plain": "Hello world",
"ssml": "Hello <readium:image id=\"image1\"/> world"
},
"children": [{
"id": "image1",
"role": ["image"],
"imgref": "img.png",
"textref": "hello.xhtml#css(body%20%3E%20section:nth-child(1)%20%3E%20p%20%3E%20img)",
"description": "An image"
}]
}]
},
{
"role": ["section"],
"textref": "hello.xhtml#s2",
"children": [{
"role": ["paragraph"],
"textref": "hello.xhtml#css(%23s2%20%3E%20p)",
"text": "Two"
}]
}`), res)
}

// Objects that carry nothing but a role are still emitted when locators are
// enabled, since the locator alone makes them useful. Pagebreaks without an id
// fall back to a css() selector too.
func TestConvertTextRefLocatorsStandalone(t *testing.T) {
res := convertBodyWithLocators(t, `
<hr/>
<span epub:type="pagebreak" title="5"></span>`)
assert.JSONEq(t, guidedJSON(`{
"role": ["separator"],
"textref": "hello.xhtml#css(body%20%3E%20hr)"
},
{
"role": ["pagebreak"],
"text": "5",
"textref": "hello.xhtml#css(body%20%3E%20span)"
}`), res)
}
71 changes: 60 additions & 11 deletions pkg/guidednavigation/converter/html.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"unicode"

"github.com/readium/go-toolkit/pkg/guidednavigation"
iutil "github.com/readium/go-toolkit/pkg/internal/util"
"github.com/readium/go-toolkit/pkg/manifest"
"github.com/readium/go-toolkit/pkg/util/url"
"golang.org/x/net/html"
Expand Down Expand Up @@ -340,9 +341,24 @@ func (a *idAllocator) allocate(prefix string) string {
// contain noterefs. Beyond it, notes are referenced by textref instead.
const maxNoterefDepth = 3

// An Option configures an [HTMLConverter].
type Option func(*HTMLConverter)

// WithTextRefLocators makes every emitted object whose element carries roles
// reference its location in the source document through textref: the element's
// fragment id when it has one (e.g. "chapter.xhtml#par1"), a css() fragment with
// a unique CSS selector otherwise (e.g. "chapter.xhtml#css(body%20%3E%20p:nth-child(3))").
// The selectors are anchored to the closest ancestor with an id when there is one.
func WithTextRefLocators() Option {
return func(c *HTMLConverter) {
c.textRefLocators = true
}
}

type HTMLConverter struct {
baseLocator manifest.Locator
xmlParsed bool // Whether the tree comes from an XML parser (self-closing tags handled correctly).
baseLocator manifest.Locator
xmlParsed bool // Whether the tree comes from an XML parser (self-closing tags handled correctly).
textRefLocators bool // Whether every object with roles gets a textref locating its source element.

segments []textSegment // Closed segments of the text flow accumulated for the current block.
textAcc strings.Builder // Text of the currently open segment, with coalesced whitespace.
Expand All @@ -362,10 +378,14 @@ type HTMLConverter struct {
allowNode *html.Node // Node exempt from suppression/visibility checks (target of a noteref sub-conversion).
}

func NewHTMLConverter(baseLocator manifest.Locator) *HTMLConverter {
return &HTMLConverter{
func NewHTMLConverter(baseLocator manifest.Locator, opts ...Option) *HTMLConverter {
c := &HTMLConverter{
baseLocator: baseLocator,
}
for _, opt := range opts {
opt(c)
}
return c
}

// Whether an element opens (and closes) a navigation object during the traversal.
Expand Down Expand Up @@ -393,6 +413,24 @@ func (c *HTMLConverter) fragmentRef(id string) url.URL {
return c.baseLocator.Href.Resolve(frag)
}

// Builds a reference locating an element of the converted resource: the element's
// fragment id when it has one, a css() fragment with a unique CSS selector otherwise,
// e.g. "chapter.xhtml#css(body%20%3E%20p:nth-child(3))".
func (c *HTMLConverter) nodeRef(n *html.Node) url.URL {
if id := getAttr(n, "id"); id != "" {
return c.fragmentRef(id)
}
sel := iutil.CSSSelector(n)
if sel == "" || c.baseLocator.Href == nil {
return nil
}
frag, err := url.URLFromGo(&nurl.URL{Fragment: "css(" + sel + ")"})
if err != nil {
return nil
}
return c.baseLocator.Href.Resolve(frag)
}

func (c *HTMLConverter) resolveHref(href string) url.URL {
u, err := url.URLFromString(href)
if err != nil || u == nil {
Expand Down Expand Up @@ -703,6 +741,11 @@ func (c *HTMLConverter) head(n *html.Node) {
if n.DataAtom == atom.Body {
// Contextualize the top-level object per the specification
cur.TextRef = c.baseLocator.Href
} else if c.textRefLocators {
// Locate every object carrying roles in the source document
if len(roles) > 0 {
cur.TextRef = c.nodeRef(n)
}
} else if len(roles) > 0 && !c.current.noText {
if id := getAttr(n, "id"); id != "" {
cur.TextRef = c.fragmentRef(id)
Expand Down Expand Up @@ -808,6 +851,11 @@ func (c *HTMLConverter) placeholderWithID(n *html.Node, tag string, object guide
// Registering it anyway would leave a dangling id in the SSML.
return
}
if c.textRefLocators && object.TextRef == nil {
// Locate the object's source element (checked after Empty so that
// skipped elements don't come back as bare locators)
object.TextRef = c.nodeRef(n)
}
child := &navigationObject{node: n, object: object}
if c.current.noText {
// The surrounding text is suppressed: keep the object, without a placeholder
Expand Down Expand Up @@ -875,13 +923,14 @@ func (c *HTMLConverter) noteref(n *html.Node, roles []guidednavigation.GuidedNav
// section) are only referenced, never embedded.
if target := c.ids[fragment]; target != nil && !isAncestorOf(target, n) && c.noterefDepth < maxNoterefDepth {
sub := &HTMLConverter{
baseLocator: c.baseLocator,
xmlParsed: c.xmlParsed,
ids: c.ids,
suppressed: c.suppressed,
idAlloc: c.idAlloc,
noterefDepth: c.noterefDepth + 1,
allowNode: target,
baseLocator: c.baseLocator,
xmlParsed: c.xmlParsed,
textRefLocators: c.textRefLocators,
ids: c.ids,
suppressed: c.suppressed,
idAlloc: c.idAlloc,
noterefDepth: c.noterefDepth + 1,
allowNode: target,
}
sub.Convert(target)
obj.Children = sub.Result()
Expand Down
47 changes: 46 additions & 1 deletion pkg/guidednavigation/fragments.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ import (
// https://idpf.org/epub/renditions/region-nav/#sec-3.5.1
// - Text: fragment ids ("chapter.html#par1") and text fragments
// https://wicg.github.io/scroll-to-text-fragment/
// - Text: css() selector fragments locating an element of an (X)HTML resource,
// e.g. "chapter.html#css(body%20%3E%20p:nth-child(3))", as emitted by the
// converter package's WithTextRefLocators option

// The unit of a spatial fragment's coordinates.
type RegionUnit string
Expand Down Expand Up @@ -161,6 +164,13 @@ func (o GuidedNavigationObject) TextFragments() []TextFragment {
return refTextFragments(o.TextRef)
}

// TextCSSSelector returns the CSS selector of the object's text reference
// (e.g. "body > p:nth-child(3)" for "chapter.html#css(body%20%3E%20p:nth-child(3))"),
// or an empty string when the reference has no css() fragment.
func (o GuidedNavigationObject) TextCSSSelector() string {
return refCSSSelector(o.TextRef)
}

// AudioFile returns the audio resource referenced by the description, without its media fragment.
func (d GuidedNavigationDescription) AudioFile() url.URL {
return refFile(d.AudioRef)
Expand Down Expand Up @@ -223,6 +233,12 @@ func (d GuidedNavigationDescription) TextFragments() []TextFragment {
return refTextFragments(d.TextRef)
}

// TextCSSSelector returns the CSS selector of the description's text reference,
// or an empty string when the reference has no css() fragment.
func (d GuidedNavigationDescription) TextCSSSelector() string {
return refCSSSelector(d.TextRef)
}

// The reference without its fragment.
func refFile(ref url.URL) url.URL {
if ref == nil {
Expand Down Expand Up @@ -503,11 +519,40 @@ func refFragmentID(ref url.URL) string {
// A fragment directive doesn't belong to the fragment id preceding it
fragment, _, _ = strings.Cut(fragment, textDirectiveDelimiter)
if decoded, err := nurl.PathUnescape(fragment); err == nil {
return decoded
fragment = decoded
}
if _, ok := cssSelectorFragment(fragment); ok {
// A css() fragment is a selector, not an id
return ""
}
return fragment
}

// The selector of a css() fragment, e.g. "css(body > p)" -> "body > p".
// The fragment is expected in decoded form.
func cssSelectorFragment(fragment string) (string, bool) {
inner, ok := strings.CutPrefix(fragment, "css(")
if !ok {
return "", false
}
inner, ok = strings.CutSuffix(inner, ")")
if !ok {
return "", false
}
return inner, true
}

func refCSSSelector(ref url.URL) string {
if ref == nil {
return ""
}
fragment := ref.Fragment()
// A fragment directive doesn't belong to the fragment preceding it
fragment, _, _ = strings.Cut(fragment, textDirectiveDelimiter)
sel, _ := cssSelectorFragment(fragment)
return sel
}

func refTextFragments(ref url.URL) []TextFragment {
if ref == nil {
return nil
Expand Down
24 changes: 24 additions & 0 deletions pkg/guidednavigation/fragments_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,8 @@ func TestTextFragmentID(t *testing.T) {
{"chapter1.html#id:~:text=highlighted", "id"},
{"chapter1.html#:~:text=highlighted", ""},
{"chapter1.html", ""},
// A css() fragment is a selector, not an id
{"chapter1.html#css(body%20%3E%20p:nth-child(3))", ""},
}
for _, tt := range tests {
obj := GuidedNavigationObject{TextRef: url.MustURLFromString(tt.ref)}
Expand All @@ -208,6 +210,28 @@ func TestTextFragmentID(t *testing.T) {
assert.Empty(t, GuidedNavigationObject{}.TextFragmentID())
}

func TestTextCSSSelector(t *testing.T) {
tests := []struct {
ref string
selector string
}{
{"chapter1.html#css(body%20%3E%20p:nth-child(3))", "body > p:nth-child(3)"},
{"chapter1.html#css(%23s2%20%3E%20p)", "#s2 > p"},
{"chapter1.html#css(body%20%3E%20img)", "body > img"},
{"chapter1.html#start", ""},
{"chapter1.html#css(unterminated", ""},
{"chapter1.html", ""},
}
for _, tt := range tests {
obj := GuidedNavigationObject{TextRef: url.MustURLFromString(tt.ref)}
assert.Equal(t, tt.selector, obj.TextCSSSelector(), tt.ref)
desc := GuidedNavigationDescription{TextRef: url.MustURLFromString(tt.ref)}
assert.Equal(t, tt.selector, desc.TextCSSSelector(), tt.ref)
}
assert.Empty(t, GuidedNavigationObject{}.TextCSSSelector())
assert.Empty(t, GuidedNavigationDescription{}.TextCSSSelector())
}

func TestTextFragments(t *testing.T) {
// Example from the specification's README
obj := GuidedNavigationObject{TextRef: url.MustURLFromString(
Expand Down
13 changes: 8 additions & 5 deletions pkg/internal/util/css.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (

"github.com/agext/regexp"
"github.com/andybalholm/cascadia"
"github.com/pkg/errors"
"golang.org/x/net/html"
"golang.org/x/net/html/atom"
)
Expand Down Expand Up @@ -101,11 +100,15 @@ func CSSSelector(n *html.Node) string {
return selector.String()
}

s, err := cascadia.Parse(selector.String())
if err != nil {
panic(errors.Wrap(err, "failed parsing generated CSS selector"))
// Check whether the selector needs disambiguation among the parent's contents.
// When the rudimentary escaping above produces a selector cascadia fails to
// parse, matches cannot be counted: conservatively pin the element's position,
// a redundant nth-child is still correct.
ambiguous := true
if s, err := cascadia.Parse(selector.String()); err == nil {
ambiguous = len(cascadia.QueryAll(n.Parent, s)) > 1
}
if nodes := cascadia.QueryAll(n.Parent, s); len(nodes) > 1 {
if ambiguous {
// Figure out the index of this node among its siblings
idx := 1
for ps := n.PrevSibling; ps != nil; ps = ps.PrevSibling {
Expand Down
Loading