Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 12 additions & 10 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,17 @@ go 1.26.1

require (
github.com/Masterminds/semver/v3 v3.5.0
github.com/MontFerret/contrib/modules/csv v1.0.0-rc.4
github.com/MontFerret/contrib/modules/db/sqlite v1.0.0-rc.2
github.com/MontFerret/contrib/modules/security/jwt v1.0.0-rc.2
github.com/MontFerret/contrib/modules/toml v1.0.0-rc.4
github.com/MontFerret/contrib/modules/web/article v1.0.0-rc.5
github.com/MontFerret/contrib/modules/web/html v1.0.0-rc.8
github.com/MontFerret/contrib/modules/web/robots v1.0.0-rc.4
github.com/MontFerret/contrib/modules/web/sitemap v1.0.0-rc.4
github.com/MontFerret/contrib/modules/xml v1.0.0-rc.4
github.com/MontFerret/contrib/modules/yaml v1.0.0-rc.4
github.com/MontFerret/contrib/modules/csv v1.0.0-rc.6
github.com/MontFerret/contrib/modules/db/sqlite v1.0.0-rc.4
github.com/MontFerret/contrib/modules/net/rest v1.0.0-rc.2
github.com/MontFerret/contrib/modules/security/jwt v1.0.0-rc.4
github.com/MontFerret/contrib/modules/toml v1.0.0-rc.6
github.com/MontFerret/contrib/modules/web/article v1.0.0-rc.7
github.com/MontFerret/contrib/modules/web/html v1.0.0-rc.10
github.com/MontFerret/contrib/modules/web/robots v1.0.0-rc.6
github.com/MontFerret/contrib/modules/web/sitemap v1.0.0-rc.6
github.com/MontFerret/contrib/modules/xml v1.0.0-rc.6
github.com/MontFerret/contrib/modules/yaml v1.0.0-rc.6
github.com/MontFerret/ferret/v2 v2.0.0-alpha.25
github.com/chzyer/readline v1.5.1
github.com/go-waitfor/waitfor v1.1.0
Expand All @@ -31,6 +32,7 @@ require (
github.com/BurntSushi/toml v1.6.0 // indirect
github.com/JohannesKaufmann/dom v0.3.1 // indirect
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 // indirect
github.com/MontFerret/contrib/pkg/common v0.1.1 // indirect
github.com/MontFerret/cssx v0.2.0 // indirect
github.com/PuerkitoBio/goquery v1.12.0 // indirect
github.com/andybalholm/cascadia v1.3.4 // indirect
Expand Down
44 changes: 24 additions & 20 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -8,26 +8,30 @@ github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 h1:XFJZFWESIWlUEHHjzBuv8R
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2/go.mod h1:BHWO8lJzttJLqwuV8Rb1B3OG2OSzLbssZDI1FRg2eAA=
github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE=
github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
github.com/MontFerret/contrib/modules/csv v1.0.0-rc.4 h1:SFlZQq7gjKq1gM+cKp+b4zW+eNZmsxN5t1Z89cdPBeQ=
github.com/MontFerret/contrib/modules/csv v1.0.0-rc.4/go.mod h1:5YB64P+mrwzHtS6DYWumRLZZdY9JBBuV5fud10VxY34=
github.com/MontFerret/contrib/modules/db/sqlite v1.0.0-rc.2 h1:sAgF/2ljDizfJvyRa5/elMUgbrPsV21SG/lDvvrTiQY=
github.com/MontFerret/contrib/modules/db/sqlite v1.0.0-rc.2/go.mod h1:X3cTsLMYWZs/d8qHsT0nuvT1wb5KL5BGQ6qvVrnawZM=
github.com/MontFerret/contrib/modules/security/jwt v1.0.0-rc.2 h1:3/L9zX1k7nhJZN2BZJA0znr/SfR9HssGTjGfIOWk9Mo=
github.com/MontFerret/contrib/modules/security/jwt v1.0.0-rc.2/go.mod h1:m8sqDatCGaIdgCsOkotvaWJmfUJRHbO9rSccjDpkfoU=
github.com/MontFerret/contrib/modules/toml v1.0.0-rc.4 h1:Q7Ck4Uk5tsA7tgI3f+5hLZ+EGjjN0VEUCP1dxKt34+w=
github.com/MontFerret/contrib/modules/toml v1.0.0-rc.4/go.mod h1:9pP9GMHsDYIn9b+TE45ntDSxL22EgqcA5U0UQ/dpHOg=
github.com/MontFerret/contrib/modules/web/article v1.0.0-rc.5 h1:gGfEj5+pd6GAZgYYUvF8XDaXAfgAO5MOs1Njni30GlY=
github.com/MontFerret/contrib/modules/web/article v1.0.0-rc.5/go.mod h1:WNQ0tQhLsi7URLYajggdZtGb0YcsJ1puIhJDtnbLlyU=
github.com/MontFerret/contrib/modules/web/html v1.0.0-rc.8 h1:RB82v46m0Sb6axIoOTkplCDT8YA/QfxQ273TGYadLPk=
github.com/MontFerret/contrib/modules/web/html v1.0.0-rc.8/go.mod h1:i5weUFsPLtgL2KdtSYKIbuOotlTjjRauIEAio+7y0Ro=
github.com/MontFerret/contrib/modules/web/robots v1.0.0-rc.4 h1:MwKcqLQQ341TH70O9QLh3DSamSrALcJ2EstWIFcKpnU=
github.com/MontFerret/contrib/modules/web/robots v1.0.0-rc.4/go.mod h1:AibYgVv4+XXkmZZHcFvi0IwQvd7DaFblUST/Z/jyDzA=
github.com/MontFerret/contrib/modules/web/sitemap v1.0.0-rc.4 h1:GBhDeV+NhcKiLFeJSlwdlhGENu2k+p5vWHPKovpRHGE=
github.com/MontFerret/contrib/modules/web/sitemap v1.0.0-rc.4/go.mod h1:1JqsUQiLyP6ZR9MH55aSB7gLJ2gDUwegkb+EMstPZEE=
github.com/MontFerret/contrib/modules/xml v1.0.0-rc.4 h1:PjT+hr/ZDRBiNbyePLP1fv4tjoIaLS/ADdrhLWR/6kw=
github.com/MontFerret/contrib/modules/xml v1.0.0-rc.4/go.mod h1:eMl++62DU9ejC2sFyCldTi2fXZTxX21l7S+u0XmmTZc=
github.com/MontFerret/contrib/modules/yaml v1.0.0-rc.4 h1:5vVxjrFKDCvAvUnQCZYJ+6YnjouSeTB1o+kgvB/Yxl0=
github.com/MontFerret/contrib/modules/yaml v1.0.0-rc.4/go.mod h1:T/pQe51gAbWk+nvXjsjk1QTivlfsVfZ3juOQVrHRnKw=
github.com/MontFerret/contrib/modules/csv v1.0.0-rc.6 h1:iXY/7/u47KRrcxdq/1CqtRRpvlHy7hGeacVUJqmNUnE=
github.com/MontFerret/contrib/modules/csv v1.0.0-rc.6/go.mod h1:GKx3zKQGcWIdB7BYUM/yKY+s+C2WPJWWiowPCl3hUBQ=
github.com/MontFerret/contrib/modules/db/sqlite v1.0.0-rc.4 h1:uPaX0JA5plRSNrfV48wamNKaivEB9/I/mUWZv5DUvbc=
github.com/MontFerret/contrib/modules/db/sqlite v1.0.0-rc.4/go.mod h1:Nq5GoZkAkpalBu5rH5gBhMGqF4s5F50sWcCY/vG/YhA=
github.com/MontFerret/contrib/modules/net/rest v1.0.0-rc.2 h1:50Qpx13rXZfmuwe0E+/EjoT93O7naaE6q1/N9b1KvPM=
github.com/MontFerret/contrib/modules/net/rest v1.0.0-rc.2/go.mod h1:rBLl+oruPTSu/IaONli5P3TH7APWFyCU3TItcEdiB2U=
github.com/MontFerret/contrib/modules/security/jwt v1.0.0-rc.4 h1:IpAYXY0u3rf3+ktz94DeRAj7rEsZ+HJyA4MGr1y89CY=
github.com/MontFerret/contrib/modules/security/jwt v1.0.0-rc.4/go.mod h1:WizwOVqaRtlPTkay126L4GPMotZSU4HLPKXvRpfY+3k=
github.com/MontFerret/contrib/modules/toml v1.0.0-rc.6 h1:XDMiuX6zDigvFxpfqqsJkIlaOScfQpsZkRmx965S2W8=
github.com/MontFerret/contrib/modules/toml v1.0.0-rc.6/go.mod h1:DhKnW7MPZNUKRI8HLYn7oyR8fIxWxZkb+DiIcbe69WI=
github.com/MontFerret/contrib/modules/web/article v1.0.0-rc.7 h1:Kb7d4kaYb09XQ3tnucrSCAKdRiK7wl44i+DJ7s5J7qI=
github.com/MontFerret/contrib/modules/web/article v1.0.0-rc.7/go.mod h1:bmtwL4l2kv6tWMPqDNaZyQ61aePlnl2DMpASUxEzOts=
github.com/MontFerret/contrib/modules/web/html v1.0.0-rc.10 h1:2ucNg6fAbg8dclJzlUyKcC3Dx1my59lv+vgnB9t7V9Q=
github.com/MontFerret/contrib/modules/web/html v1.0.0-rc.10/go.mod h1:QWaiNrJyZA4ftj021u5GuJ1ArSKGSnXBMdMiJ7YkLO0=
github.com/MontFerret/contrib/modules/web/robots v1.0.0-rc.6 h1:ilzerXcr2+BPoSxlC/DKvEvMMmV5vM79p5UUubxQaOc=
github.com/MontFerret/contrib/modules/web/robots v1.0.0-rc.6/go.mod h1:Vns8AVGMYQuY77rdEbVGA5GQUL8M5yZnUN17iYvlndI=
github.com/MontFerret/contrib/modules/web/sitemap v1.0.0-rc.6 h1:OkiQwPv5XCY4N4byj8RXI1bbV/QBmExZLSAmw9g8sfo=
github.com/MontFerret/contrib/modules/web/sitemap v1.0.0-rc.6/go.mod h1:LNv2FLxVKHMGMO8GaZAXILmf3CrYEakPm15yCNd5Vi4=
github.com/MontFerret/contrib/modules/xml v1.0.0-rc.6 h1:fPuan3cnzs8o275ty8Qo5Qmuq2vcpxfIQMWgFxer0Ck=
github.com/MontFerret/contrib/modules/xml v1.0.0-rc.6/go.mod h1:fQyVGq16aTq8/YRnaDv7oDGJtmxwLTWnF94tBIeYUTQ=
github.com/MontFerret/contrib/modules/yaml v1.0.0-rc.6 h1:D2l5YmQ7RpThRdJt8HAIRP8A98cw80A104noEabqlEw=
github.com/MontFerret/contrib/modules/yaml v1.0.0-rc.6/go.mod h1:15QVJzey4GG0ZLBasjaqu6aw6lGIM8k937pRYc8vlZ8=
github.com/MontFerret/contrib/pkg/common v0.1.1 h1:qA2URATIO+EVLH3BNFKYupLx97ftotyGWnKtJJhtR0Y=
github.com/MontFerret/contrib/pkg/common v0.1.1/go.mod h1:wiMLQnE46YGci3qMzA3ExArKomQDXR5DvNSE7HnP0ik=
github.com/MontFerret/cssx v0.2.0 h1:De0C6Irbg+qgFPXgWmPpVnwD4RRYUBQSbIYFTUVCNWU=
github.com/MontFerret/cssx v0.2.0/go.mod h1:fmGtRUNVaeJYpiPSDlNIbbYzb3+K8NxmNmJOYqlHATU=
github.com/MontFerret/ferret/v2 v2.0.0-alpha.25 h1:MYmfyKqAXatfJzwcqgkHjRdpDl6YLy0T4XihtSVR/cI=
Expand Down
61 changes: 45 additions & 16 deletions pkg/runtime/builtin_modules.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,37 +5,50 @@ import (

"github.com/MontFerret/contrib/modules/csv"
"github.com/MontFerret/contrib/modules/db/sqlite"
"github.com/MontFerret/contrib/modules/net/rest"
"github.com/MontFerret/contrib/modules/security/jwt"
"github.com/MontFerret/contrib/modules/toml"
"github.com/MontFerret/contrib/modules/web/article"
"github.com/MontFerret/contrib/modules/web/html"
"github.com/MontFerret/contrib/modules/web/html/drivers/cdp"
"github.com/MontFerret/contrib/modules/web/html/drivers/memory"
"github.com/MontFerret/contrib/modules/web/robots"
"github.com/MontFerret/contrib/modules/web/sitemap"
"github.com/MontFerret/contrib/modules/xml"
"github.com/MontFerret/contrib/modules/yaml"
"github.com/MontFerret/ferret/v2/pkg/module"

"github.com/MontFerret/contrib/modules/web/html"
"github.com/MontFerret/contrib/modules/web/html/drivers/cdp"
"github.com/MontFerret/contrib/modules/web/html/drivers/memory"
)

type namespaceInitializer func(opts Options) ([]module.Module, error)

func newModules(opts Options) ([]module.Module, error) {
webmods, err := newWebModules(opts)
return initModules(
opts,
webMods,
dataMods,
dbMods,
securityMods,
networkMods,
)
}

if err != nil {
return nil, err
}
func initModules(opts Options, initializers ...namespaceInitializer) ([]module.Module, error) {
var merged []module.Module

datamods, err := newDataModules()
for _, r := range initializers {
modules, err := r(opts)

if err != nil {
return nil, err
if err != nil {
return nil, err
}

merged = append(merged, modules...)
}

return append(webmods, datamods...), nil
return merged, nil
}

func newWebModules(opts Options) ([]module.Module, error) {
func webMods(opts Options) ([]module.Module, error) {
htmlmod, err := html.New(
html.WithDefaultDriver(memory.New(opts.ToInMemory()...)),
html.WithDrivers(
Expand All @@ -52,16 +65,32 @@ func newWebModules(opts Options) ([]module.Module, error) {
sitemap.New(),
article.New(),
robots.New(),
jwt.New(),
sqlite.New(),
}, nil
}

func newDataModules() ([]module.Module, error) {
func dataMods(_ Options) ([]module.Module, error) {
return []module.Module{
csv.New(),
toml.New(),
xml.New(),
yaml.New(),
}, nil
}

func dbMods(_ Options) ([]module.Module, error) {
return []module.Module{
sqlite.New(),
}, nil
}

func securityMods(_ Options) ([]module.Module, error) {
return []module.Module{
jwt.New(),
}, nil
}

func networkMods(_ Options) ([]module.Module, error) {
return []module.Module{
rest.New(),
}, nil
}
Loading