diff --git a/go.mod b/go.mod index 33a3617..a6dd312 100644 --- a/go.mod +++ b/go.mod @@ -4,16 +4,17 @@ go 1.26.1 require ( github.com/Masterminds/semver/v3 v3.5.0 - github.com/MontFerret/contrib/modules/csv v1.0.0-rc.4 - github.com/MontFerret/contrib/modules/db/sqlite v1.0.0-rc.2 - github.com/MontFerret/contrib/modules/security/jwt v1.0.0-rc.2 - github.com/MontFerret/contrib/modules/toml v1.0.0-rc.4 - github.com/MontFerret/contrib/modules/web/article v1.0.0-rc.5 - github.com/MontFerret/contrib/modules/web/html v1.0.0-rc.8 - github.com/MontFerret/contrib/modules/web/robots v1.0.0-rc.4 - github.com/MontFerret/contrib/modules/web/sitemap v1.0.0-rc.4 - github.com/MontFerret/contrib/modules/xml v1.0.0-rc.4 - github.com/MontFerret/contrib/modules/yaml v1.0.0-rc.4 + github.com/MontFerret/contrib/modules/csv v1.0.0-rc.6 + github.com/MontFerret/contrib/modules/db/sqlite v1.0.0-rc.4 + github.com/MontFerret/contrib/modules/net/rest v1.0.0-rc.2 + github.com/MontFerret/contrib/modules/security/jwt v1.0.0-rc.4 + github.com/MontFerret/contrib/modules/toml v1.0.0-rc.6 + github.com/MontFerret/contrib/modules/web/article v1.0.0-rc.7 + github.com/MontFerret/contrib/modules/web/html v1.0.0-rc.10 + github.com/MontFerret/contrib/modules/web/robots v1.0.0-rc.6 + github.com/MontFerret/contrib/modules/web/sitemap v1.0.0-rc.6 + github.com/MontFerret/contrib/modules/xml v1.0.0-rc.6 + github.com/MontFerret/contrib/modules/yaml v1.0.0-rc.6 github.com/MontFerret/ferret/v2 v2.0.0-alpha.25 github.com/chzyer/readline v1.5.1 github.com/go-waitfor/waitfor v1.1.0 @@ -31,6 +32,7 @@ require ( github.com/BurntSushi/toml v1.6.0 // indirect github.com/JohannesKaufmann/dom v0.3.1 // indirect github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 // indirect + github.com/MontFerret/contrib/pkg/common v0.1.1 // indirect github.com/MontFerret/cssx v0.2.0 // indirect github.com/PuerkitoBio/goquery v1.12.0 // indirect github.com/andybalholm/cascadia v1.3.4 // indirect diff --git a/go.sum b/go.sum index 4e827a1..953f563 100644 --- a/go.sum +++ b/go.sum @@ -8,26 +8,30 @@ github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 h1:XFJZFWESIWlUEHHjzBuv8R github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2/go.mod h1:BHWO8lJzttJLqwuV8Rb1B3OG2OSzLbssZDI1FRg2eAA= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/MontFerret/contrib/modules/csv v1.0.0-rc.4 h1:SFlZQq7gjKq1gM+cKp+b4zW+eNZmsxN5t1Z89cdPBeQ= -github.com/MontFerret/contrib/modules/csv v1.0.0-rc.4/go.mod h1:5YB64P+mrwzHtS6DYWumRLZZdY9JBBuV5fud10VxY34= -github.com/MontFerret/contrib/modules/db/sqlite v1.0.0-rc.2 h1:sAgF/2ljDizfJvyRa5/elMUgbrPsV21SG/lDvvrTiQY= -github.com/MontFerret/contrib/modules/db/sqlite v1.0.0-rc.2/go.mod h1:X3cTsLMYWZs/d8qHsT0nuvT1wb5KL5BGQ6qvVrnawZM= -github.com/MontFerret/contrib/modules/security/jwt v1.0.0-rc.2 h1:3/L9zX1k7nhJZN2BZJA0znr/SfR9HssGTjGfIOWk9Mo= -github.com/MontFerret/contrib/modules/security/jwt v1.0.0-rc.2/go.mod h1:m8sqDatCGaIdgCsOkotvaWJmfUJRHbO9rSccjDpkfoU= -github.com/MontFerret/contrib/modules/toml v1.0.0-rc.4 h1:Q7Ck4Uk5tsA7tgI3f+5hLZ+EGjjN0VEUCP1dxKt34+w= -github.com/MontFerret/contrib/modules/toml v1.0.0-rc.4/go.mod h1:9pP9GMHsDYIn9b+TE45ntDSxL22EgqcA5U0UQ/dpHOg= -github.com/MontFerret/contrib/modules/web/article v1.0.0-rc.5 h1:gGfEj5+pd6GAZgYYUvF8XDaXAfgAO5MOs1Njni30GlY= -github.com/MontFerret/contrib/modules/web/article v1.0.0-rc.5/go.mod h1:WNQ0tQhLsi7URLYajggdZtGb0YcsJ1puIhJDtnbLlyU= -github.com/MontFerret/contrib/modules/web/html v1.0.0-rc.8 h1:RB82v46m0Sb6axIoOTkplCDT8YA/QfxQ273TGYadLPk= -github.com/MontFerret/contrib/modules/web/html v1.0.0-rc.8/go.mod h1:i5weUFsPLtgL2KdtSYKIbuOotlTjjRauIEAio+7y0Ro= -github.com/MontFerret/contrib/modules/web/robots v1.0.0-rc.4 h1:MwKcqLQQ341TH70O9QLh3DSamSrALcJ2EstWIFcKpnU= -github.com/MontFerret/contrib/modules/web/robots v1.0.0-rc.4/go.mod h1:AibYgVv4+XXkmZZHcFvi0IwQvd7DaFblUST/Z/jyDzA= -github.com/MontFerret/contrib/modules/web/sitemap v1.0.0-rc.4 h1:GBhDeV+NhcKiLFeJSlwdlhGENu2k+p5vWHPKovpRHGE= -github.com/MontFerret/contrib/modules/web/sitemap v1.0.0-rc.4/go.mod h1:1JqsUQiLyP6ZR9MH55aSB7gLJ2gDUwegkb+EMstPZEE= -github.com/MontFerret/contrib/modules/xml v1.0.0-rc.4 h1:PjT+hr/ZDRBiNbyePLP1fv4tjoIaLS/ADdrhLWR/6kw= -github.com/MontFerret/contrib/modules/xml v1.0.0-rc.4/go.mod h1:eMl++62DU9ejC2sFyCldTi2fXZTxX21l7S+u0XmmTZc= -github.com/MontFerret/contrib/modules/yaml v1.0.0-rc.4 h1:5vVxjrFKDCvAvUnQCZYJ+6YnjouSeTB1o+kgvB/Yxl0= -github.com/MontFerret/contrib/modules/yaml v1.0.0-rc.4/go.mod h1:T/pQe51gAbWk+nvXjsjk1QTivlfsVfZ3juOQVrHRnKw= +github.com/MontFerret/contrib/modules/csv v1.0.0-rc.6 h1:iXY/7/u47KRrcxdq/1CqtRRpvlHy7hGeacVUJqmNUnE= +github.com/MontFerret/contrib/modules/csv v1.0.0-rc.6/go.mod h1:GKx3zKQGcWIdB7BYUM/yKY+s+C2WPJWWiowPCl3hUBQ= +github.com/MontFerret/contrib/modules/db/sqlite v1.0.0-rc.4 h1:uPaX0JA5plRSNrfV48wamNKaivEB9/I/mUWZv5DUvbc= +github.com/MontFerret/contrib/modules/db/sqlite v1.0.0-rc.4/go.mod h1:Nq5GoZkAkpalBu5rH5gBhMGqF4s5F50sWcCY/vG/YhA= +github.com/MontFerret/contrib/modules/net/rest v1.0.0-rc.2 h1:50Qpx13rXZfmuwe0E+/EjoT93O7naaE6q1/N9b1KvPM= +github.com/MontFerret/contrib/modules/net/rest v1.0.0-rc.2/go.mod h1:rBLl+oruPTSu/IaONli5P3TH7APWFyCU3TItcEdiB2U= +github.com/MontFerret/contrib/modules/security/jwt v1.0.0-rc.4 h1:IpAYXY0u3rf3+ktz94DeRAj7rEsZ+HJyA4MGr1y89CY= +github.com/MontFerret/contrib/modules/security/jwt v1.0.0-rc.4/go.mod h1:WizwOVqaRtlPTkay126L4GPMotZSU4HLPKXvRpfY+3k= +github.com/MontFerret/contrib/modules/toml v1.0.0-rc.6 h1:XDMiuX6zDigvFxpfqqsJkIlaOScfQpsZkRmx965S2W8= +github.com/MontFerret/contrib/modules/toml v1.0.0-rc.6/go.mod h1:DhKnW7MPZNUKRI8HLYn7oyR8fIxWxZkb+DiIcbe69WI= +github.com/MontFerret/contrib/modules/web/article v1.0.0-rc.7 h1:Kb7d4kaYb09XQ3tnucrSCAKdRiK7wl44i+DJ7s5J7qI= +github.com/MontFerret/contrib/modules/web/article v1.0.0-rc.7/go.mod h1:bmtwL4l2kv6tWMPqDNaZyQ61aePlnl2DMpASUxEzOts= +github.com/MontFerret/contrib/modules/web/html v1.0.0-rc.10 h1:2ucNg6fAbg8dclJzlUyKcC3Dx1my59lv+vgnB9t7V9Q= +github.com/MontFerret/contrib/modules/web/html v1.0.0-rc.10/go.mod h1:QWaiNrJyZA4ftj021u5GuJ1ArSKGSnXBMdMiJ7YkLO0= +github.com/MontFerret/contrib/modules/web/robots v1.0.0-rc.6 h1:ilzerXcr2+BPoSxlC/DKvEvMMmV5vM79p5UUubxQaOc= +github.com/MontFerret/contrib/modules/web/robots v1.0.0-rc.6/go.mod h1:Vns8AVGMYQuY77rdEbVGA5GQUL8M5yZnUN17iYvlndI= +github.com/MontFerret/contrib/modules/web/sitemap v1.0.0-rc.6 h1:OkiQwPv5XCY4N4byj8RXI1bbV/QBmExZLSAmw9g8sfo= +github.com/MontFerret/contrib/modules/web/sitemap v1.0.0-rc.6/go.mod h1:LNv2FLxVKHMGMO8GaZAXILmf3CrYEakPm15yCNd5Vi4= +github.com/MontFerret/contrib/modules/xml v1.0.0-rc.6 h1:fPuan3cnzs8o275ty8Qo5Qmuq2vcpxfIQMWgFxer0Ck= +github.com/MontFerret/contrib/modules/xml v1.0.0-rc.6/go.mod h1:fQyVGq16aTq8/YRnaDv7oDGJtmxwLTWnF94tBIeYUTQ= +github.com/MontFerret/contrib/modules/yaml v1.0.0-rc.6 h1:D2l5YmQ7RpThRdJt8HAIRP8A98cw80A104noEabqlEw= +github.com/MontFerret/contrib/modules/yaml v1.0.0-rc.6/go.mod h1:15QVJzey4GG0ZLBasjaqu6aw6lGIM8k937pRYc8vlZ8= +github.com/MontFerret/contrib/pkg/common v0.1.1 h1:qA2URATIO+EVLH3BNFKYupLx97ftotyGWnKtJJhtR0Y= +github.com/MontFerret/contrib/pkg/common v0.1.1/go.mod h1:wiMLQnE46YGci3qMzA3ExArKomQDXR5DvNSE7HnP0ik= github.com/MontFerret/cssx v0.2.0 h1:De0C6Irbg+qgFPXgWmPpVnwD4RRYUBQSbIYFTUVCNWU= github.com/MontFerret/cssx v0.2.0/go.mod h1:fmGtRUNVaeJYpiPSDlNIbbYzb3+K8NxmNmJOYqlHATU= github.com/MontFerret/ferret/v2 v2.0.0-alpha.25 h1:MYmfyKqAXatfJzwcqgkHjRdpDl6YLy0T4XihtSVR/cI= diff --git a/pkg/runtime/builtin_modules.go b/pkg/runtime/builtin_modules.go index b1f82a6..98fb37f 100644 --- a/pkg/runtime/builtin_modules.go +++ b/pkg/runtime/builtin_modules.go @@ -5,37 +5,50 @@ import ( "github.com/MontFerret/contrib/modules/csv" "github.com/MontFerret/contrib/modules/db/sqlite" + "github.com/MontFerret/contrib/modules/net/rest" "github.com/MontFerret/contrib/modules/security/jwt" "github.com/MontFerret/contrib/modules/toml" "github.com/MontFerret/contrib/modules/web/article" + "github.com/MontFerret/contrib/modules/web/html" + "github.com/MontFerret/contrib/modules/web/html/drivers/cdp" + "github.com/MontFerret/contrib/modules/web/html/drivers/memory" "github.com/MontFerret/contrib/modules/web/robots" "github.com/MontFerret/contrib/modules/web/sitemap" "github.com/MontFerret/contrib/modules/xml" "github.com/MontFerret/contrib/modules/yaml" "github.com/MontFerret/ferret/v2/pkg/module" - - "github.com/MontFerret/contrib/modules/web/html" - "github.com/MontFerret/contrib/modules/web/html/drivers/cdp" - "github.com/MontFerret/contrib/modules/web/html/drivers/memory" ) +type namespaceInitializer func(opts Options) ([]module.Module, error) + func newModules(opts Options) ([]module.Module, error) { - webmods, err := newWebModules(opts) + return initModules( + opts, + webMods, + dataMods, + dbMods, + securityMods, + networkMods, + ) +} - if err != nil { - return nil, err - } +func initModules(opts Options, initializers ...namespaceInitializer) ([]module.Module, error) { + var merged []module.Module - datamods, err := newDataModules() + for _, r := range initializers { + modules, err := r(opts) - if err != nil { - return nil, err + if err != nil { + return nil, err + } + + merged = append(merged, modules...) } - return append(webmods, datamods...), nil + return merged, nil } -func newWebModules(opts Options) ([]module.Module, error) { +func webMods(opts Options) ([]module.Module, error) { htmlmod, err := html.New( html.WithDefaultDriver(memory.New(opts.ToInMemory()...)), html.WithDrivers( @@ -52,12 +65,10 @@ func newWebModules(opts Options) ([]module.Module, error) { sitemap.New(), article.New(), robots.New(), - jwt.New(), - sqlite.New(), }, nil } -func newDataModules() ([]module.Module, error) { +func dataMods(_ Options) ([]module.Module, error) { return []module.Module{ csv.New(), toml.New(), @@ -65,3 +76,21 @@ func newDataModules() ([]module.Module, error) { yaml.New(), }, nil } + +func dbMods(_ Options) ([]module.Module, error) { + return []module.Module{ + sqlite.New(), + }, nil +} + +func securityMods(_ Options) ([]module.Module, error) { + return []module.Module{ + jwt.New(), + }, nil +} + +func networkMods(_ Options) ([]module.Module, error) { + return []module.Module{ + rest.New(), + }, nil +}