From 5387bd7bbc456e5c5181c32d49df6a1b33788651 Mon Sep 17 00:00:00 2001 From: Lorris Saint-Genez Date: Thu, 30 Jul 2026 13:04:37 -0700 Subject: [PATCH 1/2] fix(crawler): accept all renderJavaScript shapes --- api/crawler/types.go | 66 +++++++++++++-- api/crawler/types_test.go | 174 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 233 insertions(+), 7 deletions(-) create mode 100644 api/crawler/types_test.go diff --git a/api/crawler/types.go b/api/crawler/types.go index df455f0a..ab33aafe 100644 --- a/api/crawler/types.go +++ b/api/crawler/types.go @@ -1,6 +1,9 @@ package crawler import ( + "bytes" + "encoding/json" + "fmt" "time" "github.com/algolia/algoliasearch-client-go/v4/algolia/search" @@ -64,13 +67,13 @@ type Config struct { StartUrls []string `json:"startUrls,omitempty"` Sitemaps []string `json:"sitemaps,omitempty"` - ExclusionPatterns []string `json:"exclusionPatterns,omitempty"` - IgnoreQueryParams []string `json:"ignoreQueryParams,omitempty"` - RenderJavaScript bool `json:"renderJavaScript,omitempty"` - RateLimit int `json:"rateLimit,omitempty"` - ExtraUrls []string `json:"extraUrls,omitempty"` - MaxDepth int `json:"maxDepth,omitempty"` - MaxURLs int `json:"maxUrls,omitempty"` + ExclusionPatterns []string `json:"exclusionPatterns,omitempty"` + IgnoreQueryParams []string `json:"ignoreQueryParams,omitempty"` + RenderJavaScript *RenderJavaScript `json:"renderJavaScript,omitempty"` + RateLimit int `json:"rateLimit,omitempty"` + ExtraUrls []string `json:"extraUrls,omitempty"` + MaxDepth int `json:"maxDepth,omitempty"` + MaxURLs int `json:"maxUrls,omitempty"` IgnoreRobotsTxtRules bool `json:"ignoreRobotsTxtRules,omitempty"` IgnoreNoIndex bool `json:"ignoreNoIndex,omitempty"` @@ -83,6 +86,55 @@ type Config struct { Actions []*Action `json:"actions,omitempty"` } +type RenderJavaScript struct { + Enabled bool + + raw json.RawMessage +} + +func (r *RenderJavaScript) UnmarshalJSON(data []byte) error { + trimmed := bytes.TrimSpace(data) + if len(trimmed) == 0 || bytes.Equal(trimmed, []byte("null")) { + return nil + } + + switch trimmed[0] { + case 't', 'f': + var enabled bool + if err := json.Unmarshal(trimmed, &enabled); err != nil { + return err + } + r.Enabled = enabled + case '[': + r.Enabled = true + case '{': + var object struct { + Enabled *bool `json:"enabled"` + } + if err := json.Unmarshal(trimmed, &object); err != nil { + return err + } + r.Enabled = object.Enabled == nil || *object.Enabled + default: + return fmt.Errorf( + "renderJavaScript must be a boolean, an array of URL patterns or an object, got %s", + trimmed, + ) + } + + r.raw = bytes.Clone(trimmed) + + return nil +} + +func (r *RenderJavaScript) MarshalJSON() ([]byte, error) { + if len(r.raw) > 0 { + return bytes.Clone(r.raw), nil + } + + return json.Marshal(r.Enabled) +} + // Action is a Crawler configuration action. type Action struct { IndexName string `json:"indexName"` diff --git a/api/crawler/types_test.go b/api/crawler/types_test.go new file mode 100644 index 00000000..dcb145e0 --- /dev/null +++ b/api/crawler/types_test.go @@ -0,0 +1,174 @@ +package crawler + +import ( + "encoding/json" + "reflect" + "strings" + "testing" +) + +func TestRenderJavaScript_UnmarshalJSON(t *testing.T) { + tests := []struct { + name string + payload string + want bool + wantErr bool + }{ + {name: "bool true", payload: `true`, want: true}, + {name: "bool false", payload: `false`, want: false}, + { + name: "array of patterns", + payload: `["https://example.com/docs/**","https://example.com/api/**"]`, + want: true, + }, + { + name: "object with enabled false", + payload: `{"enabled":false,"waitTime":{"min":1000,"max":5000}}`, + want: false, + }, + { + name: "object with enabled true", + payload: `{"enabled":true,"adblock":true,"patterns":["https://example.com/**"]}`, + want: true, + }, + { + name: "object without enabled", + payload: `{"waitTime":{"min":1000,"max":5000}}`, + want: true, + }, + {name: "number", payload: `42`, wantErr: true}, + {name: "string", payload: `"yes"`, wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got RenderJavaScript + err := json.Unmarshal([]byte(tt.payload), &got) + + if tt.wantErr { + if err == nil { + t.Fatalf("expected an error for payload %s, got nil", tt.payload) + } + return + } + if err != nil { + t.Fatalf("unexpected error for payload %s: %v", tt.payload, err) + } + if got.Enabled != tt.want { + t.Errorf("expected Enabled %t, got %t", tt.want, got.Enabled) + } + }) + } +} + +func TestRenderJavaScript_RoundTrip(t *testing.T) { + tests := []struct { + name string + payload string + }{ + {name: "bool", payload: `{"renderJavaScript":true}`}, + { + name: "array", + payload: `{"renderJavaScript":["https://example.com/docs/**","https://example.com/api/**"]}`, + }, + { + name: "object", + payload: `{"renderJavaScript":{"enabled":true,"waitTime":{"min":1000,"max":5000},"adblock":true,"patterns":["https://example.com/**"]}}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var config Config + if err := json.Unmarshal([]byte(tt.payload), &config); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + got, err := json.Marshal(&config) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + var want, have interface{} + if err := json.Unmarshal([]byte(tt.payload), &want); err != nil { + t.Fatalf("unmarshal expected payload: %v", err) + } + if err := json.Unmarshal(got, &have); err != nil { + t.Fatalf("unmarshal marshalled payload: %v", err) + } + if !reflect.DeepEqual(want, have) { + t.Errorf("expected %s, got %s", tt.payload, got) + } + }) + } +} + +func TestCrawler_UnmarshalJSON_RenderJavaScriptPatterns(t *testing.T) { + payload := `{ + "id": "1a2b3c4d-1234-5678-90ab-cdef12345678", + "name": "my-crawler", + "running": true, + "createdAt": "2024-01-02T03:04:05.000Z", + "config": { + "appId": "APP_ID", + "indexPrefix": "crawler_", + "startUrls": ["https://example.com"], + "renderJavaScript": ["https://example.com/docs/**"], + "rateLimit": 8, + "actions": [ + { + "indexName": "docs", + "pathsToMatch": ["https://example.com/docs/**"], + "recordExtractor": {"__type": "function", "source": "() => {}"} + } + ] + } + }` + + var crawler Crawler + if err := json.Unmarshal([]byte(payload), &crawler); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if crawler.Config == nil { + t.Fatal("expected a config, got nil") + } + if crawler.Config.RenderJavaScript == nil { + t.Fatal("expected a renderJavaScript value, got nil") + } + if !crawler.Config.RenderJavaScript.Enabled { + t.Error("expected Enabled true, got false") + } +} + +func TestConfig_MarshalJSON_WithoutRenderJavaScript(t *testing.T) { + got, err := json.Marshal(&Config{AppID: "APP_ID"}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if strings.Contains(string(got), "renderJavaScript") { + t.Errorf("expected no renderJavaScript key, got %s", got) + } +} + +func TestRenderJavaScript_MarshalJSON_FromGo(t *testing.T) { + tests := []struct { + name string + value *RenderJavaScript + want string + }{ + {name: "enabled", value: &RenderJavaScript{Enabled: true}, want: `true`}, + {name: "disabled", value: &RenderJavaScript{}, want: `false`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := json.Marshal(tt.value) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if string(got) != tt.want { + t.Errorf("expected %s, got %s", tt.want, got) + } + }) + } +} From a8135965583041e24119d4d8b55341170399fa66 Mon Sep 17 00:00:00 2001 From: Lorris Saint-Genez Date: Thu, 30 Jul 2026 13:53:06 -0700 Subject: [PATCH 2/2] fix(crawler): refine renderJavaScript Enabled semantics --- api/crawler/types.go | 7 ++++++- api/crawler/types_test.go | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/api/crawler/types.go b/api/crawler/types.go index ab33aafe..5892daa5 100644 --- a/api/crawler/types.go +++ b/api/crawler/types.go @@ -105,8 +105,13 @@ func (r *RenderJavaScript) UnmarshalJSON(data []byte) error { return err } r.Enabled = enabled + return nil case '[': - r.Enabled = true + var patterns []json.RawMessage + if err := json.Unmarshal(trimmed, &patterns); err != nil { + return err + } + r.Enabled = len(patterns) > 0 case '{': var object struct { Enabled *bool `json:"enabled"` diff --git a/api/crawler/types_test.go b/api/crawler/types_test.go index dcb145e0..5a41cf1b 100644 --- a/api/crawler/types_test.go +++ b/api/crawler/types_test.go @@ -16,11 +16,13 @@ func TestRenderJavaScript_UnmarshalJSON(t *testing.T) { }{ {name: "bool true", payload: `true`, want: true}, {name: "bool false", payload: `false`, want: false}, + {name: "null", payload: `null`, want: false}, { name: "array of patterns", payload: `["https://example.com/docs/**","https://example.com/api/**"]`, want: true, }, + {name: "empty array", payload: `[]`, want: false}, { name: "object with enabled false", payload: `{"enabled":false,"waitTime":{"min":1000,"max":5000}}`,