Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 65 additions & 0 deletions demos/e2e-tests/e2e-tests.nlogox
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,70 @@ to test-usage
report-totals
end

;; ---------------------------------------------------------------------------
;; T10 — per-agent profiles (#68)
;;
;; A second configuration is derived from the active one with a different
;; model, loaded as a profile, and bound to one turtle. That turtle must
;; report and call the alternate model while the observer stays on the
;; default, and the real reply must come back.
;; ---------------------------------------------------------------------------

to test-profiles
log-line "T10 per-agent profiles"
let default-model item 1 llm:active
let alt alt-model
let alt-config (word "config-" provider-under-test "-alt.txt")
write-config-with-model active-config alt-config alt
carefully
[ llm:load-profile "alt" alt-config ]
[ log-line (word " load-profile failed: " error-message) ]

crt 1 [ set label "profile-turtle" ]
let t one-of turtles with [label = "profile-turtle"]
ask t [ llm:use-profile "alt" ]

let reply ""
carefully
[ set reply [llm:chat "Reply with exactly the word: OK"] of t ]
[ set reply (word "ERROR: " error-message) ]
log-line (word " turtle on " [llm:active] of t " -> " reply)

assert "profile is listed" (member? "alt" llm:profiles)
assert "turtle reports its profile" ([llm:profile] of t = "alt")
assert "turtle's active model is the profile's model" ([item 1 llm:active] of t = alt)
assert "observer stays on the default model" (item 1 llm:active = default-model)
assert "observer reports the default profile" (llm:profile = "default")
assert "call through the profile returned" (is-string? reply and not member? "ERROR:" reply)
assert "usage was credited to the turtle" ([llm:get llm:usage "calls"] of t = 1)

ask t [ die ]
carefully [ file-delete alt-config ] [ ]
report-totals
end

;; A second model on the provider under test, so a profile can be told apart
;; from the default. Ollama has no guaranteed second model pulled, so it
;; reuses the active one and the test still proves routing and bookkeeping.
to-report alt-model
if provider-under-test = "groq" [ report "openai/gpt-oss-120b" ]
if provider-under-test = "anthropic" [ report "claude-3-5-haiku-latest" ]
report item 1 llm:active
end

;; Copy a config file with its model line replaced.
to write-config-with-model [src dst model-name]
let lines []
file-open src
while [ not file-at-end? ] [ set lines lput file-read-line lines ]
file-close
carefully [ file-delete dst ] [ ]
file-open dst
foreach filter [ l -> not (length l >= 6 and substring l 0 6 = "model=") ] lines [ l -> file-print l ]
file-print (word "model=" model-name)
file-close
end

;; Run everything in sequence.
to run-headless
carefully [ file-delete "e2e-results.txt" ] [ ]
Expand All @@ -326,6 +390,7 @@ to test-all
test-choose
test-throttling
test-usage
test-profiles
output-print ""
log-line (word "TOTAL passed " pass-count " failed " fail-count)
end
Expand Down
6 changes: 6 additions & 0 deletions demos/test-config-fast-timeout
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Test fixture: global config with a one-second wait budget and no retry allowance
provider=openai
openai_api_key=test-key
model=model-fast-timeout
timeout_seconds=1
retry_max_elapsed_seconds=0
4 changes: 4 additions & 0 deletions demos/test-profile-a
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Test fixture: profile A for per-agent model selection tests
provider=openai
openai_api_key=test-key
model=model-a
3 changes: 3 additions & 0 deletions demos/test-profile-anthropic
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Test fixture: Anthropic profile with no model, so the provider default must apply
provider=anthropic
anthropic_api_key=test-key
4 changes: 4 additions & 0 deletions demos/test-profile-b
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Test fixture: profile B for per-agent model selection tests
provider=openai
openai_api_key=test-key
model=model-b
6 changes: 6 additions & 0 deletions demos/test-profile-slow
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# Test fixture: profile with a one-second wait budget and no retry allowance
provider=openai
openai_api_key=test-key
model=model-slow
timeout_seconds=1
retry_max_elapsed_seconds=0
96 changes: 96 additions & 0 deletions docs/API-REFERENCE.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ The NetLogo Multi-LLM Extension provides a unified interface for multiple Large
| `llm:set-history list` | History | Set conversation history for current agent |
| `llm:clear-history` | History | Clear conversation history for current agent |
| `llm:load-config filename` | Configuration | Load settings from file |
| `llm:load-profile name filename` | Profiles | Load a second configuration under a name |
| `llm:use-profile name` | Profiles | Route the calling agent's calls through that profile |
| `llm:profile` | Profiles | The calling agent's profile name (`"default"` if none) |
| `llm:profiles` | Profiles | Names of the loaded profiles |
| `llm:set-provider name` | Configuration | Set active provider (openai, anthropic, gemini, ollama, openrouter, together) |
| `llm:set-api-key key` | Configuration | Set API key for current provider |
| `llm:set-model name` | Configuration | Set model to use for current provider |
Expand Down Expand Up @@ -860,6 +864,98 @@ print llm:list-models ; Shows all providers, with Anthropic marked as ACTIVE
- Custom models added via `models-override.yaml` are marked with `[custom]`
- The currently active provider and model are marked with `[ACTIVE]`

## Profiles: different models for different agents

By default every agent shares one configuration: one provider, one key, one
model. A profile is a second configuration loaded under a name. Any agent can
be bound to a profile, and from then on its calls go through that profile's
provider and model. Agents that are not bound keep using the default
configuration exactly as before.

```netlogo
to setup
clear-all
llm:load-config "config.txt" ;; the default, as before
llm:load-profile "llama" "config-groq.txt" ;; a second provider and key
llm:load-profile "sonnet" "config-anthropic.txt"
llm:load-profile "local" "config-ollama.txt"

create-turtles 30
ask turtles with [who mod 3 = 0] [ llm:use-profile "llama" ]
ask turtles with [who mod 3 = 1] [ llm:use-profile "sonnet" ]
ask turtles with [who mod 3 = 2] [ llm:use-profile "local" ]
end

to go
ask turtles [
let reply llm:chat "One word: forage or rest?" ;; each goes to its own model
]
end
```

Each turtle keeps its own history and its own `llm:usage`, so three models can
be compared inside one run under identical conditions.

### llm:load-profile

**Syntax**: `llm:load-profile name filename`

**Description**: Loads a configuration file under a name. The file has the same
format and keys as `llm:load-config`, including thinking, retry and throttling
settings, and is validated the same way: an unknown provider, a missing key, or
an unreachable local server rejects the load. Loading a name that already
exists replaces it. A rejected reload leaves the existing profile untouched.

**Parameters**:

- `name` (string): Any name except `"default"`, which is reserved for the
global configuration. Names are case-insensitive.
- `filename` (string): Config file, resolved like `llm:load-config`

**Notes**:

- Profiles survive `clear-all`, like the global configuration does. Agent
bindings do not, since the agents themselves are gone.
- `llm:set-model`, `llm:set-provider`, `llm:set-api-key` and the thinking
setters act on the default configuration only. A profile is what its file
says. To change a profile, edit the file and load it again.
- Two profiles on the same provider and endpoint share one request throttle.
If their `max_concurrent_requests` or `min_request_interval_ms` differ, the
shared throttle takes whichever setting the most recent call carried, without
losing count of requests already in flight. Give profiles on one endpoint the
same throttling settings.
- A profile's `timeout_seconds` and retry settings govern calls made through
it. An async call keeps the budget in force when it was launched.

### llm:use-profile

**Syntax**: `llm:use-profile name`

**Description**: Routes the calling agent's calls through the named profile.
`llm:use-profile "default"` returns the agent to the global configuration.
An unknown name is an error that lists the loaded profiles.

**Notes**:

- An async call already in flight keeps the provider it started with. Rebinding
an agent affects its next call, never a pending one.
- Any agent can be bound: turtles, patches, links, or the observer.

### llm:profile

**Syntax**: `llm:profile`

**Returns**: String - the calling agent's profile name, or `"default"`

### llm:profiles

**Syntax**: `llm:profiles`

**Returns**: List - the loaded profile names, sorted

With a profile bound, `llm:active` and `llm:config` report the calling agent's
effective provider, model and configuration rather than the global ones.

## Token Usage

Every provider reports how many tokens a call consumed. The extension records
Expand Down
20 changes: 20 additions & 0 deletions docs/CONFIGURATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,26 @@ model=gpt-4o-mini
# llm:set-model "claude-3-5-sonnet-20241022"
```

### Profiles: one config file per model, different agents on different models

`llm:load-config` sets the one configuration every agent shares. To run several
models in one simulation, load extra config files as named profiles and bind
agents to them. Each profile is an ordinary config file with its own provider,
key and model:

```
llm:load-config "config.txt" ;; default for everyone
llm:load-profile "llama" "config-groq.txt"
llm:load-profile "sonnet" "config-anthropic.txt"
ask turtles with [role = "scout"] [ llm:use-profile "llama" ]
ask turtles with [role = "leader"] [ llm:use-profile "sonnet" ]
```

A profile file is validated on load the same way as `config.txt`. Thinking,
retry and throttling keys in the file apply to that profile. See the API
reference for `llm:load-profile`, `llm:use-profile`, `llm:profile` and
`llm:profiles`.

### Request Throttling (staying inside a rate limit)
A model that calls the LLM once per agent per tick sends one request per agent
simultaneously. On a free tier that exceeds the quota on the first tick.
Expand Down
Loading
Loading