Skip to content

Commit 0b7133c

Browse files
BazookaMusicBazookaMusic
committed
JS: Add prompt injection detection (CWE-1427) for OpenAI, Anthropic, and Google GenAI SDKs
Add experimental CodeQL query detecting prompt injection vulnerabilities in JavaScript/TypeScript applications using AI SDK libraries. Modeled frameworks: - openai (OpenAI, AzureOpenAI): responses, chat.completions, completions, images, embeddings, beta.assistants, beta.threads, audio APIs - @openai/agents: Agent instructions, handoffDescription, run/Runner.run, asTool, tool() - @anthropic-ai/sdk: messages.create, beta.messages.create, beta.agents.create/update - @google/genai (GoogleGenAI): generateContent, generateContentStream, generateImages, editImage, chats, live.connect Includes role-based filtering (system/developer/assistant/model roles) and constant-comparison sanitizer guard.
1 parent 154d213 commit 0b7133c

15 files changed

Lines changed: 1382 additions & 0 deletions

File tree

‎javascript/ql/lib/semmle/javascript/Concepts.qll‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,3 +226,28 @@ module Cryptography {
226226

227227
class CryptographicAlgorithm = SC::CryptographicAlgorithm;
228228
}
229+
230+
/**
231+
* A data-flow node that prompts an AI model.
232+
*
233+
* Extend this class to refine existing API models. If you want to model new APIs,
234+
* extend `AIPrompt::Range` instead.
235+
*/
236+
class AIPrompt extends DataFlow::Node instanceof AIPrompt::Range {
237+
/** Gets an input that is used as AI prompt. */
238+
DataFlow::Node getAPrompt() { result = super.getAPrompt() }
239+
}
240+
241+
/** Provides a class for modeling new AI prompting mechanisms. */
242+
module AIPrompt {
243+
/**
244+
* A data-flow node that prompts an AI model.
245+
*
246+
* Extend this class to model new APIs. If you want to refine existing API models,
247+
* extend `AIPrompt` instead.
248+
*/
249+
abstract class Range extends DataFlow::Node {
250+
/** Gets an input that is used as AI prompt. */
251+
abstract DataFlow::Node getAPrompt();
252+
}
253+
}

‎javascript/ql/src/experimental/Security/CWE-1427/PromptInjection.qhelp‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<!DOCTYPE qhelp PUBLIC
2+
"-//Semmle//qhelp//EN"
3+
"qhelp.dtd">
4+
<qhelp>
5+
6+
<overview>
7+
<p>Prompts can be constructed to bypass the original purposes of an agent and lead to sensitive data leak or
8+
operations that were not intended.</p>
9+
</overview>
10+
11+
<recommendation>
12+
<p>Sanitize user input and also avoid using user input in developer or system level prompts.</p>
13+
</recommendation>
14+
15+
<example>
16+
<p>In the following examples, the cases marked GOOD show secure prompt construction; whereas in the case marked BAD they may be susceptible to prompt injection.</p>
17+
<sample src="examples/example.py" />
18+
</example>
19+
20+
<references>
21+
<li>OpenAI: <a href="https://openai.github.io/openai-guardrails-python">Guardrails</a>.</li>
22+
</references>
23+
24+
</qhelp>

‎javascript/ql/src/experimental/Security/CWE-1427/PromptInjection.ql‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/**
2+
* @name Prompt injection
3+
* @kind path-problem
4+
* @problem.severity error
5+
* @security-severity 5.0
6+
* @precision high
7+
* @id js/prompt-injection
8+
* @tags security
9+
* experimental
10+
* external/cwe/cwe-1427
11+
*/
12+
13+
import javascript
14+
import experimental.semmle.javascript.security.PromptInjection.PromptInjectionQuery
15+
import PromptInjectionFlow::PathGraph
16+
17+
from PromptInjectionFlow::PathNode source, PromptInjectionFlow::PathNode sink
18+
where PromptInjectionFlow::flowPath(source, sink)
19+
select sink.getNode(), source, sink, "This prompt construction depends on a $@.", source.getNode(),
20+
"user-provided value"

‎javascript/ql/src/experimental/Security/CWE-1427/examples/example.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
from flask import Flask, request
2+
from agents import Agent
3+
from guardrails import GuardrailAgent
4+
5+
@app.route("/parameter-route")
6+
def get_input():
7+
input = request.args.get("input")
8+
9+
goodAgent = GuardrailAgent( # GOOD: Agent created with guardrails automatically configured.
10+
config=Path("guardrails_config.json"),
11+
name="Assistant",
12+
instructions="This prompt is customized for " + input)
13+
14+
badAgent = Agent(
15+
name="Assistant",
16+
instructions="This prompt is customized for " + input # BAD: user input in agent instruction.
17+
)

‎javascript/ql/src/experimental/semmle/javascript/frameworks/Anthropic.qll‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/**
2+
* Provides classes modeling security-relevant aspects of the `@anthropic-ai/sdk` package.
3+
* See https://github.com/anthropics/anthropic-sdk-typescript
4+
*/
5+
6+
private import javascript
7+
8+
module Anthropic {
9+
/** Gets a reference to the `Anthropic` client instance. */
10+
API::Node classRef() {
11+
// Default export: import Anthropic from '@anthropic-ai/sdk'; new Anthropic()
12+
result = API::moduleImport("@anthropic-ai/sdk").getInstance()
13+
}
14+
15+
16+
/** Gets a reference to a sink for the system prompt in the Anthropic messages API. */
17+
API::Node getContentNode() {
18+
exists(API::Node createParams |
19+
// client.messages.create({ ... })
20+
createParams = classRef()
21+
.getMember("messages")
22+
.getMember("create")
23+
.getParameter(0)
24+
or
25+
// client.beta.messages.create({ ... })
26+
createParams = classRef()
27+
.getMember("beta")
28+
.getMember("messages")
29+
.getMember("create")
30+
.getParameter(0)
31+
|
32+
// system: "string"
33+
result = createParams.getMember("system")
34+
or
35+
// system: [{ type: "text", text: "..." }]
36+
result = createParams.getMember("system").getArrayElement().getMember("text")
37+
or
38+
// messages: [{ role: "assistant", content: "..." }]
39+
// Injecting content into what the model said from external sources is very likely an injection.
40+
exists(API::Node msg |
41+
msg = createParams.getMember("messages").getArrayElement() and
42+
msg.getMember("role").asSink().mayHaveStringValue("assistant")
43+
|
44+
result = msg.getMember("content")
45+
)
46+
)
47+
or
48+
// client.beta.agents.create({ system: "..." })
49+
result = classRef()
50+
.getMember("beta")
51+
.getMember("agents")
52+
.getMember("create")
53+
.getParameter(0)
54+
.getMember("system")
55+
or
56+
// client.beta.agents.update(agentId, { system: "..." })
57+
result = classRef()
58+
.getMember("beta")
59+
.getMember("agents")
60+
.getMember("update")
61+
.getParameter(1)
62+
.getMember("system")
63+
}
64+
}

‎javascript/ql/src/experimental/semmle/javascript/frameworks/GoogleGenAI.qll‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/**
2+
* Provides classes modeling security-relevant aspects of the `@google/genai` package.
3+
* See https://github.com/googleapis/js-genai
4+
*/
5+
6+
private import javascript
7+
8+
module GoogleGenAI {
9+
/** Gets a reference to the `GoogleGenAI` client instance. */
10+
API::Node clientRef() {
11+
// import { GoogleGenAI } from '@google/genai'; const ai = new GoogleGenAI(...)
12+
result =
13+
API::moduleImport("@google/genai").getMember("GoogleGenAI").getInstance()
14+
}
15+
16+
/** Gets a reference to a sink for prompt content in the Google GenAI SDK. */
17+
API::Node getContentNode() {
18+
exists(API::Node params |
19+
// ai.models.generateContent({ contents, config })
20+
// ai.models.generateContentStream({ contents, config })
21+
params =
22+
clientRef()
23+
.getMember("models")
24+
.getMember(["generateContent", "generateContentStream"])
25+
.getParameter(0)
26+
|
27+
// config.systemInstruction
28+
result = params.getMember("config").getMember("systemInstruction")
29+
or
30+
// contents: [{ role: "model", parts: [{ text: "..." }] }]
31+
// Gemini uses "model" role instead of "assistant"
32+
exists(API::Node msg |
33+
msg = params.getMember("contents").getArrayElement() and
34+
msg.getMember("role").asSink().mayHaveStringValue("model")
35+
|
36+
result = msg.getMember("parts").getArrayElement().getMember("text")
37+
)
38+
)
39+
or
40+
// ai.models.generateImages({ prompt, config })
41+
result =
42+
clientRef()
43+
.getMember("models")
44+
.getMember("generateImages")
45+
.getParameter(0)
46+
.getMember("prompt")
47+
or
48+
// ai.models.editImage({ prompt, referenceImages, config })
49+
result =
50+
clientRef()
51+
.getMember("models")
52+
.getMember("editImage")
53+
.getParameter(0)
54+
.getMember("prompt")
55+
or
56+
// ai.chats.create({ config: { systemInstruction: ... } })
57+
result =
58+
clientRef()
59+
.getMember("chats")
60+
.getMember("create")
61+
.getParameter(0)
62+
.getMember("config")
63+
.getMember("systemInstruction")
64+
or
65+
// chat.sendMessage({ config: { systemInstruction: ... } })
66+
result =
67+
clientRef()
68+
.getMember("chats")
69+
.getMember("create")
70+
.getReturn()
71+
.getMember("sendMessage")
72+
.getParameter(0)
73+
.getMember("config")
74+
.getMember("systemInstruction")
75+
or
76+
// ai.live.connect({ config: { systemInstruction: ... } })
77+
result =
78+
clientRef()
79+
.getMember("live")
80+
.getMember("connect")
81+
.getParameter(0)
82+
.getMember("config")
83+
.getMember("systemInstruction")
84+
}
85+
}

0 commit comments

Comments
 (0)