Repository navigation
Expand file tree
/
Copy pathpackage.json
More file actions
178 lines (178 loc) · 5.58 KB
/
Copy pathpackage.json
File metadata and controls
178 lines (178 loc) · 5.58 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
{
"name": "deval",
"displayName": "Deval - Agent Evaluator ✅",
"description": "Run DeepEval TypeScript test suites to evaluate your agent behavior.",
"version": "0.0.2",
"publisher": "lanly-dev",
"sponsor": {
"url": "https://github.com/sponsors/lanly-dev"
},
"engines": {
"vscode": "^1.140.0"
},
"categories": [
"AI",
"Other",
"Testing"
],
"keywords": [
"agent",
"benchmark",
"deepeval",
"evaluation",
"llm",
"vitest"
],
"icon": "media/deval.png",
"galleryBanner": {
"color": "white",
"theme": "light"
},
"main": "./dist/extension.js",
"contributes": {
"commands": [
{
"command": "deval.runAgent",
"title": "Deval: Run Agent"
},
{
"command": "deval.evaluateAgentRun",
"title": "Deval: Score Agent Run"
},
{
"command": "deval.benchmarkChatModel",
"title": "Deval: Benchmark Model"
},
{
"command": "deval.showScoreboard",
"title": "Deval: Show Scoreboard"
}
],
"languageModelTools": [
{
"name": "deval_readFile",
"displayName": "Read file",
"modelDescription": "Reads the full text contents of a file in the workspace. Input: { path: workspace-relative file path }.",
"inputSchema": {
"type": "object",
"properties": {
"path": {
"type": "string",
"description": "Workspace-relative path of the file to read, e.g. \"src/extension.ts\"."
}
},
"required": [
"path"
]
}
},
{
"name": "deval_listFiles",
"displayName": "List files",
"modelDescription": "Lists the files and directories directly inside a workspace folder, non-recursive. Input: { directory?: workspace-relative directory, defaults to the workspace root }.",
"inputSchema": {
"type": "object",
"properties": {
"directory": {
"type": "string",
"description": "Workspace-relative directory to list. Defaults to the workspace root."
}
},
"required": []
}
},
{
"name": "deval_grep",
"displayName": "Search file contents",
"modelDescription": "Searches file contents for a text pattern across the workspace and returns matching lines. Input: { pattern: text or regular expression to search for, directory?: workspace-relative directory to search }.",
"inputSchema": {
"type": "object",
"properties": {
"pattern": {
"type": "string",
"description": "The text or regular expression to search for."
},
"directory": {
"type": "string",
"description": "Workspace-relative directory to search. Defaults to the workspace root."
}
},
"required": [
"pattern"
]
}
}
],
"configuration": {
"title": "DeepEval",
"properties": {
"deval.deepevalCommand": {
"type": "string",
"default": "npx",
"description": "Executable used to launch the DeepEval TypeScript test runner."
},
"deval.packageManager": {
"type": "string",
"enum": [
"bun",
"npm",
"pnpm",
"yarn"
],
"default": "npm",
"description": "Package manager the Scaffold Sample Benchmark command uses to install deepeval and vitest."
},
"deval.samplesPerPrompt": {
"type": "integer",
"default": 1,
"minimum": 1,
"maximum": 5,
"description": "How many times each prompt is asked in 'Deval: Benchmark Model'. Values above 1 enable pass@k scoring (pass@1 / pass@k per case) at the cost of proportionally more model calls."
}
}
}
},
"scripts": {
"postinstall": "node scripts/patch-deepeval.mjs",
"vscode:prepublish": "npm run package",
"compile": "npm run check-types && npm run lint && node esbuild.js",
"watch": "npm-run-all -p watch:*",
"watch:esbuild": "node esbuild.js --watch",
"watch:tsc": "tsc --noEmit --watch --project tsconfig.json",
"package": "npm run check-types && npm run lint && node esbuild.js --production",
"compile-tests": "node -e \"const fs=require('fs'); fs.rmSync('out', { recursive: true, force: true });\" && tsc -p . --outDir out",
"watch-tests": "tsc -p . -w --outDir out",
"pretest": "npm run compile-tests && npm run compile && npm run lint",
"check-types": "tsc --noEmit -p tsconfig.json && tsc --noEmit -p tsconfig.benchmarks.json",
"lint": "eslint src benchmarks resources vitest.config.mts",
"benchmark": "deepeval test run benchmarks",
"benchmark:captured": "deepeval test run benchmarks/captured-run-benchmark.test.ts",
"test": "vscode-test",
"vsce-package": "vsce package"
},
"devDependencies": {
"@sentry/node": "^11.6.0",
"@types/mocha": "^10.0.10",
"@types/node": "26.x",
"@types/vscode": "^1.140.0",
"@vscode/test-cli": "^0.0.15",
"@vscode/test-electron": "^3.1.0",
"deepeval": "^0.9.22",
"esbuild": "^0.28.2",
"eslint": "^10.12.0",
"npm-run-all": "^4.1.5",
"ovsx": "^1.2.0",
"typescript": "^6.0.3",
"typescript-eslint": "^8.71.1",
"vitest": "^5.0.3"
},
"overrides": {
"diff": "^8.0.4",
"serialize-javascript": "^7.0.6"
},
"repository": {
"type": "git",
"url": "https://github.com/lanly-dev/vscode-deval.git"
},
"license": "MIT"
}