Skip to content

Commit 8376cdb

Browse files
committed
Clean up unreferenced prompt storage files when truncating history and extend upload lifespan to 12 months
- make history not bound to 25 entries limit - create an util, that on extension startup that removes entries in HISTORY_ASK_ABOUT_FILES_STATE_KEY and HISTORY_EDIT_FILES_STATE_KEY above the limit of 25. Doing so, look for symbols #Website #PastedText #Image and remove their files from the persistent storage. You can do it through [Fragment] if feasible. - place [Pasted text] in values └── apps └── editor └── src ├── constants │ ├── state-keys.ts │ └── values.ts ├── extension.ts ├── utils │ ├── natural-sort.ts │ ├── normalize-path.ts │ └── path-sanitizer.ts └── views └── prompt ├── backend │ ├── message-handlers │ │ ├── handle-get-history.ts │ │ └── handle-save-history.ts │ └── utils │ ├── index.ts │ ├── persistent-prompt-storage.ts │ └── symbols │ ├── image │ │ └── replace-image-symbol.ts │ ├── pasted-text │ │ └── replace-pasted-text-symbol.ts │ └── website │ └── replace-website-symbol.ts ├── frontend │ └── Main │ └── Main.tsx └── types └── messages.ts
1 parent b26c557 commit 8376cdb

4 files changed

Lines changed: 117 additions & 8 deletions

File tree

‎apps/editor/src/constants/values.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
export const CHECKPOINT_LIFESPAN = 7 * 24 * 60 * 60 * 1000 // 7 days
2-
export const PROMPT_UPLOADS_LIFESPAN = 6 * 30 * 24 * 60 * 60 * 1000 // 6 months
2+
export const PROMPT_UPLOADS_LIFESPAN = 12 * 30 * 24 * 60 * 60 * 1000 // 12 months
3+
export const PROMPT_HISTORY_ENTRIES_LIMIT = 25
34
export const MAX_FILE_TOKENS_FOR_COMMIT_MESSAGE = 20000
45
export const MAX_PROMPT_CHARS_IN_COMMIT_MESSAGE = 1000
56
export const GIT_LOG_SINCE_DURATION = '6 months ago'

‎apps/editor/src/views/prompt/backend/message-handlers/handle-save-history.ts‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,23 +41,19 @@ export const handle_save_history = async (
4141

4242
const new_history: HistoryEntry[] = []
4343

44-
// Using a Set to ensure no duplicates are added, though frontend should handle this.
4544
const processedTexts = new Set<string>()
4645

4746
for (const text of text_history) {
4847
if (processedTexts.has(text)) continue
4948
processedTexts.add(text)
5049

51-
// The first item in text_history is the one that was just used.
52-
// Its timestamp is updated to reflect recent use.
5350
if (text === text_history[0]) {
5451
new_history.push({ text, createdAt: Date.now() })
5552
} else {
5653
const old_entry = old_history_map.get(text)
5754
if (old_entry) {
5855
new_history.push(old_entry)
5956
} else {
60-
// Fallback for entries not in old history
6157
new_history.push({ text, createdAt: Date.now() })
6258
}
6359
}

‎apps/editor/src/views/prompt/backend/utils/persistent-prompt-storage.ts‎

Lines changed: 114 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,15 @@ import * as fs from 'fs'
22
import * as path from 'path'
33
import * as vscode from 'vscode'
44
import * as crypto from 'crypto'
5-
import { PROMPT_UPLOADS_LIFESPAN } from '@/constants/values'
5+
import {
6+
PROMPT_UPLOADS_LIFESPAN,
7+
PROMPT_HISTORY_ENTRIES_LIMIT
8+
} from '@/constants/values'
9+
import {
10+
HISTORY_ASK_ABOUT_FILES_STATE_KEY,
11+
HISTORY_EDIT_FILES_STATE_KEY,
12+
HistoryEntry
13+
} from '@/constants/state-keys'
614

715
export class PersistentPromptStorage<T extends 'text' | 'image'> {
816
private dir_path: string
@@ -67,6 +75,17 @@ export class PersistentPromptStorage<T extends 'text' | 'image'> {
6775
return path.join(this.dir_path, filename)
6876
}
6977

78+
async delete(id: string): Promise<void> {
79+
try {
80+
const file_path = this.get_file_path(id)
81+
if (fs.existsSync(file_path)) {
82+
await fs.promises.unlink(file_path)
83+
}
84+
} catch {
85+
// Ignore
86+
}
87+
}
88+
7089
static async cleanup(extension_context: vscode.ExtensionContext) {
7190
const text_storage = new PersistentPromptStorage(extension_context, 'text')
7291
const image_storage = new PersistentPromptStorage(
@@ -76,10 +95,103 @@ export class PersistentPromptStorage<T extends 'text' | 'image'> {
7695

7796
await Promise.all([
7897
text_storage.garbage_collect(),
79-
image_storage.garbage_collect()
98+
image_storage.garbage_collect(),
99+
PersistentPromptStorage.cleanup_history(
100+
extension_context,
101+
text_storage,
102+
image_storage
103+
)
80104
])
81105
}
82106

107+
private static async cleanup_history(
108+
extension_context: vscode.ExtensionContext,
109+
text_storage: PersistentPromptStorage<'text'>,
110+
image_storage: PersistentPromptStorage<'image'>
111+
) {
112+
const ask_history = extension_context.workspaceState.get<HistoryEntry[]>(
113+
HISTORY_ASK_ABOUT_FILES_STATE_KEY,
114+
[]
115+
)
116+
const edit_history = extension_context.workspaceState.get<HistoryEntry[]>(
117+
HISTORY_EDIT_FILES_STATE_KEY,
118+
[]
119+
)
120+
121+
const process_history = async (history: HistoryEntry[], key: string) => {
122+
if (!Array.isArray(history)) return { kept: [], removed: [] }
123+
124+
if (history.length <= PROMPT_HISTORY_ENTRIES_LIMIT) {
125+
return { kept: history, removed: [] }
126+
}
127+
128+
const kept = history.slice(0, PROMPT_HISTORY_ENTRIES_LIMIT)
129+
const removed = history.slice(PROMPT_HISTORY_ENTRIES_LIMIT)
130+
131+
await extension_context.workspaceState.update(key, kept)
132+
133+
return { kept, removed }
134+
}
135+
136+
const { kept: kept_ask, removed: removed_ask } = await process_history(
137+
ask_history,
138+
HISTORY_ASK_ABOUT_FILES_STATE_KEY
139+
)
140+
const { kept: kept_edit, removed: removed_edit } = await process_history(
141+
edit_history,
142+
HISTORY_EDIT_FILES_STATE_KEY
143+
)
144+
145+
const all_kept = [...kept_ask, ...kept_edit]
146+
.map((h: any) => (typeof h === 'string' ? h : h?.text))
147+
.filter(Boolean) as string[]
148+
const all_removed = [...removed_ask, ...removed_edit]
149+
.map((h: any) => (typeof h === 'string' ? h : h?.text))
150+
.filter(Boolean) as string[]
151+
152+
const extract_hashes = (texts: string[]) => {
153+
const image_hashes = new Set<string>()
154+
const text_hashes = new Set<string>()
155+
156+
for (const text of texts) {
157+
if (!text) continue
158+
159+
for (const match of text.matchAll(/#Image\(([a-fA-F0-9]+)\)/g)) {
160+
image_hashes.add(match[1])
161+
}
162+
163+
for (const match of text.matchAll(
164+
/#PastedText\(([a-fA-F0-9]+)(?::\d+)?\)/g
165+
)) {
166+
text_hashes.add(match[1])
167+
}
168+
169+
for (const match of text.matchAll(
170+
/#Website\([^ ]+ ([a-fA-F0-9]+)(?::\d+)?(?: "((?:\\.|[^"\\])*)")?\)/g
171+
)) {
172+
text_hashes.add(match[1])
173+
}
174+
}
175+
176+
return { image_hashes, text_hashes }
177+
}
178+
179+
const kept_hashes = extract_hashes(all_kept)
180+
const removed_hashes = extract_hashes(all_removed)
181+
182+
for (const hash of removed_hashes.image_hashes) {
183+
if (!kept_hashes.image_hashes.has(hash)) {
184+
await image_storage.delete(hash)
185+
}
186+
}
187+
188+
for (const hash of removed_hashes.text_hashes) {
189+
if (!kept_hashes.text_hashes.has(hash)) {
190+
await text_storage.delete(hash)
191+
}
192+
}
193+
}
194+
83195
private async garbage_collect() {
84196
try {
85197
if (!fs.existsSync(this.dir_path)) {

‎apps/editor/src/views/prompt/frontend/Main/Main.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,7 @@ export const Main: React.FC<Props> = (props) => {
253253
history && history.length > 0 && history[0] === trimmed_instruction
254254

255255
if (!is_duplicate) {
256-
const new_history = [trimmed_instruction, ...(history || [])].slice(0, 25)
256+
const new_history = [trimmed_instruction, ...(history || [])]
257257
set_history(new_history)
258258

259259
post_message(props.vscode, {

0 commit comments

Comments
 (0)