-
Notifications
You must be signed in to change notification settings - Fork 0
Show total size on the Storage and Database pages #781
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,7 @@ | ||
| import { useState, useCallback } from 'react'; | ||
| import { createFileRoute } from '@tanstack/react-router'; | ||
| import { Trash2Icon, ChevronLeftIcon, ChevronRightIcon, FileIcon } from 'lucide-react'; | ||
| import { useStorageDocuments } from '@/hooks/useAdminQueries'; | ||
| import { useStorageDocuments, useAdminStorageSummary } from '@/hooks/useAdminQueries'; | ||
| import { deleteStorageDocuments } from '@/stores/adminStore'; | ||
| import { useDebouncedValue } from '@/hooks/useDebouncedValue'; | ||
| import { showToast } from '@/lib/toast'; | ||
|
|
@@ -21,6 +21,8 @@ import { | |
| AdminPage, | ||
| AdminPanel, | ||
| AdminSearch, | ||
| AdminStat, | ||
| AdminStatRow, | ||
| ADMIN_TH, | ||
| ADMIN_TD, | ||
| ADMIN_TD_MUTED, | ||
|
|
@@ -65,6 +67,10 @@ function StorageManagementPage() { | |
| const documentsData = documentsDataQuery.data; | ||
| const documents = documentsData?.documents ?? []; | ||
|
|
||
| const summaryQuery = useAdminStorageSummary(); | ||
| const summary = summaryQuery.data; | ||
| const scannedHint = summary?.truncated ? `first ${summary.scanCap} objects` : undefined; | ||
|
|
||
| const resetPaging = () => { | ||
| setCursor(null); | ||
| setCursorHistory([]); | ||
|
|
@@ -159,6 +165,35 @@ function StorageManagementPage() { | |
| title='Storage' | ||
| description='PDFs in R2. Files marked orphaned exist in R2 but are not tracked in the mediaFiles table, usually from a failed cleanup, and are safe to delete.' | ||
| > | ||
| <AdminStatRow> | ||
| <AdminStat | ||
| label='Total size' | ||
| value={formatFileSize(summary?.totalBytes ?? 0)} | ||
| hint={scannedHint ?? `${formatFileSize(summary?.documentBytes ?? 0)} in documents`} | ||
| loading={summaryQuery.isLoading} | ||
|
Comment on lines
+171
to
+173
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win 🔎 Supported by static analysis🌐 Web query:
💡 Result: <search_synthesis> <source_evidence> Citations:
🏁 Script executed: #!/bin/bash
set -eu
file='packages/web/src/routes/_app/_protected/admin/storage.tsx'
printf '%s\n' '--- summary query references ---'
rg -n -C 8 'summaryQuery|summary\b|useQuery' "$file"
printf '%s\n' '--- rendered metrics ---'
sed -n '155,185p' "$file"
printf '%s\n' '--- file outline ---'
ast-grep outline "$file" --view compactRepository: InfinityBowman/corates Length of output: 3915 🤖 get_repo_knowledge executed:
Length of output: 5172 🏁 Script executed: #!/bin/bash
set -eu
printf '%s\n' '--- hook definition ---'
rg -n -C 12 'useAdminStorageSummary' packages/web/src
printf '%s\n' '--- AdminStat definition ---'
rg -n -C 16 'function AdminStat|const AdminStat|export.*AdminStat' packages/web/srcRepository: InfinityBowman/corates Length of output: 23026 Show an unavailable state when When the summary query rejects, 🤖 Prompt for AI Agents |
||
| /> | ||
| <AdminStat | ||
| label='Objects' | ||
| value={(summary?.objectCount ?? 0).toLocaleString()} | ||
| hint={scannedHint ?? `${(summary?.documentCount ?? 0).toLocaleString()} documents`} | ||
| loading={summaryQuery.isLoading} | ||
| /> | ||
| <AdminStat | ||
| label='Orphaned' | ||
| value={(summary?.orphanedCount ?? 0).toLocaleString()} | ||
| hint='Documents missing from mediaFiles' | ||
| tone={summary?.orphanedCount ? 'warning' : 'default'} | ||
| loading={summaryQuery.isLoading} | ||
| /> | ||
| <AdminStat | ||
| label='Orphaned size' | ||
| value={formatFileSize(summary?.orphanedBytes ?? 0)} | ||
| hint='Reclaimable by deleting' | ||
| tone={summary?.orphanedBytes ? 'warning' : 'default'} | ||
| loading={summaryQuery.isLoading} | ||
| /> | ||
| </AdminStatRow> | ||
|
|
||
| <div className='flex flex-col gap-3 sm:flex-row'> | ||
| <AdminSearch | ||
| value={search} | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,6 @@ | ||
| import type { Database } from '@corates/db/client'; | ||
| import { dbSchema, mediaFiles, organization, projects, user } from '@corates/db/schema'; | ||
| import { and, asc, count, desc, eq } from 'drizzle-orm'; | ||
| import { and, asc, count, desc, eq, sql } from 'drizzle-orm'; | ||
| import { throwDomainError, AUTH_ERRORS } from '@corates/shared'; | ||
| import { isAdminUser } from '@corates/workers/auth-admin'; | ||
| import { ALLOWED_TABLES, isAllowedTable, type AllowedTableName } from '@/server/lib/dbTables'; | ||
|
|
@@ -29,7 +29,17 @@ export async function listAdminDatabaseTables(session: Session, db: Database) { | |
| }), | ||
| ); | ||
|
|
||
| return { tables: tables.filter(t => t !== null) }; | ||
| const counted = tables.filter(t => t !== null); | ||
|
|
||
| // D1 exposes no size API and blocks the page_count pragma, but every query's | ||
| // meta carries the database size, so a no-op statement is the cheapest read. | ||
| const sizeProbe = await db.run(sql`SELECT 1`); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🩺 Stability & Availability | 🟠 Major | ⚡ Quick win Handle size-probe failures without dropping the table list. When 🤖 Prompt for AI Agents |
||
|
|
||
| return { | ||
| tables: counted, | ||
| totalRows: counted.reduce((sum, t) => sum + t.rowCount, 0), | ||
| databaseSizeBytes: sizeProbe.meta?.size_after ?? 0, | ||
| }; | ||
| } | ||
|
|
||
| interface DrizzleColumn { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -149,6 +149,62 @@ export async function listAdminStorageDocuments( | |
| return response; | ||
| } | ||
|
|
||
| // R2 has no aggregate API, so a total means walking the bucket a page at a | ||
| // time. The cap keeps one page load bounded; past it the figures are floors. | ||
| const SUMMARY_SCAN_CAP = 50000; | ||
|
|
||
| export async function getAdminStorageSummary(session: Session, db: Database) { | ||
| assertAdmin(session); | ||
|
|
||
| const trackedKeys = await db.select({ bucketKey: mediaFiles.bucketKey }).from(mediaFiles); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🚀 Performance & Scalability | 🟠 Major | ⚡ Quick win 🔎 Supported by static analysis🌐 Web query:
💡 Result: <search_synthesis> <source_evidence> Citations:
🏁 Script executed: #!/bin/bash
set -e
printf '%s\n' '--- function outline ---'
ast-grep outline packages/web/src/server/functions/admin-storage.server.ts
printf '%s\n' '--- function source ---'
sed -n '120,225p' packages/web/src/server/functions/admin-storage.server.ts
printf '%s\n' '--- direct bindings ---'
rg -n -C 3 'mediaFiles|trackedKeys|getAdminStorageSummary|db\.select' packages/web/src/server/functions packages/web/src/server packages/web/src/db packages/web/src 2>/dev/null | head -n 240
printf '%s\n' '--- package versions ---'
rg -n -C 2 '"drizzle-orm"|drizzle-orm' package.json packages/*/package.json pnpm-lock.yaml package-lock.json yarn.lock 2>/dev/null | head -n 120Repository: InfinityBowman/corates Length of output: 37618 🤖 get_repo_knowledge executed:
Length of output: 2405 Bound the database read by the scan budget. Drizzle ORM 0.45.2 does not add an implicit limit to 🤖 Prompt for AI Agents |
||
| const trackedKeysSet = new Set(trackedKeys.map(row => row.bucketKey)); | ||
|
|
||
| let objectCount = 0; | ||
| let totalBytes = 0; | ||
| let documentCount = 0; | ||
| let documentBytes = 0; | ||
| let orphanedCount = 0; | ||
| let orphanedBytes = 0; | ||
| let cursor: string | undefined = undefined; | ||
| let truncated = false; | ||
|
|
||
| while (objectCount < SUMMARY_SCAN_CAP) { | ||
| const listOptions: { limit: number; cursor?: string } = { limit: 1000 }; | ||
| if (cursor) listOptions.cursor = cursor; | ||
| const listed = await env.PDF_BUCKET.list(listOptions); | ||
|
|
||
| for (const obj of listed.objects) { | ||
| objectCount += 1; | ||
| totalBytes += obj.size; | ||
|
|
||
| // The bucket also holds avatars, which mediaFiles never tracks. Only | ||
| // study documents can be orphaned, so only they are classified. | ||
| if (!parseKey(obj.key)) continue; | ||
| documentCount += 1; | ||
| documentBytes += obj.size; | ||
| if (!trackedKeysSet.has(obj.key)) { | ||
| orphanedCount += 1; | ||
| orphanedBytes += obj.size; | ||
| } | ||
| } | ||
|
|
||
| if (!listed.truncated) break; | ||
| cursor = listed.cursor; | ||
| if (objectCount >= SUMMARY_SCAN_CAP) truncated = true; | ||
| } | ||
|
|
||
| return { | ||
| objectCount, | ||
| totalBytes, | ||
| documentCount, | ||
| documentBytes, | ||
| orphanedCount, | ||
| orphanedBytes, | ||
| truncated, | ||
| scanCap: SUMMARY_SCAN_CAP, | ||
| }; | ||
| } | ||
|
|
||
| export async function deleteAdminStorageDocuments(session: Session, params: { keys: string[] }) { | ||
| assertAdmin(session); | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win
Show unavailable database metrics when the tables query fails.
When
listAdminDatabaseTablesActionrejects and no previous data exists,useAdminDatabaseTablesleavesdataundefined. The?? 0fallbacks then display zero fordatabaseSizeBytesandtotalRows, while the page has notablesQuery.isErroror unavailable state. Render an error or unavailable value for these metrics instead.🤖 Prompt for AI Agents