Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 36 additions & 14 deletions scripts/seed.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import { embedMany } from "ai";

import { adminDb } from "../src/middleware/firebase";
import { embeddingModel } from "../src/middleware/genai";
import { embeddingModel, googleOptions } from "../src/middleware/genai";

async function deleteCollection(collectionPath: string) {
if (!adminDb) throw new Error("adminDb is null");
Expand Down Expand Up @@ -43,23 +43,26 @@
if (typeof obj === "string") return obj;
if (typeof obj === "number" || typeof obj === "boolean") return String(obj);

// Skip React elements
if (obj && typeof obj === "object" && obj.$$typeof) return "";

if (Array.isArray(obj)) {
if (obj.every((i) => typeof i === "string")) {
return obj.join(", ");
}
return obj.map((i) => stringify(i, indent + " ")).join("\n" + indent);
return obj
.map((i) => stringify(i, indent + " "))
.filter((s) => s !== "")
.join("\n" + indent);
}

if (typeof obj === "object") {
return Object.entries(obj)
.filter(
([, v]) =>
v !== null &&
v !== undefined &&
v !== "" &&
(Array.isArray(v) ? v.length > 0 : true),
)
.map(([k, v]) => `${indent}${k}: ${stringify(v, indent + " ")}`)
.map(([k, v]) => {
const s = stringify(v, indent + " ");
return s ? `${indent}${k}: ${s}` : "";
})
.filter((s) => s !== "")
.join("\n");
}

Expand All @@ -77,9 +80,15 @@
return chunks;
}

async function main() {

Check failure on line 83 in scripts/seed.ts

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Refactor this function to reduce its Cognitive Complexity from 33 to the 15 allowed.

See more on https://sonarcloud.io/project/issues?id=amrabed.github.io&issues=AZ6j_mvqaAGyPx85Dvca&open=AZ6j_mvqaAGyPx85Dvca&pullRequest=74
const reset = process.argv.includes("--reset");
if (reset) {
const dryRun = process.argv.includes("--dry-run");

if (dryRun) {
console.log("Dry run enabled. No changes will be made to the database.");
}

if (reset && !dryRun) {
console.log("Resetting knowledge collection...");
await deleteCollection("knowledge");
}
Expand Down Expand Up @@ -108,12 +117,16 @@
data = dataModule.default || Object.values(dataModule)[0];
}

if (!Array.isArray(data)) {
console.log(`Skipping ${file}: not an array.`);
let items: any[];
if (Array.isArray(data)) {
items = data;
} else if (data && typeof data === "object") {
items = Object.entries(data);
} else {
console.log(`Skipping ${file}: not an array or object.`);
continue;
}

const items = data;
totalItems += items.length;

for (let i = 0; i < items.length; i++) {
Expand All @@ -124,9 +137,18 @@
for (let j = 0; j < chunks.length; j += 10) {
const batchChunks = chunks.slice(j, j + 10);

if (dryRun) {
totalChunks += batchChunks.length;
console.log(
`[DRY RUN] Would process ${batchChunks.length} chunks for item ${i} from ${file}`,
);
continue;
}

const { embeddings } = await embedMany({
model: embeddingModel,
values: batchChunks,
providerOptions: googleOptions,
});

if (!adminDb) throw new Error("adminDb is null");
Expand Down
11 changes: 2 additions & 9 deletions src/app/api/chat/route.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
import { convertToModelMessages, embed, streamText } from "ai";

import { GoogleEmbeddingModelOptions } from "@ai-sdk/google";

import { findSimilarChunks } from "@/middleware/firebase";
import { chatModel, embeddingModel } from "@/middleware/genai";
import { chatModel, embeddingModel, googleOptions } from "@/middleware/genai";
import { ratelimit } from "@/middleware/upstash";

export async function POST(req: Request) {
Expand Down Expand Up @@ -43,12 +41,7 @@ export async function POST(req: Request) {
const { embedding } = await embed({
model: embeddingModel,
value: userQuery,
providerOptions: {
google: {
outputDimensionality: 1536,
taskType: "SEMANTIC_SIMILARITY",
} satisfies GoogleEmbeddingModelOptions,
},
providerOptions: googleOptions,
});

const chunks = await findSimilarChunks(embedding, 4);
Expand Down
16 changes: 13 additions & 3 deletions src/middleware/genai.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,20 @@
import { createGoogleGenerativeAI } from "@ai-sdk/google";
import {
createGoogleGenerativeAI,
GoogleEmbeddingModelOptions,
} from "@ai-sdk/google";

const google = createGoogleGenerativeAI({
apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY!,
});

const embeddingModel = google.embeddingModel("gemini-embedding-001");
const chatModel = google("gemini-3.1-flash-lite");
const chatModel = google("gemini-flash-lite-latest");

export { google, embeddingModel, chatModel };
const googleOptions = {
google: {
outputDimensionality: 1536,
taskType: "SEMANTIC_SIMILARITY",
} satisfies GoogleEmbeddingModelOptions,
};

export { google, googleOptions, embeddingModel, chatModel };