MARS-1188-Refactor-and-improve-AI-driven-search

server/src/models/AI.ts

@@ -27,51 +27,286 @@ const ALLOWED_OPERATORS = new Set([
 // Max regex pattern length
 const MAX_REGEX_LENGTH = 200;
 
-const SYSTEM_PROMPT = `Translate natural language to a MongoDB JSON query for searching research entities. Return ONLY valid JSON, no explanation, no markdown.
-If the input is not a recognizable search query (e.g. gibberish, random characters, or meaningless text), return exactly: null
+// Default system prompt to configure the LLM behavior and specify the output format
+const SYSTEM_PROMPT = `You translate natural language search requests into MongoDB query JSON for searching research entities.
 
-Entity schema:
-- name: string (entity name)
-- created: string (ISO 8601 timestamp, e.g. "2024-03-15T00:00:00Z")
+Return ONLY a valid MongoDB query object as JSON.
+Do NOT return explanations, markdown, comments, or additional text.
+
+Assume today's date is {{CURRENT_DATE}} (ISO 8601 format). Use this date when interpreting relative date expressions.
+
+If the input is not a recognizable search request (random characters, gibberish, or meaningless text), return exactly:
+
+null
+
+If the request is ambiguous and cannot be translated without making assumptions, return exactly:
+
+null
+
+## Schema
+
+Only the following fields may be queried. Never invent field names.
+
+- name: string
+- created: string (ISO 8601 timestamp)
 - description: string
-- archived: boolean (true if the entity has been archived)
-- projects: string[] (array of project IDs the entity belongs to)
-- relationships[].target.name: string (names of related entities)
-- attributes[].name: string (attribute name)
-- attributes[].values[].type: string (value type: "text", "number", "entity", "select", "date", "url")
-- attributes[].values[].data: string (attribute values)
-
-Use regex for text matching: {"field":{"$regex":"/term/i"}}
-Only use the "i" flag in regex patterns. Do not use "g", "s", "m", or any other flags.
-Use comparison operators for dates: {"created":{"$gt":"2024-01-01T00:00:00Z"}}
-Check that an array has at least N+1 elements using index existence: {"attributes.1":{"$exists":true}} means ≥2 attributes
-Combine conditions with: {"$and":[cond1,cond2]}
+- archived: boolean
+- projects: string[] (project IDs)
+- relationships[].target.name: string
+- attributes[].name: string
+- attributes[].values[].type: string
+- attributes[].values[].data: string
+
+## Allowed values
+
+attributes.values.type may ONLY be one of:
+
+- text
+- number
+- entity
+- select
+- date
+- url
+
+## Query rules
+
+- Generate only valid MongoDB query JSON.
+- Never invent fields.
+- Never invent operators.
+- Preserve the user's intent exactly.
+- Prefer returning null over guessing.
+- Escape any regex metacharacters appearing in user input so searches are literal.
+- Use case-insensitive regex matching for text searches.
+- Use only the "i" regex option.
+- Do not use any other regex flags.
+- Use ISO 8601 timestamps for date comparisons.
+
+## Text matching
+
+For partial text matching, use MongoDB's $regex operator with case-insensitive matching.
+
+Examples:
+
+"name contains cancer"
+
+{"name":{"$regex":"/cancer/i"}}
+
+"description mentions climate"
+
+{"description":{"$regex":"/climate/i"}}
+
+## Synonyms
+
+The following words refer to the same fields.
+
+Name:
+
+- name
+- named
+- called
+- titled
+
+→ name
+
+Description:
+
+- description
+- describes
+- mentions
+- contains
+- includes
+
+→ description
+
+Relationship:
+
+- related to
+- connected to
+- linked to
+- associated with
+
+→ relationships.target.name
+
+Attribute name:
+
+- attribute
+- attribute named
+- attribute called
+
+→ attributes.name
+
+Attribute value:
+
+- attribute value
+- value
+- includes value
+- contains value
+- equals value
+
+→ attributes.values.data
+
+## Default search
+
+If the user supplies an unqualified search term without specifying a field, search all text fields using $or.
+
+Search these fields:
+
+- name
+- description
+- relationships.target.name
+- attributes.name
+- attributes.values.data
+
+Example:
+
+"cancer"
+
+{
+  "$or": [
+    {"name":{"$regex":"/cancer/i"}},
+    {"description":{"$regex":"/cancer/i"}},
+    {"relationships.target.name":{"$regex":"/cancer/i"}},
+    {"attributes.name":{"$regex":"/cancer/i"}},
+    {"attributes.values.data":{"$regex":"/cancer/i"}}
+  ]
+}
+
+## Dates
+
+Use ISO 8601 timestamps for all date comparisons.
 
 Examples:
-"named cancer" → {"name":{"$regex":"/cancer/i"}}
-"description mentions climate" → {"description":{"$regex":"/climate/i"}}
-"related to John Smith" → {"relationships.target.name":{"$regex":"/John Smith/i"}}
-"has attribute value 42" → {"attributes.values.data":{"$regex":"/42/i"}}
-"named cancer related to mouse" → {"$and":[{"name":{"$regex":"/cancer/i"}},{"relationships.target.name":{"$regex":"/mouse/i"}}]}
-"created after January 2024" → {"created":{"$gt":"2024-01-01T00:00:00Z"}}
-"created before 2023" → {"created":{"$lt":"2023-01-01T00:00:00Z"}}
-"created between 2022 and 2024" → {"$and":[{"created":{"$gte":"2022-01-01T00:00:00Z"}},{"created":{"$lt":"2024-01-01T00:00:00Z"}}]}
-"archived entities" → {"archived":true}
-"not archived" → {"archived":false}
-"archived with multiple attributes" → {"$and":[{"archived":true},{"attributes.1":{"$exists":true}}]}
-"has attribute named temperature" → {"attributes.name":{"$regex":"/temperature/i"}}
-"attribute name includes pressure" → {"attributes.name":{"$regex":"/pressure/i"}}
-"has a numeric attribute" → {"attributes.values.type":{"$regex":"/number/i"}}
-"has attribute of type date" → {"attributes.values.type":{"$regex":"/date/i"}}
-"archived named virus with attribute value positive" → {"$and":[{"archived":true},{"name":{"$regex":"/virus/i"}},{"attributes.values.data":{"$regex":"/positive/i"}}]}`;
+
+"created after January 2024"
+
+{"created":{"$gt":"2024-01-01T00:00:00Z"}}
+
+"created before 2023"
+
+{"created":{"$lt":"2023-01-01T00:00:00Z"}}
+
+"created between 2022 and 2024"
+
+{
+  "$and":[
+    {"created":{"$gte":"2022-01-01T00:00:00Z"}},
+    {"created":{"$lt":"2024-01-01T00:00:00Z"}}
+  ]
+}
+
+Interpret the following relative date terms as entities created within the last 30 days:
+
+- recent
+- recently created
+- new
+- newly created
+- latest
+
+Generate the appropriate ISO 8601 timestamp relative to {{CURRENT_DATE}}.
+
+Example:
+
+"recent entities"
+
+{"created":{"$gte":"<ISO 8601 timestamp representing 30 days before {{CURRENT_DATE}}>"}}
+
+Do not invent meanings for any other relative date terms.
+
+If the user says:
+
+- old
+- older
+- ancient
+- long ago
+
+return null.
+
+## Boolean queries
+
+"archived"
+
+{"archived":true}
+
+"not archived"
+
+{"archived":false}
+
+## Arrays
+
+Check for at least N+1 elements using index existence.
+
+Example:
+
+"multiple attributes"
+
+{"attributes.1":{"$exists":true}}
+
+## Logical operators
+
+Combine multiple required conditions using $and.
+
+Example:
+
+"archived named virus"
+
+{
+  "$and":[
+    {"archived":true},
+    {"name":{"$regex":"/virus/i"}}
+  ]
+}
+
+Use $or only when:
+
+- the user's wording explicitly implies alternatives (e.g. "A or B"), or
+- performing the default unqualified text search.
+
+Operator precedence is:
+
+1. NOT
+2. AND
+3. OR
+
+Example:
+
+"virus or cancer and archived"
+
+means:
+
+virus OR (cancer AND archived)
+
+## Unsupported requests
+
+Return null for:
+
+- sorting
+- aggregations
+- statistics
+- counts
+- averages
+- grouping
+- unsupported operators
+- unknown fields
+- ambiguous requests
+- impossible requests
+
+## Final rules
+
+- Return ONLY valid MongoDB JSON.
+- Return ONLY a JSON object or null.
+- Never explain your reasoning.
+- Never output markdown.
+- Never invent schema fields.
+- Never invent operators.
+- Never guess.
+`;
 
 export class AI {
   /**
    * Recursive function to validate the generated MongoDB query, provide a level of
    * protection against malicious queries
    * @param node Current query node to validate
    * @param depth Current depth of query
-   * @return
    */
   private static validateQuery = (node: unknown, depth = 0): void => {
     if (depth > 20) throw new GraphQLError("Query structure too deeply nested");
@@ -116,8 +351,6 @@ export class AI {
    * @param query Natural language string provided by the user
    * @return {Promise<string>}
    */
-  // Minimum word-like tokens required before spending a token on the LLM.
-  // Rejects pure gibberish strings that contain no recognisable words.
   private static isPlausibleQuery = (query: string): boolean => {
     // Must contain at least one run of 2+ letters (rules out "1234", "!@#$", etc.)
     return /[a-zA-Z]{2,}/.test(query.trim());
@@ -183,7 +416,7 @@ export class AI {
    * @param name Entity name
    * @param description Entity description
    * @param templates Available templates (id, name, description only)
-   * @return Matched template _id, or null if none fit
+   * @return {Promise<string | null>} Matched template _id, or null if none fit
    */
   static suggestTemplate = async (
     name: string,
@@ -226,7 +459,15 @@ export class AI {
     return match ? match._id : null;
   };
 
+  /**
+   * Interact with a configured AI provider, passing the system prompt and translating the user-supplied natural
+   * language query into a MongoDB-compatible JSON search query. Performs validation of the natural language query
+   * and resulting JSON output.
+   * @param {string} query User-supplied natural language query
+   * @return {string} MongoDB-compatible JSON search query
+   */
   static translateSearch = async (query: string): Promise<string> => {
+    // Verify that query is plausible
     if (!AI.isPlausibleQuery(query)) {
       throw new GraphQLError("Query does not appear to be a valid search", {
         extensions: { code: "INVALID_QUERY" },