Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions packages/elf-service/src/search/ranking.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,4 @@ pub(super) use self::{
tokenize_query,
},
};
#[cfg(test)]
pub(super) use self::{
policy::types::BlendSegment,
text::{lexical_overlap_ratio, scope_description_boost},
};
#[cfg(test)] pub(super) use self::{policy::types::BlendSegment, text::lexical_overlap_ratio};
73 changes: 3 additions & 70 deletions packages/elf-service/src/search/ranking/text.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
mod scope;

use std::collections::{HashMap, HashSet};

use time::OffsetDateTime;

use crate::search::DeterministicRankingTerms;
use elf_config::{Config, Context};
use elf_domain::english_gate;

pub fn build_dense_embedding_input(
query: &str,
Expand All @@ -24,75 +25,7 @@ pub fn build_scope_context_boost_by_scope<'a>(
tokens: &[String],
context: Option<&'a Context>,
) -> HashMap<&'a str, f32> {
let Some(context) = context else { return HashMap::new() };
let Some(weight) = context.scope_boost_weight else { return HashMap::new() };

if weight <= 0.0 || tokens.is_empty() {
return HashMap::new();
}

let Some(descriptions) = context.scope_descriptions.as_ref() else { return HashMap::new() };
let mut out = HashMap::new();

for (scope, description) in descriptions {
let boost = scope_description_boost(tokens, description, weight);

if boost > 0.0 {
out.insert(scope.as_str(), boost);
}
}

out
}

pub fn scope_description_boost(tokens: &[String], description: &str, weight: f32) -> f32 {
if weight <= 0.0 || tokens.is_empty() {
return 0.0;
}

let trimmed = description.trim();

if trimmed.is_empty() || !english_gate::is_english_natural_language(trimmed) {
return 0.0;
}

let mut normalized = String::with_capacity(trimmed.len());

for ch in trimmed.chars() {
if ch.is_ascii_alphanumeric() {
normalized.push(ch.to_ascii_lowercase());
} else {
normalized.push(' ');
}
}

let mut description_tokens = HashSet::new();

for token in normalized.split_whitespace() {
if token.len() < 2 {
continue;
}

description_tokens.insert(token);
}

if description_tokens.is_empty() {
return 0.0;
}

let mut matched = 0_usize;

for token in tokens {
if description_tokens.contains(token.as_str()) {
matched += 1;
}
}

if matched == 0 {
return 0.0;
}

weight * (matched as f32 / tokens.len() as f32)
scope::build_scope_context_boost_by_scope(tokens, context)
}

pub fn tokenize_query(query: &str, max_terms: usize) -> Vec<String> {
Expand Down
100 changes: 100 additions & 0 deletions packages/elf-service/src/search/ranking/text/scope.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
use std::collections::{HashMap, HashSet};

use elf_config::Context;
use elf_domain::english_gate;

pub(super) fn build_scope_context_boost_by_scope<'a>(
tokens: &[String],
context: Option<&'a Context>,
) -> HashMap<&'a str, f32> {
let Some(context) = context else { return HashMap::new() };
let Some(weight) = context.scope_boost_weight else { return HashMap::new() };

if weight <= 0.0 || tokens.is_empty() {
return HashMap::new();
}

let Some(descriptions) = context.scope_descriptions.as_ref() else { return HashMap::new() };
let mut out = HashMap::new();

for (scope, description) in descriptions {
let boost = scope_description_boost(tokens, description, weight);

if boost > 0.0 {
out.insert(scope.as_str(), boost);
}
}

out
}

fn scope_description_boost(tokens: &[String], description: &str, weight: f32) -> f32 {
if weight <= 0.0 || tokens.is_empty() {
return 0.0;
}

let trimmed = description.trim();

if trimmed.is_empty() || !english_gate::is_english_natural_language(trimmed) {
return 0.0;
}

let mut normalized = String::with_capacity(trimmed.len());

for ch in trimmed.chars() {
if ch.is_ascii_alphanumeric() {
normalized.push(ch.to_ascii_lowercase());
} else {
normalized.push(' ');
}
}

let mut description_tokens = HashSet::new();

for token in normalized.split_whitespace() {
if token.len() < 2 {
continue;
}

description_tokens.insert(token);
}

if description_tokens.is_empty() {
return 0.0;
}

let mut matched = 0_usize;

for token in tokens {
if description_tokens.contains(token.as_str()) {
matched += 1;
}
}

if matched == 0 {
return 0.0;
}

weight * (matched as f32 / tokens.len() as f32)
}

#[cfg(test)]
mod tests {
use crate::search::ranking::text::scope;

#[test]
fn scope_description_boost_matches_whole_tokens_only() {
let tokens = vec!["go".to_string()];
let boost = scope::scope_description_boost(&tokens, "MongoDB operational notes.", 0.1);

assert_eq!(boost, 0.0);
}

#[test]
fn scope_description_boost_scales_by_fraction_of_matched_tokens() {
let tokens = vec!["security".to_string(), "policy".to_string(), "deployment".to_string()];
let boost = scope::scope_description_boost(&tokens, "Security policy notes.", 0.12);

assert!((boost - 0.08).abs() < 1e-4, "Unexpected boost: {boost}");
}
}
16 changes: 0 additions & 16 deletions packages/elf-service/src/search/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,22 +31,6 @@ fn dense_embedding_input_skips_empty_project_context() {
assert_eq!(input, "Find payments code.");
}

#[test]
fn scope_description_boost_matches_whole_tokens_only() {
let tokens = vec!["go".to_string()];
let boost = ranking::scope_description_boost(&tokens, "MongoDB operational notes.", 0.1);

assert_eq!(boost, 0.0);
}

#[test]
fn scope_description_boost_scales_by_fraction_of_matched_tokens() {
let tokens = vec!["security".to_string(), "policy".to_string(), "deployment".to_string()];
let boost = ranking::scope_description_boost(&tokens, "Security policy notes.", 0.12);

assert!((boost - 0.08).abs() < 1e-4, "Unexpected boost: {boost}");
}

#[test]
fn normalize_queries_includes_original_and_dedupes() {
let queries = vec!["alpha".to_string(), "beta".to_string(), "alpha".to_string()];
Expand Down