diff --git a/inc/API.php b/inc/API.php index d7d5b8e..2937415 100644 --- a/inc/API.php +++ b/inc/API.php @@ -1806,15 +1806,27 @@ private function search_knowledge_base_qdrant( $message_vector, $similarity_scor } $source_scores = []; + $cutoff = null; foreach ( $knowledge_points as $point ) { if ( empty( $point['post_title'] ) || empty( $point['post_content'] ) || empty( $point['token_count'] ) ) { continue; } + // Qdrant returns points best-first, so the first scored point is + // the top match; weaker stragglers below its relative cutoff stay + // out of the context (see relative_score_cutoff()). + if ( isset( $point['score'] ) ) { + if ( null === $cutoff ) { + $cutoff = self::relative_score_cutoff( (float) $point['score'] ); + } elseif ( (float) $point['score'] < $cutoff ) { + break; + } + } + $tokens_count = intval( $point['token_count'] ); - if ( $tokens_threshold <= ( $current_token_count + $tokens_count ) ) { + if ( ( $current_token_count + $tokens_count ) > $tokens_threshold ) { continue; } @@ -1893,18 +1905,7 @@ private function search_knowledge_base_wp( $message_vector, $similarity_score_th if ( $current_token_count > $tokens_threshold ) { // Sort by score and drop the ones that do not fit in the context. - usort( - $matched_articles, - function ( $a, $b ) { - if ( $a['score'] < $b['score'] ) { - return 1; - } elseif ( $a['score'] > $b['score'] ) { - return -1; - } else { - return 0; - } - } - ); + usort( $matched_articles, [ __CLASS__, 'compare_by_score' ] ); while ( $current_token_count > $tokens_threshold ) { $article = array_pop( $matched_articles ); @@ -1923,6 +1924,24 @@ function ( $a, $b ) { return $articles_embedded_data; } + // The strongest matches go first in the context blob even when + // everything fit the budget; otherwise the order is whatever the + // database returned, and models weight early context more. + usort( $matched_articles, [ __CLASS__, 'compare_by_score' ] ); + + // Weak stragglers that merely cleared the noise floor stay out of the + // context; only chunks competitive with the best match get in. + $cutoff = self::relative_score_cutoff( $matched_articles[0]['score'] ); + + $matched_articles = array_values( + array_filter( + $matched_articles, + function ( $article ) use ( $cutoff ) { + return $article['score'] >= $cutoff; + } + ) + ); + $source_scores = []; foreach ( $matched_articles as $article ) { @@ -1945,6 +1964,57 @@ function ( $a, $b ) { return $articles_embedded_data; } + /** + * Compare two matched articles by score, highest first. + * + * @since 1.5.1 + * + * @param array $a First article, with a `score` key. + * @param array $b Second article, with a `score` key. + * + * @return int + */ + private static function compare_by_score( $a, $b ) { + return $b['score'] <=> $a['score']; + } + + /** + * The minimum score a chunk must reach to share the context with the best + * match for this query. + * + * An absolute score cannot tell an answerable question from an + * unanswerable one — a specific question about one table row scores lower + * against its (correct) page than an off-topic question scores against a + * tangential one. So within the absolute noise floor, chunks are kept + * relative to the best match for this query, and the model judges from + * the retrieved text itself whether the question can be answered. + * + * @since 1.5.1 + * + * @param float $top_score The best similarity score for this query. + * + * @return float + */ + private static function relative_score_cutoff( $top_score ) { + /** + * Filters how close to the best match a chunk must score, as a ratio + * of the top score (0–1), to be included in the chat context. A higher + * value keeps only near-equal matches; 0 keeps everything above the + * similarity score threshold. + * + * @since 1.5.1 + * + * @param float $ratio The minimum score ratio relative to the best match. Default 0.6. + */ + $ratio = (float) apply_filters( 'hyve_context_score_ratio', 0.6 ); + + if ( $ratio <= 0 ) { + return 0.0; + } + + return $top_score * min( 1.0, $ratio ); + } + /** * Order de-duplicated sources by relevance, drop weak matches and return * their post IDs. @@ -2001,9 +2071,21 @@ function ( $score ) use ( $threshold ) { * * @return string The articles blob data that match the given message vector. */ - public function search_knowledge_base( $message_vector, $similarity_score_threshold = 0.4, $tokens_threshold = 2000 ) { + public function search_knowledge_base( $message_vector, $similarity_score_threshold = 0.25, $tokens_threshold = 2000 ) { $this->source_post_ids = []; + /** + * Filters the token budget for the knowledge base context sent with a + * chat message. Matched chunks are added best-first until the budget + * is full; a larger budget lets more of the knowledge base reach the + * model at a higher API cost per message. + * + * @since 1.5.1 + * + * @param int $tokens_threshold Maximum context tokens. Default 2000. + */ + $tokens_threshold = (int) apply_filters( 'hyve_chat_context_token_limit', $tokens_threshold ); + if ( Qdrant_API::is_active() ) { return $this->search_knowledge_base_qdrant( $message_vector, $similarity_score_threshold, $tokens_threshold ); } @@ -2016,13 +2098,18 @@ public function search_knowledge_base( $message_vector, $similarity_score_thresh * * Retrieval embeds this text and searches the knowledge base with it. For the * first message it is just the question. For follow-ups it also blends in the - * most recent turns of the conversation, so a topic-less question such as - * "How difficult is it?" still carries the subject ("pickleball") into the + * visitor's most recent turns, so a topic-less question such as "How + * difficult is it?" still carries the subject ("pickleball") into the * search and matches the relevant content, instead of embedding a query with * no topic that finds nothing. The model already receives the conversation * history through the OpenAI conversation; this closes the same gap for * retrieval. * + * Only the visitor's turns are blended. Bot replies either echo knowledge + * base text (inflating every follow-up's similarity scores) or are fallback + * apologies ("Sorry, I'm not able to help with that") that drag the query + * off-topic, so they carry no retrieval signal of their own. + * * @param string $message The current user message. * @param int|string $record_id The thread post ID, when the conversation exists. * @param string $thread_id The OpenAI conversation ID, when one exists. @@ -2041,14 +2128,15 @@ private function build_retrieval_query( $message, $record_id, $thread_id = '' ) } /** - * Filters how many recent messages are blended into the retrieval query. + * Filters how many recent visitor messages are blended into the + * retrieval query. * * Set to 0 to disable conversation-aware retrieval and search with the * current message only. * * @since 1.5.0 * - * @param int $count Number of most recent messages to include. Default 6. + * @param int $count Number of most recent visitor messages to include. Default 6. * @param string $thread_id The OpenAI conversation ID, when one exists. */ $count = (int) apply_filters( 'hyve_retrieval_history_count', 6, $thread_id ); @@ -2067,7 +2155,14 @@ private function build_retrieval_query( $message, $record_id, $thread_id = '' ) $parts = []; if ( $count > 0 && ! empty( $history ) ) { - $recent = array_slice( $history, - $count ); + $user_turns = array_filter( + $history, + function ( $entry ) { + return isset( $entry['sender'] ) && 'user' === $entry['sender']; + } + ); + + $recent = array_slice( $user_turns, - $count ); foreach ( $recent as $entry ) { if ( empty( $entry['message'] ) ) { @@ -2676,14 +2771,17 @@ private function prepare_chat( $request ) { * * The similarity score threshold determines the minimum cosine similarity * required for an article to be considered relevant to the user's query. - * A higher value means stricter matching, while a lower value allows for - * broader results. + * This is a noise floor: within it, only chunks scoring close to the best + * match are kept (see `hyve_context_score_ratio`), and the model judges + * from the retrieved text whether the question can be answered. A higher + * value means stricter matching, while a lower value allows for broader + * results. * * @since 1.4.0 * - * @param float $similarity_score_threshold The similarity score threshold. Default 0.4. + * @param float $similarity_score_threshold The similarity score threshold. Default 0.25. */ - $similarity_score_threshold = apply_filters( 'hyve_similarity_score_threshold', 0.4 ); + $similarity_score_threshold = apply_filters( 'hyve_similarity_score_threshold', 0.25 ); $article_context = $this->search_knowledge_base( $message_vector, $similarity_score_threshold ); diff --git a/inc/DB_Table.php b/inc/DB_Table.php index 2d2eb71..4cd548f 100644 --- a/inc/DB_Table.php +++ b/inc/DB_Table.php @@ -957,8 +957,11 @@ private function connect_document( $post_id, $doc ) { 'type' => $this->connect_source_type( $post_id ), 'title' => (string) $doc['title'], 'url' => $url ? $url : null, - // The plugin extracts text; the platform chunks it. - 'content' => wp_strip_all_tags( (string) $doc['content'] ), + // The plugin extracts text; the platform chunks it. Structured + // extraction keeps tables as "label | value" lines and headings on + // their own line, so hosted retrieval sees the same clean text as + // self-hosted (see Tokenizer::html_to_text()). + 'content' => Tokenizer::html_to_text( (string) $doc['content'] ), ]; } diff --git a/inc/Main.php b/inc/Main.php index e547a02..54852c4 100644 --- a/inc/Main.php +++ b/inc/Main.php @@ -481,7 +481,7 @@ public static function get_default_settings() { 'chat_model' => 'gpt-5.4-nano', 'welcome_message' => '', 'default_message' => '', - 'similarity_score_threshold' => 0.4, + 'similarity_score_threshold' => 0.25, 'post_row_addon_enabled' => true, 'sound_enabled' => true, 'show_timestamp' => true, diff --git a/inc/Page_Context.php b/inc/Page_Context.php index ac5221a..596bb91 100644 --- a/inc/Page_Context.php +++ b/inc/Page_Context.php @@ -431,14 +431,6 @@ private function trim_to_limit( $text ) { return ''; } - $trimmed = trim( $chunks[0] ); - - // create_chunks re-appends the sentence separator, so text that already - // ended in a period comes back with two. - if ( '..' === substr( $trimmed, -2 ) && '...' !== substr( $trimmed, -3 ) ) { - $trimmed = substr( $trimmed, 0, -1 ); - } - - return $trimmed; + return trim( $chunks[0] ); } } diff --git a/inc/Tokenizer.php b/inc/Tokenizer.php index 6f641a8..3770689 100644 --- a/inc/Tokenizer.php +++ b/inc/Tokenizer.php @@ -1,7 +1,7 @@ /i', + // A cell boundary becomes a separator, so a value keeps its label. + '/<\/(td|th)>/i', + // Closing a block-level element ends the line. + '/<\/(tr|table|thead|tbody|tfoot|caption|p|div|h[1-6]|li|ul|ol|dl|dt|dd|blockquote|pre|figure|figcaption|section|article|header|footer|aside|address|details|summary)>/i', + ], + [ "\n", ' | ', "\n" ], + $content + ); + + // Drops the remaining tags, plus script/style/comments with their content. + $content = wp_strip_all_tags( $content ); + + $content = html_entity_decode( $content, ENT_QUOTES, 'UTF-8' ); + $content = str_replace( "\xc2\xa0", ' ', $content ); + + // Tidy up: no separator dangling at a row's end, no whitespace runs. + $content = (string) preg_replace( + [ '/[ \t]*\|[ \t]*(?=\n|$)/', '/[ \t]+/', '/ ?\n ?/', '/\n{2,}/' ], + [ '', ' ', "\n", "\n" ], + $content + ); + + return trim( $content ); + } + /** * Tokenize data. - * + * * @param array $post Post data. - * + * * @return array> */ public static function tokenize( $post ) { @@ -25,23 +77,39 @@ public static function tokenize( $post ) { $provider->setVocabCache( get_temp_dir() ); $encoder = $provider->get( 'cl100k_base' ); - $content = preg_replace( '/<[^>]+>/', '', $post['content'] ); + + // Chunks are sized, embedded and sent to the model as this cleaned + // text, so sizes are measured on what is actually stored rather than + // on markup that gets stripped later. + $content = self::html_to_text( $post['content'] ); $tokens = $encoder->encode( $content ); $article = [ 'post_id' => $post['ID'] ?? null, 'post_title' => $post['title'], - 'post_content' => $post['content'], + 'post_content' => $content, 'tokens' => $tokens, ]; $data = []; - $chunked_token_size = 1000; + /** + * Filters the maximum size of a knowledge base chunk, in tokens. + * + * Content longer than this is split into multiple chunks, each embedded + * and retrieved on its own. Smaller chunks give sharper matches for + * questions about one detail of a long page, at the cost of more + * embeddings; larger chunks keep more surrounding context together. + * + * @since 1.5.1 + * + * @param int $chunked_token_size Maximum chunk size in tokens. Default 1000. + */ + $chunked_token_size = max( 100, (int) apply_filters( 'hyve_chunk_token_size', 1000 ) ); $token_length = count( $tokens ); if ( $token_length > $chunked_token_size ) { - $shortened_sentences = self::create_chunks( $article['post_content'], $chunked_token_size ); + $shortened_sentences = self::create_chunks( $content, $chunked_token_size ); foreach ( $shortened_sentences as $shortened_sentence ) { $chunked_tokens = $encoder->encode( $post['title'] . ' ' . $shortened_sentence ); @@ -71,43 +139,71 @@ public static function tokenize( $post ) { /** * Create Chunks. - * + * + * Splits on sentence ends and line breaks, keeping each segment's own + * punctuation, and packs segments into chunks of at most `$size` tokens. + * A single segment larger than the whole budget (a table flattened to one + * line, minified markup) is hard-split by tokens rather than dropped, so + * no content silently disappears from the knowledge base. + * * @param string $text Text to chunk. * @param int $size Chunk size. - * + * * @return array */ public static function create_chunks( $text, $size = 1000 ) { + $size = max( 1, (int) $size ); + $provider = new EncoderProvider(); $provider->setVocabCache( get_temp_dir() ); - + $encoder = $provider->get( 'cl100k_base' ); - $sentences = explode( '. ', $text ); + $segments = preg_split( '/(?<=[.!?])[ \t]+|\n+/u', (string) $text, -1, PREG_SPLIT_NO_EMPTY ); + + if ( ! is_array( $segments ) ) { + return []; + } $chunks = []; $tokens_so_far = 0; $chunk = []; - foreach ( $sentences as $sentence ) { - $token_length = count( $encoder->encode( ' ' . $sentence ) ); + foreach ( $segments as $segment ) { + $token_length = count( $encoder->encode( ' ' . $segment ) ); + + if ( $token_length > $size ) { + if ( 0 < count( $chunk ) ) { + $chunks[] = implode( "\n", $chunk ); + $chunk = []; + $tokens_so_far = 0; + } + + foreach ( array_chunk( $encoder->encode( $segment ), $size ) as $piece ) { + // A hard token split can land mid-character; drop the + // stray bytes at the seam rather than storing broken text. + $piece_text = wp_check_invalid_utf8( $encoder->decode( $piece ), true ); + + if ( '' !== trim( $piece_text ) ) { + $chunks[] = trim( $piece_text ); + } + } + + continue; + } if ( $tokens_so_far + $token_length > $size ) { - $chunks[] = implode( '. ', $chunk ) . '.'; + $chunks[] = implode( "\n", $chunk ); $chunk = []; $tokens_so_far = 0; } - if ( $token_length > $size ) { - continue; - } - - $chunk[] = $sentence; + $chunk[] = $segment; $tokens_so_far += $token_length + 1; } if ( 0 < count( $chunk ) ) { - $chunks[] = implode( '. ', $chunk ) . '.'; + $chunks[] = implode( "\n", $chunk ); } return $chunks; diff --git a/tests/php/unit/tests/test-retrieval.php b/tests/php/unit/tests/test-retrieval.php new file mode 100644 index 0000000..15bfabe --- /dev/null +++ b/tests/php/unit/tests/test-retrieval.php @@ -0,0 +1,179 @@ +table = new DB_Table(); + } + + /** + * Reset filters between tests. + */ + protected function tearDown(): void { + remove_all_filters( 'hyve_chat_context_token_limit' ); + remove_all_filters( 'hyve_context_score_ratio' ); + parent::tearDown(); + } + + /** + * Insert a processed, embedded chunk row. + * + * @param string $title Chunk title. + * @param array $embeddings Embedding vector. + * @param int $token_count Token count. + * + * @return int Row ID. + */ + private function seed_chunk( $title, $embeddings, $token_count = 100 ) { + return $this->table->insert( + [ + 'post_id' => 1000 + wp_rand( 1, 999 ), + 'post_title' => $title, + 'post_content' => $title . ' content.', + 'embeddings' => wp_json_encode( $embeddings ), + 'token_count' => $token_count, + 'post_status' => 'processed', + ] + ); + } + + /** + * The strongest match leads the context blob even when everything fits + * the budget. It used to be database order, which buried the best match. + */ + public function test_wp_search_orders_context_by_score() { + // Inserted weakest-first on purpose. + $this->seed_chunk( 'WeakerMatch', [ 0.5, 0.5, 0.0 ] ); + $this->seed_chunk( 'StrongestMatch', [ 1.0, 0.0, 0.0 ] ); + $this->seed_chunk( 'Irrelevant', [ 0.0, 1.0, 0.0 ] ); + + $context = API::instance()->search_knowledge_base( [ 1.0, 0.0, 0.0 ], 0.3, 2000 ); + + $this->assertStringContainsString( 'StrongestMatch', $context ); + $this->assertStringContainsString( 'WeakerMatch', $context ); + $this->assertStringNotContainsString( 'Irrelevant', $context ); + $this->assertLessThan( + strpos( $context, 'WeakerMatch' ), + strpos( $context, 'StrongestMatch' ) + ); + } + + /** + * When the budget is exceeded, the weakest matches are the ones dropped, + * and the hyve_chat_context_token_limit filter controls the budget. + */ + public function test_context_budget_drops_weakest_and_is_filterable() { + $this->seed_chunk( 'WeakerMatch', [ 0.5, 0.5, 0.0 ], 150 ); + $this->seed_chunk( 'StrongestMatch', [ 1.0, 0.0, 0.0 ], 150 ); + + add_filter( + 'hyve_chat_context_token_limit', + function () { + return 200; + } + ); + + $context = API::instance()->search_knowledge_base( [ 1.0, 0.0, 0.0 ], 0.3, 2000 ); + + $this->assertStringContainsString( 'StrongestMatch', $context ); + $this->assertStringNotContainsString( 'WeakerMatch', $context ); + } + + /** + * Chunks scoring far below this query's best match stay out of the + * context even when they clear the absolute noise floor, so tangential + * pages do not dilute a strong answer. + */ + public function test_weak_stragglers_stay_out_of_context() { + $this->seed_chunk( 'StrongestMatch', [ 1.0, 0.0, 0.0 ] ); + // Above the 0.3 floor, but below 0.6 × the top score (~0.995). + $this->seed_chunk( 'TangentialMatch', [ 0.4, 0.9, 0.0 ] ); + + $context = API::instance()->search_knowledge_base( [ 1.0, 0.0, 0.0 ], 0.3, 2000 ); + + $this->assertStringContainsString( 'StrongestMatch', $context ); + $this->assertStringNotContainsString( 'TangentialMatch', $context ); + + // A ratio of 0 disables the band: everything above the floor is kept. + add_filter( 'hyve_context_score_ratio', '__return_zero' ); + + $context = API::instance()->search_knowledge_base( [ 1.0, 0.0, 0.0 ], 0.3, 2000 ); + + $this->assertStringContainsString( 'TangentialMatch', $context ); + } + + /** + * Build the retrieval query through the private method. + * + * @param string $message Current message. + * @param int|string $record_id Thread post ID. + * + * @return string + */ + private function build_query( $message, $record_id ) { + $method = new ReflectionMethod( API::instance(), 'build_retrieval_query' ); + $method->setAccessible( true ); + + return $method->invoke( API::instance(), $message, $record_id ); + } + + /** + * Follow-up retrieval blends the visitor's earlier turns (so the topic + * carries over) but never the bot's replies: fallback apologies drag the + * query off-topic and answered replies echo knowledge base text, which + * inflates every follow-up's similarity scores. + */ + public function test_retrieval_query_blends_visitor_turns_only() { + $record_id = Threads::create_thread( + 'Do you offer pickleball lessons?', + [ + 'thread_id' => 'conv_q1', + 'sender' => 'user', + 'message' => 'Do you offer pickleball lessons?', + ] + ); + + Threads::add_message( + $record_id, + [ + 'thread_id' => 'conv_q1', + 'sender' => 'bot', + 'message' => "Sorry, I'm not able to help with that.", + ] + ); + + $query = $this->build_query( 'How difficult is it?', $record_id ); + + $this->assertStringContainsString( 'pickleball', $query ); + $this->assertStringNotContainsString( 'Sorry', $query ); + + // The current question comes last, where it weighs the most. + $this->assertStringEndsWith( 'How difficult is it?', $query ); + } +} diff --git a/tests/php/unit/tests/test-tokenizer.php b/tests/php/unit/tests/test-tokenizer.php index fb4b48d..84603c7 100644 --- a/tests/php/unit/tests/test-tokenizer.php +++ b/tests/php/unit/tests/test-tokenizer.php @@ -51,4 +51,89 @@ public function testTokenizeLongContent() { $this->assertArrayHasKey( 'token_count', $chunk ); } } + + /** + * Table cells keep their label ("Laundry | $32.00/hr") instead of fusing + * into "Laundry$32.00/hr", and rows stay on separate lines. + */ + public function test_html_to_text_preserves_table_structure() { + $html = '
' . + '
ServiceRate
Laundry$32.00/hr
Ironing$28.00/hr
'; + + $text = Tokenizer::html_to_text( $html ); + + $this->assertStringContainsString( 'Service | Rate', $text ); + $this->assertStringContainsString( 'Laundry | $32.00/hr', $text ); + $this->assertStringContainsString( "Laundry | \$32.00/hr\nIroning", $text ); + $this->assertStringNotContainsString( 'Laundry$32', $text ); + } + + /** + * Headings do not fuse with the paragraph below, and entities decode so + * the stored text matches what a visitor would type. + */ + public function test_html_to_text_separates_blocks_and_decodes_entities() { + $html = '

Our Services

Care O’Clock offers home care & support.

'; + + $text = Tokenizer::html_to_text( $html ); + + $this->assertSame( "Our Services\nCare O’Clock offers home care & support.", $text ); + } + + /** + * A table larger than the chunk budget has no ". " sentence boundaries; it + * used to be dropped entirely, leaving the content unsearchable. It must + * chunk instead, with every row surviving somewhere in the output. + */ + public function test_tokenize_keeps_oversized_sentence_less_content() { + $rows = ''; + + for ( $i = 1; $i <= 400; $i++ ) { + $rows .= sprintf( 'Specialized long-running service number %d$%d.00/hr', $i, $i ); + } + + $post = [ + 'ID' => 1, + 'title' => 'Services & Rates', + 'content' => '' . $rows . '
', + ]; + + $result = Tokenizer::tokenize( $post ); + + $this->assertGreaterThan( 1, count( $result ) ); + + $all_text = implode( "\n", array_column( $result, 'post_content' ) ); + + $this->assertStringContainsString( 'service number 1 | $1.00/hr', $all_text ); + $this->assertStringContainsString( 'service number 400 | $400.00/hr', $all_text ); + + foreach ( $result as $chunk ) { + $this->assertLessThanOrEqual( 1100, $chunk['token_count'] ); + } + } + + /** + * A single boundary-less blob larger than the chunk size is hard-split by + * tokens, not discarded. + */ + public function test_create_chunks_hard_splits_oversized_segment() { + $blob = 'START' . str_repeat( 'x7f9q2 ', 2000 ) . 'END'; + + $chunks = Tokenizer::create_chunks( $blob, 100 ); + + $this->assertGreaterThan( 1, count( $chunks ) ); + $this->assertStringContainsString( 'START', $chunks[0] ); + $this->assertStringContainsString( 'END', end( $chunks ) ); + } + + /** + * Sentences keep their own punctuation: no doubled periods, no invented + * separators. + */ + public function test_create_chunks_keeps_punctuation() { + $chunks = Tokenizer::create_chunks( 'First sentence. Second one! Third?', 1000 ); + + $this->assertCount( 1, $chunks ); + $this->assertSame( "First sentence.\nSecond one!\nThird?", $chunks[0] ); + } }