diff --git a/frontend/src/global_styles/openproject.sass b/frontend/src/global_styles/openproject.sass index f3271f1dc015..72b27557f660 100644 --- a/frontend/src/global_styles/openproject.sass +++ b/frontend/src/global_styles/openproject.sass @@ -26,6 +26,7 @@ @import "../../../modules/documents/app/components/_index.sass" @import "../../../modules/gitlab_integration/app/components/_index.sass" @import "../../../modules/grids/app/components/_index.sass" +@import "../../../modules/llm/app/components/_index.sass" @import "../../../modules/meeting/app/components/_index.sass" @import "../../../modules/overviews/app/components/_index.sass" @import "../../../modules/resource_management/app/components/_index.sass" diff --git a/modules/llm/app/components/_index.sass b/modules/llm/app/components/_index.sass new file mode 100644 index 000000000000..21e214112700 --- /dev/null +++ b/modules/llm/app/components/_index.sass @@ -0,0 +1 @@ +@import "llm_connections/semantic_indexes/box_component" diff --git a/modules/llm/app/components/llm_connections/feature_binding_component.html.erb b/modules/llm/app/components/llm_connections/feature_binding_component.html.erb index 5189690fcd16..9c1352815093 100644 --- a/modules/llm/app/components/llm_connections/feature_binding_component.html.erb +++ b/modules/llm/app/components/llm_connections/feature_binding_component.html.erb @@ -33,52 +33,26 @@ See COPYRIGHT and LICENSE files for more details. <%= render(Primer::Beta::Text.new(tag: :p, color: :muted, mb: 2)) { feature.caption } %> <% end %> - <% if dangling? %> - <%= render(Primer::Alpha::Banner.new(scheme: :warning, mb: 2, icon: :alert)) do %> - <%= t("admin.llm_feature_bindings.dangling", model: binding.resolved_model_id) %> - <% end %> - <% end %> - - <% if locked? %> - <%= render(Primer::Alpha::Banner.new(scheme: :warning, mb: 2, icon: :lock)) do %> - <%= t("admin.llm_feature_bindings.locked", model: binding.model_id) %> + <% unless feature.embedding? %> + <% if dangling? %> + <%= render(Primer::Alpha::Banner.new(scheme: :warning, mb: 2, icon: :alert)) do %> + <%= t("admin.llm_feature_bindings.dangling", model: binding.resolved_model_id) %> + <% end %> <% end %> - <% end %> - <%= primer_form_with(model: form_model, url: form_url, method: :patch, scope: :llm_feature_binding) do |f| %> - <%= render( - LlmConnections::FeatureBindingForm.new( - f, - options: model_options, - inherit_label:, - feature_key: feature.key, - locked: locked?, - embedding: feature.embedding?, - models_left_out: models_left_out?, - selected_model_id: binding&.model_id, - dimensions_hint: probed_dimensions - ) - ) %> - <% end %> - - <% if locked? && feature.embedding? %> - <%# Rendered as text rather than disabled inputs: a disabled input submits - nothing, so the values would arrive blank and wipe the columns. %> - <%= render(Primer::Beta::Text.new(tag: :p, font_weight: :bold, mt: 2, mb: 1)) do %> - <%= t("admin.llm_feature_bindings.locked_values_heading") %> - <% end %> - <% locked_values.each do |label, value| %> - <%= render(Primer::Beta::Text.new(tag: :p, color: :muted, mb: 0)) do %> - <%= "#{label}: #{value}" %> - <% end %> + <%= primer_form_with(model: form_model, url: form_url, method: :patch, scope: :llm_feature_binding) do |f| %> + <%= render( + LlmConnections::FeatureBindingForm.new( + f, + options: model_options, + inherit_label:, + feature_key: feature.key, + models_left_out: models_left_out?, + selected_model_id: binding&.model_id + ) + ) %> <% end %> <% end %> - <% if feature.embedding? %> - <%= button_to t("admin.work_package_embeddings.reindex.button"), - reindex_work_package_embeddings_path, - method: :post, - class: "Button Button--secondary Button--small mt-3" %> - <%= render(LlmConnections::IndexingStatusComponent.new) %> - <% end %> + <%= content %> <% end %> diff --git a/modules/llm/app/components/llm_connections/feature_binding_component.rb b/modules/llm/app/components/llm_connections/feature_binding_component.rb index 70b9febf5817..7a7ac7c8272d 100644 --- a/modules/llm/app/components/llm_connections/feature_binding_component.rb +++ b/modules/llm/app/components/llm_connections/feature_binding_component.rb @@ -63,28 +63,8 @@ def inherit_label end end - def locked? = binding&.locked? - def dangling? = binding&.dangling? - # What the embeddings probe last saw, offered as information. Never filled - # into the field: the server decides the vector size at index time. - def probed_dimensions - return unless feature.embedding? - - model_id = binding&.resolved_model_id - return if model_id.blank? - - connection.capability_verdicts.for_model(model_id).for_capability(:embeddings).first&.dimensions - end - - def locked_values - [ - [LlmFeatureBinding.human_attribute_name(:model_id), binding.model_id], - [LlmFeatureBinding.human_attribute_name(:dimensions), binding.dimensions || "—"] - ] - end - private attr_reader :feature, :connection, :binding diff --git a/modules/llm/app/components/llm_connections/indexing_status_component.html.erb b/modules/llm/app/components/llm_connections/indexing_status_component.html.erb deleted file mode 100644 index 23d2a0a4a55d..000000000000 --- a/modules/llm/app/components/llm_connections/indexing_status_component.html.erb +++ /dev/null @@ -1,52 +0,0 @@ -<%#-- copyright -OpenProject is an open source project management software. -Copyright (C) the OpenProject GmbH - -This program is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License version 3. - -OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: -Copyright (C) 2006-2013 Jean-Philippe Lang -Copyright (C) 2010-2013 the ChiliProject Team - -This program is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License -as published by the Free Software Foundation; either version 2 -of the License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - -See COPYRIGHT and LICENSE files for more details. - -++#%> -<%= component_wrapper(**wrapper_data_attrs) do %> - <%= render(Primer::Box.new(mt: 3)) do %> - <%= render(Primer::Box.new(display: :flex, align_items: :center, mb: 2)) do %> - <%= render(Primer::Box.new(display: :flex, mr: 2, test_selector: "reindex-state-#{state}")) do %> - <% case state %> - <% when :running %> - <%= render(Primer::Beta::Spinner.new(size: :small, "aria-label": t("admin.work_package_embeddings.status.state.running"))) %> - <% when :succeeded %> - <%= render(Primer::Beta::Octicon.new(icon: :"check-circle-fill", color: :success, "aria-label": t("admin.work_package_embeddings.status.state.succeeded"))) %> - <% when :failed %> - <%= render(Primer::Beta::Octicon.new(icon: :"x-circle-fill", color: :danger, "aria-label": t("admin.work_package_embeddings.status.state.failed"))) %> - <% end %> - <% end %> - - <%= render(Primer::Beta::ProgressBar.new(flex: 1)) { |bar| bar.with_item(percentage:, bg: progress_color) } %> - <% end %> - - <% if failed_count.positive? %> - <%= render(Primer::Alpha::Banner.new(scheme: :danger, mb: 2, icon: :alert)) do %> - <%= t("admin.work_package_embeddings.status.failed", count: failed_count) %> - <% end %> - <% end %> - <% end %> -<% end %> diff --git a/modules/llm/app/components/llm_connections/semantic_indexes/activate_dialog_component.html.erb b/modules/llm/app/components/llm_connections/semantic_indexes/activate_dialog_component.html.erb new file mode 100644 index 000000000000..8cb9f1e498ad --- /dev/null +++ b/modules/llm/app/components/llm_connections/semantic_indexes/activate_dialog_component.html.erb @@ -0,0 +1,66 @@ +<%#-- copyright +OpenProject is an open source project management software. +Copyright (C) the OpenProject GmbH + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License version 3. + +OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +Copyright (C) 2006-2013 Jean-Philippe Lang +Copyright (C) 2010-2013 the ChiliProject Team + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License +as published by the Free Software Foundation; either version 2 +of the License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +See COPYRIGHT and LICENSE files for more details. + +++#%> + +<%= + render( + Primer::Alpha::Dialog.new( + title: t("admin.semantic_indexes.activate.title", title: semantic_index.title), + size: :medium, + id: DIALOG_ID, + test_selector: DIALOG_ID + ) + ) do |d| + d.with_header(variant: :large) + + d.with_body do + form_with(url: activate_semantic_index_path(semantic_index), method: :post, id: FORM_ID) do + render(Primer::Beta::Text.new(tag: :p)) { body } + end + end + + d.with_footer do + component_collection do |buttons| + buttons.with_component(Primer::Beta::Button.new(data: { close_dialog_id: DIALOG_ID })) do + t(:button_cancel) + end + + buttons.with_component( + Primer::Beta::Button.new( + scheme: :primary, + form: FORM_ID, + type: :submit, + test_selector: "llm-semantic-index--activate-submit" + ) + ) do + t("admin.semantic_indexes.activate.confirm") + end + end + end + end +%> diff --git a/modules/llm/app/components/llm_connections/semantic_indexes/activate_dialog_component.rb b/modules/llm/app/components/llm_connections/semantic_indexes/activate_dialog_component.rb new file mode 100644 index 000000000000..7a614578ca40 --- /dev/null +++ b/modules/llm/app/components/llm_connections/semantic_indexes/activate_dialog_component.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module LlmConnections + module SemanticIndexes + class ActivateDialogComponent < ApplicationComponent + include OpTurbo::Streamable + include OpPrimer::ComponentHelpers + + DIALOG_ID = "llm-semantic-index-activate-dialog" + FORM_ID = "llm-semantic-index-activate-form" + + alias_method :semantic_index, :model + + options :current + + private + + def body + if current + t("admin.semantic_indexes.activate.body_switch_html", current: current.title, title: semantic_index.title) + else + t("admin.semantic_indexes.activate.body_html", title: semantic_index.title) + end + end + end + end +end diff --git a/modules/llm/app/components/llm_connections/semantic_indexes/box_component.html.erb b/modules/llm/app/components/llm_connections/semantic_indexes/box_component.html.erb index b16951d2c3e2..55f1c5f508c5 100644 --- a/modules/llm/app/components/llm_connections/semantic_indexes/box_component.html.erb +++ b/modules/llm/app/components/llm_connections/semantic_indexes/box_component.html.erb @@ -27,31 +27,57 @@ See COPYRIGHT and LICENSE files for more details. ++#%> -<%= render(Primer::Box.new(border: true, border_radius: 2, p: 3, mb: 3, test_selector: "llm-semantic-index--box")) do %> - <%= - flex_layout(justify_content: :space_between, align_items: :flex_start, mb: 3) do |header| - header.with_column do - concat(render(Primer::Beta::Text.new(tag: :h3, font_size: 4, font_weight: :bold, mb: 1)) { t("admin.semantic_indexes.box.title") }) - concat(render(Primer::Beta::Text.new(tag: :p, color: :muted, mb: 0)) { t("admin.semantic_indexes.box.caption") }) - end +<%= component_wrapper(**wrapper_data_attrs) do %> + <%= render(Primer::Box.new(mt: 4, test_selector: "llm-semantic-index--box")) do %> + <%= + flex_layout(justify_content: :space_between, align_items: :flex_start, mb: 3) do |header| + header.with_column do + concat(render(Primer::Beta::Text.new(tag: :h4, font_weight: :bold, mb: 1)) { t("admin.semantic_indexes.box.title") }) + concat(render(Primer::Beta::Text.new(tag: :p, color: :muted, mb: 0)) { t("admin.semantic_indexes.box.caption") }) + end - header.with_column(ml: 3) do - render( - Primer::Beta::Button.new( - scheme: :primary, - size: :small, - tag: :a, - href: new_dialog_semantic_indexes_path, - data: { controller: "async-dialog" }, - test_selector: "llm-semantic-index--new-button" - ) - ) do |button| - button.with_leading_visual_icon(icon: :plus) - t("admin.semantic_indexes.box.new") + header.with_column(ml: 3) do + render( + Primer::Beta::Button.new( + scheme: :primary, + size: :small, + tag: :a, + href: new_dialog_semantic_indexes_path, + data: { controller: "async-dialog" }, + test_selector: "llm-semantic-index--new-button" + ) + ) do |button| + button.with_leading_visual_icon(icon: :plus) + t("admin.semantic_indexes.box.new") + end end end - end - %> + %> + + <% if active_semantic_index %> + <%= render(Primer::Box.new(mb: 4, classes: "op-semantic-index-active-card", test_selector: "llm-semantic-index--active")) do %> + <%= + flex_layout( + justify_content: :space_between, align_items: :center, flex_wrap: :wrap, + px: 3, py: 2, classes: "op-semantic-index-active-card--header" + ) do |header| + header.with_column(font_weight: :bold) do + concat(render(Primer::Beta::Octicon.new(icon: :"check-circle-fill", mr: 2))) + concat(t("admin.semantic_indexes.box.active_heading")) + end + header.with_column(font_size: :small) { t("admin.semantic_indexes.box.active_caption") } + end + %> + <%= render(LlmConnections::SemanticIndexes::TableComponent.new(rows: [active_semantic_index], active: true)) %> + <% end %> + <% else %> + <%= render(Primer::Box.new(mb: 4, test_selector: "llm-semantic-index--no-active")) do %> + <%= render(LlmConnections::SemanticIndexes::TableComponent.new(rows: [], active: true)) %> + <% end %> + <% end %> - <%= render(LlmConnections::SemanticIndexes::TableComponent.new(rows: semantic_indexes)) %> + <% if other_semantic_indexes.any? %> + <%= render(LlmConnections::SemanticIndexes::TableComponent.new(rows: other_semantic_indexes)) %> + <% end %> + <% end %> <% end %> diff --git a/modules/llm/app/components/llm_connections/semantic_indexes/box_component.rb b/modules/llm/app/components/llm_connections/semantic_indexes/box_component.rb index 5e870c67334a..12ed94f41706 100644 --- a/modules/llm/app/components/llm_connections/semantic_indexes/box_component.rb +++ b/modules/llm/app/components/llm_connections/semantic_indexes/box_component.rb @@ -32,8 +32,37 @@ module LlmConnections module SemanticIndexes class BoxComponent < ApplicationComponent include OpPrimer::ComponentHelpers + include OpTurbo::Streamable + + POLL_INTERVAL_MS = 2000 options :semantic_indexes + + def active_semantic_index + semantic_indexes.find { it.id == active_semantic_index_id } + end + + def other_semantic_indexes + semantic_indexes.reject { it.id == active_semantic_index_id } + end + + def wrapper_data_attrs + return {} unless semantic_indexes.any?(&:indexing?) + + { + data: { + controller: "poll-for-changes", + poll_for_changes_url_value: status_semantic_indexes_path, + poll_for_changes_interval_value: POLL_INTERVAL_MS + } + } + end + + private + + def active_semantic_index_id + @active_semantic_index_id ||= SemanticIndex.active_id + end end end end diff --git a/modules/llm/app/components/llm_connections/semantic_indexes/box_component.sass b/modules/llm/app/components/llm_connections/semantic_indexes/box_component.sass new file mode 100644 index 000000000000..b8f49b495459 --- /dev/null +++ b/modules/llm/app/components/llm_connections/semantic_indexes/box_component.sass @@ -0,0 +1,23 @@ +.op-semantic-index-active-card + --op-semantic-index-active-emphasis: var(--borderColor-success-emphasis) + --op-semantic-index-active-muted: var(--bgColor-success-muted) + --op-semantic-index-active-fg: var(--fgColor-success) + + border: var(--borderWidth-thin) solid var(--op-semantic-index-active-emphasis) + border-left-width: 4px + border-radius: var(--borderRadius-medium) + box-shadow: 0 0 0 3px var(--op-semantic-index-active-muted) + overflow: hidden + + &--header + background: var(--op-semantic-index-active-muted) + color: var(--op-semantic-index-active-fg) + + .Box + border: 0 + border-radius: 0 + + &--marker + background: var(--op-semantic-index-active-muted) + border-color: transparent + color: var(--op-semantic-index-active-fg) diff --git a/modules/llm/app/components/llm_connections/semantic_indexes/form_component.html.erb b/modules/llm/app/components/llm_connections/semantic_indexes/form_component.html.erb index 58c724ac532e..047e32ff1ee5 100644 --- a/modules/llm/app/components/llm_connections/semantic_indexes/form_component.html.erb +++ b/modules/llm/app/components/llm_connections/semantic_indexes/form_component.html.erb @@ -27,8 +27,12 @@ See COPYRIGHT and LICENSE files for more details. ++#%> -<%= - component_wrapper do +<%= component_wrapper do %> + <% if locked? %> + <%= render(Primer::Alpha::Banner.new(scheme: :warning, icon: :lock, mb: 3)) { t("admin.semantic_indexes.form.locked_once_indexed") } %> + <% end %> + + <%= primer_form_with( id: FORM_ID, model:, @@ -37,7 +41,7 @@ See COPYRIGHT and LICENSE files for more details. method: http_method, data: { turbo: true } ) do |f| - render(::SemanticIndexes::Form.new(f, model_options:)) + render(::SemanticIndexes::Form.new(f, model_options:, locked: locked?)) end - end -%> + %> +<% end %> diff --git a/modules/llm/app/components/llm_connections/semantic_indexes/form_component.rb b/modules/llm/app/components/llm_connections/semantic_indexes/form_component.rb index ab52b553bb74..51174d6531f9 100644 --- a/modules/llm/app/components/llm_connections/semantic_indexes/form_component.rb +++ b/modules/llm/app/components/llm_connections/semantic_indexes/form_component.rb @@ -38,6 +38,8 @@ class FormComponent < ApplicationComponent options :connection + def locked? = model.parameters_locked? + private def http_method diff --git a/modules/llm/app/components/llm_connections/semantic_indexes/row_component.rb b/modules/llm/app/components/llm_connections/semantic_indexes/row_component.rb index 3f70394d5331..03221114701d 100644 --- a/modules/llm/app/components/llm_connections/semantic_indexes/row_component.rb +++ b/modules/llm/app/components/llm_connections/semantic_indexes/row_component.rb @@ -40,7 +40,7 @@ def row_data def title safe_join( [ - render(Primer::Beta::Text.new(font_weight: :bold)) { semantic_index.title }, + render(Primer::Beta::Text.new(font_weight: :bold, font_size: active? ? 4 : nil)) { semantic_index.title }, description ].compact ) @@ -48,18 +48,66 @@ def title delegate :model_id, :context_token_length, :embedding_vector_dimensions, to: :semantic_index + STATE_LABEL_SCHEMES = { + "not_indexed" => :secondary, + "indexing" => :attention, + "ready" => :accent, + "failed" => :danger + }.freeze + + def column_css_class(column) + return "#{super} -no-ellipsis" if column == :status + + super + end + def status - render(Primer::Beta::Label.new(scheme: :secondary)) do - I18n.t("admin.semantic_indexes.states.#{semantic_index.state}") - end + safe_join( + [ + render(Primer::Beta::Label.new(scheme: STATE_LABEL_SCHEMES.fetch(semantic_index.state))) do + I18n.t("admin.semantic_indexes.states.#{semantic_index.state}") + end, + indexing_progress, + error_message + ].compact + ) end def button_links - [action_menu] + [start_indexing_button, activation, action_menu].compact end private + def active? = table.active + + def activation + if active? + active_marker + elsif semantic_index.ready? + activate_button + end + end + + def active_marker + render(Primer::Beta::Label.new(size: :large, classes: "op-semantic-index-active-card--marker", + test_selector: "llm-semantic-index--active-marker")) do + safe_join([render(Primer::Beta::Octicon.new(icon: :check, mr: 1)), I18n.t("admin.semantic_indexes.active")]) + end + end + + def activate_button + render( + Primer::Beta::Button.new( + size: :small, + tag: :a, + href: activation_dialog_semantic_index_path(semantic_index), + data: { controller: "async-dialog" }, + test_selector: "llm-semantic-index--activate-#{semantic_index.id}" + ) + ) { I18n.t("admin.semantic_indexes.activate.button") } + end + def description return if semantic_index.description.blank? @@ -68,29 +116,92 @@ def description end end + def indexing_progress + return unless semantic_index.indexing? + + batch = semantic_index.indexing_batch + percentage = batch ? Llm::IndexingProgress.new(batch).percentage : 0 + + render(Primer::Box.new(mt: 1, test_selector: "llm-semantic-index--progress-#{semantic_index.id}")) do + safe_join( + [ + render(Primer::Beta::ProgressBar.new(size: :small)) do |bar| + bar.with_item(percentage:, bg: :success_emphasis) + end, + render(Primer::Beta::Text.new(tag: :div, color: :subtle, font_size: :small, mt: 1)) do + I18n.t("admin.semantic_indexes.indexing.progress", percentage:) + end + ] + ) + end + end + + def start_indexing_button + return unless semantic_index.indexable? + + helpers.form_with(url: start_indexing_semantic_index_path(semantic_index), + method: :post, + class: "d-inline-block") do + render( + Primer::Beta::Button.new( + type: :submit, + size: :small, + test_selector: "llm-semantic-index--start-indexing-#{semantic_index.id}" + ) + ) do |button| + button.with_leading_visual_icon(icon: semantic_index.not_indexed? ? :play : :sync) + I18n.t(semantic_index.state, scope: "admin.semantic_indexes.start_indexing.button") + end + end + end + + def error_message + return unless semantic_index.failed? && semantic_index.error_message.present? + + render(Primer::Beta::Text.new(tag: :div, color: :danger, font_size: :small, mt: 1)) do + semantic_index.error_message + end + end + def action_menu render(Primer::Alpha::ActionMenu.new(anchor_align: :end, test_selector: "llm-semantic-index--menu-#{semantic_index.id}")) do |menu| menu.with_show_button(icon: "kebab-horizontal", "aria-label": I18n.t(:label_more), scheme: :invisible) - menu.with_item( - label: I18n.t(:button_edit), - tag: :a, - href: edit_dialog_semantic_index_path(semantic_index), - content_arguments: { data: { controller: "async-dialog" } } - ) do |item| - item.with_leading_visual_icon(icon: :pencil) - end + edit_item(menu) + menu.with_divider + active? ? locked_delete_item(menu) : delete_item(menu) + end + end - menu.with_item( - label: I18n.t("admin.semantic_indexes.menu.delete"), - scheme: :danger, - tag: :a, - href: deletion_dialog_semantic_index_path(semantic_index), - content_arguments: { data: { controller: "async-dialog" } } - ) do |item| - item.with_leading_visual_icon(icon: :trash) - end + def edit_item(menu) + menu.with_item( + label: I18n.t(semantic_index.parameters_locked? ? "admin.semantic_indexes.menu.edit_locked" : :button_edit), + tag: :a, + href: edit_dialog_semantic_index_path(semantic_index), + content_arguments: { data: { controller: "async-dialog" } } + ) do |item| + item.with_leading_visual_icon(icon: :pencil) + end + end + + def locked_delete_item(menu) + menu.with_item(label: I18n.t("admin.semantic_indexes.menu.delete"), scheme: :danger, disabled: true) do |item| + item.with_leading_visual_icon(icon: :trash) + item.with_description { I18n.t("admin.semantic_indexes.menu.locked_while_active") } + end + end + + def delete_item(menu) + menu.with_item( + label: I18n.t("admin.semantic_indexes.menu.delete"), + scheme: :danger, + tag: :a, + href: deletion_dialog_semantic_index_path(semantic_index), + disabled: semantic_index.indexing?, + content_arguments: { data: { controller: "async-dialog" } } + ) do |item| + item.with_leading_visual_icon(icon: :trash) end end end diff --git a/modules/llm/app/components/llm_connections/semantic_indexes/table_component.rb b/modules/llm/app/components/llm_connections/semantic_indexes/table_component.rb index bc9ebd662465..545e8acfaa57 100644 --- a/modules/llm/app/components/llm_connections/semantic_indexes/table_component.rb +++ b/modules/llm/app/components/llm_connections/semantic_indexes/table_component.rb @@ -39,15 +39,23 @@ class TableComponent < OpPrimer::BorderBoxTableComponent mobile_labels :status + options active: false + def has_actions? = true def has_footer? = false - def mobile_title = I18n.t("admin.semantic_indexes.box.title") + def blank_title = I18n.t("admin.semantic_indexes.box.no_active_title") + + def blank_description = I18n.t("admin.semantic_indexes.box.no_active_description") + + def blank_icon = :search + + def mobile_title = title_caption def headers [ - [:title, { caption: SemanticIndex.human_attribute_name(:title) }], + [:title, { caption: title_caption }], [:model_id, { caption: SemanticIndex.human_attribute_name(:model_id) }], [:context_token_length, { caption: SemanticIndex.human_attribute_name(:context_token_length) }], [:embedding_vector_dimensions, { caption: SemanticIndex.human_attribute_name(:embedding_vector_dimensions) }], @@ -55,11 +63,17 @@ def headers ] end - def blank_title = I18n.t("admin.semantic_indexes.box.blank_title") - - def blank_description = I18n.t("admin.semantic_indexes.box.blank_description") + private - def blank_icon = :database + def title_caption + if !active + I18n.t("admin.semantic_indexes.box.others_heading") + elsif rows.empty? + I18n.t("admin.semantic_indexes.box.active_heading") + else + SemanticIndex.human_attribute_name(:title) + end + end end end end diff --git a/modules/llm/app/contracts/semantic_indexes/base_contract.rb b/modules/llm/app/contracts/semantic_indexes/base_contract.rb index fb5bb6c7b69b..634bc1f69a5b 100644 --- a/modules/llm/app/contracts/semantic_indexes/base_contract.rb +++ b/modules/llm/app/contracts/semantic_indexes/base_contract.rb @@ -37,5 +37,15 @@ class BaseContract < ::ModelContract attribute :model_id attribute :embedding_vector_dimensions attribute :context_token_length + + validate :model_offered, if: -> { model.model_id.present? && model.model_id_changed? } + + private + + def model_offered + return if LlmConnection.active_connection&.embedding_model_ids&.include?(model.model_id) + + errors.add(:model_id, :not_offered) + end end end diff --git a/modules/llm/app/contracts/semantic_indexes/delete_contract.rb b/modules/llm/app/contracts/semantic_indexes/delete_contract.rb index b0fead7d9964..9a76d4904a4f 100644 --- a/modules/llm/app/contracts/semantic_indexes/delete_contract.rb +++ b/modules/llm/app/contracts/semantic_indexes/delete_contract.rb @@ -31,5 +31,18 @@ module SemanticIndexes class DeleteContract < ::DeleteContract delete_permission :admin + + validate :not_indexing + validate :not_active + + private + + def not_indexing + errors.add(:base, :locked_while_indexing) if model.indexing? + end + + def not_active + errors.add(:base, :active) if model.active? + end end end diff --git a/modules/llm/app/contracts/semantic_indexes/update_contract.rb b/modules/llm/app/contracts/semantic_indexes/update_contract.rb index bfd79d15b150..17b7529591a3 100644 --- a/modules/llm/app/contracts/semantic_indexes/update_contract.rb +++ b/modules/llm/app/contracts/semantic_indexes/update_contract.rb @@ -30,5 +30,16 @@ module SemanticIndexes class UpdateContract < BaseContract + validate :parameters_unchanged_once_indexed + + private + + def parameters_unchanged_once_indexed + return unless model.parameters_locked? + + SemanticIndex::PARAMETERS.each do |attribute| + errors.add(attribute, :locked_once_indexed) if model.public_send(:"#{attribute}_changed?") + end + end end end diff --git a/modules/llm/app/controllers/admin/llm_feature_bindings_controller.rb b/modules/llm/app/controllers/admin/llm_feature_bindings_controller.rb index 0beace0fec97..d72de5fb448c 100644 --- a/modules/llm/app/controllers/admin/llm_feature_bindings_controller.rb +++ b/modules/llm/app/controllers/admin/llm_feature_bindings_controller.rb @@ -107,20 +107,11 @@ def confirm_assignment(feature, verdict) def build_binding(feature) binding = binding_for(feature) binding.model_id = binding_params[:model_id].presence - - # Only ever accepted for the kind of feature it describes; the model - # rejects it elsewhere, and it is not read at all for a chat feature. - assign_embedding_settings(binding) if feature.embedding? - binding end - def assign_embedding_settings(binding) - binding.dimensions = binding_params[:dimensions].presence - end - def binding_params - @binding_params ||= params.expect(llm_feature_binding: %i[model_id dimensions]) + @binding_params ||= params.expect(llm_feature_binding: %i[model_id]) end # The verdict that actually matters is the one for the model an administrator diff --git a/modules/llm/app/controllers/admin/semantic_indexes_controller.rb b/modules/llm/app/controllers/admin/semantic_indexes_controller.rb index 08460b5a5aad..1181b432bdfd 100644 --- a/modules/llm/app/controllers/admin/semantic_indexes_controller.rb +++ b/modules/llm/app/controllers/admin/semantic_indexes_controller.rb @@ -39,7 +39,8 @@ class SemanticIndexesController < ApplicationController before_action :require_admin before_action :set_connection before_action :require_enabled_connection - before_action :find_semantic_index, only: %i[edit_dialog update deletion_dialog destroy] + before_action :find_semantic_index, + only: %i[edit_dialog update deletion_dialog destroy start_indexing activation_dialog activate] def new_dialog respond_with_dialog LlmConnections::SemanticIndexes::DialogComponent.new(SemanticIndex.new, connection: @connection) @@ -83,6 +84,47 @@ def destroy redirect_to llm_feature_bindings_path, status: :see_other end + def start_indexing + result = ::SemanticIndexes::StartIndexingService + .new(user: current_user, model: @semantic_index) + .call + + if result.success? + flash[:notice] = t(".started", title: @semantic_index.title) + else + flash[:error] = result.message + end + + redirect_to llm_feature_bindings_path, status: :see_other + end + + def activation_dialog + respond_with_dialog LlmConnections::SemanticIndexes::ActivateDialogComponent.new( + @semantic_index, current: SemanticIndex.active + ) + end + + def activate + result = ::SemanticIndexes::ActivateService + .new(user: current_user, model: @semantic_index) + .call + + if result.success? + flash[:notice] = t(".success", title: @semantic_index.title) + else + flash[:error] = result.message + end + + redirect_to llm_feature_bindings_path, status: :see_other + end + + def status + replace_via_turbo_stream( + component: LlmConnections::SemanticIndexes::BoxComponent.new(semantic_indexes: SemanticIndex.order(:created_at)) + ) + respond_with_turbo_streams + end + private def respond_to_save(result, notice:) diff --git a/modules/llm/app/forms/llm_connections/feature_binding_form.rb b/modules/llm/app/forms/llm_connections/feature_binding_form.rb index aad6966b82fc..d376af158bdb 100644 --- a/modules/llm/app/forms/llm_connections/feature_binding_form.rb +++ b/modules/llm/app/forms/llm_connections/feature_binding_form.rb @@ -31,20 +31,14 @@ module LlmConnections # The model select for one registered feature. class FeatureBindingForm < ApplicationForm - include Redmine::I18n - # Primer::Forms::Base.new assigns the builder itself and calls this with the # remaining keywords, so the builder must not appear in the signature. - def initialize(options:, inherit_label:, feature_key:, locked: false, embedding: false, dimensions_hint: nil, - selected_model_id: nil, models_left_out: false) + def initialize(options:, inherit_label:, feature_key:, selected_model_id: nil, models_left_out: false) super() @selected_model_id = selected_model_id @model_options = options @inherit_label = inherit_label @feature_key = feature_key - @locked = locked - @embedding = embedding - @dimensions_hint = dimensions_hint @models_left_out = models_left_out end @@ -53,7 +47,6 @@ def initialize(options:, inherit_label:, feature_key:, locked: false, embedding: name: :model_id, label: LlmFeatureBinding.human_attribute_name(:model_id), caption: model_caption, - disabled: locked, autocomplete_options: { decorated: true, inputValue: selected_model_id, @@ -70,56 +63,20 @@ def initialize(options:, inherit_label:, feature_key:, locked: false, embedding: end end - # Only for an embedding feature, and only while unlocked. A locked binding - # renders it as text instead: a disabled input submits nothing, so the - # value would arrive blank and wipe the column. - if embedding && !locked - f.text_field( - name: :dimensions, - type: :number, - min: 1, - label: LlmFeatureBinding.human_attribute_name(:dimensions), - caption: dimensions_caption, - input_width: :small, - data: { test_selector: "llm-feature-binding--dimensions-#{feature_key}" } - ) - end - - unless locked - f.submit( - name: :submit, - label: I18n.t(:button_save), - scheme: :secondary, - data: { test_selector: "llm-feature-binding--submit-#{feature_key}" } - ) - end + f.submit( + name: :submit, + label: I18n.t(:button_save), + scheme: :secondary, + data: { test_selector: "llm-feature-binding--submit-#{feature_key}" } + ) end private - attr_reader :model_options, :inherit_label, :feature_key, :locked, :embedding, :dimensions_hint, - :selected_model_id, :models_left_out - - # Blank is the right default: the server decides the vector size, and baking - # in a number it may contradict helps nobody. Where the probe has already - # seen a vector, its size is offered as information rather than filled in. - def dimensions_caption - return I18n.t("admin.llm_feature_bindings.form.dimensions_caption") if dimensions_hint.blank? + attr_reader :model_options, :inherit_label, :feature_key, :selected_model_id, :models_left_out - I18n.t("admin.llm_feature_bindings.form.dimensions_caption_probed", dimensions: dimensions_hint) - end - - # Says why only a few of the stored models are on offer, for an administrator - # who does not know what an embedding model is. + # Says why only a few of the stored models are on offer. def model_caption - return ruled_out_caption unless embedding - - link_translate("admin.llm_feature_bindings.form.model_caption_embedding", - links: { docs_url: %i[embeddings_explanation] }, - external: true) - end - - def ruled_out_caption I18n.t("admin.llm_feature_bindings.form.model_caption_ruled_out") if models_left_out end diff --git a/modules/llm/app/forms/semantic_indexes/form.rb b/modules/llm/app/forms/semantic_indexes/form.rb index b01bed081c33..2262a19a484f 100644 --- a/modules/llm/app/forms/semantic_indexes/form.rb +++ b/modules/llm/app/forms/semantic_indexes/form.rb @@ -32,9 +32,10 @@ module SemanticIndexes class Form < ApplicationForm # Primer::Forms::Base.new assigns the builder itself and calls this with the # remaining keywords, so the builder must not appear in the signature. - def initialize(model_options:) + def initialize(model_options:, locked: false) super() @model_options = model_options + @locked = locked end form do |f| @@ -51,23 +52,14 @@ def initialize(model_options:) rows: 3 ) - f.autocompleter( - name: :model_id, - label: SemanticIndex.human_attribute_name(:model_id), - caption: I18n.t("admin.semantic_indexes.form.model_caption"), - required: true, - autocomplete_options: { - decorated: true, - inputValue: selected_model_id, - append_to: "##{LlmConnections::SemanticIndexes::DialogComponent::DIALOG_ID}", - data: { test_selector: "llm-semantic-index--model" } - } - ) do |list| - model_options.each do |option| - list.option(label: option_label(option), - value: option.model_id, - selected: selected_model_id == option.model_id) - end + if locked + f.text_field( + name: :model_id, + label: SemanticIndex.human_attribute_name(:model_id), + disabled: true + ) + else + model_autocompleter(f) end f.text_field( @@ -77,6 +69,7 @@ def initialize(model_options:) label: SemanticIndex.human_attribute_name(:context_token_length), caption: I18n.t("admin.semantic_indexes.form.context_token_length_caption"), required: true, + disabled: locked, input_width: :small ) @@ -87,13 +80,35 @@ def initialize(model_options:) label: SemanticIndex.human_attribute_name(:embedding_vector_dimensions), caption: I18n.t("admin.semantic_indexes.form.embedding_vector_dimensions_caption"), required: true, + disabled: locked, input_width: :small ) end private - attr_reader :model_options + attr_reader :model_options, :locked + + def model_autocompleter(form) + form.autocompleter( + name: :model_id, + label: SemanticIndex.human_attribute_name(:model_id), + caption: I18n.t("admin.semantic_indexes.form.model_caption"), + required: true, + autocomplete_options: { + decorated: true, + inputValue: selected_model_id, + append_to: "##{LlmConnections::SemanticIndexes::DialogComponent::DIALOG_ID}", + data: { test_selector: "llm-semantic-index--model" } + } + ) do |list| + model_options.each do |option| + list.option(label: option_label(option), + value: option.model_id, + selected: selected_model_id == option.model_id) + end + end + end def selected_model_id = model.model_id diff --git a/modules/llm/app/models/llm_feature_binding.rb b/modules/llm/app/models/llm_feature_binding.rb index 6b2cec4fd2ef..dc8d00fdf1aa 100644 --- a/modules/llm/app/models/llm_feature_binding.rb +++ b/modules/llm/app/models/llm_feature_binding.rb @@ -35,23 +35,12 @@ # does not lose the administrator's choice. class LlmFeatureBinding < ApplicationRecord belongs_to :llm_connection, optional: false - - # Settings that describe how vectors are written, and so only mean anything - # for an embedding feature. - EMBEDDING_SETTINGS = %i[dimensions].freeze - - # Everything a stored index depends on. Changing any of it invalidates the - # vectors already written, not just the model. - LOCKED_SETTINGS = ([:model_id] + EMBEDDING_SETTINGS).freeze - - before_save :freeze_on_lock + belongs_to :semantic_index, optional: true validates :feature_key, presence: true, uniqueness: { scope: :llm_connection_id } - validates :dimensions, numericality: { only_integer: true, greater_than: 0 }, allow_nil: true validate :feature_registered validate :model_offered_by_connection, if: :model_id_changed? - validate :embedding_settings_only_for_embedding_features - validate :locked_settings_unchanged + validate :semantic_index_activatable, if: :semantic_index_id_changed? def feature OpenProject::Llm::Features[feature_key] @@ -71,30 +60,11 @@ def inherits_default? = model_id.blank? # catalogue changed. def dangling? resolved = resolved_model_id - resolved.present? && (from_another_deployment? || llm_connection.available_model_ids.exclude?(resolved)) + resolved.present? && llm_connection.available_model_ids.exclude?(resolved) end - # The same model name on another server need not be the same model, so a lock - # only holds for the deployment it was taken on. - def from_another_deployment? - locked? && locked_deployment_fingerprint != llm_connection.settings_fingerprint - end - - def locked? = locked_at.present? - private - # A lock freezes what the vectors were written with. A binding that inherits - # the connection default would keep following it after the lock, so the - # resolved model is written down at the moment of locking, together with the - # deployment it was resolved on. - def freeze_on_lock - return unless locked_at.present? && locked_at_changed? && locked_at_was.nil? - - self.locked_deployment_fingerprint = llm_connection.settings_fingerprint - self.model_id = default_model_id if model_id.blank? - end - def default_model_id return if feature.nil? @@ -116,32 +86,13 @@ def model_offered_by_connection errors.add(:model_id, :not_available) end - def embedding_settings_only_for_embedding_features - return if feature.nil? || feature.embedding? - - EMBEDDING_SETTINGS.each do |attribute| - next if public_send(attribute).blank? - - errors.add(attribute, :not_for_chat_feature) - end - end + def semantic_index_activatable + return if semantic_index.nil? - # Vectors written under one embedding model are meaningless under another, and - # the dimension count is baked into the index, so a locked binding can only be - # changed by an explicit re-index. - # - # TODO(#69620): re-indexing is what clears locked_at. Until that job exists a - # locked binding can only be changed in the database. - def locked_settings_unchanged - # Only constrains later edits. On the save that records the lock -- and on - # create -- every attribute reads as changed from nil, and there is nothing - # indexed yet for them to contradict. - return unless locked? && locked_at_was.present? - - LOCKED_SETTINGS.each do |attribute| - next unless public_send(:"#{attribute}_changed?") - - errors.add(attribute, :locked) + if !feature&.embedding? + errors.add(:semantic_index, :not_for_chat_feature) + elsif !semantic_index.ready? + errors.add(:semantic_index, :not_ready) end end end diff --git a/modules/llm/app/models/queries/work_packages/filter/semantic_search_filter.rb b/modules/llm/app/models/queries/work_packages/filter/semantic_search_filter.rb index 31df1e3c2f08..0b456025914e 100644 --- a/modules/llm/app/models/queries/work_packages/filter/semantic_search_filter.rb +++ b/modules/llm/app/models/queries/work_packages/filter/semantic_search_filter.rb @@ -40,9 +40,13 @@ def human_name end def where + semantic_index = Llm::WorkPackages::SemanticSearch.active_semantic_index vector = Llm::WorkPackages::SemanticSearch.query_vector(values.first) - return "1=0" if vector.nil? + return "1=0" if semantic_index.nil? || vector.nil? - "#{WorkPackage.table_name}.id IN (#{WorkPackageEmbedding.work_package_ids_within_search_distance_sql(vector)})" + matching_ids_sql = WorkPackageEmbedding.work_package_ids_within_search_distance_sql( + vector, semantic_index_id: semantic_index.id + ) + "#{WorkPackage.table_name}.id IN (#{matching_ids_sql})" end end diff --git a/modules/llm/app/models/queries/work_packages/selects/semantic_similarity_select.rb b/modules/llm/app/models/queries/work_packages/selects/semantic_similarity_select.rb index 8ed428bb47a3..e62d7bae8026 100644 --- a/modules/llm/app/models/queries/work_packages/selects/semantic_similarity_select.rb +++ b/modules/llm/app/models/queries/work_packages/selects/semantic_similarity_select.rb @@ -44,9 +44,10 @@ def self.similarity_order_sql(query) filter = query&.filters&.find { |f| f.field.to_s == "semantic_search" } return NO_SIMILARITY_SQL unless filter + semantic_index = Llm::WorkPackages::SemanticSearch.active_semantic_index vector = Llm::WorkPackages::SemanticSearch.query_vector(filter.values.first) - return NO_SIMILARITY_SQL if vector.nil? + return NO_SIMILARITY_SQL if semantic_index.nil? || vector.nil? - WorkPackageEmbedding.distance_to_work_package_sql(vector) + WorkPackageEmbedding.distance_to_work_package_sql(vector, semantic_index_id: semantic_index.id) end end diff --git a/modules/llm/app/models/semantic_index.rb b/modules/llm/app/models/semantic_index.rb index 27f359db6b50..107ce15dfd81 100644 --- a/modules/llm/app/models/semantic_index.rb +++ b/modules/llm/app/models/semantic_index.rb @@ -29,20 +29,76 @@ #++ class SemanticIndex < ApplicationRecord - enum :state, { not_indexed: "not_indexed" } + class InvalidTransitionError < StandardError; end + + INDEXABLE_STATES = %w[not_indexed failed ready].freeze + PARAMETERS = %w[model_id embedding_vector_dimensions context_token_length].freeze + + has_many :llm_feature_bindings, dependent: :restrict_with_error + + enum :state, { + not_indexed: "not_indexed", + indexing: "indexing", + ready: "ready", + failed: "failed" + } validates :title, presence: true validates :model_id, presence: true, length: { maximum: LlmModel::MAX_EXTERNAL_ID_LENGTH } validates :embedding_vector_dimensions, :context_token_length, numericality: { only_integer: true, greater_than: 0 } - validate :model_offered, if: -> { model_id.present? && model_id_changed? } + def self.semantic_search_binding + semantic_search_bindings.first_or_initialize + end + + def self.semantic_search_bindings + LlmConnection.active_connection.feature_bindings.where(feature_key: "semantic_search") + end + + def self.active + joins(:llm_feature_bindings).merge(semantic_search_bindings).first + end + + def self.active_id + semantic_search_bindings.pick(:semantic_index_id) + end + + def active? = persisted? && self.class.active_id == id + + def indexing_batch + return if indexing_batch_id.nil? + + GoodJob::BatchRecord.find_by(id: indexing_batch_id) + end + + def indexable? = INDEXABLE_STATES.include?(state) + + def parameters_locked? = !not_indexed? + + def start_indexing! + with_lock do + raise InvalidTransitionError, "cannot start indexing from #{state}" unless indexable? + + update!(state: :indexing, error_message: nil) + end + end + + def finish_indexing! + transition_from_indexing!(state: :ready) + end + + def fail_indexing!(error_message) + transition_from_indexing!(state: :failed, error_message:) + end private - def model_offered - return if LlmConnection.active_connection.embedding_model_ids.include?(model_id) + def transition_from_indexing!(**attributes) + with_lock do + raise InvalidTransitionError, "cannot finish indexing from #{state}" unless indexing? - errors.add(:model_id, :not_offered) + update!(**attributes) + end end end diff --git a/modules/llm/app/models/work_package_embedding.rb b/modules/llm/app/models/work_package_embedding.rb index 4eed35985354..61890aa00c40 100644 --- a/modules/llm/app/models/work_package_embedding.rb +++ b/modules/llm/app/models/work_package_embedding.rb @@ -31,6 +31,7 @@ class WorkPackageEmbedding < ApplicationRecord MAX_SEARCH_DISTANCE = 0.5 + belongs_to :semantic_index belongs_to :work_package def self.vector_literal(vector) @@ -41,17 +42,19 @@ def embedding=(vector) super(self.class.vector_literal(vector)) end - def self.work_package_ids_within_search_distance_sql(vector_literal) + def self.work_package_ids_within_search_distance_sql(vector_literal, semantic_index_id:) <<~SQL.squish SELECT work_package_id FROM #{table_name} - WHERE embedding <=> '#{vector_literal}' < #{MAX_SEARCH_DISTANCE} + WHERE semantic_index_id = #{Integer(semantic_index_id)} + AND embedding <=> '#{vector_literal}' < #{MAX_SEARCH_DISTANCE} SQL end - def self.distance_to_work_package_sql(vector_literal) + def self.distance_to_work_package_sql(vector_literal, semantic_index_id:) <<~SQL.squish (SELECT embedding <=> '#{vector_literal}' FROM #{table_name} - WHERE #{table_name}.work_package_id = #{WorkPackage.table_name}.id) + WHERE #{table_name}.semantic_index_id = #{Integer(semantic_index_id)} + AND #{table_name}.work_package_id = #{WorkPackage.table_name}.id) SQL end end diff --git a/modules/llm/app/components/llm_connections/indexing_status_component.rb b/modules/llm/app/services/llm/indexing_progress.rb similarity index 63% rename from modules/llm/app/components/llm_connections/indexing_status_component.rb rename to modules/llm/app/services/llm/indexing_progress.rb index edaf660a7d22..61b3d8adff49 100644 --- a/modules/llm/app/components/llm_connections/indexing_status_component.rb +++ b/modules/llm/app/services/llm/indexing_progress.rb @@ -23,58 +23,32 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # # See COPYRIGHT and LICENSE files for more details. #++ -module LlmConnections - class IndexingStatusComponent < ApplicationComponent - include OpTurbo::Streamable - - POLL_INTERVAL_MS = 2000 - - def render? = batch.present? - - def state - return :running unless batch.finished? - - batch.discarded? ? :failed : :succeeded - end - - def running? = state == :running - - def progress_color = state == :failed ? :danger_emphasis : :success_emphasis - - def failed_count - @failed_count ||= embed_jobs.where.not(finished_at: nil).where.not(error: nil).count +module Llm + class IndexingProgress + def initialize(batch) + @batch = batch end def percentage - return state == :succeeded ? 100 : 0 if total_count.zero? + return 0 if total_count.zero? finished_count * 100 / total_count end - def wrapper_data_attrs - return {} unless running? - - { - data: { - controller: "poll-for-changes", - poll_for_changes_url_value: status_work_package_embeddings_path, - poll_for_changes_interval_value: POLL_INTERVAL_MS - } - } + def failed_count + @failed_count ||= embed_jobs.where.not(finished_at: nil).where.not(error: nil).count end private - def batch - @batch ||= Llm::IndexWorkPackagesJob.latest_batch - end + attr_reader :batch - def embed_jobs = batch.jobs.where(job_class: Llm::EmbedWorkPackagesJob.name) + def embed_jobs = GoodJob::Job.where(batch_id: batch.id, job_class: Llm::EmbedWorkPackagesJob.name) def total_count @total_count ||= embed_jobs.count diff --git a/modules/llm/app/services/llm/runtime.rb b/modules/llm/app/services/llm/runtime.rb index 480c1b95ff26..afc7742758ff 100644 --- a/modules/llm/app/services/llm/runtime.rb +++ b/modules/llm/app/services/llm/runtime.rb @@ -134,7 +134,7 @@ def effective_override end def usable?(model_id) - connection.available_model_ids.include?(model_id) && !(binding_model_id == model_id && binding.from_another_deployment?) + connection.available_model_ids.include?(model_id) end def binding diff --git a/modules/llm/app/services/llm/validators/feature_validator.rb b/modules/llm/app/services/llm/validators/feature_validator.rb index a503c34f4ce8..b8975ed36dbb 100644 --- a/modules/llm/app/services/llm/validators/feature_validator.rb +++ b/modules/llm/app/services/llm/validators/feature_validator.rb @@ -47,10 +47,10 @@ def self.key = :features private def validate - register_checks(:bindings_resolvable, :locked_bindings_intact) + register_checks(:bindings_resolvable, :active_semantic_index_intact) bindings_resolvable - locked_bindings_intact + active_semantic_index_intact end def bindings_resolvable @@ -73,21 +73,22 @@ def bindings_resolvable pass_check(:bindings_resolvable) end - # A locked binding is the record that a vector index exists and which model - # and dimension it was written under. If that model has left the catalogue - # the index can no longer be extended or queried consistently, which is a - # data problem rather than a configuration one. - def locked_bindings_intact - broken = subject.feature_bindings.select { |binding| binding.locked? && binding.dangling? } + # If the model an index was built with has left the catalogue, queries can + # no longer be embedded consistently with the stored vectors. + def active_semantic_index_intact + semantic_index = subject.feature_bindings.find_by(feature_key: "semantic_search")&.semantic_index - return pass_check(:locked_bindings_intact) if broken.empty? + if semantic_index.nil? || subject.available_model_ids.include?(semantic_index.model_id) + return pass_check(:active_semantic_index_intact) + end - fail_check(:locked_bindings_intact, :locked_model_missing, - context: { features: broken.filter_map { |b| b.feature&.label }.join(", ") }) + fail_check(:active_semantic_index_intact, :semantic_index_model_missing, + context: { title: semantic_index.title, model: semantic_index.model_id }) end def resolutions - @resolutions ||= OpenProject::Llm::Features.available.map do |feature| + # Embedding features take their model from the active Semantic Index. + @resolutions ||= OpenProject::Llm::Features.available.reject(&:embedding?).map do |feature| Llm::Runtime.for(feature.key) end end diff --git a/modules/llm/app/services/llm/work_packages/embed_service.rb b/modules/llm/app/services/llm/work_packages/embed_service.rb index 70e06dbb1290..a601041bba12 100644 --- a/modules/llm/app/services/llm/work_packages/embed_service.rb +++ b/modules/llm/app/services/llm/work_packages/embed_service.rb @@ -31,53 +31,45 @@ module Llm module WorkPackages class EmbedService - def initialize(work_packages) + def initialize(semantic_index, work_packages) + @semantic_index = semantic_index @work_packages = Array(work_packages) end def call - raise Llm::Errors::ConfigurationError, "binding not ready" unless binding_ready? + raise Llm::Errors::ConfigurationError, "no active connection" if connection.nil? - result = session.embed(texts, model: binding.resolved_model_id, dimensions: binding.dimensions) + result = session.embed(texts, + model: semantic_index.model_id, + dimensions: semantic_index.embedding_vector_dimensions) store(result.vectors) - lock_binding_on_first_write end private - attr_reader :work_packages + attr_reader :semantic_index, :work_packages - def binding - @binding ||= LlmConnection.active_connection - &.feature_bindings - &.find_or_initialize_by(feature_key: "semantic_search") + def connection + @connection ||= LlmConnection.active_connection end - def binding_ready? - binding.present? && binding.resolved_model_id.present? - end - - def session = Llm::Session.for(binding.llm_connection) + def session = Llm::Session.for(connection) def texts - context_window = binding.llm_connection.models.find_by(external_id: binding.resolved_model_id)&.context_window - - work_packages.map { |wp| EmbeddableText.for(wp, context_window: context_window) } + work_packages.map { |wp| EmbeddableText.for(wp, context_window: semantic_index.context_token_length) } end def store(vectors) rows = work_packages.zip(vectors).map do |work_package, vector| - { work_package_id: work_package.id, embedding: WorkPackageEmbedding.vector_literal(vector) } + { + semantic_index_id: semantic_index.id, + work_package_id: work_package.id, + embedding: WorkPackageEmbedding.vector_literal(vector) + } end - WorkPackageEmbedding.upsert_all(rows, unique_by: :work_package_id) - end - - def lock_binding_on_first_write - return if binding.locked? - - binding.update!(locked_at: Time.current) + WorkPackageEmbedding.upsert_all(rows, unique_by: %i[semantic_index_id work_package_id]) end end end diff --git a/modules/llm/app/services/llm/work_packages/semantic_search.rb b/modules/llm/app/services/llm/work_packages/semantic_search.rb index 1fb88011601d..7cc81e5321ef 100644 --- a/modules/llm/app/services/llm/work_packages/semantic_search.rb +++ b/modules/llm/app/services/llm/work_packages/semantic_search.rb @@ -34,7 +34,11 @@ module SemanticSearch def self.available? return false unless OpenProject::FeatureDecisions.semantic_search_active? - active_binding&.locked? || false + active_semantic_index.present? + end + + def self.active_semantic_index + RequestStore.fetch(:semantic_search_active_index) { SemanticIndex.active } end # Memoized per request: the query filter and the similarity sort both need it. @@ -45,21 +49,17 @@ def self.query_vector(query) private_class_method def self.fetch_query_vector(query) return if query.blank? || !OpenProject::FeatureDecisions.semantic_search_active? - binding = active_binding - return unless binding&.locked? - - embed(query, binding) - end + semantic_index = active_semantic_index + return if semantic_index.nil? - private_class_method def self.active_binding - LlmConnection.active_connection - &.feature_bindings - &.find_by(feature_key: "semantic_search") + embed(query, semantic_index) end - private_class_method def self.embed(query, binding) - session = Llm::Session.for(binding.llm_connection) - vector = session.embed(query, model: binding.resolved_model_id, dimensions: binding.dimensions).vectors + private_class_method def self.embed(query, semantic_index) + session = Llm::Session.for(LlmConnection.active_connection) + vector = session.embed(query, + model: semantic_index.model_id, + dimensions: semantic_index.embedding_vector_dimensions).vectors WorkPackageEmbedding.vector_literal(vector) if vector.present? rescue Llm::Errors::Error nil diff --git a/modules/llm/app/services/llm_connections/sync_models_service.rb b/modules/llm/app/services/llm_connections/sync_models_service.rb index 195b1ebd4094..946c3005aa66 100644 --- a/modules/llm/app/services/llm_connections/sync_models_service.rb +++ b/modules/llm/app/services/llm_connections/sync_models_service.rb @@ -186,7 +186,7 @@ def forget_the_previous_deployment def discard_discovered_models(discarded) connection.models.discovered.delete_all connection.capability_verdicts.where.not(source: "admin").delete_all - connection.feature_bindings.where(model_id: discarded, locked_at: nil).delete_all + connection.feature_bindings.where(model_id: discarded, semantic_index_id: nil).delete_all end # The administrator's context-window override is theirs, and a routine diff --git a/modules/llm/app/controllers/admin/work_package_embeddings_controller.rb b/modules/llm/app/services/semantic_indexes/activate_service.rb similarity index 61% rename from modules/llm/app/controllers/admin/work_package_embeddings_controller.rb rename to modules/llm/app/services/semantic_indexes/activate_service.rb index 1ba9256106b6..6be8b3748a39 100644 --- a/modules/llm/app/controllers/admin/work_package_embeddings_controller.rb +++ b/modules/llm/app/services/semantic_indexes/activate_service.rb @@ -28,36 +28,31 @@ # See COPYRIGHT and LICENSE files for more details. #++ -module Admin - class WorkPackageEmbeddingsController < ApplicationController - include OpTurbo::ComponentStream - - layout "admin" +module SemanticIndexes + class ActivateService < ::BaseServices::BaseCallable + attr_reader :user, :model + + def initialize(user:, model:) + super() + @user = user + @model = model + end - before_action :require_admin - before_action :require_feature + protected - def reindex - if Llm::IndexWorkPackagesJob.running? - flash[:warning] = t(".already_running") - else - Llm::IndexWorkPackagesJob.enqueue_batch - flash[:notice] = t(".enqueued") + def perform + unless user.admin? && user.active? + return ServiceResult.failure(result: model, message: I18n.t(:notice_not_authorized)) end - redirect_to llm_feature_bindings_path, status: :see_other - end + binding = SemanticIndex.semantic_search_binding + binding.semantic_index = model - def status - replace_via_turbo_stream(component: LlmConnections::IndexingStatusComponent.new) - respond_with_turbo_streams - end - - private - - def require_feature - render_404 unless OpenProject::FeatureDecisions.llm_connection_active? && - OpenProject::FeatureDecisions.semantic_search_active? + if binding.save + ServiceResult.success(result: model) + else + ServiceResult.failure(result: model, errors: binding.errors) + end end end end diff --git a/modules/llm/app/services/semantic_indexes/start_indexing_service.rb b/modules/llm/app/services/semantic_indexes/start_indexing_service.rb new file mode 100644 index 000000000000..41a2bb5bf80d --- /dev/null +++ b/modules/llm/app/services/semantic_indexes/start_indexing_service.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module SemanticIndexes + class StartIndexingService < ::BaseServices::BaseCallable + attr_reader :user, :model + + def initialize(user:, model:) + super() + @user = user + @model = model + end + + protected + + def perform + return failure(I18n.t(:notice_not_authorized)) unless user.admin? && user.active? + + model.start_indexing! + batch = Llm::IndexWorkPackagesJob.enqueue_for(model) + model.update!(indexing_batch_id: batch.id) + + ServiceResult.success(result: model) + rescue SemanticIndex::InvalidTransitionError + failure(I18n.t("admin.semantic_indexes.start_indexing.not_indexable")) + end + + private + + def failure(message) = ServiceResult.failure(result: model, message:) + end +end diff --git a/modules/llm/app/views/admin/llm_feature_bindings/index.html.erb b/modules/llm/app/views/admin/llm_feature_bindings/index.html.erb index e0c2a22855e4..35627266798c 100644 --- a/modules/llm/app/views/admin/llm_feature_bindings/index.html.erb +++ b/modules/llm/app/views/admin/llm_feature_bindings/index.html.erb @@ -55,9 +55,9 @@ See COPYRIGHT and LICENSE files for more details. connection: @connection, binding: @bindings[feature.key.to_s] ) - ) %> -<% end %> - -<% if OpenProject::FeatureDecisions.semantic_search_active? %> - <%= render(LlmConnections::SemanticIndexes::BoxComponent.new(semantic_indexes: SemanticIndex.order(:created_at))) %> + ) do %> + <% if feature.key.to_s == "semantic_search" %> + <%= render(LlmConnections::SemanticIndexes::BoxComponent.new(semantic_indexes: SemanticIndex.order(:created_at))) %> + <% end %> + <% end %> <% end %> diff --git a/modules/llm/app/workers/llm/embed_work_packages_job.rb b/modules/llm/app/workers/llm/embed_work_packages_job.rb index c28b5c767593..3081d2b7d6c5 100644 --- a/modules/llm/app/workers/llm/embed_work_packages_job.rb +++ b/modules/llm/app/workers/llm/embed_work_packages_job.rb @@ -33,11 +33,14 @@ class EmbedWorkPackagesJob < ApplicationJob queue_with_priority :low retry_on Llm::Errors::RateLimitedError, wait: :polynomially_longer, attempts: 10 - def perform(work_package_ids) + def perform(semantic_index_id, work_package_ids) return unless OpenProject::FeatureDecisions.semantic_search_active? + semantic_index = SemanticIndex.find_by(id: semantic_index_id) + return unless semantic_index&.indexing? + work_packages = WorkPackage.where(id: work_package_ids) - Llm::WorkPackages::EmbedService.new(work_packages).call + Llm::WorkPackages::EmbedService.new(semantic_index, work_packages).call end end end diff --git a/modules/llm/app/workers/llm/finish_semantic_indexing_job.rb b/modules/llm/app/workers/llm/finish_semantic_indexing_job.rb new file mode 100644 index 000000000000..8a56c6d8ad19 --- /dev/null +++ b/modules/llm/app/workers/llm/finish_semantic_indexing_job.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +module Llm + class FinishSemanticIndexingJob < ApplicationJob + queue_with_priority :low + + def perform(batch, _context) + semantic_index = SemanticIndex.find_by(id: batch.properties[:semantic_index_id]) + return unless semantic_index&.indexing? + + progress = Llm::IndexingProgress.new(batch) + if progress.failed_count.positive? + semantic_index.fail_indexing!( + I18n.t("admin.semantic_indexes.indexing.failed_requests", count: progress.failed_count) + ) + else + semantic_index.finish_indexing! + end + rescue SemanticIndex::InvalidTransitionError + nil + end + end +end diff --git a/modules/llm/app/workers/llm/index_work_packages_job.rb b/modules/llm/app/workers/llm/index_work_packages_job.rb index 2d1f0edb71fd..0ecbb44fff95 100644 --- a/modules/llm/app/workers/llm/index_work_packages_job.rb +++ b/modules/llm/app/workers/llm/index_work_packages_job.rb @@ -32,29 +32,20 @@ module Llm class IndexWorkPackagesJob < ApplicationJob include GoodJob::ActiveJobExtensions::Batches - BATCH_DESCRIPTION = name - queue_with_priority :low - def self.enqueue_batch - GoodJob::Batch.enqueue(description: BATCH_DESCRIPTION) { perform_later } - end - - def self.latest_batch - GoodJob::BatchRecord.where(description: BATCH_DESCRIPTION).order(created_at: :desc).first - end - - def self.running? - batch = latest_batch - batch.present? && !batch.finished? + def self.enqueue_for(semantic_index) + GoodJob::Batch.enqueue(on_finish: Llm::FinishSemanticIndexingJob, semantic_index_id: semantic_index.id) do + perform_later(semantic_index.id) + end end - def perform + def perform(semantic_index_id) return unless OpenProject::FeatureDecisions.semantic_search_active? batch.add do WorkPackage.find_in_batches(batch_size: 100) do |work_packages| - Llm::EmbedWorkPackagesJob.perform_later(work_packages.map(&:id)) + Llm::EmbedWorkPackagesJob.perform_later(semantic_index_id, work_packages.map(&:id)) end end end diff --git a/modules/llm/config/locales/en.yml b/modules/llm/config/locales/en.yml index 05aba141aaac..acb5dfa97056 100644 --- a/modules/llm/config/locales/en.yml +++ b/modules/llm/config/locales/en.yml @@ -18,10 +18,10 @@ en: env_provisioned_at: "Applied from the environment" llm_features_enabled: "Enable LLMs for this instance" llm_feature_binding: - dimensions: "Dimensions" feature_key: "Feature" # ActiveRecord::Base.human_attribute_name strips the _id suffix. model: "Model" + semantic_index: "Semantic Index" llm_model: admin_context_window: "Context window" display_name: "Display name" @@ -31,6 +31,7 @@ en: context_token_length: "Context length" description: "Description" embedding_vector_dimensions: "Dimensions" + llm_feature_bindings: "Feature configuration" # ActiveRecord::Base.human_attribute_name strips the _id suffix. model: "Model" state: "Status" @@ -64,29 +65,43 @@ en: requires_connection: "cannot be turned on before a connection has been configured." llm_feature_binding: attributes: - dimensions: - locked: "cannot be changed while data indexed with it still exists. Re-index to change it." - not_for_chat_feature: "only applies to features that create embeddings." feature_key: not_registered: "does not belong to a known AI feature." model_id: - locked: "cannot be changed while data indexed with it still exists. Re-index to switch models." not_available: "is not offered by the configured LLM server." + semantic_index: + not_for_chat_feature: "only applies to features that create embeddings." + not_ready: "must be ready before it can be activated." llm_model: attributes: external_id: conflicting_capabilities: "has capability assertions stored under it that conflict with those of this model." semantic_index: + locked_once_indexed: "cannot be changed once the Semantic Index has been indexed." attributes: + base: + active: "The Semantic Index is active. Activate another Semantic Index first." + locked_while_indexing: "cannot be changed while it is being indexed." model_id: not_offered: "is not an embedding model offered by the configured LLM server." admin: semantic_indexes: + activate: + body_html: "Semantic search will use %{title} immediately." + body_switch_html: "Semantic search switches from %{current} to %{title} immediately. %{current} stays ready and can be reactivated." + button: "Activate" + confirm: "Activate" + success: "\"%{title}\" is now the active Semantic Index." + title: "Activate \"%{title}\"?" + active: "Active" box: - blank_description: "Create a Semantic Index to choose the model, context length and dimensions used to index work packages." - blank_title: "No Semantic Indexes yet" + active_caption: "Semantic search uses this index right now" + active_heading: "Active Semantic Index" caption: "Each Semantic Index describes one way of indexing work packages for semantic search." new: "New Semantic Index" + no_active_description: "Create a new Semantic Index or choose one from the list below." + no_active_title: "No active Semantic Index" + others_heading: "Other Semantic Indexes" title: "Semantic Indexes" destroy: description: "The Semantic Index and its settings will be removed. This cannot be undone." @@ -98,12 +113,30 @@ en: form: context_token_length_caption: "Maximum number of tokens embedded per work package." embedding_vector_dimensions_caption: "How many numbers each vector has." + locked_once_indexed: "This Semantic Index has been indexed. Model, context length and dimensions can't be changed. Create a new Semantic Index to use different parameters." model_caption: "Only models known to create embeddings are offered." option_unqualified: "%{model} (no longer eligible)" + indexing: + failed_requests: + one: "1 request failed." + other: "%{count} requests failed." + progress: "%{percentage}% done" menu: delete: "Delete…" + edit_locked: "Edit name & description" + locked_while_active: "Activate another Semantic Index first" + start_indexing: + button: + failed: "Retry indexing" + not_indexed: "Start indexing" + ready: "Re-index" + not_indexable: "This Semantic Index is already being indexed." + started: "Indexing of \"%{title}\" has started." states: + failed: "Failed" + indexing: "Indexing" not_indexed: "Not indexed" + ready: "Ready" llm_connections: delete_api_key: configured_from_env: "This connection is configured through the environment and cannot be changed here." @@ -153,33 +186,15 @@ en: llm_feature_bindings: dangling: "%{model} is no longer offered by the LLM server. This feature will not run until another model is selected." form: - dimensions_caption: "How many numbers each vector has. Leave blank to use whatever the server returns." - dimensions_caption_probed: "The server returned %{dimensions}-dimension vectors for this model. Leave blank to use whatever it returns at index time." - model_caption_embedding: "Only models known to create embeddings are offered. [Read more](docs_url) about model types." model_caption_ruled_out: "Models known to lack a capability this feature needs are not offered." index: description: "Choose which model each AI feature uses. Features without a choice use the default models set on the [Models](models_url) tab." inherit_with_default: "Use the default (%{model})" inherit_without_default: "Use the default (none set)" - locked: "%{model} is in use by indexed data and cannot be changed here. Re-index to switch models." - locked_values_heading: "Values fixed by the existing index" option_unqualified: "%{model} (no longer eligible for this feature)" update: model_incapable: "The model for %{feature} has been saved, but the server just reported that it does not support %{capability}. Pick a different model, or assert the capability on the model if you know better." success: "The model for %{feature} has been saved." - work_package_embeddings: - reindex: - already_running: "An index operation is already running. Wait for it to finish before starting another one." - button: "Index work packages" - enqueued: "Work package indexing has been enqueued. Embeddings will be updated in the background." - status: - failed: - one: "1 request failed." - other: "%{count} requests failed." - state: - failed: "Indexing failed" - running: "Indexing running" - succeeded: "Indexing finished" llm_health_status: create: success: "The checks have run: %{passed} passed, %{warnings} warnings, %{failures} failed." @@ -318,7 +333,6 @@ en: features_model_missing: "%{features} point at a model the server no longer offers." features_unbound: "%{features} have no model chosen yet." invalid_api_key: "The server rejected the stored API key." - locked_model_missing: "%{features} indexed data with a model the server no longer offers. The index cannot be extended until it is rebuilt." model_not_served: "The server has no route for %{model} (404). Check that the model is still offered to your account and that the host URL includes the API version segment." models_endpoint_missing: "The server has no model list at this address, so the API key could not be verified either. Check that the URL includes the API version segment (for example https://example.com/v1)." no_model_to_test: "No model is available to test with. Add a model or choose a default first." @@ -327,13 +341,14 @@ en: not_configured: "No endpoint has been configured." not_openai_compatible: "The endpoint did not return a valid model list. Please ensure it points at an OpenAI-API-compatible endpoint, including the API version segment." request_timed_out: "The server did not respond in time." + semantic_index_model_missing: "The active Semantic Index \"%{title}\" was built with %{model}, which the server no longer offers. Activate another Semantic Index to keep search by meaning working." server_error: "The server responded with status %{status}." ssrf_filtered: "The endpoint resolves to a blocked address." unsupported_api_format: "%{api_format} needs credentials that OpenProject cannot store." features: + active_semantic_index_intact: "The active Semantic Index still has its model" bindings_resolvable: "Every feature has a usable model" header: "AI features" - locked_bindings_intact: "Indexed data still has its model" inference: chat_round_trip: "The server answers a request" header: "Inference" diff --git a/modules/llm/config/routes.rb b/modules/llm/config/routes.rb index 1873d8601623..53942b5829bf 100644 --- a/modules/llm/config/routes.rb +++ b/modules/llm/config/routes.rb @@ -62,18 +62,15 @@ resources :semantic_indexes, only: %i[create update destroy], controller: "admin/semantic_indexes" do collection do get :new_dialog, defaults: { format: :turbo_stream } + get :status, defaults: { format: :turbo_stream } end member do get :edit_dialog, defaults: { format: :turbo_stream } get :deletion_dialog, defaults: { format: :turbo_stream } - end - end - - resources :work_package_embeddings, only: [], controller: "admin/work_package_embeddings" do - collection do - post :reindex - get :status + post :start_indexing + get :activation_dialog, defaults: { format: :turbo_stream } + post :activate end end end diff --git a/modules/llm/db/migrate/20261016131029_add_error_message_to_semantic_indexes.rb b/modules/llm/db/migrate/20261016131029_add_error_message_to_semantic_indexes.rb new file mode 100644 index 000000000000..a237331b6cc3 --- /dev/null +++ b/modules/llm/db/migrate/20261016131029_add_error_message_to_semantic_indexes.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class AddErrorMessageToSemanticIndexes < ActiveRecord::Migration[8.1] + def change + add_column :semantic_indexes, :error_message, :text + end +end diff --git a/modules/llm/db/migrate/20261016131200_add_semantic_index_to_llm_feature_bindings.rb b/modules/llm/db/migrate/20261016131200_add_semantic_index_to_llm_feature_bindings.rb new file mode 100644 index 000000000000..c3133416876e --- /dev/null +++ b/modules/llm/db/migrate/20261016131200_add_semantic_index_to_llm_feature_bindings.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class AddSemanticIndexToLlmFeatureBindings < ActiveRecord::Migration[8.1] + def change + add_reference :llm_feature_bindings, :semantic_index, foreign_key: true + end +end diff --git a/modules/llm/db/migrate/20261016150000_add_semantic_index_to_work_package_embeddings.rb b/modules/llm/db/migrate/20261016150000_add_semantic_index_to_work_package_embeddings.rb new file mode 100644 index 000000000000..5aa30d28ca55 --- /dev/null +++ b/modules/llm/db/migrate/20261016150000_add_semantic_index_to_work_package_embeddings.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class AddSemanticIndexToWorkPackageEmbeddings < ActiveRecord::Migration[8.1] + def up + remove_index :work_package_embeddings, :work_package_id + add_reference :work_package_embeddings, :semantic_index, + null: false, index: false, foreign_key: { on_delete: :cascade } + add_index :work_package_embeddings, %i[semantic_index_id work_package_id], unique: true + add_index :work_package_embeddings, :work_package_id + end + + def down + remove_index :work_package_embeddings, :work_package_id + remove_reference :work_package_embeddings, :semantic_index, foreign_key: true, index: false + add_index :work_package_embeddings, :work_package_id, unique: true + end +end diff --git a/modules/llm/db/migrate/20261016150100_add_indexing_batch_id_to_semantic_indexes.rb b/modules/llm/db/migrate/20261016150100_add_indexing_batch_id_to_semantic_indexes.rb new file mode 100644 index 000000000000..97e38751325d --- /dev/null +++ b/modules/llm/db/migrate/20261016150100_add_indexing_batch_id_to_semantic_indexes.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class AddIndexingBatchIdToSemanticIndexes < ActiveRecord::Migration[8.1] + def change + add_column :semantic_indexes, :indexing_batch_id, :uuid + end +end diff --git a/modules/llm/db/migrate/20261016150200_remove_lock_and_dimensions_from_llm_feature_bindings.rb b/modules/llm/db/migrate/20261016150200_remove_lock_and_dimensions_from_llm_feature_bindings.rb new file mode 100644 index 000000000000..710c5221c0e1 --- /dev/null +++ b/modules/llm/db/migrate/20261016150200_remove_lock_and_dimensions_from_llm_feature_bindings.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +class RemoveLockAndDimensionsFromLlmFeatureBindings < ActiveRecord::Migration[8.1] + def change + change_table :llm_feature_bindings, bulk: true do |t| + t.remove :dimensions, type: :integer + t.remove :locked_at, type: :datetime + t.remove :locked_deployment_fingerprint, type: :string + end + end +end diff --git a/modules/llm/spec/contracts/semantic_indexes/create_contract_spec.rb b/modules/llm/spec/contracts/semantic_indexes/create_contract_spec.rb index a2ca2f0bf40e..002a2c211263 100644 --- a/modules/llm/spec/contracts/semantic_indexes/create_contract_spec.rb +++ b/modules/llm/spec/contracts/semantic_indexes/create_contract_spec.rb @@ -44,4 +44,15 @@ include_examples "contract reuses the model errors" do let(:current_user) { build_stubbed(:admin) } end + + context "with a chat model" do + let(:current_user) { build_stubbed(:admin) } + + before { semantic_index.model_id = "qwen3.6-27b" } + + it "is invalid" do + expect(contract.validate).to be(false) + expect(contract.errors.details[:model_id]).to include(error: :not_offered) + end + end end diff --git a/modules/llm/spec/contracts/semantic_indexes/delete_contract_spec.rb b/modules/llm/spec/contracts/semantic_indexes/delete_contract_spec.rb index 73ab8b823a3e..d2a9b027dbf3 100644 --- a/modules/llm/spec/contracts/semantic_indexes/delete_contract_spec.rb +++ b/modules/llm/spec/contracts/semantic_indexes/delete_contract_spec.rb @@ -40,4 +40,21 @@ let(:contract) { described_class.new(semantic_index, current_user) } it_behaves_like "contract is valid for active admins and invalid for regular users" + + context "when the Semantic Index is being indexed" do + let(:current_user) { build_stubbed(:admin) } + + before { semantic_index.update!(state: "indexing") } + + it_behaves_like "contract is invalid", base: :locked_while_indexing + end + + context "when the Semantic Index is active" do + let(:current_user) { build_stubbed(:admin) } + let(:semantic_index) { create(:semantic_index, state: "ready") } + + before { connection.feature_bindings.create!(feature_key: "semantic_search", semantic_index:) } + + it_behaves_like "contract is invalid", base: :active + end end diff --git a/modules/llm/spec/contracts/semantic_indexes/update_contract_spec.rb b/modules/llm/spec/contracts/semantic_indexes/update_contract_spec.rb index d58d3bbbb51d..eab61df43709 100644 --- a/modules/llm/spec/contracts/semantic_indexes/update_contract_spec.rb +++ b/modules/llm/spec/contracts/semantic_indexes/update_contract_spec.rb @@ -44,4 +44,47 @@ include_examples "contract reuses the model errors" do let(:current_user) { build_stubbed(:admin) } end + + context "when the stored model is no longer offered" do + let(:current_user) { build_stubbed(:admin) } + + before { connection.capability_verdicts.destroy_all } + + it "still allows renaming" do + semantic_index.title = "Renamed" + + expect(contract.validate).to be(true) + end + end + + context "when the Semantic Index has not been indexed" do + let(:current_user) { build_stubbed(:admin) } + + it "allows changing the parameters" do + semantic_index.context_token_length = 4096 + + expect(contract.validate).to be(true) + end + end + + %w[indexing ready failed].each do |state| + context "when the Semantic Index is #{state}" do + let(:current_user) { build_stubbed(:admin) } + + before { semantic_index.update!(state:) } + + it "allows renaming" do + semantic_index.title = "Renamed" + + expect(contract.validate).to be(true) + end + + it "locks the parameters" do + semantic_index.context_token_length = 4096 + + expect(contract.validate).to be(false) + expect(contract.errors.details[:context_token_length]).to include(error: :locked_once_indexed) + end + end + end end diff --git a/modules/llm/spec/factories/llm_feature_binding_factory.rb b/modules/llm/spec/factories/llm_feature_binding_factory.rb index 4e07d13613d2..a158434bb904 100644 --- a/modules/llm/spec/factories/llm_feature_binding_factory.rb +++ b/modules/llm/spec/factories/llm_feature_binding_factory.rb @@ -33,7 +33,5 @@ llm_connection feature_key { "description_assistant" } model_id { nil } - dimensions { nil } - locked_at { nil } end end diff --git a/modules/llm/spec/factories/work_package_embedding_factory.rb b/modules/llm/spec/factories/work_package_embedding_factory.rb index 820c5f77adcf..f5f27425db32 100644 --- a/modules/llm/spec/factories/work_package_embedding_factory.rb +++ b/modules/llm/spec/factories/work_package_embedding_factory.rb @@ -30,7 +30,8 @@ FactoryBot.define do factory :work_package_embedding do - association :work_package + work_package + semantic_index embedding { [0.1, 0.2, 0.3] } end end diff --git a/modules/llm/spec/features/admin/llm_connection_spec.rb b/modules/llm/spec/features/admin/llm_connection_spec.rb index e1cabe14dae1..ce649d531346 100644 --- a/modules/llm/spec/features/admin/llm_connection_spec.rb +++ b/modules/llm/spec/features/admin/llm_connection_spec.rb @@ -266,12 +266,12 @@ def choose_action(item) mock_llm_embeddings_response(base_url) end - it "offers the vector settings only for features that embed" do + it "offers a model selection only for features that do not embed" do visit llm_feature_bindings_path expect(page).to have_test_selector("llm-settings--tabs") - expect(page).to have_test_selector("llm-feature-binding--dimensions-semantic_search") - expect(page).to have_no_test_selector("llm-feature-binding--dimensions-description_assistant") + expect(page).to have_test_selector("llm-feature-binding--model-description_assistant") + expect(page).to have_no_test_selector("llm-feature-binding--model-semantic_search") expect(page).to be_axe_clean.within("#content") end @@ -331,6 +331,62 @@ def within_semantic_index_menu(semantic_index, &) expect_flash(message: "Successful deletion.") expect(page).to have_no_test_selector("llm-semantic-index--row-#{semantic_index.id}") end + + it "activates a ready Semantic Index and locks its parameters" do + current = create(:semantic_index, title: "Qwen3 baseline", state: "ready") + candidate = create(:semantic_index, title: "Nomic small", state: "ready") + connection.feature_bindings.create!(feature_key: "semantic_search", semantic_index: current) + + visit llm_feature_bindings_path + + find_test_selector("llm-semantic-index--activate-#{candidate.id}").click + + within_test_selector("llm-semantic-index-activate-dialog") do + expect(page).to have_text("switches from Qwen3 baseline to Nomic small") + click_on "Activate" + end + + expect_flash(message: "\"Nomic small\" is now the active Semantic Index.") + within_test_selector("llm-semantic-index--active") do + expect(page).to have_text("Nomic small") + expect(page).to have_test_selector("llm-semantic-index--active-marker") + end + + within_semantic_index_menu(candidate) { click_on "Edit name & description" } + + within_test_selector("llm-semantic-index-dialog") do + expect(page).to have_text("This Semantic Index has been indexed.") + expect(page).to have_field("Context length", disabled: true) + fill_in "Display name", with: "Nomic production" + click_on "Save" + end + + expect_flash(message: "Successful update.") + expect(candidate.reload.title).to eq("Nomic production") + end + + it "indexes a Semantic Index and shows the outcome without a reload" do + semantic_index = create(:semantic_index, title: "Nomic small") + + visit llm_feature_bindings_path + + find_test_selector("llm-semantic-index--start-indexing-#{semantic_index.id}").click + + expect_flash(message: "Indexing of \"Nomic small\" has started.") + within_test_selector("llm-semantic-index--row-#{semantic_index.id}") do + expect(page).to have_text("Indexing") + expect(page).to have_test_selector("llm-semantic-index--progress-#{semantic_index.id}", text: "0% done") + end + + semantic_index.reload.fail_indexing!("1 request failed.") + + within_test_selector("llm-semantic-index--row-#{semantic_index.id}") do + expect(page).to have_text("Failed") + expect(page).to have_text("1 request failed.") + expect(page).to have_test_selector("llm-semantic-index--start-indexing-#{semantic_index.id}", + text: "Retry indexing") + end + end end end end diff --git a/modules/llm/spec/models/llm_feature_binding_spec.rb b/modules/llm/spec/models/llm_feature_binding_spec.rb index a7f35beb33a5..26b6ce2e945b 100644 --- a/modules/llm/spec/models/llm_feature_binding_spec.rb +++ b/modules/llm/spec/models/llm_feature_binding_spec.rb @@ -29,6 +29,7 @@ #++ require "spec_helper" +require_module_spec_helper RSpec.describe LlmFeatureBinding do let(:connection) { create(:llm_connection, :with_models) } @@ -89,40 +90,34 @@ def bind(model_id) expect(binding.resolved_model_id).to eq("qwen3.6-27b") expect(binding).not_to be_dangling end - - it "writes that identifier down when the binding is locked" do - binding = connection.feature_bindings.create!(feature_key: "semantic_search") - - binding.update!(locked_at: Time.current) - - expect(binding.reload.model_id).to eq("bge-m3") - end end - describe "a binding locked on one deployment" do - let(:binding) do - bind("qwen3.6-27b").tap(&:save!).tap { it.update!(locked_at: Time.current) } - end + describe "the active Semantic Index" do + include_context "with an embedding model offered" - it "remembers which deployment it was locked on" do - expect(binding.locked_deployment_fingerprint).to eq(connection.settings_fingerprint) - end + let(:binding) { connection.feature_bindings.new(feature_key: "semantic_search") } - it "is intact while the connection still points there" do - expect(binding).not_to be_dangling + it "may be a ready one" do + binding.semantic_index = create(:semantic_index, state: "ready") + + expect(binding).to be_valid end - it "dangles once the connection points elsewhere, although the model name is still offered" do - binding - connection.update_columns(base_url: "https://elsewhere.example/v1") + %w[not_indexed indexing failed].each do |state| + it "must not be #{state.humanize(capitalize: false)}" do + binding.semantic_index = create(:semantic_index, state:) - expect(binding.reload).to be_dangling + expect(binding).not_to be_valid + expect(binding.errors.details[:semantic_index]).to include(error: :not_ready) + end end - it "dangles when the lock does not say which deployment it was taken on" do - binding.update_columns(locked_deployment_fingerprint: nil) + it "only applies to embedding features" do + binding = connection.feature_bindings.new(feature_key: "description_assistant", + semantic_index: create(:semantic_index, state: "ready")) - expect(binding.reload).to be_dangling + expect(binding).not_to be_valid + expect(binding.errors.details[:semantic_index]).to include(error: :not_for_chat_feature) end end end diff --git a/modules/llm/spec/models/queries/work_packages/filter/semantic_search_filter_spec.rb b/modules/llm/spec/models/queries/work_packages/filter/semantic_search_filter_spec.rb index 63b3b4417f7a..a8e35e0adbcf 100644 --- a/modules/llm/spec/models/queries/work_packages/filter/semantic_search_filter_spec.rb +++ b/modules/llm/spec/models/queries/work_packages/filter/semantic_search_filter_spec.rb @@ -40,14 +40,18 @@ shared_let(:similar_closed) { create(:work_package, status: closed_status) } shared_let(:unrelated) { create(:work_package, status: open_status) } shared_let(:not_embedded) { create(:work_package, status: open_status) } + shared_let(:semantic_index) { create(:semantic_index) } + shared_let(:other_index) { create(:semantic_index) } shared_let(:embeddings) do - create(:work_package_embedding, work_package: similar_open, embedding: [0.1, 0.2, 0.3]) - create(:work_package_embedding, work_package: similar_closed, embedding: [0.2, 0.2, 0.2]) - create(:work_package_embedding, work_package: unrelated, embedding: [-0.1, -0.2, -0.3]) + create(:work_package_embedding, semantic_index: other_index, work_package: unrelated, embedding: [0.1, 0.2, 0.3]) + create(:work_package_embedding, semantic_index:, work_package: similar_open, embedding: [0.1, 0.2, 0.3]) + create(:work_package_embedding, semantic_index:, work_package: similar_closed, embedding: [0.2, 0.2, 0.2]) + create(:work_package_embedding, semantic_index:, work_package: unrelated, embedding: [-0.1, -0.2, -0.3]) end before do + allow(Llm::WorkPackages::SemanticSearch).to receive(:active_semantic_index).and_return(semantic_index) allow(Llm::WorkPackages::SemanticSearch).to receive(:query_vector).with("login problems").and_return("[0.1,0.2,0.3]") end @@ -62,7 +66,7 @@ end describe "#where" do - it "restricts to the work packages within the semantic distance" do + it "restricts to the work packages within the semantic distance in the active Semantic Index" do expect(WorkPackage.where(instance.where)).to contain_exactly(similar_open, similar_closed) end @@ -71,6 +75,12 @@ expect(WorkPackage.where(instance.where)).to be_empty end + + it "matches nothing without an active Semantic Index" do + allow(Llm::WorkPackages::SemanticSearch).to receive(:active_semantic_index).and_return(nil) + + expect(WorkPackage.where(instance.where)).to be_empty + end end describe "in a query" do @@ -94,7 +104,7 @@ it "only finds work packages visible to the user" do invisible_similar = create(:work_package, status: open_status) - create(:work_package_embedding, work_package: invisible_similar, embedding: [0.1, 0.2, 0.3]) + create(:work_package_embedding, semantic_index:, work_package: invisible_similar, embedding: [0.1, 0.2, 0.3]) query.user = create(:user, member_with_permissions: { similar_open.project => %i[view_work_packages] }) login_as(query.user) diff --git a/modules/llm/spec/models/queries/work_packages/selects/semantic_similarity_select_spec.rb b/modules/llm/spec/models/queries/work_packages/selects/semantic_similarity_select_spec.rb index d24159896ad9..d7f463f1213d 100644 --- a/modules/llm/spec/models/queries/work_packages/selects/semantic_similarity_select_spec.rb +++ b/modules/llm/spec/models/queries/work_packages/selects/semantic_similarity_select_spec.rb @@ -36,11 +36,14 @@ shared_let(:first) { create(:work_package) } shared_let(:second) { create(:work_package) } shared_let(:third) { create(:work_package) } + shared_let(:semantic_index) { create(:semantic_index) } + shared_let(:other_index) { create(:semantic_index) } shared_let(:embeddings) do - create(:work_package_embedding, work_package: third, embedding: [0.1, 0.2, 0.3]) - create(:work_package_embedding, work_package: first, embedding: [0.2, 0.2, 0.2]) - create(:work_package_embedding, work_package: second, embedding: [0.3, 0.2, 0.1]) + create(:work_package_embedding, semantic_index: other_index, work_package: second, embedding: [0.1, 0.2, 0.3]) + create(:work_package_embedding, semantic_index:, work_package: third, embedding: [0.1, 0.2, 0.3]) + create(:work_package_embedding, semantic_index:, work_package: first, embedding: [0.2, 0.2, 0.2]) + create(:work_package_embedding, semantic_index:, work_package: second, embedding: [0.3, 0.2, 0.1]) end let(:query) do @@ -50,7 +53,9 @@ end before do - allow(Llm::WorkPackages::SemanticSearch).to receive_messages(available?: true, query_vector: "[0.1,0.2,0.3]") + allow(Llm::WorkPackages::SemanticSearch).to receive_messages(available?: true, + active_semantic_index: semantic_index, + query_vector: "[0.1,0.2,0.3]") login_as(admin) end @@ -58,7 +63,7 @@ expect(described_class.instances.caption).to eq("Similarity") end - it "orders by vector distance when the semantic search filter is active" do + it "orders by the vector distance in the active Semantic Index when the semantic search filter is active" do query.add_filter("semantic_search", "**", ["login problems"]) expect(query.results.work_packages.to_a).to eq([third, first, second]) diff --git a/modules/llm/spec/models/semantic_index_spec.rb b/modules/llm/spec/models/semantic_index_spec.rb index b62a4aa009f9..d98c4088cd9c 100644 --- a/modules/llm/spec/models/semantic_index_spec.rb +++ b/modules/llm/spec/models/semantic_index_spec.rb @@ -57,19 +57,61 @@ end end - it "rejects a chat model" do - semantic_index.model_id = "qwen3.6-27b" + describe "#start_indexing!" do + %w[not_indexed failed ready].each do |state| + it "starts indexing from #{state} and clears the previous error" do + semantic_index = create(:semantic_index, state:, error_message: "Some Error") - expect(semantic_index).not_to be_valid - expect(semantic_index.errors.details[:model_id]).to include(error: :not_offered) + semantic_index.start_indexing! + + expect(semantic_index.reload).to have_attributes(state: "indexing", error_message: nil) + end + end + + it "refuses to start while indexing" do + semantic_index = create(:semantic_index, state: "indexing") + + expect { semantic_index.start_indexing! }.to raise_error(SemanticIndex::InvalidTransitionError) + expect(semantic_index.reload).to be_indexing + end + end + + describe "finishing indexing" do + let(:semantic_index) { create(:semantic_index, state: "indexing") } + + it "becomes ready" do + semantic_index.finish_indexing! + + expect(semantic_index.reload).to be_ready + end + + it "fails with an error message" do + semantic_index.fail_indexing!("Some Error") + + expect(semantic_index.reload).to have_attributes(state: "failed", error_message: "Some Error") + end + + it "refuses unless indexing" do + semantic_index.update!(state: "not_indexed") + + expect { semantic_index.finish_indexing! }.to raise_error(SemanticIndex::InvalidTransitionError) + expect { semantic_index.fail_indexing!("Some Error") }.to raise_error(SemanticIndex::InvalidTransitionError) + end end - it "keeps a stored model that is no longer offered" do - semantic_index.save! - connection.capability_verdicts.destroy_all + describe "#active?" do + let(:semantic_index) { create(:semantic_index, state: "ready") } - semantic_index.title = "Renamed" + it "is active once the semantic search binding references it" do + expect { connection.feature_bindings.create!(feature_key: "semantic_search", semantic_index:) } + .to change(semantic_index, :active?).from(false).to(true) + end - expect(semantic_index).to be_valid + it "cannot be destroyed while active" do + connection.feature_bindings.create!(feature_key: "semantic_search", semantic_index:) + + expect(semantic_index.destroy).to be(false) + expect(described_class.exists?(semantic_index.id)).to be(true) + end end end diff --git a/modules/llm/spec/models/work_package_embedding_spec.rb b/modules/llm/spec/models/work_package_embedding_spec.rb index 9c981d33a59f..0865fc9c19cb 100644 --- a/modules/llm/spec/models/work_package_embedding_spec.rb +++ b/modules/llm/spec/models/work_package_embedding_spec.rb @@ -44,8 +44,9 @@ end it "stores embeddings of any width" do - create(:work_package_embedding, embedding: [0.1, 0.2, 0.3]) - create(:work_package_embedding, embedding: Array.new(4096, 0.1)) + semantic_index = create(:semantic_index) + create(:work_package_embedding, semantic_index:, embedding: [0.1, 0.2, 0.3]) + create(:work_package_embedding, semantic_index:, embedding: Array.new(4096, 0.1)) expect(described_class.count).to eq(2) end diff --git a/modules/llm/spec/requests/admin/llm_feature_bindings_spec.rb b/modules/llm/spec/requests/admin/llm_feature_bindings_spec.rb index d26529e6d9b7..350653f89443 100644 --- a/modules/llm/spec/requests/admin/llm_feature_bindings_spec.rb +++ b/modules/llm/spec/requests/admin/llm_feature_bindings_spec.rb @@ -148,33 +148,12 @@ def offered_models(feature_key) expect(response.body).to include("Use the default (Qwen 3.6)") end - # An unconfirmed capability is not a capability: offering such a model - # invites a choice that fails much later, at index time. - it "offers an embedding feature only models known to create embeddings" do - connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", - state: "supported", source: "probe", checked_at: Time.current) - + it "leaves the model of an embedding feature to its Semantic Indexes" do get llm_feature_bindings_path - expect(offered_models(:semantic_search)).to contain_exactly("bge-m3") + expect(page).to have_no_css("form[action='#{llm_feature_binding_path(:semantic_search)}']") expect(offered_models(:description_assistant)).to contain_exactly("qwen3.6-27b") - end - - it "says so, and where to read up on model types" do - get llm_feature_bindings_path - - expect(response.body).to include("Only models known to create embeddings are offered") - expect(response.body).to include("huggingface.co/blog/getting-started-with-embeddings") - end - - # Otherwise opening the page and saving it would blank a working binding. - it "keeps the bound model listed once it no longer qualifies" do - connection.feature_bindings.create!(feature_key: "semantic_search", model_id: "qwen3.6-27b") - - get llm_feature_bindings_path - - expect(offered_models(:semantic_search)).to include("qwen3.6-27b") - expect(response.body).to include("no longer eligible for this feature") + expect(response.body).to include("Semantic Indexes") end end end @@ -287,54 +266,4 @@ def offered_models(feature_key) expect(response).to have_http_status(:not_found) end end - - describe "embedding settings" do - let!(:connection) { create(:llm_connection, :with_models, base_url:) } - - before do - login_as admin - # Binding an embedding feature probes the model for a vector. - mock_llm_embeddings_response(base_url) - end - - it "stores the vector size" do - patch llm_feature_binding_path(:semantic_search), - params: { llm_feature_binding: { model_id: "bge-m3", dimensions: "1024" } } - - binding = connection.feature_bindings.find_by(feature_key: "semantic_search") - - expect(binding.dimensions).to eq(1024) - end - - it "rejects a dimension count that is not a positive integer" do - patch llm_feature_binding_path(:semantic_search), - params: { llm_feature_binding: { model_id: "bge-m3", dimensions: "0" } } - - expect(connection.feature_bindings.find_by(feature_key: "semantic_search")&.dimensions).to be_nil - end - - it "ignores vector settings sent to a chat feature" do - patch llm_feature_binding_path(:description_assistant), - params: { llm_feature_binding: { model_id: "qwen3.6-27b", dimensions: "1024" } } - - binding = connection.feature_bindings.find_by(feature_key: "description_assistant") - - expect(binding.model_id).to eq("qwen3.6-27b") - expect(binding.dimensions).to be_nil - end - - # A locked binding is the record that a vector index exists. Everything the - # index depends on is frozen, not just the model. - it "refuses to change anything a locked index depends on" do - binding = connection.feature_bindings.create!(feature_key: "semantic_search", model_id: "bge-m3", - dimensions: 1024, locked_at: Time.current) - - patch llm_feature_binding_path(:semantic_search), - params: { llm_feature_binding: { model_id: "qwen3.6-27b", dimensions: "512" } } - - binding.reload - expect(binding.model_id).to eq("bge-m3") - expect(binding.dimensions).to eq(1024) - end - end end diff --git a/modules/llm/spec/requests/admin/llm_models_spec.rb b/modules/llm/spec/requests/admin/llm_models_spec.rb index 3ea0875cdd30..5ca278158be4 100644 --- a/modules/llm/spec/requests/admin/llm_models_spec.rb +++ b/modules/llm/spec/requests/admin/llm_models_spec.rb @@ -800,15 +800,6 @@ def rendered_rows(body) = body.scan("llm-model--edit-").size expect(Llm::Runtime.for(:description_assistant).model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") end - it "follows a model a locked binding depends on" do - binding = connection.feature_bindings.first - binding.update!(locked_at: Time.current) - - patch llm_model_path(llm_model), params: { llm_model: { external_id: "qwen/qwen3.6-35b-a3b:bf16" } } - - expect(binding.reload.model_id).to eq("qwen/qwen3.6-35b-a3b:bf16") - end - # The server names its own models; renaming one here would only be undone by # the next refresh. it "refuses to rename a discovered model" do diff --git a/modules/llm/spec/requests/admin/semantic_indexes_spec.rb b/modules/llm/spec/requests/admin/semantic_indexes_spec.rb index 503f55031b67..e228afb35c64 100644 --- a/modules/llm/spec/requests/admin/semantic_indexes_spec.rb +++ b/modules/llm/spec/requests/admin/semantic_indexes_spec.rb @@ -79,6 +79,15 @@ expect(page).to have_test_selector("llm-semantic-index--box", text: "Not indexed") end + it "shows an empty state while no Semantic Index is active" do + create(:semantic_index, title: "Qwen3 baseline") + + get llm_feature_bindings_path + + expect(page).to have_test_selector("llm-semantic-index--no-active", + text: "Create a new Semantic Index or choose one from the list below.") + end + it "hides the box without the semantic search flag", with_flag: { llm_connection: true, semantic_search: false } do get llm_feature_bindings_path @@ -128,6 +137,14 @@ expect(response.body).to include("Edit "Qwen3 baseline"") end + it "locks the parameters in the edit dialog once indexed" do + semantic_index.update!(state: "failed") + + get edit_dialog_semantic_index_path(semantic_index), headers: turbo_headers + + expect(response.body).to include("Create a new Semantic Index to use different parameters.") + end + it "updates the set" do patch semantic_index_path(semantic_index), params: { semantic_index: { title: "Qwen3 production", context_token_length: 4096 } } @@ -149,5 +166,111 @@ expect(response).to redirect_to(llm_feature_bindings_path) expect(SemanticIndex.count).to eq(0) end + + context "when indexing", with_good_job_batches: [Llm::IndexWorkPackagesJob] do + def index_jobs = GoodJob::Job.where(job_class: Llm::IndexWorkPackagesJob.name) + + it "starts indexing" do + post start_indexing_semantic_index_path(semantic_index) + + expect(response).to redirect_to(llm_feature_bindings_path) + expect(flash[:notice]).to eq("Indexing of \"Qwen3 baseline\" has started.") + expect(semantic_index.reload).to be_indexing + expect(index_jobs.pluck(:batch_id)).to eq([semantic_index.indexing_batch_id]) + end + + it "re-indexes a ready Semantic Index" do + semantic_index.update!(state: "ready") + + post start_indexing_semantic_index_path(semantic_index) + + expect(semantic_index.reload).to be_indexing + expect(index_jobs.count).to eq(1) + end + + it "refuses to start indexing twice" do + semantic_index.update!(state: "indexing") + + post start_indexing_semantic_index_path(semantic_index) + + expect(flash[:error]).to eq("This Semantic Index is already being indexed.") + expect(index_jobs).to be_empty + end + end + end + + describe "activation" do + let!(:current) { create(:semantic_index, title: "Qwen3 baseline", state: "ready") } + let!(:candidate) { create(:semantic_index, title: "Nomic small", state: "ready") } + + before { connection.feature_bindings.create!(feature_key: "semantic_search", semantic_index: current) } + + it "groups the active Semantic Index above the others" do + get llm_feature_bindings_path + + expect(page).to have_test_selector("llm-semantic-index--active", text: "Qwen3 baseline") + expect(page).to have_test_selector("llm-semantic-index--active", text: "Ready") + expect(page).to have_no_test_selector("llm-semantic-index--no-active") + expect(page).to have_no_test_selector("llm-semantic-index--active", text: "Nomic small") + expect(page).to have_test_selector("llm-semantic-index--activate-#{candidate.id}") + expect(page).to have_no_test_selector("llm-semantic-index--activate-#{current.id}") + end + + it "names both Semantic Indexes in the confirmation dialog" do + get activation_dialog_semantic_index_path(candidate), headers: turbo_headers + + expect(response).to have_http_status(:ok) + expect(response.body).to include("switches from Qwen3 baseline to Nomic small") + end + + it "activates a ready Semantic Index" do + post activate_semantic_index_path(candidate) + + expect(response).to redirect_to(llm_feature_bindings_path) + expect(flash[:notice]).to eq("\"Nomic small\" is now the active Semantic Index.") + expect(SemanticIndex.semantic_search_binding.semantic_index).to eq(candidate) + end + + it "refuses a Semantic Index that is not ready" do + candidate.update!(state: "failed") + + post activate_semantic_index_path(candidate) + + expect(flash[:error]).to eq("Semantic Index must be ready before it can be activated.") + expect(SemanticIndex.semantic_search_binding.semantic_index).to eq(current) + end + + it "refuses to delete the active Semantic Index" do + delete semantic_index_path(current) + + expect(flash[:error]).to include("The Semantic Index is active. Activate another Semantic Index first.") + expect(SemanticIndex.exists?(current.id)).to be(true) + end + end + + describe "GET status" do + it "reports the progress of a Semantic Index that is indexing and keeps polling" do + batch = GoodJob::Batch.new.tap(&:save) + create(:semantic_index, state: "indexing", indexing_batch_id: batch.id) + [Time.current, nil, nil, nil].each do |finished_at| + GoodJob::Job.create!(id: SecureRandom.uuid, active_job_id: SecureRandom.uuid, batch_id: batch.id, + job_class: Llm::EmbedWorkPackagesJob.name, queue_name: "default", + serialized_params: {}, finished_at:) + end + + get status_semantic_indexes_path, headers: turbo_headers + + expect(response).to have_http_status(:ok) + expect(response.body).to include("turbo-stream", "25% done", "poll-for-changes") + end + + it "replaces the box once indexing has finished" do + create(:semantic_index, state: "failed", error_message: "Some Error") + + get status_semantic_indexes_path, headers: turbo_headers + + expect(response).to have_http_status(:ok) + expect(response.body).to include("turbo-stream", "Failed", "Some Error") + end end end diff --git a/modules/llm/spec/requests/admin/work_package_embeddings_spec.rb b/modules/llm/spec/requests/admin/work_package_embeddings_spec.rb deleted file mode 100644 index 2551fa5759d3..000000000000 --- a/modules/llm/spec/requests/admin/work_package_embeddings_spec.rb +++ /dev/null @@ -1,168 +0,0 @@ -# frozen_string_literal: true - -#-- copyright -# OpenProject is an open source project management software. -# Copyright (C) the OpenProject GmbH -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License version 3. -# -# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: -# Copyright (C) 2006-2013 Jean-Philippe Lang -# Copyright (C) 2010-2013 the ChiliProject Team -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# -# See COPYRIGHT and LICENSE files for more details. -#++ - -require "spec_helper" -require_module_spec_helper - -RSpec.describe "Admin work package embeddings", :skip_csrf, type: :rails_request, - with_flag: { llm_connection: true, semantic_search: true } do - let(:batch_description) { Llm::IndexWorkPackagesJob::BATCH_DESCRIPTION } - - def create_embed_job(batch, finished_at: nil, error: nil) - GoodJob::Job.create!(active_job_id: SecureRandom.uuid, job_class: Llm::EmbedWorkPackagesJob.name, - queue_name: "default", batch_id: batch.id, error:, - performed_at: finished_at && (finished_at - 1.second), finished_at:, - serialized_params: { "job_class" => Llm::EmbedWorkPackagesJob.name, "arguments" => [[1, 2]] }) - end - - describe "POST /admin/work_package_embeddings/reindex" do - context "as admin" do - before { login_as create(:admin) } - - it "enqueues IndexWorkPackagesJob in a indexing batch", with_good_job_batches: [Llm::IndexWorkPackagesJob] do - post reindex_work_package_embeddings_path - - batch = Llm::IndexWorkPackagesJob.latest_batch - expect(batch.jobs.pluck(:job_class)).to eq([Llm::IndexWorkPackagesJob.name]) - end - - it "does not start another indexing while one is running", with_good_job_batches: [Llm::IndexWorkPackagesJob] do - GoodJob::BatchRecord.create!(description: batch_description) - - expect { post reindex_work_package_embeddings_path }.not_to change(GoodJob::BatchRecord, :count) - expect(flash[:warning]).to eq(I18n.t("admin.work_package_embeddings.reindex.already_running")) - end - - it "redirects to the feature bindings page" do - post reindex_work_package_embeddings_path - - expect(response).to redirect_to(llm_feature_bindings_path) - end - end - - context "as a non-admin" do - before { login_as create(:user) } - - it "is forbidden" do - post reindex_work_package_embeddings_path - - expect(response).to have_http_status(:forbidden) - end - end - - context "with the llm_connection feature flag off", with_flag: { llm_connection: false } do - before { login_as create(:admin) } - - it "returns 404" do - post reindex_work_package_embeddings_path - - expect(response).to have_http_status(:not_found) - end - end - - context "with the semantic search feature flag off", with_flag: { semantic_search: false } do - before { login_as create(:admin) } - - it "returns 404" do - post reindex_work_package_embeddings_path - - expect(response).to have_http_status(:not_found) - end - end - end - - describe "GET /admin/work_package_embeddings/status" do - before { login_as create(:admin) } - - def poll - get status_work_package_embeddings_path, headers: { "Accept" => "*/*" } - end - - context "when the indexing is running" do - let(:batch) { GoodJob::BatchRecord.create!(description: batch_description, enqueued_at: Time.current) } - - before do - create_embed_job(batch, finished_at: Time.current) - create_embed_job(batch) - end - - it "reports the share of finished requests and keeps polling" do - poll - - expect(response.body).to include("width: 50%") - expect(response.body).to include('data-test-selector="reindex-state-running"') - expect(response.body).to include('data-controller="poll-for-changes"') - end - end - - context "when the indexing has succeeded" do - before do - batch = GoodJob::BatchRecord.create!(description: batch_description, finished_at: Time.current) - create_embed_job(batch, finished_at: Time.current) - end - - it "shows a success icon and stops polling" do - poll - - expect(response.body).to include('data-test-selector="reindex-state-succeeded"') - expect(response.body).not_to include('data-controller="poll-for-changes"') - end - end - - context "when the indexing has failed" do - before do - batch = GoodJob::BatchRecord.create!(description: batch_description, finished_at: Time.current, - discarded_at: Time.current) - create_embed_job(batch, finished_at: Time.current, error: "Llm::Errors::ConnectionError: timeout") - end - - it "shows an error icon and the number of failed requests and colors the progress bar" do - poll - - expect(response.body).to include("1 request failed.") - - expect(response.body).to include('data-test-selector="reindex-state-failed"') - expect(response.body).not_to include('data-controller="poll-for-changes"') - - expect(response.body).to include("color-bg-danger-emphasis") - end - end - - context "as a non-admin" do - before { login_as create(:user) } - - it "is forbidden" do - poll - - expect(response).to have_http_status(:forbidden) - end - end - end -end diff --git a/modules/llm/spec/services/llm/runtime_spec.rb b/modules/llm/spec/services/llm/runtime_spec.rb index 103dbc422f12..e86574f98441 100644 --- a/modules/llm/spec/services/llm/runtime_spec.rb +++ b/modules/llm/spec/services/llm/runtime_spec.rb @@ -73,24 +73,6 @@ expect(resolution.model_id).to be_nil end - # The same name on another server need not be the same model, and an index - # written with the first one must not be queried with the second. - it "refuses a binding locked on another deployment, although the name is still offered" do - connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") - .update!(locked_at: Time.current) - connection.update_columns(base_url: "https://elsewhere.example/v1") - - expect(resolution.status).to eq(:model_missing) - expect(resolution.model_id).to eq("qwen3.6-27b") - end - - it "resolves a locked binding while the connection still points at its deployment" do - connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") - .update!(locked_at: Time.current) - - expect(resolution.status).to eq(:ready) - end - it "falls back to the connection default" do connection.update!(default_chat_model: connection.models.find_by(external_id: "qwen3.6-27b")) diff --git a/modules/llm/spec/services/llm/validators/connection_validator_spec.rb b/modules/llm/spec/services/llm/validators/connection_validator_spec.rb index 733e4ce48074..d15881d0215b 100644 --- a/modules/llm/spec/services/llm/validators/connection_validator_spec.rb +++ b/modules/llm/spec/services/llm/validators/connection_validator_spec.rb @@ -244,77 +244,35 @@ def bind_to_withdrawn_model(feature_key, model_id, **) end context "with an embedding feature available", with_flag: { llm_connection: true, semantic_search: true } do - it "passes a default embedding model the server offers" do - connection.update!(default_embedding_model: connection.models.find_by!(external_id: "bge-m3")) - - expect(result_for(:models, :default_embedding_model).state).to eq(:success) - end - - it "names a default embedding model the server no longer offers" do - default = connection.models.find_by!(external_id: "bge-m3") - connection.update!(default_embedding_model: default) - default.update!(active: false) - - result = result_for(:models, :default_embedding_model) + it "leaves it to the active Semantic Index instead of asking for a model" do + result = result_for(:features, :bindings_resolvable) - expect(result.state).to eq(:warning) - expect(result.code).to eq(:default_model_missing) - expect(result.context[:model]).to eq("bge-m3") + expect(result.context[:features]).to eq("Description assistant") end end - end - - describe "the features group" do - before { mock_llm_models_response(base_url) } - - it "warns about features with no model chosen" do - result = result_for(:features, :bindings_resolvable) - - expect(result.state).to eq(:warning) - expect(result.code).to eq(:features_unbound) - end - - it "fails when a binding points at a model the server no longer offers" do - bind_to_withdrawn_model("description_assistant", "vanished") - - result = result_for(:features, :bindings_resolvable) - expect(result.state).to eq(:failure) - expect(result.code).to eq(:features_model_missing) - end + describe "the active Semantic Index" do + let(:semantic_index) { create(:semantic_index, state: "ready") } - context "with an embedding feature available", with_flag: { llm_connection: true, semantic_search: true } do - it "fails when the bound model lacks a capability the feature requires" do - connection.feature_bindings.create!(feature_key: "semantic_search", model_id: "bge-m3") + before do connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", - state: "unsupported", source: "probe", checked_at: Time.current) - - result = result_for(:features, :bindings_resolvable) - - expect(result.state).to eq(:failure) - expect(result.code).to eq(:features_incapable) - expect(result.context).to include(features: "Semantic search", capabilities: "Embeddings") + state: "supported", source: "probe", checked_at: Time.current) + connection.feature_bindings.create!(feature_key: "semantic_search", semantic_index:) end - end - - it "passes while the model of a locked binding is still offered" do - connection.feature_bindings.create!(feature_key: "semantic_search", model_id: "bge-m3", - locked_at: Time.current) - expect(result_for(:features, :locked_bindings_intact).state).to eq(:success) - end + it "passes while its model is still offered" do + expect(result_for(:features, :active_semantic_index_intact).state).to eq(:success) + end - # Only reachable while the feature itself is switched off: for an available - # feature the missing model already fails bindings_resolvable, which halts - # the group. - it "fails when the model of a locked binding has left the catalogue" do - bind_to_withdrawn_model("semantic_search", "retired-embedder", locked_at: Time.current) + it "fails once its model has left the catalogue" do + connection.models.find_by!(external_id: "bge-m3").update!(active: false) - result = result_for(:features, :locked_bindings_intact) + result = result_for(:features, :active_semantic_index_intact) - expect(result.state).to eq(:failure) - expect(result.code).to eq(:locked_model_missing) - expect(result.context).to include(features: "Semantic search") + expect(result.state).to eq(:failure) + expect(result.code).to eq(:semantic_index_model_missing) + expect(result.context).to include(title: semantic_index.title, model: "bge-m3") + end end end diff --git a/modules/llm/spec/services/llm/work_packages/embed_service_spec.rb b/modules/llm/spec/services/llm/work_packages/embed_service_spec.rb index eb865c95d0fe..810c974d113a 100644 --- a/modules/llm/spec/services/llm/work_packages/embed_service_spec.rb +++ b/modules/llm/spec/services/llm/work_packages/embed_service_spec.rb @@ -32,78 +32,54 @@ require_module_spec_helper RSpec.describe Llm::WorkPackages::EmbedService do + include_context "with an embedding model offered" + let(:work_package) { create(:work_package, subject: "Fix bug") } - let(:connection) { create(:llm_connection, :with_models) } - let(:binding) do - create(:llm_feature_binding, llm_connection: connection, feature_key: "semantic_search", - model_id: "bge-m3", dimensions: 3) + let(:semantic_index) do + create(:semantic_index, model_id: "bge-m3", embedding_vector_dimensions: 3, context_token_length: 512) end let(:session_double) { instance_double(Llm::Session) } let(:embedding_double) { instance_double(RubyLLM::Embedding, vectors: [[1.0, 2.0, 3.0]]) } - subject(:service) { described_class.new([work_package]) } - - context "when no binding exists" do - it "raises" do - expect { service.call }.to raise_error(Llm::Errors::ConfigurationError, "binding not ready") - end - end - - context "when a binding exists but has no resolved model" do - before do - create(:llm_feature_binding, llm_connection: connection, feature_key: "semantic_search", - model_id: nil, dimensions: nil) - end + subject(:service) { described_class.new(semantic_index, [work_package]) } - it "raises" do - expect { service.call }.to raise_error(Llm::Errors::ConfigurationError, "binding not ready") - end + before do + allow(Llm::Session).to receive(:for).with(connection).and_return(session_double) + allow(session_double).to receive(:embed).and_return(embedding_double) end - context "when no binding exists but the connection has a default embedding model" do - let(:connection) { create(:llm_connection, :with_models, default_embedding_model_identifier: "bge-m3") } + it "embeds with the parameters of the Semantic Index" do + allow(Llm::WorkPackages::EmbeddableText).to receive(:for).and_call_original - before do - allow(Llm::Session).to receive(:for).with(connection).and_return(session_double) - allow(session_double).to receive(:embed).and_return(embedding_double) - end + service.call - it "embeds with the default model and locks the binding to it" do - service.call + expect(session_double).to have_received(:embed).with(anything, model: "bge-m3", dimensions: 3) + expect(Llm::WorkPackages::EmbeddableText).to have_received(:for).with(work_package, context_window: 512) + end - expect(session_double).to have_received(:embed).with(anything, model: "bge-m3", dimensions: nil) + it "stores the embedding in the Semantic Index" do + service.call - binding = connection.feature_bindings.find_by!(feature_key: "semantic_search") - expect(binding).to be_locked - expect(binding.model_id).to eq("bge-m3") - end + record = WorkPackageEmbedding.find_by!(semantic_index:, work_package:) + expect(record.embedding).to eq("[1,2,3]") end - context "when a ready binding exists" do - before do - binding - allow(Llm::Session).to receive(:for).with(connection).and_return(session_double) - allow(session_double).to receive(:embed).and_return(embedding_double) - end - - it "stores the embedding" do - service.call + it "replaces an earlier embedding of the same work package" do + service.call + allow(session_double).to receive(:embed) + .and_return(instance_double(RubyLLM::Embedding, vectors: [[4.0, 5.0, 6.0]])) - record = WorkPackageEmbedding.find_by!(work_package:) - expect(record.embedding).to eq("[1,2,3]") - end + described_class.new(semantic_index, [work_package]).call - it "locks the binding on first write" do - expect { service.call }.to change { binding.reload.locked? }.from(false).to(true) - end + expect(WorkPackageEmbedding.where(semantic_index:).pluck(:embedding)).to eq(["[4,5,6]"]) + end - it "does not modify locked_at on a subsequent write" do - binding.update!(locked_at: 1.hour.ago) - locked_at_before = binding.locked_at + it "leaves the embeddings of other Semantic Indexes alone" do + other_index = create(:semantic_index) + create(:work_package_embedding, semantic_index: other_index, work_package:) - service.call + service.call - expect(binding.reload.locked_at).to be_within(1.second).of(locked_at_before) - end + expect(WorkPackageEmbedding.where(work_package:).count).to eq(2) end end diff --git a/modules/llm/spec/services/llm/work_packages/semantic_search_spec.rb b/modules/llm/spec/services/llm/work_packages/semantic_search_spec.rb index c48f7496db06..c04a510d28c0 100644 --- a/modules/llm/spec/services/llm/work_packages/semantic_search_spec.rb +++ b/modules/llm/spec/services/llm/work_packages/semantic_search_spec.rb @@ -32,34 +32,38 @@ require_module_spec_helper RSpec.describe Llm::WorkPackages::SemanticSearch, with_flag: { semantic_search: true } do - let(:connection) { create(:llm_connection, :with_models) } + include_context "with an embedding model offered" + let(:session_double) { instance_double(Llm::Session) } let(:embedding_double) { instance_double(RubyLLM::Embedding, vectors: [0.1, 0.2, 0.3]) } + let(:semantic_index) { create(:semantic_index, state: "ready", embedding_vector_dimensions: 3) } - def create_binding(locked_at: Time.current) - create(:llm_feature_binding, llm_connection: connection, feature_key: "semantic_search", - model_id: "bge-m3", dimensions: 3, locked_at:) + def activate(semantic_index) + create(:llm_feature_binding, llm_connection: connection, feature_key: "semantic_search", semantic_index:) end describe ".available?" do - it "is false without a binding" do + it "is false without an active Semantic Index" do + create(:llm_feature_binding, llm_connection: connection, feature_key: "semantic_search") + expect(described_class).not_to be_available end - it "is false while the binding is not locked" do - create_binding(locked_at: nil) + it "is true once a Semantic Index is active" do + activate(semantic_index) - expect(described_class).not_to be_available + expect(described_class).to be_available end - it "is true once the binding is locked" do - create_binding + it "stays true while the active Semantic Index is re-indexed" do + activate(semantic_index) + semantic_index.start_indexing! expect(described_class).to be_available end it "is false with the feature flag off", with_flag: { semantic_search: false } do - create_binding + activate(semantic_index) expect(described_class).not_to be_available end @@ -71,16 +75,14 @@ def create_binding(locked_at: Time.current) allow(session_double).to receive(:embed).and_return(embedding_double) end - it "returns nothing without a locked binding" do - create_binding(locked_at: nil) - + it "returns nothing without an active Semantic Index" do expect(described_class.query_vector("some query")).to be_nil end - context "with a locked binding" do - before { create_binding } + context "with an active Semantic Index" do + before { activate(semantic_index) } - it "embeds the query as a vector literal" do + it "embeds the query with the parameters of the Semantic Index" do expect(described_class.query_vector("some query")).to eq("[0.1,0.2,0.3]") expect(session_double).to have_received(:embed).with("some query", model: "bge-m3", dimensions: 3) end diff --git a/modules/llm/spec/services/llm_connections/sync_models_service_spec.rb b/modules/llm/spec/services/llm_connections/sync_models_service_spec.rb index bf7377f5a650..06bda199434c 100644 --- a/modules/llm/spec/services/llm_connections/sync_models_service_spec.rb +++ b/modules/llm/spec/services/llm_connections/sync_models_service_spec.rb @@ -101,28 +101,17 @@ expect(connection.feature_bindings).to be_empty end - it "keeps a locked binding, which records what an existing index was written with" do - connection.capability_verdicts.create!(model_id: "bge-m3", capability: "embeddings", - state: "supported", source: "probe", checked_at: Time.current) - binding = connection.feature_bindings.create!(feature_key: "semantic_search", model_id: "bge-m3", - dimensions: 1024, locked_at: Time.current) + it "keeps the binding that points at the active Semantic Index" do + semantic_index = SemanticIndex.new(title: "Index", model_id: "bge-m3", embedding_vector_dimensions: 1024, + context_token_length: 8192, state: :ready) + semantic_index.save!(validate: false) + binding = connection.feature_bindings.create!(feature_key: "semantic_search", model_id: "qwen3.6-27b", + semantic_index:) mock_llm_models_response("https://elsewhere.example/v1", response_code: 405) service.call - expect(binding.reload).to have_attributes(model_id: "bge-m3", dimensions: 1024) - expect(binding).to be_locked - end - - it "leaves a kept binding dangling when the new server offers a model of the same name" do - binding = connection.feature_bindings.create!(feature_key: "description_assistant", model_id: "qwen3.6-27b") - binding.update_columns(locked_at: Time.current, locked_deployment_fingerprint: connection.connection_fingerprint) - mock_llm_models_response("https://elsewhere.example/v1") - - service.call - - expect(connection.reload.available_model_ids).to include("qwen3.6-27b") - expect(binding.reload).to be_dangling + expect(binding.reload.semantic_index).to eq(semantic_index) end # on_delete: :nullify on the two default_*_model_id foreign keys. diff --git a/modules/llm/spec/services/semantic_indexes/activate_service_spec.rb b/modules/llm/spec/services/semantic_indexes/activate_service_spec.rb new file mode 100644 index 000000000000..47ddd7675230 --- /dev/null +++ b/modules/llm/spec/services/semantic_indexes/activate_service_spec.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" +require_module_spec_helper + +RSpec.describe SemanticIndexes::ActivateService do + include_context "with an embedding model offered" + + let(:semantic_index) { create(:semantic_index, state: "ready") } + let(:user) { build_stubbed(:admin) } + + subject(:result) { described_class.new(user:, model: semantic_index).call } + + def active_semantic_index = SemanticIndex.semantic_search_binding.semantic_index + + it "makes the Semantic Index the active one" do + expect(result).to be_success + expect(active_semantic_index).to eq(semantic_index) + end + + it "replaces the previously active Semantic Index" do + previous = create(:semantic_index, state: "ready") + connection.feature_bindings.create!(feature_key: "semantic_search", semantic_index: previous) + + expect(result).to be_success + expect(active_semantic_index).to eq(semantic_index) + expect(previous).not_to be_active + end + + context "when the Semantic Index is not ready" do + let(:semantic_index) { create(:semantic_index, state: "failed") } + + it "fails" do + expect(result).to be_failure + expect(result.message).to eq("Semantic Index must be ready before it can be activated.") + expect(active_semantic_index).to be_nil + end + end + + context "as a regular user" do + let(:user) { build_stubbed(:user) } + + it "fails" do + expect(result).to be_failure + expect(active_semantic_index).to be_nil + end + end +end diff --git a/modules/llm/spec/services/semantic_indexes/start_indexing_service_spec.rb b/modules/llm/spec/services/semantic_indexes/start_indexing_service_spec.rb new file mode 100644 index 000000000000..63b023d55c42 --- /dev/null +++ b/modules/llm/spec/services/semantic_indexes/start_indexing_service_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" +require_module_spec_helper + +RSpec.describe SemanticIndexes::StartIndexingService, with_good_job_batches: [Llm::IndexWorkPackagesJob] do + include_context "with an embedding model offered" + + let(:semantic_index) { create(:semantic_index) } + let(:user) { build_stubbed(:admin) } + + subject(:result) { described_class.new(user:, model: semantic_index).call } + + def index_jobs = GoodJob::Job.where(job_class: Llm::IndexWorkPackagesJob.name) + + it "starts indexing in a batch it remembers" do + expect(result).to be_success + expect(semantic_index.reload).to be_indexing + expect(index_jobs.pluck(:batch_id)).to eq([semantic_index.indexing_batch_id]) + end + + context "when the Semantic Index is already indexing" do + let(:semantic_index) { create(:semantic_index, state: "indexing") } + + it "fails without enqueuing" do + expect(result).to be_failure + expect(index_jobs).to be_empty + end + end + + context "as a regular user" do + let(:user) { build_stubbed(:user) } + + it "fails without changing the state" do + expect(result).to be_failure + expect(semantic_index.reload).to be_not_indexed + end + end +end diff --git a/modules/llm/spec/workers/llm/embed_work_packages_job_spec.rb b/modules/llm/spec/workers/llm/embed_work_packages_job_spec.rb index 73f83ac74dbb..fdc268c4a2a7 100644 --- a/modules/llm/spec/workers/llm/embed_work_packages_job_spec.rb +++ b/modules/llm/spec/workers/llm/embed_work_packages_job_spec.rb @@ -33,23 +33,39 @@ RSpec.describe Llm::EmbedWorkPackagesJob, with_flag: { semantic_search: true } do let(:work_package) { create(:work_package) } + let(:semantic_index) { create(:semantic_index, state: "indexing") } - it "calls EmbedService with the resolved work packages" do + before { allow(Llm::WorkPackages::EmbedService).to receive(:new).and_call_original } + + it "embeds the work packages into the Semantic Index" do service_double = instance_double(Llm::WorkPackages::EmbedService, call: :ok) allow(Llm::WorkPackages::EmbedService).to receive(:new).and_return(service_double) - described_class.perform_now([work_package.id]) + described_class.perform_now(semantic_index.id, [work_package.id]) - expect(Llm::WorkPackages::EmbedService).to have_received(:new) do |wps| + expect(Llm::WorkPackages::EmbedService).to have_received(:new) do |index, wps| + expect(index).to eq(semantic_index) expect(wps.map(&:id)).to contain_exactly(work_package.id) end expect(service_double).to have_received(:call) end - it "does nothing with the feature flag off", with_flag: { semantic_search: false } do - allow(Llm::WorkPackages::EmbedService).to receive(:new) + it "does nothing once the Semantic Index is no longer indexing" do + semantic_index.update!(state: "failed") + + described_class.perform_now(semantic_index.id, [work_package.id]) + + expect(Llm::WorkPackages::EmbedService).not_to have_received(:new) + end - described_class.perform_now([work_package.id]) + it "does nothing once the Semantic Index is gone" do + described_class.perform_now(0, [work_package.id]) + + expect(Llm::WorkPackages::EmbedService).not_to have_received(:new) + end + + it "does nothing with the feature flag off", with_flag: { semantic_search: false } do + described_class.perform_now(semantic_index.id, [work_package.id]) expect(Llm::WorkPackages::EmbedService).not_to have_received(:new) end diff --git a/modules/llm/spec/workers/llm/finish_semantic_indexing_job_spec.rb b/modules/llm/spec/workers/llm/finish_semantic_indexing_job_spec.rb new file mode 100644 index 000000000000..4fdcd3829404 --- /dev/null +++ b/modules/llm/spec/workers/llm/finish_semantic_indexing_job_spec.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +#-- copyright +# OpenProject is an open source project management software. +# Copyright (C) the OpenProject GmbH +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License version 3. +# +# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows: +# Copyright (C) 2006-2013 Jean-Philippe Lang +# Copyright (C) 2010-2013 the ChiliProject Team +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# See COPYRIGHT and LICENSE files for more details. +#++ + +require "spec_helper" +require_module_spec_helper + +RSpec.describe Llm::FinishSemanticIndexingJob do + let(:semantic_index) { create(:semantic_index, state: "indexing") } + let(:batch) { GoodJob::Batch.new(semantic_index_id: semantic_index.id).tap(&:save) } + + def embed_job(finished_at: Time.current, error: nil) + GoodJob::Job.create!(id: SecureRandom.uuid, active_job_id: SecureRandom.uuid, batch_id: batch.id, + job_class: Llm::EmbedWorkPackagesJob.name, queue_name: "default", + serialized_params: {}, finished_at:, error:) + end + + it "marks the Semantic Index ready when every request succeeded" do + embed_job + + described_class.perform_now(batch, { event: :finish }) + + expect(semantic_index.reload).to be_ready + end + + it "marks the Semantic Index failed with the number of failed requests" do + embed_job + embed_job(error: "Llm::Errors::ApiError: boom") + + described_class.perform_now(batch, { event: :finish }) + + expect(semantic_index.reload).to have_attributes(state: "failed", error_message: "1 request failed.") + end + + it "leaves a Semantic Index alone that is no longer indexing" do + semantic_index.update!(state: "ready") + + described_class.perform_now(batch, { event: :finish }) + + expect(semantic_index.reload).to be_ready + end +end diff --git a/modules/llm/spec/workers/llm/index_work_packages_job_spec.rb b/modules/llm/spec/workers/llm/index_work_packages_job_spec.rb index 5f8697596b61..158f6b25ce6b 100644 --- a/modules/llm/spec/workers/llm/index_work_packages_job_spec.rb +++ b/modules/llm/spec/workers/llm/index_work_packages_job_spec.rb @@ -34,37 +34,21 @@ RSpec.describe Llm::IndexWorkPackagesJob, with_flag: { semantic_search: true }, with_good_job_batches: [Llm::IndexWorkPackagesJob, Llm::EmbedWorkPackagesJob] do - describe ".enqueue_batch" do - it "enqueues the job inside a index batch" do - batch = described_class.enqueue_batch + let(:semantic_index) { create(:semantic_index, state: "indexing") } - expect(batch.description).to eq(described_class::BATCH_DESCRIPTION) - expect(GoodJob::Job.where(batch_id: batch.id).pluck(:job_class)).to eq([described_class.name]) - end - end - - describe ".running?" do - it "is false without a batch" do - expect(described_class).not_to be_running - end + describe ".enqueue_for" do + it "enqueues the job inside a batch that finishes the Semantic Index" do + batch = described_class.enqueue_for(semantic_index) - it "is true while the latest batch is unfinished" do - GoodJob::BatchRecord.create!(description: described_class::BATCH_DESCRIPTION, finished_at: 1.hour.ago) - GoodJob::BatchRecord.create!(description: described_class::BATCH_DESCRIPTION) - - expect(described_class).to be_running - end - - it "is false once the latest batch has finished" do - GoodJob::BatchRecord.create!(description: described_class::BATCH_DESCRIPTION, finished_at: Time.current) - - expect(described_class).not_to be_running + expect(batch.properties).to include(semantic_index_id: semantic_index.id) + expect(batch.on_finish).to eq(Llm::FinishSemanticIndexingJob.name) + expect(GoodJob::Job.where(batch_id: batch.id).pluck(:job_class)).to eq([described_class.name]) end end describe "#perform" do let!(:work_packages) { create_list(:work_package, 2) } - let(:batch) { GoodJob::Batch.new(description: described_class::BATCH_DESCRIPTION) } + let(:batch) { GoodJob::Batch.new } let(:job) { described_class.new } before { allow(job).to receive(:batch).and_return(batch) } @@ -73,15 +57,15 @@ def embed_jobs GoodJob::Job.where(batch_id: batch.id, job_class: Llm::EmbedWorkPackagesJob.name) end - it "adds an embedding job for the work packages to its batch" do - job.perform + it "adds an embedding job for the work packages of the Semantic Index to its batch" do + job.perform(semantic_index.id) - expect(embed_jobs.map { it.serialized_params["arguments"].first }) - .to contain_exactly(match_array(work_packages.map(&:id))) + expect(embed_jobs.map { it.serialized_params["arguments"] }) + .to contain_exactly([semantic_index.id, match_array(work_packages.map(&:id))]) end it "does nothing with the feature flag off", with_flag: { semantic_search: false } do - job.perform + job.perform(semantic_index.id) expect(embed_jobs).to be_empty end