Feature: Use alias-backed Solr targets for ontology search indexing

Gemfile

@@ -39,7 +39,7 @@ group :development do
   gem 'rubocop', require: false
 end
 # NCBO gems (can be from a local dev path or from rubygems/git)
-gem 'goo', github: 'ncbo/goo', branch: 'development'
+gem 'goo', github: 'ncbo/goo', branch: 'feature/solrcloud-alias-indexing-codex'
 gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'development'
 
 

Gemfile.lock

@@ -8,8 +8,8 @@ GIT
 
 GIT
   remote: https://github.com/ncbo/goo.git
-  revision: 90365864f67a9458011f16e1b9f44a73af0d3739
-  branch: development
+  revision: 8816ce8f8f9b270b443495b2d6caf0e00730cbcb
+  branch: feature/solrcloud-alias-indexing-codex
   specs:
     goo (0.0.2)
       addressable (~> 2.8)

config/config.rb.sample

@@ -3,6 +3,10 @@ LinkedData.config do |config|
   config.goo_host          = "localhost"
   config.search_server_url = "http://localhost:8983/solr"
   config.property_search_server_url = "http://localhost:8983/solr"
+  config.term_search_num_shards = 1
+  config.term_search_replication_factor = 1
+  config.property_search_num_shards = 1
+  config.property_search_replication_factor = 1
   config.repository_folder = "./test/data/ontology_files/repo"
   config.rest_url_prefix   = "http://localhost:9393/"
   config.enable_security   = false

config/config.test.rb

@@ -17,20 +17,24 @@
 SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : 'http://localhost:8983/solr'
 
 LinkedData.config do |config|
-  config.goo_backend_name              = GOO_BACKEND_NAME.to_s
-  config.goo_host                      = GOO_HOST.to_s
-  config.goo_port                      = GOO_PORT.to_i
-  config.goo_path_query                = GOO_PATH_QUERY.to_s
-  config.goo_path_data                 = GOO_PATH_DATA.to_s
-  config.goo_path_update               = GOO_PATH_UPDATE.to_s
-  config.goo_redis_host                = REDIS_HOST.to_s
-  config.goo_redis_port                = REDIS_PORT.to_i
-  config.http_redis_host               = REDIS_HOST.to_s
-  config.http_redis_port               = REDIS_PORT.to_i
-  config.ontology_analytics_redis_host = REDIS_HOST.to_s
-  config.ontology_analytics_redis_port = REDIS_PORT.to_i
-  config.search_server_url             = SOLR_TERM_SEARCH_URL.to_s
-  config.property_search_server_url    = SOLR_PROP_SEARCH_URL.to_s
+  config.goo_backend_name                   = GOO_BACKEND_NAME.to_s
+  config.goo_host                           = GOO_HOST.to_s
+  config.goo_port                           = GOO_PORT.to_i
+  config.goo_path_query                     = GOO_PATH_QUERY.to_s
+  config.goo_path_data                      = GOO_PATH_DATA.to_s
+  config.goo_path_update                    = GOO_PATH_UPDATE.to_s
+  config.goo_redis_host                     = REDIS_HOST.to_s
+  config.goo_redis_port                     = REDIS_PORT.to_i
+  config.http_redis_host                    = REDIS_HOST.to_s
+  config.http_redis_port                    = REDIS_PORT.to_i
+  config.ontology_analytics_redis_host      = REDIS_HOST.to_s
+  config.ontology_analytics_redis_port      = REDIS_PORT.to_i
+  config.search_server_url                  = SOLR_TERM_SEARCH_URL.to_s
+  config.property_search_server_url         = SOLR_PROP_SEARCH_URL.to_s
+  config.term_search_num_shards             = 1
+  config.term_search_replication_factor     = 1
+  config.property_search_num_shards         = 1
+  config.property_search_replication_factor = 1
 #  config.enable_notifications          = false
 
   config.oauth_providers = {

lib/ontologies_linked_data/config/config.rb

@@ -25,10 +25,17 @@ def config(&block)
     @settings.goo_path_update               ||= '/update/'
     @settings.search_server_url             ||= 'http://localhost:8983/solr'
     @settings.property_search_server_url    ||= 'http://localhost:8983/solr'
+    @settings.term_search_num_shards        ||= 1
+    @settings.term_search_replication_factor ||= 1
+    @settings.term_search_bootstrap_collection ||= :term_search_bootstrap
+    @settings.property_search_num_shards    ||= 1
+    @settings.property_search_replication_factor ||= 1
+    @settings.property_search_bootstrap_collection ||= :property_search_bootstrap
     @settings.repository_folder             ||= './test/data/ontology_files/repo'
     @settings.rest_url_prefix               ||= DEFAULT_PREFIX
     @settings.enable_security               ||= false
     @settings.enable_slices                 ||= false
+    @settings.skip_connect_goo              ||= ENV['LINKEDDATA_SKIP_CONNECT_GOO'] == 'true'
 
     # Java/JVM options
     @settings.java_max_heap_size            ||= '10240M'
@@ -95,6 +102,7 @@ def config(&block)
 
     # Override defaults
     yield @settings, overide_connect_goo if block_given?
+    configure_search_collections
 
     # Check to make sure url prefix has trailing slash
     @settings.rest_url_prefix = "#{@settings.rest_url_prefix}/" unless @settings.rest_url_prefix[-1].eql?('/')
@@ -105,7 +113,7 @@ def config(&block)
     puts "(LD) >> Using HTTP Redis instance at #{@settings.http_redis_host}:#{@settings.http_redis_port}"
     puts "(LD) >> Using Goo Redis instance at #{@settings.goo_redis_host}:#{@settings.goo_redis_port}"
 
-    connect_goo unless overide_connect_goo
+    connect_goo unless overide_connect_goo || @settings.skip_connect_goo
   end
 
   ##
@@ -137,6 +145,17 @@ def connect_goo
     end
   end
 
+  def configure_search_collections
+    Goo.set_search_collection_bootstrap(:term_search, @settings.term_search_bootstrap_collection)
+    Goo.set_search_collection_bootstrap(:property_search, @settings.property_search_bootstrap_collection)
+    Goo.set_search_collection_topology(:term_search,
+                                       num_shards: @settings.term_search_num_shards,
+                                       replication_factor: @settings.term_search_replication_factor)
+    Goo.set_search_collection_topology(:property_search,
+                                       num_shards: @settings.property_search_num_shards,
+                                       replication_factor: @settings.property_search_replication_factor)
+  end
+
   ##
   # Configure ontologies_linked_data namespaces
   # We do this at initial runtime because goo needs namespaces for its DSL

lib/ontologies_linked_data/models/class.rb

@@ -182,7 +182,11 @@ def self.index_schema(schema_generator)
         schema_generator.add_dynamic_field('definition_*', 'text_general', indexed: true, stored: true, multi_valued: true)
       end
 
-      enable_indexing(:term_search_core1) do |schema_generator|
+      enable_indexing(:term_search,
+                      :main,
+                      bootstrap_collection: LinkedData.settings.term_search_bootstrap_collection,
+                      num_shards: LinkedData.settings.term_search_num_shards,
+                      replication_factor: LinkedData.settings.term_search_replication_factor) do |schema_generator|
         index_schema(schema_generator)
       end
 

lib/ontologies_linked_data/models/properties/annotation_property.rb

@@ -36,7 +36,11 @@ class AnnotationProperty < LinkedData::Models::OntologyProperty
               LinkedData::Hypermedia::Link.new("descendants", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/descendants"}, self.uri_type),
               LinkedData::Hypermedia::Link.new("tree", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/tree"}, self.uri_type)
 
-      enable_indexing(:prop_search_core1, :property) do |schema_generator|
+      enable_indexing(:property_search,
+                      :property,
+                      bootstrap_collection: LinkedData.settings.property_search_bootstrap_collection,
+                      num_shards: LinkedData.settings.property_search_num_shards,
+                      replication_factor: LinkedData.settings.property_search_replication_factor) do |schema_generator|
         index_schema(schema_generator)
       end
     end

lib/ontologies_linked_data/models/properties/datatype_property.rb

@@ -36,7 +36,11 @@ class DatatypeProperty < LinkedData::Models::OntologyProperty
               LinkedData::Hypermedia::Link.new("descendants", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/descendants"}, self.uri_type),
               LinkedData::Hypermedia::Link.new("tree", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/tree"}, self.uri_type)
 
-      enable_indexing(:prop_search_core1, :property) do |schema_generator|
+      enable_indexing(:property_search,
+                      :property,
+                      bootstrap_collection: LinkedData.settings.property_search_bootstrap_collection,
+                      num_shards: LinkedData.settings.property_search_num_shards,
+                      replication_factor: LinkedData.settings.property_search_replication_factor) do |schema_generator|
         index_schema(schema_generator)
       end
     end

lib/ontologies_linked_data/models/properties/object_property.rb

@@ -36,7 +36,11 @@ class ObjectProperty < LinkedData::Models::OntologyProperty
               LinkedData::Hypermedia::Link.new("descendants", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/descendants"}, self.uri_type),
               LinkedData::Hypermedia::Link.new("tree", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/tree"}, self.uri_type)
 
-      enable_indexing(:prop_search_core1, :property) do |schema_generator|
+      enable_indexing(:property_search,
+                      :property,
+                      bootstrap_collection: LinkedData.settings.property_search_bootstrap_collection,
+                      num_shards: LinkedData.settings.property_search_num_shards,
+                      replication_factor: LinkedData.settings.property_search_replication_factor) do |schema_generator|
         index_schema(schema_generator)
       end
     end

lib/ontologies_linked_data/models/properties/ontology_property.rb

@@ -26,7 +26,11 @@ def self.index_schema(schema_generator)
         end
       end
 
-      enable_indexing(:prop_search_core1, :property) do |schema_generator|
+      enable_indexing(:property_search,
+                      :property,
+                      bootstrap_collection: LinkedData.settings.property_search_bootstrap_collection,
+                      num_shards: LinkedData.settings.property_search_num_shards,
+                      replication_factor: LinkedData.settings.property_search_replication_factor) do |schema_generator|
         index_schema(schema_generator)
       end
 

lib/ontologies_linked_data/models/provisional_class.rb

@@ -40,7 +40,11 @@ class ProvisionalClass < LinkedData::Models::Base
                 end
               }, Goo.vocabulary["Ontology"])
 
-      enable_indexing(:term_search_core1) do |schema_generator|
+      enable_indexing(:term_search,
+                      :main,
+                      bootstrap_collection: LinkedData.settings.term_search_bootstrap_collection,
+                      num_shards: LinkedData.settings.term_search_num_shards,
+                      replication_factor: LinkedData.settings.term_search_replication_factor) do |schema_generator|
         Class.index_schema(schema_generator)
       end
 

test/test_case.rb

@@ -54,6 +54,7 @@ module MinitestRunHooks
     def run(*args)
       LinkedData::TestCase.setup_test_repository_folder
       LinkedData::TestCase.backend_4s_delete
+      Goo.init_search_connections(true)
       super
     ensure
       LinkedData::TestCase.backend_4s_delete

test/util/test_ontology_csv_writer.rb

@@ -57,8 +57,9 @@ def enclosed_in_square_brackets_with_quotes?(string)
   end
 
   def get_csv_string_from_path(csv_path)
-    gz = Zlib::GzipReader.open(csv_path)
-    gz.read
+    Zlib::GzipReader.open(csv_path) do |gz|
+      gz.read.force_encoding(Encoding::UTF_8)
+    end
   end
 
   def test_csv_writer_valid
@@ -337,4 +338,4 @@ def test_for_non_array_values
     end
   end
 
-end
+end