diff --git a/Gemfile b/Gemfile index 88d62baf..4bf6416e 100644 --- a/Gemfile +++ b/Gemfile @@ -39,7 +39,7 @@ group :development do gem 'rubocop', require: false end # NCBO gems (can be from a local dev path or from rubygems/git) -gem 'goo', github: 'ncbo/goo', branch: 'development' +gem 'goo', github: 'ncbo/goo', branch: 'feature/solrcloud-alias-indexing-codex' gem 'sparql-client', github: 'ncbo/sparql-client', branch: 'development' diff --git a/Gemfile.lock b/Gemfile.lock index 50355fae..81f068ab 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,8 +8,8 @@ GIT GIT remote: https://github.com/ncbo/goo.git - revision: 90365864f67a9458011f16e1b9f44a73af0d3739 - branch: development + revision: 8816ce8f8f9b270b443495b2d6caf0e00730cbcb + branch: feature/solrcloud-alias-indexing-codex specs: goo (0.0.2) addressable (~> 2.8) diff --git a/config/config.rb.sample b/config/config.rb.sample index 75a6d71d..eb0251c7 100644 --- a/config/config.rb.sample +++ b/config/config.rb.sample @@ -3,6 +3,10 @@ LinkedData.config do |config| config.goo_host = "localhost" config.search_server_url = "http://localhost:8983/solr" config.property_search_server_url = "http://localhost:8983/solr" + config.term_search_num_shards = 1 + config.term_search_replication_factor = 1 + config.property_search_num_shards = 1 + config.property_search_replication_factor = 1 config.repository_folder = "./test/data/ontology_files/repo" config.rest_url_prefix = "http://localhost:9393/" config.enable_security = false diff --git a/config/config.test.rb b/config/config.test.rb index ad52fbc4..50115e1d 100644 --- a/config/config.test.rb +++ b/config/config.test.rb @@ -17,20 +17,24 @@ SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : 'http://localhost:8983/solr' LinkedData.config do |config| - config.goo_backend_name = GOO_BACKEND_NAME.to_s - config.goo_host = GOO_HOST.to_s - config.goo_port = GOO_PORT.to_i - config.goo_path_query = GOO_PATH_QUERY.to_s - config.goo_path_data = GOO_PATH_DATA.to_s - config.goo_path_update = GOO_PATH_UPDATE.to_s - config.goo_redis_host = REDIS_HOST.to_s - config.goo_redis_port = REDIS_PORT.to_i - config.http_redis_host = REDIS_HOST.to_s - config.http_redis_port = REDIS_PORT.to_i - config.ontology_analytics_redis_host = REDIS_HOST.to_s - config.ontology_analytics_redis_port = REDIS_PORT.to_i - config.search_server_url = SOLR_TERM_SEARCH_URL.to_s - config.property_search_server_url = SOLR_PROP_SEARCH_URL.to_s + config.goo_backend_name = GOO_BACKEND_NAME.to_s + config.goo_host = GOO_HOST.to_s + config.goo_port = GOO_PORT.to_i + config.goo_path_query = GOO_PATH_QUERY.to_s + config.goo_path_data = GOO_PATH_DATA.to_s + config.goo_path_update = GOO_PATH_UPDATE.to_s + config.goo_redis_host = REDIS_HOST.to_s + config.goo_redis_port = REDIS_PORT.to_i + config.http_redis_host = REDIS_HOST.to_s + config.http_redis_port = REDIS_PORT.to_i + config.ontology_analytics_redis_host = REDIS_HOST.to_s + config.ontology_analytics_redis_port = REDIS_PORT.to_i + config.search_server_url = SOLR_TERM_SEARCH_URL.to_s + config.property_search_server_url = SOLR_PROP_SEARCH_URL.to_s + config.term_search_num_shards = 1 + config.term_search_replication_factor = 1 + config.property_search_num_shards = 1 + config.property_search_replication_factor = 1 # config.enable_notifications = false config.oauth_providers = { diff --git a/lib/ontologies_linked_data/config/config.rb b/lib/ontologies_linked_data/config/config.rb index 67b6e43d..58a43faf 100644 --- a/lib/ontologies_linked_data/config/config.rb +++ b/lib/ontologies_linked_data/config/config.rb @@ -25,10 +25,17 @@ def config(&block) @settings.goo_path_update ||= '/update/' @settings.search_server_url ||= 'http://localhost:8983/solr' @settings.property_search_server_url ||= 'http://localhost:8983/solr' + @settings.term_search_num_shards ||= 1 + @settings.term_search_replication_factor ||= 1 + @settings.term_search_bootstrap_collection ||= :term_search_bootstrap + @settings.property_search_num_shards ||= 1 + @settings.property_search_replication_factor ||= 1 + @settings.property_search_bootstrap_collection ||= :property_search_bootstrap @settings.repository_folder ||= './test/data/ontology_files/repo' @settings.rest_url_prefix ||= DEFAULT_PREFIX @settings.enable_security ||= false @settings.enable_slices ||= false + @settings.skip_connect_goo ||= ENV['LINKEDDATA_SKIP_CONNECT_GOO'] == 'true' # Java/JVM options @settings.java_max_heap_size ||= '10240M' @@ -95,6 +102,7 @@ def config(&block) # Override defaults yield @settings, overide_connect_goo if block_given? + configure_search_collections # Check to make sure url prefix has trailing slash @settings.rest_url_prefix = "#{@settings.rest_url_prefix}/" unless @settings.rest_url_prefix[-1].eql?('/') @@ -105,7 +113,7 @@ def config(&block) puts "(LD) >> Using HTTP Redis instance at #{@settings.http_redis_host}:#{@settings.http_redis_port}" puts "(LD) >> Using Goo Redis instance at #{@settings.goo_redis_host}:#{@settings.goo_redis_port}" - connect_goo unless overide_connect_goo + connect_goo unless overide_connect_goo || @settings.skip_connect_goo end ## @@ -137,6 +145,17 @@ def connect_goo end end + def configure_search_collections + Goo.set_search_collection_bootstrap(:term_search, @settings.term_search_bootstrap_collection) + Goo.set_search_collection_bootstrap(:property_search, @settings.property_search_bootstrap_collection) + Goo.set_search_collection_topology(:term_search, + num_shards: @settings.term_search_num_shards, + replication_factor: @settings.term_search_replication_factor) + Goo.set_search_collection_topology(:property_search, + num_shards: @settings.property_search_num_shards, + replication_factor: @settings.property_search_replication_factor) + end + ## # Configure ontologies_linked_data namespaces # We do this at initial runtime because goo needs namespaces for its DSL diff --git a/lib/ontologies_linked_data/models/class.rb b/lib/ontologies_linked_data/models/class.rb index 874d3013..1f153367 100644 --- a/lib/ontologies_linked_data/models/class.rb +++ b/lib/ontologies_linked_data/models/class.rb @@ -182,7 +182,11 @@ def self.index_schema(schema_generator) schema_generator.add_dynamic_field('definition_*', 'text_general', indexed: true, stored: true, multi_valued: true) end - enable_indexing(:term_search_core1) do |schema_generator| + enable_indexing(:term_search, + :main, + bootstrap_collection: LinkedData.settings.term_search_bootstrap_collection, + num_shards: LinkedData.settings.term_search_num_shards, + replication_factor: LinkedData.settings.term_search_replication_factor) do |schema_generator| index_schema(schema_generator) end diff --git a/lib/ontologies_linked_data/models/properties/annotation_property.rb b/lib/ontologies_linked_data/models/properties/annotation_property.rb index 7e216717..8e4a39d8 100644 --- a/lib/ontologies_linked_data/models/properties/annotation_property.rb +++ b/lib/ontologies_linked_data/models/properties/annotation_property.rb @@ -36,7 +36,11 @@ class AnnotationProperty < LinkedData::Models::OntologyProperty LinkedData::Hypermedia::Link.new("descendants", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/descendants"}, self.uri_type), LinkedData::Hypermedia::Link.new("tree", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/tree"}, self.uri_type) - enable_indexing(:prop_search_core1, :property) do |schema_generator| + enable_indexing(:property_search, + :property, + bootstrap_collection: LinkedData.settings.property_search_bootstrap_collection, + num_shards: LinkedData.settings.property_search_num_shards, + replication_factor: LinkedData.settings.property_search_replication_factor) do |schema_generator| index_schema(schema_generator) end end diff --git a/lib/ontologies_linked_data/models/properties/datatype_property.rb b/lib/ontologies_linked_data/models/properties/datatype_property.rb index 34789dc1..325c0161 100644 --- a/lib/ontologies_linked_data/models/properties/datatype_property.rb +++ b/lib/ontologies_linked_data/models/properties/datatype_property.rb @@ -36,7 +36,11 @@ class DatatypeProperty < LinkedData::Models::OntologyProperty LinkedData::Hypermedia::Link.new("descendants", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/descendants"}, self.uri_type), LinkedData::Hypermedia::Link.new("tree", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/tree"}, self.uri_type) - enable_indexing(:prop_search_core1, :property) do |schema_generator| + enable_indexing(:property_search, + :property, + bootstrap_collection: LinkedData.settings.property_search_bootstrap_collection, + num_shards: LinkedData.settings.property_search_num_shards, + replication_factor: LinkedData.settings.property_search_replication_factor) do |schema_generator| index_schema(schema_generator) end end diff --git a/lib/ontologies_linked_data/models/properties/object_property.rb b/lib/ontologies_linked_data/models/properties/object_property.rb index 3b96216d..c2eeb4d0 100644 --- a/lib/ontologies_linked_data/models/properties/object_property.rb +++ b/lib/ontologies_linked_data/models/properties/object_property.rb @@ -36,7 +36,11 @@ class ObjectProperty < LinkedData::Models::OntologyProperty LinkedData::Hypermedia::Link.new("descendants", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/descendants"}, self.uri_type), LinkedData::Hypermedia::Link.new("tree", lambda {|m| "#{self.ontology_link(m)}/properties/#{CGI.escape(m.id.to_s)}/tree"}, self.uri_type) - enable_indexing(:prop_search_core1, :property) do |schema_generator| + enable_indexing(:property_search, + :property, + bootstrap_collection: LinkedData.settings.property_search_bootstrap_collection, + num_shards: LinkedData.settings.property_search_num_shards, + replication_factor: LinkedData.settings.property_search_replication_factor) do |schema_generator| index_schema(schema_generator) end end diff --git a/lib/ontologies_linked_data/models/properties/ontology_property.rb b/lib/ontologies_linked_data/models/properties/ontology_property.rb index 7aaa4f54..7c23b712 100644 --- a/lib/ontologies_linked_data/models/properties/ontology_property.rb +++ b/lib/ontologies_linked_data/models/properties/ontology_property.rb @@ -26,7 +26,11 @@ def self.index_schema(schema_generator) end end - enable_indexing(:prop_search_core1, :property) do |schema_generator| + enable_indexing(:property_search, + :property, + bootstrap_collection: LinkedData.settings.property_search_bootstrap_collection, + num_shards: LinkedData.settings.property_search_num_shards, + replication_factor: LinkedData.settings.property_search_replication_factor) do |schema_generator| index_schema(schema_generator) end diff --git a/lib/ontologies_linked_data/models/provisional_class.rb b/lib/ontologies_linked_data/models/provisional_class.rb index a40b936b..c4b35fc2 100644 --- a/lib/ontologies_linked_data/models/provisional_class.rb +++ b/lib/ontologies_linked_data/models/provisional_class.rb @@ -40,7 +40,11 @@ class ProvisionalClass < LinkedData::Models::Base end }, Goo.vocabulary["Ontology"]) - enable_indexing(:term_search_core1) do |schema_generator| + enable_indexing(:term_search, + :main, + bootstrap_collection: LinkedData.settings.term_search_bootstrap_collection, + num_shards: LinkedData.settings.term_search_num_shards, + replication_factor: LinkedData.settings.term_search_replication_factor) do |schema_generator| Class.index_schema(schema_generator) end diff --git a/test/test_case.rb b/test/test_case.rb index 5a93191c..dc54da99 100644 --- a/test/test_case.rb +++ b/test/test_case.rb @@ -54,6 +54,7 @@ module MinitestRunHooks def run(*args) LinkedData::TestCase.setup_test_repository_folder LinkedData::TestCase.backend_4s_delete + Goo.init_search_connections(true) super ensure LinkedData::TestCase.backend_4s_delete diff --git a/test/util/test_ontology_csv_writer.rb b/test/util/test_ontology_csv_writer.rb index b7714883..1af088ae 100644 --- a/test/util/test_ontology_csv_writer.rb +++ b/test/util/test_ontology_csv_writer.rb @@ -57,8 +57,9 @@ def enclosed_in_square_brackets_with_quotes?(string) end def get_csv_string_from_path(csv_path) - gz = Zlib::GzipReader.open(csv_path) - gz.read + Zlib::GzipReader.open(csv_path) do |gz| + gz.read.force_encoding(Encoding::UTF_8) + end end def test_csv_writer_valid @@ -337,4 +338,4 @@ def test_for_non_array_values end end -end \ No newline at end of file +end