From df4cf51331da4e97035dfd59c3d674806acb5748 Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Thu, 25 Jun 2026 17:49:39 +0200 Subject: [PATCH] Use net-http-persistent for requests to Foreman This uses persistent connections for better performance. It modifies create_get and create_post in an incompatible way and this will break plugins that rely on it, but it shows the concept. It doesn't modify other uses, like for connections to Puppet and FreeIPA. --- lib/proxy/request.rb | 17 ++++++------ modules/puppet_proxy_common/api_request.rb | 1 + modules/registration/proxy_request.rb | 26 +++++++++---------- modules/templates/proxy_request.rb | 6 ++--- smart_proxy.gemspec | 1 + test/request_test.rb | 20 +++++++------- test/templates/template_proxy_request_test.rb | 2 ++ 7 files changed, 39 insertions(+), 34 deletions(-) diff --git a/lib/proxy/request.rb b/lib/proxy/request.rb index c0fd4df9b..5a59ee956 100644 --- a/lib/proxy/request.rb +++ b/lib/proxy/request.rb @@ -1,5 +1,6 @@ require 'net/http' require 'net/https' +require 'net/http/persistent' require 'uri' require 'cgi' @@ -15,9 +16,10 @@ def query_string(input = {}) def create_get(path, query = {}, headers = {}) uri = uri(path) - req = Net::HTTP::Get.new("#{uri.path || '/'}?#{query_string(query)}") + uri.query = query_string(query) + req = Net::HTTP::Get.new(uri) req = add_headers(req, headers) - req + [uri, req] end def uri(path) @@ -39,13 +41,13 @@ def create_post(path, body, headers = {}, query = {}) req = Net::HTTP::Post.new(uri) req = add_headers(req, headers) req.body = body - req + [uri, req] end end class ForemanRequest - def send_request(request) - http.request(request) + def send_request(uri, request) + http.request(uri, request) end def request_factory @@ -63,11 +65,10 @@ def http private def http_init - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = uri.scheme == 'https' + http = Net::HTTP::Persistent.new(name: 'foreman') http.verify_mode = OpenSSL::SSL::VERIFY_NONE - if http.use_ssl? + if uri.scheme == 'https' ca_file = Proxy::SETTINGS.foreman_ssl_ca || Proxy::SETTINGS.ssl_ca_file certificate = Proxy::SETTINGS.foreman_ssl_cert || Proxy::SETTINGS.ssl_certificate private_key = Proxy::SETTINGS.foreman_ssl_key || Proxy::SETTINGS.ssl_private_key diff --git a/modules/puppet_proxy_common/api_request.rb b/modules/puppet_proxy_common/api_request.rb index e5ece8a32..6c433e53f 100644 --- a/modules/puppet_proxy_common/api_request.rb +++ b/modules/puppet_proxy_common/api_request.rb @@ -49,6 +49,7 @@ def delete(path, timeout = 60, additional_headers = {}) end def create_http_context(uri, timeout) + # TODO: Net::HTTP::Persistent.new http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = uri.scheme == 'https' http.verify_mode = OpenSSL::SSL::VERIFY_NONE diff --git a/modules/registration/proxy_request.rb b/modules/registration/proxy_request.rb index 6ad9e99ee..a42042a6c 100644 --- a/modules/registration/proxy_request.rb +++ b/modules/registration/proxy_request.rb @@ -3,11 +3,11 @@ module Proxy::Registration class ProxyRequest < ::Proxy::HttpRequest::ForemanRequest def global_register(request) - proxy_req = request_factory.create_get '/register', - request_params(request), - headers(request) + uri, proxy_req = request_factory.create_get '/register', + request_params(request), + headers(request) - send_request(proxy_req) + send_request(uri, proxy_req) end # we support two way of sending data - either a JSON or url encoded data @@ -18,19 +18,19 @@ def host_register(request) # also request.params contain the same data that is in request.body, just parsed to hash, # in case they are nested (e.g. host hash) we need this causes problem during CGI escaping # therefore we only add url, everything else should be in body in this type of request - proxy_req = request_factory.create_post '/register', - request.body.read, - headers(request).merge("Content-Type" => request.content_type), - { url: register_url(request) } + uri, proxy_req = request_factory.create_post '/register', + request.body.read, + headers(request).merge("Content-Type" => request.content_type), + { url: register_url(request) } else # the application/json request body contains the data - JSON as a string, query contains only the URL - proxy_req = request_factory.create_post '/register', - request.body.read, - headers(request), - request_params(request) + uri, proxy_req = request_factory.create_post '/register', + request.body.read, + headers(request), + request_params(request) end - send_request(proxy_req) + send_request(uri, proxy_req) end private diff --git a/modules/templates/proxy_request.rb b/modules/templates/proxy_request.rb index bbb6d4ad9..7ba39115a 100644 --- a/modules/templates/proxy_request.rb +++ b/modules/templates/proxy_request.rb @@ -38,15 +38,15 @@ def call_template(method, path, env, params, body = '') proxy_headers["X-Forwarded-For"] = env['REMOTE_ADDR'] proxy_headers["Content-Type"] = params["Content-Type"] if params["Content-Type"] if method == :get - proxy_req = request_factory.create_get(path, opts, proxy_headers) + uri, proxy_req = request_factory.create_get(path, opts, proxy_headers) elsif method == :post - proxy_req = request_factory.create_post(path, body, proxy_headers, opts) + uri, proxy_req = request_factory.create_post(path, body, proxy_headers, opts) else raise "Unknown method: #{method}" end logger.debug "Retrieving a template from %s%s" % [uri, proxy_req.path] logger.debug "HTTP headers: #{proxy_headers.inspect}" - res = send_request(proxy_req) + res = send_request(uri, proxy_req) # You get a 201 from the 'built' URL raise ::Proxy::Error::HttpError.new(res.code.to_i, nil, "Error retrieving #{path} for #{opts.inspect} from #{uri.host}: #{res.class}") unless ["200", "201"].include?(res.code) res.body diff --git a/smart_proxy.gemspec b/smart_proxy.gemspec index 4dc05065d..6a6dd2825 100644 --- a/smart_proxy.gemspec +++ b/smart_proxy.gemspec @@ -16,6 +16,7 @@ Gem::Specification.new do |s| s.add_dependency 'base64' s.add_dependency 'json' s.add_dependency 'logging' + s.add_dependency 'net-http-persistent', '~> 4.0' s.add_dependency 'ostruct' s.add_dependency 'rack', '>= 1.3' s.add_dependency 'rexml', '~> 3.2' diff --git a/test/request_test.rb b/test/request_test.rb index 81d6f53b0..f7fb9fc17 100644 --- a/test/request_test.rb +++ b/test/request_test.rb @@ -18,15 +18,15 @@ def setup def test_get stub_request(:get, @foreman_url + '/path').to_return(:status => [200, 'OK'], :body => "body") - proxy_req = @request.request_factory.create_get("/path") - result = @request.send_request(proxy_req) + uri, proxy_req = @request.request_factory.create_get("/path") + result = @request.send_request(uri, proxy_req) assert_equal("body", result.body) end def test_get_with_headers stub_request(:get, @foreman_url + '/path?a=b').with(:headers => {"h1" => "header"}).to_return(:status => [200, 'OK'], :body => "body") - proxy_req = @request.request_factory.create_get("/path", {"a" => "b"}, "h1" => "header") - result = @request.send_request(proxy_req) + uri, proxy_req = @request.request_factory.create_get("/path", {"a" => "b"}, "h1" => "header") + result = @request.send_request(uri, proxy_req) assert_equal("body", result.body) end @@ -42,15 +42,15 @@ def test_get_with_nested_params {"repo" => "repo2", "repo_gpg_key_url" => "key2"}, ], "update_packages" => "false" } - proxy_req = @request.request_factory.create_get("/register", request_params, "h1" => "header") - result = @request.send_request(proxy_req) + uri, proxy_req = @request.request_factory.create_get("/register", request_params, "h1" => "header") + result = @request.send_request(uri, proxy_req) assert_equal("body", result.body) end def test_post stub_request(:post, @foreman_url + '/path').with(:body => "body").to_return(:status => [200, 'OK'], :body => "body") - proxy_req = @request.request_factory.create_post("/path", "body") - result = @request.send_request(proxy_req) + uri, proxy_req = @request.request_factory.create_post("/path", "body") + result = @request.send_request(uri, proxy_req) assert_equal("body", result.body) end @@ -66,8 +66,8 @@ def test_post_with_nested_params {"repo" => "repo2", "repo_gpg_key_url" => "key2"}, ], "update_packages" => "false" } - proxy_req = @request.request_factory.create_post "/register", {"body" => "body"}, {"h1" => "header"}, request_params - result = @request.send_request(proxy_req) + uri, proxy_req = @request.request_factory.create_post "/register", {"body" => "body"}, {"h1" => "header"}, request_params + result = @request.send_request(uri, proxy_req) assert_equal("body", result.body) end end diff --git a/test/templates/template_proxy_request_test.rb b/test/templates/template_proxy_request_test.rb index a942f34e9..5a418d574 100644 --- a/test/templates/template_proxy_request_test.rb +++ b/test/templates/template_proxy_request_test.rb @@ -52,7 +52,9 @@ def test_template_requests_via_post headers: { 'Accept' => ['*/*', 'application/json,version=2'], 'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', + 'Connection' => 'keep-alive', 'Content-Type' => 'application/json', + 'Keep-Alive' => '30', 'User-Agent' => 'Ruby', 'X-Forwarded-For' => '1.2.3.4', }).