From 59078381ab125570e2e01c4481f8f487b4304bfe Mon Sep 17 00:00:00 2001 From: Tsutomu Katsube Date: Tue, 1 Sep 2026 19:53:07 +0900 Subject: [PATCH 01/13] Add RemoteInput.open without cache GitHub: GH-5 This patch will introduce the new high level API for IO like object (input only object, read only object). `RemoteInput.open` downloads the data lazily on the first read. The data is stored in a system temporary directory and removed on close: ```ruby RemoteInput.open("https://example.com/sample/data.csv") do |input| input.read end ``` `RemoteInput::Input#local_path` returns a temporary file path: ```ruby RemoteInput.open("https://example.com/sample/data.csv") do |input| input.local_path # => /system temporary directory/red-remote-input/1234-567/data # 1234 is the process ID # 567 is the object ID # data is the fixed file name end ``` Without a block, `RemoteInput.open` returns a `RemoteInput::Input` object that must be closed explicitly: ```ruby input = RemoteInput.open("https://example.com/sample/data.csv") input.read input.close ``` --- lib/remote_input.rb | 18 ++++++++++- lib/remote_input/input.rb | 42 +++++++++++++++++++++++++ lib/remote_input/tmp-path.rb | 23 ++++++++++++++ test/test-input.rb | 60 ++++++++++++++++++++++++++++++++++++ test/test-remote-input.rb | 43 ++++++++++++++++++++++++++ 5 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 lib/remote_input/input.rb create mode 100644 lib/remote_input/tmp-path.rb create mode 100644 test/test-input.rb create mode 100644 test/test-remote-input.rb diff --git a/lib/remote_input.rb b/lib/remote_input.rb index 0c77230..d55321e 100644 --- a/lib/remote_input.rb +++ b/lib/remote_input.rb @@ -1,6 +1,22 @@ -require_relative "remote_input/downloader" require_relative "remote_input/cache-path" +require_relative "remote_input/downloader" +require_relative "remote_input/input" +require_relative "remote_input/tmp-path" require_relative "remote_input/zip-extractor" module RemoteInput + class << self + def open(...) + input = Input.new(...) + if block_given? + begin + yield(input) + ensure + input.close + end + else + input + end + end + end end diff --git a/lib/remote_input/input.rb b/lib/remote_input/input.rb new file mode 100644 index 0000000..3de60b3 --- /dev/null +++ b/lib/remote_input/input.rb @@ -0,0 +1,42 @@ +module RemoteInput + class Input + def initialize(url, mode="r", fallback_urls: [], **http_options) + @downloader = Downloader.new(url, *fallback_urls, **http_options) + validate_mode(mode) + @mode = mode + @tmp_path = TmpPath.new("#{Process.pid}-#{object_id}") + @local_file = nil + @closed = false + end + + def read(maxlen=nil, out_string=nil) + local_file.read(maxlen, out_string) + end + + def close + @local_file.close if @local_file and not @local_file.closed? + @tmp_path.remove + @closed = true + end + + def local_path + @tmp_path.base_dir + "data" + end + + private + + def local_file + raise IOError, "closed stream" if @closed + return @local_file if @local_file + @downloader.download(local_path) + @local_file = local_path.open(@mode) + end + + def validate_mode(mode) + unless ["r", "rb", File::RDONLY].include?(mode) + message = "mode must be \"r\", \"rb\" or File::RDONLY: #{mode.inspect}" + raise ArgumentError, message + end + end + end +end diff --git a/lib/remote_input/tmp-path.rb b/lib/remote_input/tmp-path.rb new file mode 100644 index 0000000..1055715 --- /dev/null +++ b/lib/remote_input/tmp-path.rb @@ -0,0 +1,23 @@ +require "tmpdir" + +module RemoteInput + class TmpPath + def initialize(id) + @id = id + end + + def base_dir + Pathname(system_tmp_dir).expand_path + "red-remote-input" + @id + end + + def remove + FileUtils.rmtree(base_dir.to_s, secure: true) if base_dir.exist? + end + + private + + def system_tmp_dir + Dir.tmpdir + end + end +end diff --git a/test/test-input.rb b/test/test-input.rb new file mode 100644 index 0000000..75d9700 --- /dev/null +++ b/test/test-input.rb @@ -0,0 +1,60 @@ +require_relative "helper" + +class InputTest < Test::Unit::TestCase + def test_valid_mode + input = RemoteInput::Input.new("https://example.com/file", "rb") + assert_equal("rb", input.instance_variable_get(:@mode)) + end + + def test_invalid_mode + message = "mode must be \"r\", \"rb\" or File::RDONLY: \"wb\"" + assert_raise(ArgumentError.new(message)) do + RemoteInput::Input.new("https://example.com/file", "wb") + end + end + + class WithoutCacheTest < self + def setup + @input = RemoteInput::Input.new("https://example.com/file") + begin + yield + ensure + @input.close + end + end + + def test_local_path + tmp_path = RemoteInput::TmpPath.new("#{Process.pid}-#{@input.object_id}") + assert_equal(tmp_path.base_dir + "data", @input.local_path) + end + + def test_read_once + @input.local_path.parent.mkpath + @input.local_path.write("12") + assert_equal("12", @input.read) + end + + def test_read_twice + @input.local_path.parent.mkpath + @input.local_path.write("12") + assert_equal("1", @input.read(1)) + assert_equal("2", @input.read(1)) + end + + def test_read_after_close + @input.close + assert_raise(IOError.new("closed stream")) do + @input.read + end + end + + def test_close_deletes_local_path + @input.local_path.parent.mkpath + @input.local_path.write("12") + @input.close + assert do + not @input.local_path.parent.exist? + end + end + end +end diff --git a/test/test-remote-input.rb b/test/test-remote-input.rb new file mode 100644 index 0000000..4ab1abc --- /dev/null +++ b/test/test-remote-input.rb @@ -0,0 +1,43 @@ +require_relative "helper" + +class RemoteInputTest < Test::Unit::TestCase + class WithoutCacheTest < self + def test_open_with_block + local_path = nil + RemoteInput.open("https://example.com/file") do |input| + input.local_path.parent.mkpath + input.local_path.write("12") + local_path = input.local_path + end + assert do + not local_path.parent.exist? + end + end + + def test_open_with_block_raised + local_path = nil + assert_raise(RuntimeError.new("error in block")) do + RemoteInput.open("https://example.com/file") do |input| + input.local_path.parent.mkpath + input.local_path.write("12") + local_path = input.local_path + raise "error in block" + end + end + assert do + not local_path.parent.exist? + end + end + + def test_open_without_block + input = RemoteInput.open("https://example.com/file") + begin + input.local_path.parent.mkpath + input.local_path.write("12") + assert_equal("12", input.read) + ensure + input.close + end + end + end +end From c547b94f7ff6b8ba6d80e30b0b30f0e29aec2f98 Mon Sep 17 00:00:00 2001 From: Tsutomu Katsube Date: Wed, 2 Sep 2026 21:05:39 +0900 Subject: [PATCH 02/13] Make RemoteInput a class instead of a module Because `RemoteInput::Input` is redundant. --- lib/remote_input.rb | 44 +++++++++++++++++++-- lib/remote_input/cache-path.rb | 2 +- lib/remote_input/downloader.rb | 2 +- lib/remote_input/error.rb | 2 +- lib/remote_input/input.rb | 42 -------------------- lib/remote_input/tmp-path.rb | 2 +- lib/remote_input/version.rb | 2 +- lib/remote_input/zip-extractor.rb | 2 +- test/test-input.rb | 60 ---------------------------- test/test-remote-input.rb | 65 +++++++++++++++++++++++++++---- 10 files changed, 105 insertions(+), 118 deletions(-) delete mode 100644 lib/remote_input/input.rb delete mode 100644 test/test-input.rb diff --git a/lib/remote_input.rb b/lib/remote_input.rb index d55321e..dc0c0a3 100644 --- a/lib/remote_input.rb +++ b/lib/remote_input.rb @@ -1,13 +1,12 @@ require_relative "remote_input/cache-path" require_relative "remote_input/downloader" -require_relative "remote_input/input" require_relative "remote_input/tmp-path" require_relative "remote_input/zip-extractor" -module RemoteInput +class RemoteInput class << self def open(...) - input = Input.new(...) + input = new(...) if block_given? begin yield(input) @@ -19,4 +18,43 @@ def open(...) end end end + + def initialize(url, mode="r", fallback_urls: [], **http_options) + @downloader = Downloader.new(url, *fallback_urls, **http_options) + validate_mode(mode) + @mode = mode + @tmp_path = TmpPath.new("#{Process.pid}-#{object_id}") + @local_file = nil + @closed = false + end + + def read(maxlen=nil, out_string=nil) + local_file.read(maxlen, out_string) + end + + def close + @local_file.close if @local_file and not @local_file.closed? + @tmp_path.remove + @closed = true + end + + def local_path + @tmp_path.base_dir + "data" + end + + private + + def local_file + raise IOError, "closed stream" if @closed + return @local_file if @local_file + @downloader.download(local_path) + @local_file = local_path.open(@mode) + end + + def validate_mode(mode) + unless ["r", "rb", File::RDONLY].include?(mode) + message = "mode must be \"r\", \"rb\" or File::RDONLY: #{mode.inspect}" + raise ArgumentError, message + end + end end diff --git a/lib/remote_input/cache-path.rb b/lib/remote_input/cache-path.rb index c21a4c3..43a5bdd 100644 --- a/lib/remote_input/cache-path.rb +++ b/lib/remote_input/cache-path.rb @@ -1,4 +1,4 @@ -module RemoteInput +class RemoteInput class CachePath def initialize(id) @id = id diff --git a/lib/remote_input/downloader.rb b/lib/remote_input/downloader.rb index 3056779..59f164a 100644 --- a/lib/remote_input/downloader.rb +++ b/lib/remote_input/downloader.rb @@ -9,7 +9,7 @@ require_relative "error" require_relative "version" -module RemoteInput +class RemoteInput class Downloader class TooManyRedirects < Error; end diff --git a/lib/remote_input/error.rb b/lib/remote_input/error.rb index da653da..68b859c 100644 --- a/lib/remote_input/error.rb +++ b/lib/remote_input/error.rb @@ -1,4 +1,4 @@ -module RemoteInput +class RemoteInput class Error < StandardError end end diff --git a/lib/remote_input/input.rb b/lib/remote_input/input.rb deleted file mode 100644 index 3de60b3..0000000 --- a/lib/remote_input/input.rb +++ /dev/null @@ -1,42 +0,0 @@ -module RemoteInput - class Input - def initialize(url, mode="r", fallback_urls: [], **http_options) - @downloader = Downloader.new(url, *fallback_urls, **http_options) - validate_mode(mode) - @mode = mode - @tmp_path = TmpPath.new("#{Process.pid}-#{object_id}") - @local_file = nil - @closed = false - end - - def read(maxlen=nil, out_string=nil) - local_file.read(maxlen, out_string) - end - - def close - @local_file.close if @local_file and not @local_file.closed? - @tmp_path.remove - @closed = true - end - - def local_path - @tmp_path.base_dir + "data" - end - - private - - def local_file - raise IOError, "closed stream" if @closed - return @local_file if @local_file - @downloader.download(local_path) - @local_file = local_path.open(@mode) - end - - def validate_mode(mode) - unless ["r", "rb", File::RDONLY].include?(mode) - message = "mode must be \"r\", \"rb\" or File::RDONLY: #{mode.inspect}" - raise ArgumentError, message - end - end - end -end diff --git a/lib/remote_input/tmp-path.rb b/lib/remote_input/tmp-path.rb index 1055715..cdd5775 100644 --- a/lib/remote_input/tmp-path.rb +++ b/lib/remote_input/tmp-path.rb @@ -1,6 +1,6 @@ require "tmpdir" -module RemoteInput +class RemoteInput class TmpPath def initialize(id) @id = id diff --git a/lib/remote_input/version.rb b/lib/remote_input/version.rb index d25a8ed..23f7807 100644 --- a/lib/remote_input/version.rb +++ b/lib/remote_input/version.rb @@ -1,3 +1,3 @@ -module RemoteInput +class RemoteInput VERSION = "0.0.0" end diff --git a/lib/remote_input/zip-extractor.rb b/lib/remote_input/zip-extractor.rb index 3a98332..44b9f3f 100644 --- a/lib/remote_input/zip-extractor.rb +++ b/lib/remote_input/zip-extractor.rb @@ -1,6 +1,6 @@ require 'zip' -module RemoteInput +class RemoteInput class ZipExtractor def initialize(path) @path = path diff --git a/test/test-input.rb b/test/test-input.rb deleted file mode 100644 index 75d9700..0000000 --- a/test/test-input.rb +++ /dev/null @@ -1,60 +0,0 @@ -require_relative "helper" - -class InputTest < Test::Unit::TestCase - def test_valid_mode - input = RemoteInput::Input.new("https://example.com/file", "rb") - assert_equal("rb", input.instance_variable_get(:@mode)) - end - - def test_invalid_mode - message = "mode must be \"r\", \"rb\" or File::RDONLY: \"wb\"" - assert_raise(ArgumentError.new(message)) do - RemoteInput::Input.new("https://example.com/file", "wb") - end - end - - class WithoutCacheTest < self - def setup - @input = RemoteInput::Input.new("https://example.com/file") - begin - yield - ensure - @input.close - end - end - - def test_local_path - tmp_path = RemoteInput::TmpPath.new("#{Process.pid}-#{@input.object_id}") - assert_equal(tmp_path.base_dir + "data", @input.local_path) - end - - def test_read_once - @input.local_path.parent.mkpath - @input.local_path.write("12") - assert_equal("12", @input.read) - end - - def test_read_twice - @input.local_path.parent.mkpath - @input.local_path.write("12") - assert_equal("1", @input.read(1)) - assert_equal("2", @input.read(1)) - end - - def test_read_after_close - @input.close - assert_raise(IOError.new("closed stream")) do - @input.read - end - end - - def test_close_deletes_local_path - @input.local_path.parent.mkpath - @input.local_path.write("12") - @input.close - assert do - not @input.local_path.parent.exist? - end - end - end -end diff --git a/test/test-remote-input.rb b/test/test-remote-input.rb index 4ab1abc..1457182 100644 --- a/test/test-remote-input.rb +++ b/test/test-remote-input.rb @@ -1,10 +1,48 @@ require_relative "helper" class RemoteInputTest < Test::Unit::TestCase + def test_valid_mode + input = RemoteInput.new("https://example.com/file", "rb") + assert_equal("rb", input.instance_variable_get(:@mode)) + end + + def test_invalid_mode + message = "mode must be \"r\", \"rb\" or File::RDONLY: \"wb\"" + assert_raise(ArgumentError.new(message)) do + RemoteInput.new("https://example.com/file", "wb") + end + end + + def test_read_once + open_input do |input| + input.local_path.parent.mkpath + input.local_path.write("12") + assert_equal("12", input.read) + end + end + + def test_read_twice + open_input do |input| + input.local_path.parent.mkpath + input.local_path.write("12") + assert_equal("1", input.read(1)) + assert_equal("2", input.read(1)) + end + end + + def test_read_after_close + open_input do |input| + input.close + assert_raise(IOError.new("closed stream")) do + input.read + end + end + end + class WithoutCacheTest < self def test_open_with_block local_path = nil - RemoteInput.open("https://example.com/file") do |input| + open_input do |input| input.local_path.parent.mkpath input.local_path.write("12") local_path = input.local_path @@ -17,7 +55,7 @@ def test_open_with_block def test_open_with_block_raised local_path = nil assert_raise(RuntimeError.new("error in block")) do - RemoteInput.open("https://example.com/file") do |input| + open_input do |input| input.local_path.parent.mkpath input.local_path.write("12") local_path = input.local_path @@ -29,15 +67,28 @@ def test_open_with_block_raised end end - def test_open_without_block - input = RemoteInput.open("https://example.com/file") - begin + def test_local_path + open_input do |input| + tmp_path = RemoteInput::TmpPath.new("#{Process.pid}-#{input.object_id}") + assert_equal(tmp_path.base_dir + "data", input.local_path) + end + end + + def test_close_deletes_local_path + open_input do |input| input.local_path.parent.mkpath input.local_path.write("12") - assert_equal("12", input.read) - ensure input.close + assert do + not input.local_path.parent.exist? + end end end end + + private + + def open_input(&block) + RemoteInput.open("https://example.com/file", &block) + end end From 2d2a9d295ab299ecd28e505b56d9aed72dc41033 Mon Sep 17 00:00:00 2001 From: Tsutomu Katsube Date: Tue, 22 Sep 2026 06:56:19 +0900 Subject: [PATCH 03/13] Remove mode Because we only accept read only mode. Co-authored-by: Sutou Kouhei --- lib/remote_input.rb | 13 ++----------- test/test-remote-input.rb | 12 ------------ 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/lib/remote_input.rb b/lib/remote_input.rb index dc0c0a3..e2fcf9a 100644 --- a/lib/remote_input.rb +++ b/lib/remote_input.rb @@ -19,10 +19,8 @@ def open(...) end end - def initialize(url, mode="r", fallback_urls: [], **http_options) + def initialize(url, fallback_urls: [], **http_options) @downloader = Downloader.new(url, *fallback_urls, **http_options) - validate_mode(mode) - @mode = mode @tmp_path = TmpPath.new("#{Process.pid}-#{object_id}") @local_file = nil @closed = false @@ -48,13 +46,6 @@ def local_file raise IOError, "closed stream" if @closed return @local_file if @local_file @downloader.download(local_path) - @local_file = local_path.open(@mode) - end - - def validate_mode(mode) - unless ["r", "rb", File::RDONLY].include?(mode) - message = "mode must be \"r\", \"rb\" or File::RDONLY: #{mode.inspect}" - raise ArgumentError, message - end + @local_file = local_path.open end end diff --git a/test/test-remote-input.rb b/test/test-remote-input.rb index 1457182..c80fa07 100644 --- a/test/test-remote-input.rb +++ b/test/test-remote-input.rb @@ -1,18 +1,6 @@ require_relative "helper" class RemoteInputTest < Test::Unit::TestCase - def test_valid_mode - input = RemoteInput.new("https://example.com/file", "rb") - assert_equal("rb", input.instance_variable_get(:@mode)) - end - - def test_invalid_mode - message = "mode must be \"r\", \"rb\" or File::RDONLY: \"wb\"" - assert_raise(ArgumentError.new(message)) do - RemoteInput.new("https://example.com/file", "wb") - end - end - def test_read_once open_input do |input| input.local_path.parent.mkpath From f97ed5ba15b5e3c89ad9bd4df2fa7b1b592a5103 Mon Sep 17 00:00:00 2001 From: Tsutomu Katsube Date: Tue, 22 Sep 2026 08:00:48 +0900 Subject: [PATCH 04/13] Use *fallback_urls instead of fallback_urls: Because `Downloader.new` accepts fallback URLs as positional arguments. --- lib/remote_input.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/remote_input.rb b/lib/remote_input.rb index e2fcf9a..56e8542 100644 --- a/lib/remote_input.rb +++ b/lib/remote_input.rb @@ -19,7 +19,7 @@ def open(...) end end - def initialize(url, fallback_urls: [], **http_options) + def initialize(url, *fallback_urls, **http_options) @downloader = Downloader.new(url, *fallback_urls, **http_options) @tmp_path = TmpPath.new("#{Process.pid}-#{object_id}") @local_file = nil From be1293c6b8eb6a908ed38cfbd2f217f808e2283f Mon Sep 17 00:00:00 2001 From: Tsutomu Katsube Date: Tue, 22 Sep 2026 08:01:35 +0900 Subject: [PATCH 05/13] Provide encoding options like IO.new Because `mode` was removed, we need another way to read not UTF-8 encoding such as Windows-31J. Co-authored-by: Sutou Kouhei --- lib/remote_input.rb | 23 +++++++++++++++++++++-- test/test-remote-input.rb | 12 ++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/lib/remote_input.rb b/lib/remote_input.rb index 56e8542..f494bea 100644 --- a/lib/remote_input.rb +++ b/lib/remote_input.rb @@ -19,7 +19,16 @@ def open(...) end end - def initialize(url, *fallback_urls, **http_options) + def initialize(url, + *fallback_urls, + encoding: nil, + internal_encoding: nil, + external_encoding: nil, + **http_options) + validate_encoding(encoding, internal_encoding, external_encoding) + @encoding = encoding + @internal_encoding = internal_encoding + @external_encoding = external_encoding @downloader = Downloader.new(url, *fallback_urls, **http_options) @tmp_path = TmpPath.new("#{Process.pid}-#{object_id}") @local_file = nil @@ -46,6 +55,16 @@ def local_file raise IOError, "closed stream" if @closed return @local_file if @local_file @downloader.download(local_path) - @local_file = local_path.open + options = {} + options[:encoding] = @encoding if @encoding + options[:internal_encoding] = @internal_encoding if @internal_encoding + options[:external_encoding] = @external_encoding if @external_encoding + @local_file = local_path.open(**options) + end + + def validate_encoding(encoding, internal_encoding, external_encoding) + encoding.to_s.split(":") {|enc| Encoding.find(enc)} if encoding + Encoding.find(internal_encoding) if internal_encoding + Encoding.find(external_encoding) if external_encoding end end diff --git a/test/test-remote-input.rb b/test/test-remote-input.rb index c80fa07..a13188e 100644 --- a/test/test-remote-input.rb +++ b/test/test-remote-input.rb @@ -18,6 +18,14 @@ def test_read_twice end end + def test_read_with_encoding + open_input(encoding: "Windows-31J:UTF-8") do |input| + input.local_path.parent.mkpath + input.local_path.write("入力", encoding: "Windows-31J:UTF-8") + assert_equal("入力", input.read) + end + end + def test_read_after_close open_input do |input| input.close @@ -76,7 +84,7 @@ def test_close_deletes_local_path private - def open_input(&block) - RemoteInput.open("https://example.com/file", &block) + def open_input(**options, &block) + RemoteInput.open("https://example.com/file", **options, &block) end end From 0f23e1f8ebc668e3cc72df9b3bca9b9f63327783 Mon Sep 17 00:00:00 2001 From: Tsutomu Katsube Date: Tue, 22 Sep 2026 08:49:50 +0900 Subject: [PATCH 06/13] Use the same cache ID for the same URL Because we want to avoid re-download for the same URL. So, we don't remove cache file on close. Co-authored-by: Sutou Kouhei --- lib/remote_input.rb | 21 +++++++++--- lib/remote_input/tmp-path.rb | 23 ------------- test/test-remote-input.rb | 64 ++++++++++++++++-------------------- 3 files changed, 45 insertions(+), 63 deletions(-) delete mode 100644 lib/remote_input/tmp-path.rb diff --git a/lib/remote_input.rb b/lib/remote_input.rb index f494bea..14d8e67 100644 --- a/lib/remote_input.rb +++ b/lib/remote_input.rb @@ -1,6 +1,5 @@ require_relative "remote_input/cache-path" require_relative "remote_input/downloader" -require_relative "remote_input/tmp-path" require_relative "remote_input/zip-extractor" class RemoteInput @@ -26,11 +25,12 @@ def initialize(url, external_encoding: nil, **http_options) validate_encoding(encoding, internal_encoding, external_encoding) + @url = URI(url) @encoding = encoding @internal_encoding = internal_encoding @external_encoding = external_encoding @downloader = Downloader.new(url, *fallback_urls, **http_options) - @tmp_path = TmpPath.new("#{Process.pid}-#{object_id}") + @cache_path = nil @local_file = nil @closed = false end @@ -41,16 +41,29 @@ def read(maxlen=nil, out_string=nil) def close @local_file.close if @local_file and not @local_file.closed? - @tmp_path.remove @closed = true end def local_path - @tmp_path.base_dir + "data" + cache_path.base_dir + File.basename(normalize_path) end private + def cache_path + return @cache_path if @cache_path + dirname = File.dirname(normalize_path).delete_suffix("/") + cache_id = "#{@url.host}#{dirname}".tr("/", "-") + @cache_path = CachePath.new(cache_id) + end + + def normalize_path + path = @url.path + path = "/" if path.empty? + path += "data" if path.end_with?("/") + path + end + def local_file raise IOError, "closed stream" if @closed return @local_file if @local_file diff --git a/lib/remote_input/tmp-path.rb b/lib/remote_input/tmp-path.rb deleted file mode 100644 index cdd5775..0000000 --- a/lib/remote_input/tmp-path.rb +++ /dev/null @@ -1,23 +0,0 @@ -require "tmpdir" - -class RemoteInput - class TmpPath - def initialize(id) - @id = id - end - - def base_dir - Pathname(system_tmp_dir).expand_path + "red-remote-input" + @id - end - - def remove - FileUtils.rmtree(base_dir.to_s, secure: true) if base_dir.exist? - end - - private - - def system_tmp_dir - Dir.tmpdir - end - end -end diff --git a/test/test-remote-input.rb b/test/test-remote-input.rb index a13188e..829fcaf 100644 --- a/test/test-remote-input.rb +++ b/test/test-remote-input.rb @@ -35,49 +35,41 @@ def test_read_after_close end end - class WithoutCacheTest < self - def test_open_with_block - local_path = nil - open_input do |input| - input.local_path.parent.mkpath - input.local_path.write("12") - local_path = input.local_path - end - assert do - not local_path.parent.exist? - end + def test_open_with_block + opened_input = nil + open_input do |input| + opened_input = input end - - def test_open_with_block_raised - local_path = nil - assert_raise(RuntimeError.new("error in block")) do - open_input do |input| - input.local_path.parent.mkpath - input.local_path.write("12") - local_path = input.local_path - raise "error in block" - end - end - assert do - not local_path.parent.exist? - end + assert_raise(IOError.new("closed stream")) do + opened_input.read end + end - def test_local_path + def test_open_with_block_raised + opened_input = nil + assert_raise(RuntimeError.new("error in block")) do open_input do |input| - tmp_path = RemoteInput::TmpPath.new("#{Process.pid}-#{input.object_id}") - assert_equal(tmp_path.base_dir + "data", input.local_path) + opened_input = input + raise "error in block" end end + assert_raise(IOError.new("closed stream")) do + opened_input.read + end + end - def test_close_deletes_local_path - open_input do |input| - input.local_path.parent.mkpath - input.local_path.write("12") - input.close - assert do - not input.local_path.parent.exist? - end + data("no path", ["/example.com/data", "https://example.com"]) + data("root", ["/example.com/data", "https://example.com/"]) + data("file", ["/example.com/file", "https://example.com/file"]) + data("query", ["/example.com/file", "https://example.com/file?a=b"]) + data("directory", ["/example.com-a/data", "https://example.com/a/"]) + data("nested file", ["/example.com-a/file", "https://example.com/a/file"]) + data("deeply nested", ["/example.com-a-b/file", "https://example.com/a/b/file"]) + def test_local_path(data) + expected, url = data + RemoteInput.open(url) do |input| + assert do + input.local_path.to_s.end_with?(expected) end end end From 3d1c22a8db7d3e4fd7adf3a3a94489276c592a18 Mon Sep 17 00:00:00 2001 From: Tsutomu Katsube Date: Wed, 23 Sep 2026 18:00:32 +0900 Subject: [PATCH 07/13] Add clear_cache Because we don't remove the cache file on close yet, we need a way to remove it explicitly. --- lib/remote_input.rb | 4 ++++ test/test-remote-input.rb | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/lib/remote_input.rb b/lib/remote_input.rb index 14d8e67..5199651 100644 --- a/lib/remote_input.rb +++ b/lib/remote_input.rb @@ -44,6 +44,10 @@ def close @closed = true end + def clear_cache + cache_path.remove + end + def local_path cache_path.base_dir + File.basename(normalize_path) end diff --git a/test/test-remote-input.rb b/test/test-remote-input.rb index 829fcaf..9662b02 100644 --- a/test/test-remote-input.rb +++ b/test/test-remote-input.rb @@ -74,6 +74,17 @@ def test_local_path(data) end end + def test_clear_cache + open_input do |input| + input.local_path.parent.mkpath + input.local_path.write("1") + input.clear_cache + assert do + not input.local_path.parent.exist? + end + end + end + private def open_input(**options, &block) From 1a9ecd656f532857f1aa9378fb8ae77cb91abfb1 Mon Sep 17 00:00:00 2001 From: Tsutomu Katsube Date: Wed, 23 Sep 2026 20:02:52 +0900 Subject: [PATCH 08/13] test: clear garbage cache --- test/test-remote-input.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/test-remote-input.rb b/test/test-remote-input.rb index 9662b02..d5359f5 100644 --- a/test/test-remote-input.rb +++ b/test/test-remote-input.rb @@ -87,7 +87,13 @@ def test_clear_cache private - def open_input(**options, &block) - RemoteInput.open("https://example.com/file", **options, &block) + def open_input(**options) + input = RemoteInput.open("https://example.com/file", **options) + begin + yield(input) + ensure + input.close + input.clear_cache + end end end From 4d2ee34bedd2e83d440483e45e6faef90abc63fa Mon Sep 17 00:00:00 2001 From: Tsutomu Katsube Date: Fri, 25 Sep 2026 20:34:56 +0900 Subject: [PATCH 09/13] Remove validate_encoding Because we don't want to implement encoding validation in our side. Co-authored-by: Sutou Kouhei --- lib/remote_input.rb | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lib/remote_input.rb b/lib/remote_input.rb index 5199651..3c15983 100644 --- a/lib/remote_input.rb +++ b/lib/remote_input.rb @@ -24,7 +24,6 @@ def initialize(url, internal_encoding: nil, external_encoding: nil, **http_options) - validate_encoding(encoding, internal_encoding, external_encoding) @url = URI(url) @encoding = encoding @internal_encoding = internal_encoding @@ -78,10 +77,4 @@ def local_file options[:external_encoding] = @external_encoding if @external_encoding @local_file = local_path.open(**options) end - - def validate_encoding(encoding, internal_encoding, external_encoding) - encoding.to_s.split(":") {|enc| Encoding.find(enc)} if encoding - Encoding.find(internal_encoding) if internal_encoding - Encoding.find(external_encoding) if external_encoding - end end From 66b48a5c82ad04319b238f60ecd5e29cbb5c6ef4 Mon Sep 17 00:00:00 2001 From: Tsutomu Katsube Date: Sat, 26 Sep 2026 07:11:42 +0900 Subject: [PATCH 10/13] Make local_path private API Because it returns nonexistent path when `read` isn't called yet. Co-authored-by: Sutou Kouhei --- lib/remote_input.rb | 4 ++-- test/test-remote-input.rb | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/lib/remote_input.rb b/lib/remote_input.rb index 3c15983..5b4f21a 100644 --- a/lib/remote_input.rb +++ b/lib/remote_input.rb @@ -47,12 +47,12 @@ def clear_cache cache_path.remove end + private + def local_path cache_path.base_dir + File.basename(normalize_path) end - private - def cache_path return @cache_path if @cache_path dirname = File.dirname(normalize_path).delete_suffix("/") diff --git a/test/test-remote-input.rb b/test/test-remote-input.rb index d5359f5..5cb9acb 100644 --- a/test/test-remote-input.rb +++ b/test/test-remote-input.rb @@ -3,16 +3,16 @@ class RemoteInputTest < Test::Unit::TestCase def test_read_once open_input do |input| - input.local_path.parent.mkpath - input.local_path.write("12") + input.send(:local_path).parent.mkpath + input.send(:local_path).write("12") assert_equal("12", input.read) end end def test_read_twice open_input do |input| - input.local_path.parent.mkpath - input.local_path.write("12") + input.send(:local_path).parent.mkpath + input.send(:local_path).write("12") assert_equal("1", input.read(1)) assert_equal("2", input.read(1)) end @@ -20,8 +20,8 @@ def test_read_twice def test_read_with_encoding open_input(encoding: "Windows-31J:UTF-8") do |input| - input.local_path.parent.mkpath - input.local_path.write("入力", encoding: "Windows-31J:UTF-8") + input.send(:local_path).parent.mkpath + input.send(:local_path).write("入力", encoding: "Windows-31J:UTF-8") assert_equal("入力", input.read) end end @@ -69,18 +69,18 @@ def test_local_path(data) expected, url = data RemoteInput.open(url) do |input| assert do - input.local_path.to_s.end_with?(expected) + input.send(:local_path).to_s.end_with?(expected) end end end def test_clear_cache open_input do |input| - input.local_path.parent.mkpath - input.local_path.write("1") + input.send(:local_path).parent.mkpath + input.send(:local_path).write("1") input.clear_cache assert do - not input.local_path.parent.exist? + not input.send(:local_path).parent.exist? end end end From aca2d489ff1c5e9487879f422fb5d208e597e307 Mon Sep 17 00:00:00 2001 From: Tsutomu Katsube Date: Sat, 26 Sep 2026 07:16:09 +0900 Subject: [PATCH 11/13] Use path instead of local_path Because `IO#path` exists. Co-authored-by: Sutou Kouhei --- lib/remote_input.rb | 6 +++--- test/test-remote-input.rb | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/remote_input.rb b/lib/remote_input.rb index 5b4f21a..324c2b7 100644 --- a/lib/remote_input.rb +++ b/lib/remote_input.rb @@ -49,7 +49,7 @@ def clear_cache private - def local_path + def path cache_path.base_dir + File.basename(normalize_path) end @@ -70,11 +70,11 @@ def normalize_path def local_file raise IOError, "closed stream" if @closed return @local_file if @local_file - @downloader.download(local_path) + @downloader.download(path) options = {} options[:encoding] = @encoding if @encoding options[:internal_encoding] = @internal_encoding if @internal_encoding options[:external_encoding] = @external_encoding if @external_encoding - @local_file = local_path.open(**options) + @local_file = path.open(**options) end end diff --git a/test/test-remote-input.rb b/test/test-remote-input.rb index 5cb9acb..65726fb 100644 --- a/test/test-remote-input.rb +++ b/test/test-remote-input.rb @@ -3,16 +3,16 @@ class RemoteInputTest < Test::Unit::TestCase def test_read_once open_input do |input| - input.send(:local_path).parent.mkpath - input.send(:local_path).write("12") + input.send(:path).parent.mkpath + input.send(:path).write("12") assert_equal("12", input.read) end end def test_read_twice open_input do |input| - input.send(:local_path).parent.mkpath - input.send(:local_path).write("12") + input.send(:path).parent.mkpath + input.send(:path).write("12") assert_equal("1", input.read(1)) assert_equal("2", input.read(1)) end @@ -20,8 +20,8 @@ def test_read_twice def test_read_with_encoding open_input(encoding: "Windows-31J:UTF-8") do |input| - input.send(:local_path).parent.mkpath - input.send(:local_path).write("入力", encoding: "Windows-31J:UTF-8") + input.send(:path).parent.mkpath + input.send(:path).write("入力", encoding: "Windows-31J:UTF-8") assert_equal("入力", input.read) end end @@ -69,18 +69,18 @@ def test_local_path(data) expected, url = data RemoteInput.open(url) do |input| assert do - input.send(:local_path).to_s.end_with?(expected) + input.send(:path).to_s.end_with?(expected) end end end def test_clear_cache open_input do |input| - input.send(:local_path).parent.mkpath - input.send(:local_path).write("1") + input.send(:path).parent.mkpath + input.send(:path).write("1") input.clear_cache assert do - not input.send(:local_path).parent.exist? + not input.send(:path).parent.exist? end end end From a6e5e42143b75dc248bcebd938bbdec5dee7b09d Mon Sep 17 00:00:00 2001 From: Tsutomu Katsube Date: Sat, 26 Sep 2026 07:49:27 +0900 Subject: [PATCH 12/13] Fix missing rename --- test/test-remote-input.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-remote-input.rb b/test/test-remote-input.rb index 65726fb..0b71b3a 100644 --- a/test/test-remote-input.rb +++ b/test/test-remote-input.rb @@ -65,7 +65,7 @@ def test_open_with_block_raised data("directory", ["/example.com-a/data", "https://example.com/a/"]) data("nested file", ["/example.com-a/file", "https://example.com/a/file"]) data("deeply nested", ["/example.com-a-b/file", "https://example.com/a/b/file"]) - def test_local_path(data) + def test_path(data) expected, url = data RemoteInput.open(url) do |input| assert do From 834f2068be25632d1a77dc7ae7f56531add04c8c Mon Sep 17 00:00:00 2001 From: Tsutomu Katsube Date: Sat, 26 Sep 2026 07:51:30 +0900 Subject: [PATCH 13/13] Use url_path instead of path Because `RemoteInput#path` already exists and we may confuse them. --- lib/remote_input.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/remote_input.rb b/lib/remote_input.rb index 324c2b7..b0fe0e9 100644 --- a/lib/remote_input.rb +++ b/lib/remote_input.rb @@ -61,10 +61,10 @@ def cache_path end def normalize_path - path = @url.path - path = "/" if path.empty? - path += "data" if path.end_with?("/") - path + url_path = @url.path + url_path = "/" if url_path.empty? + url_path += "data" if url_path.end_with?("/") + url_path end def local_file