diff --git a/lib/remote_input.rb b/lib/remote_input.rb index 122d597..e5ac0e4 100644 --- a/lib/remote_input.rb +++ b/lib/remote_input.rb @@ -1,3 +1,4 @@ +require_relative "remote_input/cache-entry" require_relative "remote_input/cache-path" require_relative "remote_input/downloader" require_relative "remote_input/zip-extractor" @@ -20,16 +21,20 @@ def open(...) def initialize(url, *fallback_urls, + cache_path: nil, encoding: nil, internal_encoding: nil, external_encoding: nil, **http_options) - @url = URI(url) @encoding = encoding @internal_encoding = internal_encoding @external_encoding = external_encoding @downloader = Downloader.new(url, *fallback_urls, **http_options) - @cache_path = nil + if cache_path + @cache_entry = CacheEntry.new(Pathname(cache_path)) + else + @cache_entry = CacheEntry.new(URI(url)) + end @local_file = nil @closed = false end @@ -44,34 +49,13 @@ def close end def clear_cache - cache_path.remove + @cache_entry.remove end private def path - cache_path.base_dir + File.basename(normalize_path) - end - - def cache_path - return @cache_path if @cache_path - dirname = File.dirname(normalize_path).delete_suffix("/") - cache_id = to_cache_id("#{@url.host}#{dirname}") - query = @url.query - cache_id += "+#{to_cache_id(query)}" if query and not query.empty? - @cache_path = CachePath.new(cache_id) - end - - def to_cache_id(s) - allow_list = "0-9A-Za-z._~=-" - s.tr("^#{allow_list}", "-") - end - - def normalize_path - url_path = @url.path - url_path = "/" if url_path.empty? - url_path += "data" if url_path.end_with?("/") - url_path + @cache_entry.path end def local_file diff --git a/lib/remote_input/cache-entry.rb b/lib/remote_input/cache-entry.rb new file mode 100644 index 0000000..8a16bf6 --- /dev/null +++ b/lib/remote_input/cache-entry.rb @@ -0,0 +1,71 @@ +require_relative "cache-path" + +class RemoteInput + # @api private + class CacheEntry + def initialize(target) + if target.is_a?(URI) + initialize_url(target) + elsif target.is_a?(Pathname) + initialize_path(target) + else + raise ArgumentError, + "target must be URI or Pathname: #{target.inspect}" + end + @cache_path = CachePath.new(@id) + end + + def path + @cache_path.base_dir + @path_in_cache + end + + def remove + @cache_path.remove + end + + def ==(other) + other.is_a?(self.class) and + @id == other.id and + @path_in_cache == other.path_in_cache + end + + protected + attr_reader :id + attr_reader :path_in_cache + + private + def initialize_url(url) + url_path = url.path + url_path = "/" if url_path.empty? + url_path += "data" if url_path.end_with?("/") + dirname = File.dirname(url_path).delete_suffix("/") + @id = to_id("#{url.host}#{dirname}") + query = url.query + @id += "+#{to_id(query)}" if query and not query.empty? + @path_in_cache = File.basename(url_path) + end + + def initialize_path(path) + if path.absolute? + raise ArgumentError, + "cache path must be relative: #{path.inspect}" + end + @id, *rest_filenames = path.each_filename.to_a + if [@id, *rest_filenames].intersect?([".", ".."]) + raise ArgumentError, + "cache path must not include '.' or '..': #{path.inspect}" + end + if rest_filenames.empty? + raise ArgumentError, + "cache path must be /: " + + path.inspect + end + @path_in_cache = File.join(*rest_filenames) + end + + def to_id(s) + allow_list = "0-9A-Za-z._~=-" + s.tr("^#{allow_list}", "-") + end + end +end diff --git a/test/test-cache-entry.rb b/test/test-cache-entry.rb new file mode 100644 index 0000000..d111fb3 --- /dev/null +++ b/test/test-cache-entry.rb @@ -0,0 +1,40 @@ +require_relative "helper" + +class CacheEntryTest < Test::Unit::TestCase + data("no path", ["example.com/data", "https://example.com"]) + data("root", ["example.com/data", "https://example.com/"]) + data("file", ["example.com/file", "https://example.com/file"]) + data("query", ["example.com+a=-/file", "https://example.com/file?a=+"]) + data("directory", ["example.com-a/data", "https://example.com/a/"]) + data("nested file", ["example.com-a/file", "https://example.com/a/file"]) + data("nested dir", ["example.com-a-b/file", "https://example.com/a/b/file"]) + def test_url_equal_path(data) + cache_path, url = data + assert_equal(RemoteInput::CacheEntry.new(Pathname(cache_path)), + RemoteInput::CacheEntry.new(URI(url))) + end + + def test_path_split + cache_path = Pathname("example/sub-directory/data.csv") + cache_entry = RemoteInput::CacheEntry.new(cache_path) + assert_equal(["example", "sub-directory/data.csv"], + [cache_entry.send(:id), cache_entry.send(:path_in_cache)]) + end + + data("absolute", "/data.csv") + data("no cache ID", "data.csv") + data("no path in cache", "example/") + data("root cache", "./data.csv") + data("path traversal", "../example/data.csv") + def test_invalid_path(cache_path) + assert_raise(ArgumentError) do + RemoteInput::CacheEntry.new(Pathname(cache_path)) + end + end + + def test_invalid_target + assert_raise(ArgumentError) do + RemoteInput::CacheEntry.new("https://example.com/file") + end + end +end diff --git a/test/test-remote-input.rb b/test/test-remote-input.rb index d115aa7..f62838b 100644 --- a/test/test-remote-input.rb +++ b/test/test-remote-input.rb @@ -58,18 +58,11 @@ def test_open_with_block_raised end end - data("no path", ["/example.com/data", "https://example.com"]) - data("root", ["/example.com/data", "https://example.com/"]) - data("file", ["/example.com/file", "https://example.com/file"]) - data("query", ["/example.com+a=-/file", "https://example.com/file?a=+"]) - data("directory", ["/example.com-a/data", "https://example.com/a/"]) - data("nested file", ["/example.com-a/file", "https://example.com/a/file"]) - data("deeply nested", ["/example.com-a-b/file", "https://example.com/a/b/file"]) - def test_path(data) - expected, url = data - RemoteInput.open(url) do |input| + def test_cache_path + RemoteInput.open("https://example.com/file", + cache_path: "example/sub-directory/data.csv") do |input| assert do - input.send(:path).to_s.end_with?(expected) + input.send(:path).to_s.end_with?("/example/sub-directory/data.csv") end end end