Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions lib/docs/filters/node/clean_html.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@ module Docs
class Node
class CleanHtmlFilter < Filter
def call
unless at_css('h1')
at_css('h2').name = 'h1'
end

css('hr').remove

css('pre').each do |node|
Expand Down
2 changes: 1 addition & 1 deletion lib/docs/filters/node/entries.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ def get_name
end

def get_type
at_css('h2').content.strip
at_css('h1, h2').content.strip
end

def additional_entries
Expand Down
2 changes: 1 addition & 1 deletion lib/docs/scrapers/nginx.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ module Docs
class Nginx < UrlScraper
self.name = 'nginx'
self.type = 'nginx'
self.release = '1.30.0'
self.release = '1.31.1'
self.base_url = 'https://nginx.org/en/docs/'
self.links = {
home: 'https://nginx.org/',
Expand Down
2 changes: 1 addition & 1 deletion lib/docs/scrapers/node.rb
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ class Node < UrlScraper
HTML

version do
self.release = '25.8.0'
self.release = '26.2.0'
self.base_url = 'https://nodejs.org/api/'
end

Expand Down
28 changes: 22 additions & 6 deletions lib/tasks/docs.thor
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,7 @@ class DocsCLI < Thor
puts 'Docs -- BEGIN'

require 'open-uri'
require 'net/http'
require 'thread'

docs = Docs.all_versions
Expand All @@ -250,15 +251,30 @@ class DocsCLI < Thor
['index.json', 'meta.json'].each do |filename|
json = "https://documents.devdocs.io/#{doc.path}/#{filename}?#{time}"
begin
URI.open(json, "Accept-Encoding" => "identity") do |file|
mutex.synchronize do
path = File.join(dir, filename)
File.write(path, file.read)
attempts = 0

begin
attempts += 1

URI.open(json, "Accept-Encoding" => "identity") do |file|
mutex.synchronize do
path = File.join(dir, filename)
File.write(path, file.read)
end
end
rescue Net::OpenTimeout, Net::ReadTimeout => e
if attempts <= 3
wait_seconds = 2**(attempts - 1)
puts "Docs -- Retrying #{json} in #{wait_seconds}s (#{e.class}: #{e.message})"
sleep(wait_seconds)
retry
end

raise
end
rescue => e
puts "Docs -- Failed to download #{json}!"
throw e
puts "Docs -- Failed to download #{json} after #{attempts} attempts!"
raise
end
end

Expand Down
Loading