diff --git a/.gitignore b/.gitignore index 3c48c8f..33b78a3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ /.vs /.history .github/copilot-instructions.md +Gemfile.lock diff --git a/_plugins/copy_markdown_files.rb b/_plugins/copy_markdown_files.rb index d7abed9..bb66f99 100644 --- a/_plugins/copy_markdown_files.rb +++ b/_plugins/copy_markdown_files.rb @@ -41,6 +41,9 @@ # folder_to_index: true # /dir/ -> /dir/index.md when a twin exists # rewrite_domains: [dynamsoft.com] # hosts treated as internal docs # domain: https://www.dynamsoft.com # prefix used to absolutize links +# link_check: true # run the .md link checker after build +# orphan_check: true # report pages no other .md links to +# orphan_entries: [index.md] # entry pages ignored by orphan report # # A single page can opt out of publishing its .md twin with # `copy_markdown: false` in its front matter. @@ -56,6 +59,9 @@ module CopyMarkdownFiles "folder_to_index" => true, "rewrite_domains" => ["dynamsoft.com"], "domain" => nil, + "link_check" => true, + "orphan_check" => true, + "orphan_entries" => ["index.md"], }.freeze # Inline markdown link / image: ![alt](url "title") or [text](url). @@ -67,6 +73,11 @@ module CopyMarkdownFiles FENCE_RE = /\A(\s*)(`{3,}|~{3,})/ + # Alternation used by Rewriter#rewrite_line: a whole markdown link/image + # (whose label may itself contain backticks, e.g. [`setMaxFrames`](url)) + # wins over a backtick run, so code spans are only toggled outside links. + LINK_OR_TICK_RE = /#{INLINE_LINK_RE.source}|(`+)/ + YAML_FRONT_MATTER_RE = /\A(---\s*\r?\n)(.*?)^(---|\.\.\.)\s*\r?\n/m class Processor @@ -79,11 +90,15 @@ def initialize(site) @domain = (@cfg["domain"] || site.config["url"] || "https://www.dynamsoft.com").to_s.sub(%r{/+\z}, "") @baseurl = (site.config["baseurl"] || "").to_s @baseurl = "" if @baseurl == "/" + @baseurl = "/#{@baseurl}" unless @baseurl.empty? || @baseurl.start_with?("/") + @baseurl = @baseurl.sub(%r{/+\z}, "") @rewrite_domains = Array(@cfg["rewrite_domains"]) @page_map = {} @md_set = Set.new + @md_ci = {} # downcased rel => [published rels sharing that spelling] @html_to_md = {} @rewritten = 0 + @case_normalized = 0 @warn_counts = Hash.new(0) @warn_first = {} end @@ -99,6 +114,7 @@ def run build_indexes(rels) rels.each { |rel| process_file(rel) } + LinkChecker.new(self).run if @cfg["link_check"] log_summary(rels.size) end @@ -135,6 +151,8 @@ def build_page_map def build_indexes(rels) rels.each do |rel| @md_set << rel + @md_ci[rel.downcase] ||= [] + @md_ci[rel.downcase] << rel page = @page_map[rel] next unless page @@ -198,7 +216,11 @@ def render_liquid(body, rel, page) } template = site.liquid_renderer.file(rel).parse(body) - template.render!(payload, info) + rendered = template.render!(payload, info) + if rendered.include?("{{") || rendered.include?("{%") + warn_for(rel, "unresolved Liquid syntax remains in #{rel}") + end + rendered rescue StandardError => e warn_for(rel, "Liquid render failed for #{rel}, published raw instead: #{e.message}") body @@ -235,9 +257,29 @@ def abs_path_for_rel(rel) "#{baseurl_prefix}#{rel}" end + # Web URLs are case-insensitive on the production server, so a link may + # reference a published page with different casing than the actual file + # (e.g. .../auxiliary-DatamatrixDetails.html vs the real + # auxiliary-DataMatrixDetails.md). Return the actually published rel for + # a case-insensitive match of `rel`; nil when absent or ambiguous + # (two real files differing only in case). + def twin_md(rel) + return rel if @md_set.include?(rel) + + cands = @md_ci[rel.downcase] + cands && cands.size == 1 ? cands.first : nil + end + + def count_normalized + @case_normalized += 1 + end + def internal_host?(host) - host == URI.parse(@domain).host || @rewrite_domains.any? do |d| - host == d || host.end_with?(".#{d}") + normalized_host = host.to_s.split("@", 2).last.to_s.split(":", 2).first.downcase + domain_host = URI.parse(@domain).host.to_s.downcase + normalized_domains = @rewrite_domains.map { |d| d.to_s.downcase.sub(%r{/+\z}, "") } + normalized_host == domain_host || normalized_domains.any? do |d| + normalized_host == d || normalized_host.end_with?(".#{d}") end end @@ -274,18 +316,20 @@ def initialize(processor, rel) def rewrite(text) out = +"" - fence_char = nil + fence_marker = nil text.each_line do |line| - if fence_char + if fence_marker out << line - fence_char = nil if line =~ FENCE_RE && Regexp.last_match(2).start_with?(fence_char) + if (closing = FENCE_RE.match(line)) && closing[2][0] == fence_marker[0] && closing[2].length >= fence_marker.length + fence_marker = nil + end next end m = FENCE_RE.match(line) if m out << line - fence_char = m[2][0] + fence_marker = m[2] next end @@ -298,17 +342,51 @@ def rewrite(text) private # Rewrite links of one line while protecting inline code spans. + # Tokens are scanned left to right: a complete markdown link/image is + # handled as one unit (its label may contain backticks, e.g. + # [`method`](page.html#anchor)); backtick runs only toggle code-span + # state when they occur outside of such a link. def rewrite_line(line) out = +"" in_code = false seg_start = 0 - line.to_enum(:scan, /(`+)/).each do + line.to_enum(:scan, LINK_OR_TICK_RE).each do m = Regexp.last_match + seg = line[seg_start...m.begin(0)] out << (in_code ? seg : rewrite_plain(seg)) - out << m[0] - in_code = !in_code + + if m[1] + # Whole link/image token; rewrite its URL unless we are inside a + # code span or the URL is an anchor-only fragment. + url = m[2] || m[3] + label_code = m[1].include?("`") + if url.nil? || url.start_with?("#") + out << m[0] + elsif in_code && !label_code + # Genuinely inside a code span (`[x](u)`) - keep as-is. + out << m[0] + else + new_url = rewrite_url(url) + if new_url + @p.rewritten += 1 if new_url != url + rendered_url = m[2] ? "<#{new_url}>" : new_url + out << "#{m[1]}(#{rendered_url}#{m[4]})" + else + out << m[0] + end + # A link whose label is wrapped in backticks ([`x`](url)) + # consumed those backticks as part of the label, so a code + # span that seemed to start right before it ends here. + in_code = false if in_code && label_code + end + else + # Backtick run outside a link: toggle code-span state. + out << m[0] + in_code = !in_code + end + seg_start = m.end(0) end @@ -325,7 +403,8 @@ def rewrite_plain(seg) new_url = rewrite_url(url) if new_url @p.rewritten += 1 if new_url != url - "#{m[1]}(#{new_url}#{m[4]})" + rendered_url = m[2] ? "<#{new_url}>" : new_url + "#{m[1]}(#{rendered_url}#{m[4]})" else m[0] end @@ -338,7 +417,8 @@ def rewrite_plain(seg) url = m[2].delete_prefix("<").delete_suffix(">") new_url = url.start_with?("#") ? nil : rewrite_url(url) if new_url - "#{m[1]}#{new_url}#{m[3]}" + rendered_url = m[2].start_with?("<") ? "<#{new_url}>" : new_url + "#{m[1]}#{rendered_url}#{m[3]}" else seg end @@ -363,28 +443,47 @@ def rewrite_url(raw) origin = "https://#{host}" path = "/#{tail}" elsif raw =~ %r{\A([a-z][a-z0-9+.\-]*)://([^/]+)(/.*)?\z}i - return nil unless Regexp.last_match(1) == "http" || Regexp.last_match(1) == "https" + scheme = Regexp.last_match(1).downcase + return nil unless %w[http https].include?(scheme) return nil unless @p.internal_host?(Regexp.last_match(2)) - origin = "#{Regexp.last_match(1)}://#{Regexp.last_match(2)}" + origin = "#{scheme}://#{Regexp.last_match(2)}" path = Regexp.last_match(3) || "/" elsif raw.start_with?("/") path = raw else - path = resolve_relative(raw) + path = resolve_ref(raw) end path, query, fragment = split_url(path) + path = normalize_abs(path) handle(path, query, fragment, origin) end + # Collapse "." and ".." segments of an absolute path (/a/../b == /b). + def normalize_abs(path) + return path unless path.start_with?("/") + + parts = [] + path.split("/").each do |seg| + next if seg.empty? || seg == "." + + if seg == ".." + parts.pop unless parts.empty? + else + parts << seg + end + end + parts.empty? ? "/" : "/#{parts.join('/')}" + end + def split_url(path) path, fragment = path.split("#", 2) path, query = path.split("?", 2) [path, query, fragment] end - def resolve_relative(ref) + def resolve_ref(ref) parts = @dir_parts.dup ref.split("/").each do |seg| next if seg.empty? || seg == "." @@ -410,7 +509,7 @@ def handle(path, query, fragment, origin) elsif path.end_with?(".html", ".htm") handle_html(path, query, fragment, base) else - assemble(base, path, true, query, fragment) + handle_extensionless(path, query, fragment, base) end end @@ -418,8 +517,9 @@ def handle_folder(path, query, fragment, base) if @p.cfg["folder_to_index"] if @p.under_base?(path) rel = "#{@p.rel_under_base(path)}index.md" - if @p.md_set.include?(rel) - return assemble(base, @p.abs_path_for_rel(rel), false, nil, fragment) + if (actual = @p.twin_md(rel)) + @p.count_normalized if actual != rel + return assemble(base, @p.abs_path_for_rel(actual), false, nil, fragment) end warn("folder link has no published index.md twin: #{path}") elsif @p.docs_like?(path) @@ -430,10 +530,19 @@ def handle_folder(path, query, fragment, base) end def handle_md(path, query, fragment, base) - if @p.under_base?(path) && !@p.md_set.include?(@p.rel_under_base(path)) - warn("link target has no published .md twin: #{path}") - end strip = @p.cfg["strip_query"] + if @p.under_base?(path) + rel = @p.rel_under_base(path) + actual = @p.twin_md(rel) + if actual + if actual != rel + @p.count_normalized + return assemble(base, @p.abs_path_for_rel(actual), !strip, query, fragment) + end + else + warn("link target has no published .md twin: #{path}") + end + end assemble(base, path, !strip, query, fragment) end @@ -443,8 +552,9 @@ def handle_html(path, query, fragment, base) if same_repo rel = @p.rel_under_base(md_candidate) - if @p.md_set.include?(rel) - return assemble(base, md_candidate, false, nil, fragment) + if (actual = @p.twin_md(rel)) + @p.count_normalized if actual != rel + return assemble(base, @p.abs_path_for_rel(actual), false, nil, fragment) end if (hit = @p.html_to_md[path]) @@ -462,6 +572,40 @@ def handle_html(path, query, fragment, base) end end + # Extension-less page URLs (e.g. + # .../programming/javascript/samples-demos) point at the directory + # default page, which is rendered from /index.md. Resolve them + # to the .md twin when possible. + def handle_extensionless(path, query, fragment, base) + if @p.under_base?(path) + rel = @p.rel_under_base(path) + + if @p.cfg["folder_to_index"] + cand = "#{rel}/index.md" + if (actual = @p.twin_md(cand)) + @p.count_normalized if actual != cand + return assemble(base, @p.abs_path_for_rel(actual), false, nil, fragment) + end + end + + cand_file = "#{rel}.md" + if (actual = @p.twin_md(cand_file)) + @p.count_normalized if actual != cand_file + return assemble(base, @p.abs_path_for_rel(actual), false, nil, fragment) + end + + # No page twin; keep the URL and let the link checker classify + # (html-only page vs genuinely missing). + assemble(base, path, true, query, fragment) + elsif @p.docs_like?(path) && @p.cfg["folder_to_index"] + # Other product's docs on the same domain follow the same + # directory convention: /index.md. + assemble(base, "#{path}/index.md", false, nil, fragment) + else + assemble(base, path, true, query, fragment) + end + end + def assemble(base, path, keep_query, query, fragment) suffix = +"" suffix << "?#{query}" if keep_query && query @@ -473,6 +617,268 @@ def warn(message) @p.warn_for(@rel, message) end end + + # --------------------------------------------------------------------- + # Link checker + # --------------------------------------------------------------------- + # Walks every published .md twin after it is written and reports link + # problems between them - the Markdown equivalent of an HTML link + # checker (e.g. html-proofer): + # + # ERROR : a linked .md page was not published (no twin), or a + # same-site .html / folder / asset target does not exist in + # the generated output at all. + # WARN : a same-site link still points to an .html page - either the + # page has a .md twin (the link should point to it) or it has + # no twin at all (the AI will receive HTML instead of md). + # INFO : pages that no other published .md links to (orphans) - they + # are unreachable through the Markdown web. Entry pages listed + # in `orphan_entries` are ignored. + # + # Links to external hosts are skipped; links that leave the current + # docs root (other products on the same domain, e.g. /capture-vision/.. + # from a barcode-reader page) cannot be verified inside this repo's + # build and are counted only. Fenced code blocks / inline code and + # anchor-only (#...) links are ignored. Anchor (heading) existence is + # not checked yet. + class LinkChecker + def initialize(processor) + @p = processor + @errors = [] # [source_rel, target] + @warns = [] # [source_rel, target, detail] + @inbound = Hash.new(0) + @total = 0 + @cross = 0 + @external = 0 + end + + def run + @p.md_set.to_a.sort.each { |rel| check_file(rel) } + report + end + + private + + def check_file(rel) + dest = File.join(@p.site.dest, *rel.split("/")) + return unless File.file?(dest) + + fence_marker = nil + File.foreach(dest, encoding: "UTF-8") do |line| + if fence_marker + if (closing = FENCE_RE.match(line)) && closing[2][0] == fence_marker[0] && closing[2].length >= fence_marker.length + fence_marker = nil + end + next + end + + if (m = FENCE_RE.match(line)) + fence_marker = m[2] + next + end + + check_line(rel, line) + end + end + + def check_line(src, line) + pending = false + line.to_enum(:scan, LINK_OR_TICK_RE).each do + m = Regexp.last_match + if m[1] + target = m[2] || m[3] + if target && !target.start_with?("#") + check_url(src, target) unless pending + end + else + pending = !pending + end + end + return if pending + + ref = REF_DEF_RE.match(line) + check_url(src, ref[2].delete_prefix("<").delete_suffix(">")) if ref + end + + def check_url(src, raw) + url = raw.to_s.strip + return if url.empty? || url.start_with?("#") + return if url.start_with?("mailto:", "tel:", "javascript:", "data:") + # Leftover Liquid from a page that failed to render. + return if url.include?("{{") || url.include?("{%") + + @total += 1 + path = nil + + if url.start_with?("//") + host, _, tail = url.sub(%r{\A//}, "").partition("/") + unless @p.internal_host?(host) + @external += 1 + return + end + path = "/#{tail}" + elsif (m = %r{\A([a-z][a-z0-9+.\-]*)://([^/]+)(/.*)?\z}i.match(url)) + unless %w[http https].include?(m[1].downcase) + @external += 1 + return + end + unless @p.internal_host?(m[2]) + @external += 1 + return + end + path = m[3] || "/" + elsif url.start_with?("/") + path = url + else + path = resolve_relative(src, url) + end + + path, = split_pqf(path) + verify_target(src, path) + end + + # [path, query, fragment] + def split_pqf(path) + path, fragment = path.split("#", 2) + path, query = path.split("?", 2) + [path, query, fragment] + end + + def resolve_relative(src, ref) + parts = (@p.baseurl.split("/") + src.split("/")[0...-1]).reject(&:empty?) + ref.split("/").each do |seg| + next if seg.empty? || seg == "." + + if seg == ".." + parts.pop unless parts.empty? + else + parts << seg + end + end + parts.empty? ? "/" : "/#{parts.join('/')}" + end + + def verify_target(src, path) + unless @p.under_base?(path) + @cross += 1 + return + end + + rel = @p.rel_under_base(path) + if path.end_with?("/") || rel.empty? + verify_folder(src, rel) + elsif rel.end_with?(".md", ".markdown") + verify_md(src, rel) + elsif rel.end_with?(".html", ".htm") + verify_html(src, rel) + else + verify_other(src, rel) + end + end + + def verify_md(src, rel) + if (actual = @p.twin_md(rel)) + @inbound[actual] += 1 + return + end + + @errors << [src, @p.abs_path_for_rel(rel), "linked .md page has no published twin"] + end + + def verify_html(src, rel) + md_rel = rel.sub(/\.html?\z/, ".md") + if @p.twin_md(md_rel) + @warns << [src, @p.abs_path_for_rel(rel), ".html target has a .md twin; link should point to the twin"] + return + end + + if dest_file?(rel) + @warns << [src, @p.abs_path_for_rel(rel), ".html target has no .md twin (AI will get HTML)"] + else + @errors << [src, @p.abs_path_for_rel(rel), ".html target not found in output"] + end + end + + def verify_folder(src, rel_dir) + candidate = "#{rel_dir}index.md" + if (actual = @p.twin_md(candidate)) + @inbound[actual] += 1 + return + end + + html = "#{rel_dir}index.html" + if dest_file?(html) + @warns << [src, @p.abs_path_for_rel(html), "folder link has no .md index twin (AI will get HTML)"] + else + @errors << [src, @p.abs_path_for_rel(html), "folder link target not found in output"] + end + end + + def verify_other(src, rel) + return if dest_file?(rel) + + if dest_file?("#{rel}.html") + if @p.twin_md("#{rel}.md") + @warns << [src, @p.abs_path_for_rel(rel), "extensionless target has a .md twin; link should point to the twin"] + else + @warns << [src, @p.abs_path_for_rel(rel), "extensionless target has no .md twin (AI will get HTML)"] + end + return + end + + if dest_dir?(rel) + verify_folder(src, "#{rel}/") + return + end + + @errors << [src, @p.abs_path_for_rel(rel), "linked file not found in output"] + end + + def dest_file?(rel) + path = dest_resolve(rel) + path ? File.file?(path) : false + end + + def dest_dir?(rel) + path = dest_resolve(rel) + path ? File.directory?(path) : false + end + + def dest_resolve(rel) + full = File.join(@p.site.dest, *rel.split("/")) + return full if File.exist?(full) + + dir = File.dirname(full) + leaf = File.basename(full) + return nil unless File.directory?(dir) + + hit = Dir.children(dir).find { |c| c.casecmp?(leaf) } + hit ? File.join(dir, hit) : nil + end + + def report + @errors.each do |src, target, why| + Jekyll.logger.error("MD Link Check:", "broken link #{target} (#{why}) -- referenced from #{src}") + end + @warns.each do |src, target, why| + Jekyll.logger.warn("MD Link Check:", "#{target} (#{why}) -- referenced from #{src}") + end + + if @p.cfg["orphan_check"] + excluded = Array(@p.cfg["orphan_entries"]) + orphans = @p.md_set.to_a.reject { |rel| @inbound.key?(rel) || excluded.include?(rel) }.sort + if orphans.any? + list = orphans.size > 25 ? "#{orphans.first(25).join(', ')}, ... (#{orphans.size - 25} more)" : orphans.join(", ") + Jekyll.logger.info("MD Link Check:", "#{orphans.size} page(s) are not linked by any other published Markdown: #{list}") + end + end + + msg = +"checked #{@total} internal link(s) across #{@p.md_set.size} .md file(s): " \ + "#{@errors.size} broken, #{@warns.size} warnings, #{@external} external skipped, " \ + "#{@cross} cross-repo unverifiable" + Jekyll.logger.info("MD Link Check:", msg) + end + end end end end diff --git a/assets/js/docsLangLatestVersion.js b/assets/js/docsLangLatestVersion.js index ce2b395..1605da3 100644 --- a/assets/js/docsLangLatestVersion.js +++ b/assets/js/docsLangLatestVersion.js @@ -10,7 +10,7 @@ var docsLangLatestVersion = { ios: "11.6.2000", javascript: "11.6.3200", maui: "11.4.3000", - flutter: "11.4.3000", + flutter: "11.6.2000", reactNative: "11.6.2000", xamarin: "11.0.5200", cordova: "11.0.5200" @@ -23,7 +23,7 @@ var docsLangLatestVersion = { ios: "3.6.2000", javascript: "3.6.3200", xamarin: "1.0.5", - flutter: "3.4.3000", + flutter: "3.6.2000", cordova: "1.0.5", reactNative: "3.6.2000", maui: "3.4.3000",