Report skipped requests and responses in coverage report

CHANGELOG.md

@@ -20,6 +20,7 @@ No stricter / less strict request validation. Mostly internal stuff. Plus a Sina
 - Removed: `OpenapiFirst::Test::Coverage::TerminalReporter#format`, deprecated since 3.4.0. Use `#report` instead.
 
 #### Added
+- API coverage now reports skipped requests and responses.
 - Added: OpenAPI 3.2 documents are accepted, but not fully supported yet. They are handled using the OpenAPI 3.1 rules, so features introduced in 3.2 may be ignored. Loading such a document prints a warning. Operations defined under `additionalOperations` are routed. See #469.
 - Added: `OpenapiFirst::Request#parameters` returns the parameters that are defined for a request as `OpenapiFirst::Parameter` objects, which expose `name`, `location`, `schema`, `required?`, `deprecated?`, `style`, `explode?` and `media_type`. It used to return an internal object with a different interface.
 - Added: Show all covered endpoints in HTML coverage reporter and filter covered/uncovered endpoints

Gemfile.lock

@@ -273,6 +273,7 @@ GEM
     zeitwerk (2.8.2)
 
 PLATFORMS
+  arm64-darwin-23
   arm64-darwin-24
   arm64-darwin-25
   x86_64-linux

lib/openapi_first/test/coverage.rb

@@ -4,6 +4,7 @@
 require_relative 'coverage/tracker'
 require_relative 'coverage/covered_request'
 require_relative 'coverage/covered_response'
+require_relative 'coverage/skipped_summary'
 require 'drb'
 
 module OpenapiFirst
@@ -15,7 +16,15 @@ module Coverage
       autoload :TerminalReporter, 'openapi_first/test/coverage/terminal_reporter'
       autoload :HtmlReporter, 'openapi_first/test/coverage/html_reporter'
 
-      Result = Data.define(:plans, :coverage)
+      Result = Data.define(:plans, :coverage) do
+        def skipped_requests_count
+          plans.sum(&:skipped_requests_count)
+        end
+
+        def skipped_responses_count
+          plans.sum(&:skipped_responses_count)
+        end
+      end
 
       class << self
         def start(skip_response: nil, skip_route: nil)

lib/openapi_first/test/coverage/html_reporter.css

@@ -28,6 +28,7 @@
   --color-error: #f87171;
   --color-info: #7dd3fc;
   --color-warning: #fbbf24;
+  --color-skipped: #fb923c;
 }
 
 body {
@@ -112,17 +113,36 @@ body:has(#filter-covered:not(:checked)) details.route.is-covered {
 body:has(#filter-uncovered:not(:checked)) details.route.is-uncovered {
   display: none;
 }
+body:has(#filter-skipped-requests:not(:checked)) details.route.is-skipped {
+  display: none;
+}
 
 /* Hide a plan once none of its routes remain visible. */
 body:has(#filter-covered:not(:checked))
-  details.plan:not(:has(details.route.is-uncovered)) {
-  display: none;
-}
+  details.plan:not(:has(details.route.is-uncovered)):not(
+    :has(details.route.is-skipped)
+  ),
 body:has(#filter-uncovered:not(:checked))
-  details.plan:not(:has(details.route.is-covered)) {
-  display: none;
-}
+  details.plan:not(:has(details.route.is-covered)):not(
+    :has(details.route.is-skipped)
+  ),
+body:has(#filter-skipped-requests:not(:checked))
+  details.plan:not(:has(details.route.is-covered)):not(
+    :has(details.route.is-uncovered)
+  ),
 body:has(#filter-covered:not(:checked)):has(#filter-uncovered:not(:checked))
+  details.plan:not(:has(details.route.is-skipped)),
+body:has(#filter-covered:not(:checked)):has(
+    #filter-skipped-requests:not(:checked)
+  )
+  details.plan:not(:has(details.route.is-uncovered)),
+body:has(#filter-uncovered:not(:checked)):has(
+    #filter-skipped-requests:not(:checked)
+  )
+  details.plan:not(:has(details.route.is-covered)),
+body:has(#filter-covered:not(:checked)):has(
+    #filter-uncovered:not(:checked)
+  ):has(#filter-skipped-requests:not(:checked))
   details.plan {
   display: none;
 }
@@ -261,6 +281,21 @@ li {
   color: var(--color-success);
 }
 
+.skipped {
+  color: var(--color-skipped);
+}
+
+.skipped-note,
+.skipped-summary {
+  color: var(--color-skipped);
+  font-size: var(--font-size-xs);
+}
+
+.skipped-summary {
+  margin-inline-start: 0.75rem;
+  white-space: nowrap;
+}
+
 .status {
   color: var(--text-primary);
   min-width: 3ch;

lib/openapi_first/test/coverage/html_reporter.html.erb

@@ -10,9 +10,13 @@
     <h1 class="page-title">API Coverage: <%= coverage.round(1) %>%</h1>
     <div>
         <span class="generated-at">Generated at: <%= h generated_at %></span>
+        <% if any_skipped? -%>
+        <span class="skipped-summary"><%= skipped_requests_count %> request(s) and <%= skipped_responses_count %> response(s) skipped</span>
+        <% end -%>
         <div class="filters">
             <label class="filter"><input type="checkbox" id="filter-covered" checked> Covered operations</label>
             <label class="filter"><input type="checkbox" id="filter-uncovered" checked> Uncovered operations</label>
+            <label class="filter"><input type="checkbox" id="filter-skipped-requests" checked> Skipped requests</label>
         </div>
     </div>
   </header>
@@ -29,18 +33,23 @@
     <% visible_routes(plan).each do |route| -%>
     <% requests = request_items(route) -%>
     <% responses = response_items(route) -%>
-    <details class="route <%= route.finished? ? 'is-covered' : 'is-uncovered' %>"<%= ' open' if any_request_made?(route) && !route.finished? %>>
+    <details class="route <%= route_class(route) %>"<%= ' open' if any_request_made?(route) && !route.finished? %>>
       <summary>
         <span class="method"><%= h route.request_method.upcase %></span>
         <span class="path"><%= h route.path %></span>
         <% status = route_status(route) -%>
-        <% if status == :request_problem -%>
-        <span class="request-status problem">❌ <span class="problem"><%= h explain_unfinished_request(route.requests.first) %></span></span>
+        <% if status == :skipped -%>
+        <span class="request-status skipped">Skipped</span>
+        <% elsif status == :request_problem -%>
+        <span class="request-status problem">❌ <span class="problem"><%= h explain_unfinished_request(route.tracked_requests.first) %></span></span>
         <% elsif status == :responses_problem -%>
         <span class="response-summary problem">⚠️ <%= uncovered_responses_count(route) %> response(s) not covered</span>
         <% else -%>
         <span class="request-status covered">✅</span>
         <% end -%>
+        <% if route.skipped_responses.any? && status != :skipped -%>
+        <span class="response-summary skipped"><%= route.skipped_responses.count %> response(s) skipped</span>
+        <% end -%>
         <% if (label = route.summary) -%>
         <span class="operation-label"><%= h label %></span>
         <% end -%>
@@ -49,7 +58,12 @@
         <% unless requests.empty? -%>
         <ul class="tasks">
           <% requests.each do |request| -%>
-          <% if request.finished? -%>
+          <% if request.skipped? -%>
+          <li class="skipped">
+            <% if request.content_type -%><span class="content-type"><%= h request.content_type %></span><% end -%>
+            <span class="skipped-note">Skipped, not covered by any test</span>
+          </li>
+          <% elsif request.finished? -%>
           <li class="covered">
             <span>✅</span>
             <% if request.content_type -%><span class="content-type"><%= h request.content_type %></span><% end -%>
@@ -67,7 +81,13 @@
         <% unless responses.empty? -%>
         <ul class="tasks responses">
           <% responses.each do |response| -%>
-          <% if response.finished? -%>
+          <% if response.skipped? -%>
+          <li class="skipped">
+            <span class="status"><%= h response.status %></span>
+            <% if response.content_type -%><span class="content-type"><%= h response.content_type %></span><% end -%>
+            <span class="skipped-note">Skipped</span>
+          </li>
+          <% elsif response.finished? -%>
           <li class="covered">
             <span>✅</span>
             <span class="status"><%= h response.status %></span>

lib/openapi_first/test/coverage/html_reporter.rb

@@ -8,6 +8,8 @@ module Test
     module Coverage
       # Writes a self-contained HTML coverage report to a file.
       class HtmlReporter
+        include SkippedSummary
+
         def initialize(output: 'coverage/openapi_coverage.html', verbose: false, logger: Test.logger)
           @output = output
           @verbose = verbose
@@ -21,11 +23,14 @@ def report(coverage_result)
           coverage_result.plans.each do |plan|
             next if plan.coverage >= 100
 
-            @logger.info "API validation coverage for #{plan.api_identifier}: #{plan.coverage.round(4)}%"
+            logger.info "API validation coverage for #{plan.api_identifier}: #{plan.coverage.round(4)}%"
           end
-          @logger.info "API coverage report written to #{@output}"
+          log_skipped_summary(coverage_result)
+          logger.info "API coverage report written to #{@output}"
         end
 
+        private attr_reader :logger
+
         TEMPLATE_PATH = File.join(__dir__, 'html_reporter.html.erb')
         TEMPLATE = ERB.new(File.read(TEMPLATE_PATH), trim_mode: '-')
         TEMPLATE.filename = TEMPLATE_PATH

lib/openapi_first/test/coverage/html_reporter/context.rb

@@ -12,11 +12,14 @@ class Context
             'API Coverage did not detect any API requests for the registered ' \
             'API descriptions. Make sure to observe your application using OpenapiFirst::Test.'
 
-          attr_reader :coverage, :plans, :verbose, :generated_at
+          attr_reader :coverage, :plans, :verbose, :generated_at,
+                      :skipped_requests_count, :skipped_responses_count
 
           def initialize(coverage_result, verbose, generated_at: Time.now)
             @coverage = coverage_result.coverage
             @plans = coverage_result.plans
+            @skipped_requests_count = coverage_result.skipped_requests_count
+            @skipped_responses_count = coverage_result.skipped_responses_count
             @verbose = verbose
             @generated_at = generated_at.strftime('%Y-%m-%d %H:%M:%S %z')
           end
@@ -27,6 +30,10 @@ def get_binding # rubocop:disable Naming/AccessorMethodName
             binding
           end
 
+          def any_skipped?
+            skipped_requests_count.positive? || skipped_responses_count.positive?
+          end
+
           def visible_routes(plan)
             plan.routes
           end
@@ -36,17 +43,25 @@ def any_request_made?(route)
           end
 
           def route_status(route)
-            return :request_problem if route.requests.none?(&:finished?)
-            return :responses_problem if any_request_made?(route) && route.responses.any? { |r| !r.finished? }
+            return :skipped if route.skipped?
+            return :request_problem if route.tracked_requests.none?(&:finished?)
+            return :responses_problem if any_request_made?(route) && route.tracked_responses.any? { |r| !r.finished? }
 
             :ok
           end
 
+          def route_class(route)
+            return 'is-skipped' if route.skipped?
+
+            route.finished? ? 'is-covered' : 'is-uncovered'
+          end
+
           def uncovered_responses_count(route)
-            route.responses.count { |r| !r.finished? }
+            route.tracked_responses.count { |r| !r.finished? }
           end
 
           def request_items(route)
+            return route.requests if route.skipped?
             return [] unless any_request_made?(route) && route.requests.any?(&:content_type)
 
             route.requests

lib/openapi_first/test/coverage/plan.rb

@@ -14,14 +14,13 @@ class UnknownRequestError < StandardError; end
 
         def self.for(oad, skip_response: nil, skip_route: nil)
           plan = new(definition_key: oad.key, filepath: oad.filepath, title: oad.title)
-          routes = oad.routes
-          routes = routes.reject { |route| skip_route[route.path, route.request_method] } if skip_route
-          routes.each do |route|
-            responses = skip_response ? route.responses.reject(&skip_response) : route.responses
+          oad.routes.each do |route|
             plan.add_route request_method: route.request_method,
                            path: route.path,
                            requests: route.requests,
-                           responses:
+                           responses: route.responses,
+                           skipped: skip_route ? skip_route[route.path, route.request_method] : false,
+                           skip_response:
           end
           plan
         end
@@ -63,15 +62,36 @@ def tasks
           index.values
         end
 
-        def add_route(request_method:, path:, requests:, responses:)
-          request_tasks = requests.to_a.map do |request|
-            index[request.key] = RequestTask.new(request)
-          end
+        def skipped_requests
+          routes.flat_map(&:skipped_requests)
+        end
+
+        def skipped_responses
+          routes.flat_map(&:skipped_responses)
+        end
+
+        def skipped_requests_count
+          skipped_requests.count
+        end
+
+        def skipped_responses_count
+          skipped_responses.count
+        end
+
+        def add_route(request_method:, path:, requests:, responses:, skipped: false, skip_response: nil)
+          request_tasks = requests.to_a.map { |request| add_task RequestTask.new(request, skipped:) }
           response_tasks = responses.to_a.map do |response|
-            index[response.key] = ResponseTask.new(response)
+            add_task ResponseTask.new(response, skipped: skipped || (skip_response ? skip_response[response] : false))
           end
           @routes << RouteTask.new(path:, request_method:, requests: request_tasks, responses: response_tasks)
         end
+
+        private
+
+        def add_task(task)
+          index[task.key] = task unless task.skipped?
+          task
+        end
       end
     end
   end

lib/openapi_first/test/coverage/request_task.rb

@@ -9,10 +9,11 @@ module Coverage
       class RequestTask
         extend Forwardable
 
-        def_delegators :@request, :path, :request_method, :content_type
+        def_delegators :@request, :key, :path, :request_method, :content_type
 
-        def initialize(request_definition)
+        def initialize(request_definition, skipped: false)
           @request = request_definition
+          @skipped = skipped
           @requested = false
           @last_error_message = nil
         end
@@ -25,6 +26,10 @@ def track(validated_request)
           @last_error_message = validated_request.error.exception_message unless validated_request.valid?
         end
 
+        def skipped?
+          @skipped == true
+        end
+
         def requested?
           @requested == true
         end

lib/openapi_first/test/coverage/response_task.rb

@@ -11,8 +11,9 @@ class ResponseTask
 
         def_delegators :@response, :status, :content_type, :key
 
-        def initialize(response_definition)
+        def initialize(response_definition, skipped: false)
           @response = response_definition
+          @skipped = skipped
           @responded = false
           @last_error_message = nil
         end
@@ -25,6 +26,10 @@ def track(validated_response)
           @last_error_message = validated_response.error.exception_message unless validated_response.valid?
         end
 
+        def skipped?
+          @skipped == true
+        end
+
         def responded?
           @responded == true
         end