Add HtmlCheck for check type html (DPP-SEC-014)

README.md

@@ -83,6 +83,14 @@ and do not establish a presumption of conformity.
 - Criteria with `check.type: links` check every `RelatedResource` of the
   passport for the required attributes and send a HEAD request to its URL
   (at most 20 URLs per passport). A URL that does not answer gives a warning.
+- Criteria with `check.type: html` (DPP-SEC-014) request the product
+  identifier with `Accept: text/html` and list the URLs a browser loads without
+  user action while it displays the HTML view (scripts, style sheets, fonts,
+  images, frames, preloads, CSS `url()` and `@import`, meta refresh). With
+  `external_resources: forbidden` every such URL on another host than the
+  passport's fails the criterion. Without an HTML view the result is
+  `skipped` (`no_evidence`). Resources added by scripts at run time are not
+  seen.
 - Criteria whose condition does not hold are reported as `skipped`.
 - Every `skipped` criterion carries a `reason` and a `reason_code`
   (`not_applicable`, `no_evidence`, `not_evaluated`, … as defined under

app/services/html_check.rb

@@ -0,0 +1,129 @@
+require "nokogiri"
+
+# Runs a criterion with check.type html: the HTML view of the passport.
+#
+# The product identifier is requested with Accept: text/html (redirects
+# followed). Without a final 200 text/html response there is nothing to
+# check; whether the HTML view exists is rated by DPP-DAT-016.
+#
+# external_resources: forbidden collects the URLs a browser fetches without
+# user action while it displays the document (see "Check types", html, in
+# CRITERIA-FORMAT.md of dpp-criteria) and reports every http(s) URL whose host
+# differs from the host of the final response URL. Links the user has to
+# follow are not counted; resources added by scripts at run time cannot be
+# seen in the document.
+class HtmlCheck
+  Outcome = Struct.new(:skipped, :code, :messages, keyword_init: true)
+
+  SRC_ELEMENTS = %w[script img iframe frame embed audio video source track].freeze
+  SRCSET_ELEMENTS = %w[img source].freeze
+  LINK_RELS = %w[stylesheet icon apple-touch-icon mask-icon manifest preload modulepreload
+                 prefetch prerender preconnect dns-prefetch].freeze
+  CSS_URL = /url\(\s*(?:"([^"]*)"|'([^']*)'|([^)"'\s]*))\s*\)/i
+  CSS_IMPORT = /@import\s+(?:"([^"]*)"|'([^']*)')/i
+  MAX_LISTED = 10
+
+  def initialize(check, product_id, resolver)
+    @check = check
+    @product_id = product_id
+    @resolver = resolver
+  end
+
+  def outcome
+    res = @resolver.get(@product_id, accept: "text/html")
+    unless res.error.nil? && res.status == 200 && res.media_type == "text/html"
+      return Outcome.new(skipped: "no HTML view (#{describe(res)}); rated by DPP-DAT-016", code: "no_evidence")
+    end
+
+    messages = []
+    messages << external_message(res) if @check["external_resources"] == "forbidden"
+    Outcome.new(messages: messages.compact)
+  end
+
+  # The URLs a browser loads without user action, resolved, with the element
+  # or construct they come from: [[url, source], …].
+  def self.loaded_urls(html, page_url)
+    doc = Nokogiri::HTML5(html.to_s)
+    base = absolute(doc.at_css("base[href]")&.[]("href"), page_url) || page_url
+    found = []
+    add = ->(value, source) { (u = absolute(value, base)) && found << [u, source] }
+
+    doc.css(SRC_ELEMENTS.join(",")).each { |e| add.(e["src"], e.name) if e["src"] }
+    doc.css('input[src]').each { |e| add.(e["src"], "input") if e["type"].to_s.casecmp?("image") }
+    doc.css(SRCSET_ELEMENTS.join(",")).each do |e|
+      srcset(e["srcset"]).each { |u| add.(u, "#{e.name} srcset") }
+    end
+    doc.css("video[poster]").each { |e| add.(e["poster"], "video poster") }
+    doc.css("object[data]").each { |e| add.(e["data"], "object") }
+    doc.xpath("//*[local-name()='image']").each do |e|
+      (e["href"] || e["xlink:href"] || e.attribute_with_ns("href", "http://www.w3.org/1999/xlink")&.value)
+        .then { |v| add.(v, "svg image") if v }
+    end
+    doc.css("link[href]").each do |e|
+      rels = e["rel"].to_s.downcase.split
+      next if (rels & LINK_RELS).empty?
+
+      add.(e["href"], "link rel=#{rels.join(' ')}")
+    end
+    doc.css("style").each { |e| css_urls(e.content).each { |u| add.(u, "style") } }
+    doc.css("[style]").each { |e| css_urls(e["style"]).each { |u| add.(u, "style attribute") } }
+    doc.css("meta[http-equiv]").each do |e|
+      next unless e["http-equiv"].to_s.casecmp?("refresh")
+
+      target = e["content"].to_s[/url\s*=\s*['"]?([^'"\s]+)/i, 1]
+      add.(target, "meta refresh") if target
+    end
+    found.uniq
+  end
+
+  def self.absolute(value, base)
+    value = value.to_s.strip
+    return nil if value.empty?
+
+    URI.join(base.to_s, value).to_s
+  rescue URI::Error, ArgumentError
+    # A URL the URI library cannot parse (spaces, non-ASCII characters) still
+    # counts if its host can be read.
+    host = value[%r{\A(?:https?:)?//([^/?#\s:]+)}i, 1]
+    host && "https://#{host}/"
+  end
+
+  def self.srcset(value)
+    value.to_s.split(",").map { |c| c.strip.split(/\s+/).first }.compact.reject(&:empty?)
+  end
+
+  def self.css_urls(css)
+    css.to_s.scan(CSS_URL).map { |m| m.compact.first } + css.to_s.scan(CSS_IMPORT).map { |m| m.compact.first }
+  end
+
+  # Host of an http(s) URL, lower case, or nil for other schemes.
+  def self.web_host(url)
+    uri = URI.parse(url)
+    return nil unless %w[http https].include?(uri.scheme&.downcase)
+
+    uri.host&.downcase&.delete_prefix("[")&.delete_suffix("]")
+  rescue URI::Error
+    nil
+  end
+
+  private
+
+  def external_message(res)
+    own = self.class.web_host(res.url.to_s)
+    external = self.class.loaded_urls(res.body, res.url.to_s).select do |url, _|
+      (host = self.class.web_host(url)) && host != own
+    end
+    return nil if external.empty?
+
+    listed = external.first(MAX_LISTED).map { |url, source| "#{url} (#{source})" }
+    more = external.size > MAX_LISTED ? " and #{external.size - MAX_LISTED} more" : ""
+    { severity: "violation",
+      message: "the HTML view loads #{external.size} resource(s) from other hosts than #{own}: #{listed.join(', ')}#{more}" }
+  end
+
+  def describe(res)
+    return res.error if res.error
+
+    "HTTP status #{res.status}, Content-Type #{res.content_type.presence || 'missing'}"
+  end
+end

app/services/passport_linter.rb

@@ -7,7 +7,9 @@
 # therefore need one (GET /api/v1/validate/<product identifier>). Criteria with
 # check.type did hand the DIDs of the passport to didlint. Criteria with
 # check.type proof verify an integrity proof in the passport; criteria with
-# check.type links check the related resources it links to.
+# check.type links check the related resources it links to; criteria with
+# check.type html check the HTML view of the passport (needs a product
+# identifier, like resolve).
 class PassportLinter
   SH = "http://www.w3.org/ns/shacl#".freeze
   REASON_CODES = %w[not_applicable not_implemented no_evidence needs_credentials not_sent unreachable not_evaluated].freeze
@@ -72,6 +74,7 @@ def evaluate(criterion, passport, reports)
     check = criterion["check"] || {}
 
     return resolve(base, check) if check["type"] == "resolve"
+    return html(base, check) if check["type"] == "html"
     unless %w[shacl did proof links].include?(check["type"])
       return skip(base, "check type #{check['type']} is not implemented in this version", "not_evaluated")
     end
@@ -112,6 +115,15 @@ def resolve(base, check)
     base.merge(result: result_for(messages), messages: messages)
   end
 
+  def html(base, check)
+    return skip(base, "needs a product identifier", "not_evaluated") if @product_id.blank?
+
+    outcome = HtmlCheck.new(check, @product_id, @resolver).outcome
+    return skip(base, outcome.skipped, outcome.code) if outcome.skipped
+
+    base.merge(result: result_for(outcome.messages), messages: outcome.messages)
+  end
+
   def did(base, check, passport)
     did_check = DidCheck.new(check, passport, didlint: @didlint, resolver: @resolver)
     return skip(base, "no DID in #{check['paths'].join(', ')}", "not_applicable") if did_check.dids.empty?

test/integration/validations_test.rb

@@ -43,7 +43,7 @@ def criterion(result, id) = result["criteria"].find { |c| c["id"] == id }
 
   test "checks that request the product identifier need one" do
     result = lint(reference)
-    %w[DPP-ID-001 DPP-ID-013 DPP-DAT-003 DPP-DAT-016].each do |id|
+    %w[DPP-ID-001 DPP-ID-013 DPP-DAT-003 DPP-DAT-016 DPP-SEC-014].each do |id|
       assert_equal "skipped", criterion(result, id)["result"], id
     end
     assert_equal "rated by dpp-validator from its daily runs", criterion(result, "DPP-ID-002")["reason"]

test/services/html_check_test.rb

@@ -0,0 +1,103 @@
+require "test_helper"
+
+class HtmlCheckTest < ActiveSupport::TestCase
+  UPI = "https://dpp.example.org/01/09520123456788/21/0001".freeze
+  CHECK = { "type" => "html", "external_resources" => "forbidden" }.freeze
+
+  # Answers every GET with one response; records the Accept header.
+  class FakeResolver
+    attr_reader :accept
+
+    def initialize(status, type, body, url: UPI, error: nil) = (@status, @type, @body, @url, @error = status, type, body, url, error)
+
+    def get(_url, accept: nil)
+      @accept = accept
+      return HttpResolver::Response.new(url: @url, error: @error) if @error
+
+      HttpResolver::Response.new(url: @url, status: @status, content_type: @type, body: @body, headers: {})
+    end
+  end
+
+  def outcome(body, status: 200, type: "text/html; charset=utf-8", url: UPI, error: nil)
+    HtmlCheck.new(CHECK, UPI, FakeResolver.new(status, type, body, url: url, error: error)).outcome
+  end
+
+  def page(head = "", body = "") = "<!doctype html><html><head>#{head}</head><body>#{body}</body></html>"
+
+  test "requests the HTML view" do
+    resolver = FakeResolver.new(200, "text/html", page)
+    HtmlCheck.new(CHECK, UPI, resolver).outcome
+    assert_equal "text/html", resolver.accept
+  end
+
+  test "page with own and relative resources passes" do
+    o = outcome(page('<link rel="stylesheet" href="/app.css"><script src="https://dpp.example.org/app.js"></script>',
+                     '<img src="logo.png"><img src="data:image/png;base64,AAAA"><a href="https://other.example/info">info</a>'))
+    assert_nil o.skipped
+    assert_empty o.messages
+  end
+
+  test "external script, font and image fail" do
+    o = outcome(page('<script src="https://cdn.example.net/lib.js"></script>' \
+                     '<link rel="preconnect" href="https://fonts.gstatic.com">' \
+                     '<style>@import "https://fonts.googleapis.com/css2?family=Inter"; body { background: url(//img.example.com/bg.png) }</style>',
+                     '<img srcset="/a.png 1x, https://pix.example.com/b.png 2x">'))
+    assert_equal ["violation"], o.messages.map { |m| m[:severity] }
+    msg = o.messages.first[:message]
+    %w[cdn.example.net fonts.gstatic.com fonts.googleapis.com img.example.com pix.example.com].each { |h| assert_includes msg, h }
+    assert_match(/5 resource\(s\)/, msg)
+  end
+
+  test "another host of the same operator counts as external" do
+    o = outcome(page('<link rel="icon" href="https://static.example.org/favicon.ico">'))
+    assert_match(/static\.example\.org/, o.messages.first[:message])
+  end
+
+  test "host comparison ignores case and port" do
+    assert_empty outcome(page('<script src="https://DPP.example.org:443/a.js"></script>')).messages
+  end
+
+  test "own host is the host of the final URL after redirects" do
+    o = outcome(page('<script src="https://view.example.com/a.js"></script>'), url: "https://view.example.com/p/1")
+    assert_empty o.messages
+  end
+
+  test "base href is used for relative URLs" do
+    o = outcome(page('<base href="https://cdn.example.net/"><script src="a.js"></script>'))
+    assert_match(%r{https://cdn\.example\.net/a\.js \(script\)}, o.messages.first[:message])
+  end
+
+  test "style attribute, meta refresh, object, video poster and svg image count" do
+    o = outcome(page('<meta http-equiv="Refresh" content="0; url=https://track.example.com/r">',
+                     '<div style="background-image: url(\'https://a.example.com/x.png\')"></div>' \
+                     '<object data="https://b.example.com/o.svg"></object><video poster="https://c.example.com/p.jpg"></video>' \
+                     '<svg><image href="https://d.example.com/i.png"/></svg>'))
+    msg = o.messages.first[:message]
+    %w[track.example.com a.example.com b.example.com c.example.com d.example.com].each { |h| assert_includes msg, h }
+  end
+
+  test "link relations that do not load are not counted" do
+    o = outcome(page('<link rel="alternate" type="application/json" href="https://api.example.com/p.json">' \
+                     '<link rel="canonical" href="https://www.example.com/p">'))
+    assert_empty o.messages
+  end
+
+  test "more than ten external URLs are summarised" do
+    imgs = (1..12).map { |i| %(<img src="https://h#{i}.example.com/i.png">) }.join
+    msg = outcome(page("", imgs)).messages.first[:message]
+    assert_match(/12 resource\(s\)/, msg)
+    assert_match(/and 2 more\z/, msg)
+  end
+
+  test "an unparsable external URL still counts" do
+    assert_match(/bad\.example\.com/, outcome(page('<img src="https://bad.example.com/a b|c.png">')).messages.first[:message])
+  end
+
+  test "without an HTML view the check is skipped" do
+    o = outcome('{"a":1}', type: "application/json")
+    assert_equal "no_evidence", o.code
+    assert_match(/DPP-DAT-016/, o.skipped)
+    assert_equal "no_evidence", outcome(page, status: 406).code
+    assert_equal "no_evidence", outcome(nil, error: "retrieval failed: timeout (test)").code
+  end
+end

test/services/passport_linter_test.rb

@@ -55,6 +55,26 @@ def lint(responses, check = CHECK)
     assert_equal %w[active proposed deprecated], report[:criteria].map { |c| c[:status] }
   end
 
+  HTML_CHECK = { "type" => "html", "external_resources" => "forbidden" }.freeze
+
+  test "html criterion fails on an external script" do
+    page = [200, "text/html", '<html><head><script src="https://cdn.example.net/a.js"></script></head></html>']
+    result = lint({ "text/html" => page }, HTML_CHECK)[:criteria].first
+    assert_equal "failed", result[:result]
+    assert_match(/cdn\.example\.net/, result[:messages].first[:message])
+  end
+
+  test "html criterion without an HTML view is skipped with no_evidence" do
+    result = lint({ "text/html" => JSON_OK }, HTML_CHECK)[:criteria].first
+    assert_equal %w[skipped no_evidence], [result[:result], result[:reason_code]]
+  end
+
+  test "html criterion without a product identifier is skipped" do
+    report = PassportLinter.new(catalogue: FakeCatalogue.new([criterion(HTML_CHECK)]), resolver: FakeResolver.new({}))
+                           .run(passport: { "a" => 1 }, product_id: nil)
+    assert_equal %w[skipped not_evaluated], [report[:criteria].first[:result], report[:criteria].first[:reason_code]]
+  end
+
   test "no failing check passes" do
     report = lint("text/html" => HTML, "*/*" => JSON_OK)
     assert_equal "passed", report[:criteria].first[:result]