Count only active criteria in N of M, report proposed ones separately

app/services/passport_linter.rb

@@ -24,19 +24,14 @@ def run(passport:, product_id: nil, retrieval: nil, jws: nil, jws_only: false, r
     @jws = jws
     @jws_only = jws_only
     reports = {}
-    criteria = @catalogue.passport_criteria.map { |c| evaluate(c, passport, reports) }
-    counted = criteria.select { |c| %w[passed warning failed].include?(c[:result]) }
-    passed = counted.count { |c| c[:result] != "failed" }
+    criteria = @catalogue.passport_criteria.map do |c|
+      evaluate(c, passport, reports).merge(status: c["status"], counted: c["status"] == "active")
+    end
+    active, proposed = criteria.partition { |c| c[:counted] }
     {
       productId: product_id || passport&.dig("uniqueProductIdentifier"),
       retrieval: retrieval,
-      summary: {
-        text: "#{passed} of #{counted.size} automated checks passed",
-        passed: passed,
-        failed: counted.size - passed,
-        warnings: criteria.count { |c| c[:result] == "warning" },
-        skipped: criteria.count { |c| c[:result] == "skipped" }
-      },
+      summary: summary(active).merge(proposed_not_counted: summary(proposed.select { |c| c[:status] == "proposed" })),
       criteria: criteria,
       notice: "Results of automated checks only. They are no certification and establish no presumption of conformity.",
       "dpp-criteria": Rails.configuration.x.dpplint.criteria_ref
@@ -45,6 +40,21 @@ def run(passport:, product_id: nil, retrieval: nil, jws: nil, jws_only: false, r
 
   private
 
+  # Only criteria with status active count in "N of M" (CRITERIA-FORMAT.md,
+  # "Results"); proposed criteria are summarised separately and not counted.
+  # passed and warning count as passed, skipped is not counted.
+  def summary(criteria)
+    counted = criteria.select { |c| %w[passed warning failed].include?(c[:result]) }
+    passed = counted.count { |c| c[:result] != "failed" }
+    {
+      text: "#{passed} of #{counted.size} automated checks passed",
+      passed: passed,
+      failed: counted.size - passed,
+      warnings: criteria.count { |c| c[:result] == "warning" },
+      skipped: criteria.count { |c| c[:result] == "skipped" }
+    }
+  end
+
   def evaluate(criterion, passport, reports)
     base = { id: criterion["id"], title: criterion["title"], level: criterion["level"] }
     check = criterion["check"] || {}

app/views/home/show.html.erb

@@ -73,12 +73,14 @@
     const retrieval = d.retrieval || {};
     const rows = [...d.criteria].sort((a, b) => order[a.result] - order[b.result] || a.id.localeCompare(b.id)).map((c) => {
       const notes = (c.messages || []).map((m) => `<li>${esc(m.message)}</li>`).join("") + (c.reason ? `<li>${esc(c.reason)}</li>` : "");
-      return `<tr><td class="id">${esc(c.id)}</td><td>${esc(c.title)}${notes ? `<ul class="msg">${notes}</ul>` : ""}</td>
+      const proposed = c.counted === false ? ` <span class="notice">(${esc(c.status)}, not counted)</span>` : "";
+      return `<tr><td class="id">${esc(c.id)}</td><td>${esc(c.title)}${proposed}${notes ? `<ul class="msg">${notes}</ul>` : ""}</td>
               <td>${esc(c.level)}</td><td><span class="badge ${esc(c.result)}">${esc(c.result)}</span></td></tr>`;
     }).join("");
     resultEl.innerHTML = `
       <div class="summary"><strong>${esc(d.summary.text)}</strong>
         (${d.summary.failed} failed, ${d.summary.warnings} with warnings, ${d.summary.skipped} not applicable or not checked)
+        ${d.summary.proposed_not_counted ? `<div class="notice">Proposed criteria, not counted: ${esc(d.summary.proposed_not_counted.text)}</div>` : ""}
         <div class="notice" style="overflow-wrap:anywhere">Retrieved: ${esc(retrieval.url)} · HTTP ${esc(retrieval.status)} · ${esc(retrieval.contentType)}${retrieval.error ? " · " + esc(retrieval.error) : ""}</div>
         <div class="notice">${esc(d.notice)}</div></div>
       <div class="scroll"><table><thead><tr><th>Criterion</th><th>Requirement and findings</th><th>Level</th><th>Result</th></tr></thead><tbody>${rows}</tbody></table></div>`;

swagger/v1/swagger.yaml

@@ -72,12 +72,22 @@ components:
             contentType: { type: string }
         summary:
           type: object
+          description: Only criteria with status active count. passed and warning count as passed, skipped is not counted.
           properties:
             text: { type: string, example: "7 of 7 automated checks passed" }
             passed: { type: integer }
             failed: { type: integer }
             warnings: { type: integer }
             skipped: { type: integer }
+            proposed_not_counted:
+              type: object
+              description: The same figures for criteria with status proposed, which are not counted.
+              properties:
+                text: { type: string }
+                passed: { type: integer }
+                failed: { type: integer }
+                warnings: { type: integer }
+                skipped: { type: integer }
         criteria:
           type: array
           items:
@@ -87,6 +97,8 @@ components:
               title: { type: string }
               level: { type: string, enum: [MUST, SHOULD, MAY] }
               result: { type: string, enum: [passed, warning, failed, skipped] }
+              status: { type: string, enum: [proposed, active] }
+              counted: { type: boolean, description: true if the criterion counts in summary.text (status active) }
               reason: { type: string }
               messages:
                 type: array

test/services/passport_linter_test.rb

@@ -27,7 +27,7 @@ def get(_url, accept: nil)
                              "expect" => { "status" => [200], "content_type" => "application/json" } }]
   }.freeze
 
-  def criterion(check = CHECK) = { "id" => "DPP-DAT-016", "title" => "t", "level" => "MUST", "target" => "passport", "method" => "automated", "check" => check }
+  def criterion(check = CHECK) = { "id" => "DPP-DAT-016", "title" => "t", "level" => "MUST", "target" => "passport", "method" => "automated", "status" => "active", "check" => check }
 
   def lint(responses, check = CHECK)
     PassportLinter.new(catalogue: FakeCatalogue.new([criterion(check)]), resolver: FakeResolver.new(responses))
@@ -37,6 +37,18 @@ def lint(responses, check = CHECK)
   HTML = [200, "text/html", "<html></html>", { "vary" => ["Accept"] }].freeze
   JSON_OK = [200, "application/json", '{"uniqueProductIdentifier":"x"}'].freeze
 
+  test "only active criteria count; proposed ones are summarised separately" do
+    proposed = criterion.merge("id" => "DPP-DAT-099", "status" => "proposed")
+    deprecated_like = criterion.merge("id" => "DPP-DAT-098", "status" => "deprecated")
+    report = PassportLinter.new(catalogue: FakeCatalogue.new([criterion, proposed, deprecated_like]),
+                                resolver: FakeResolver.new("text/html" => HTML, "*/*" => JSON_OK))
+                           .run(passport: nil, product_id: UPI)
+    assert_equal "1 of 1 automated checks passed", report[:summary][:text]
+    assert_equal "1 of 1 automated checks passed", report[:summary][:proposed_not_counted][:text]
+    assert_equal [true, false, false], report[:criteria].map { |c| c[:counted] }
+    assert_equal %w[active proposed deprecated], report[:criteria].map { |c| c[:status] }
+  end
+
   test "no failing check passes" do
     report = lint("text/html" => HTML, "*/*" => JSON_OK)
     assert_equal "passed", report[:criteria].first[:result]
@@ -73,7 +85,7 @@ def lint(responses, check = CHECK)
   end
 
   test "applies_if with a matches pattern that is not valid ECMA-262 skips the criterion with a reason" do
-    criterion = { "id" => "DPP-ID-099", "title" => "t", "level" => "MUST", "target" => "passport", "method" => "automated",
+    criterion = { "id" => "DPP-ID-099", "title" => "t", "level" => "MUST", "target" => "passport", "method" => "automated", "status" => "active",
                   "applies_if" => [{ "path" => "$.granularity", "matches" => "a**" }], "check" => { "type" => "did", "paths" => ["$.facilityId"] } }
     report = PassportLinter.new(catalogue: FakeCatalogue.new([criterion]), resolver: FakeResolver.new({}))
                            .run(passport: { "granularity" => "item" })
@@ -83,7 +95,7 @@ def lint(responses, check = CHECK)
   end
 
   test "applies_if with ^ in search() skips the criterion with a reason instead of evaluating it" do
-    criterion = { "id" => "DPP-BAT-002", "title" => "t", "level" => "MUST", "target" => "passport", "method" => "automated",
+    criterion = { "id" => "DPP-BAT-002", "title" => "t", "level" => "MUST", "target" => "passport", "method" => "automated", "status" => "active",
                   "applies_if" => [{ "path" => "$.contentSpecificationIds[?search(@, '^[Bb]atter')]", "exists" => false }],
                   "check" => { "type" => "did", "paths" => ["$.facilityId"] } }
     report = PassportLinter.new(catalogue: FakeCatalogue.new([criterion]), resolver: FakeResolver.new({}))
@@ -95,7 +107,7 @@ def lint(responses, check = CHECK)
   end
 
   test "applies_if with search() whose condition does not hold skips the criterion" do
-    criterion = { "id" => "DPP-BAT-002", "title" => "t", "level" => "MUST", "target" => "passport", "method" => "automated",
+    criterion = { "id" => "DPP-BAT-002", "title" => "t", "level" => "MUST", "target" => "passport", "method" => "automated", "status" => "active",
                   "applies_if" => [{ "path" => "$.contentSpecificationIds[?search(@, '[Bb]atter')]", "exists" => true }],
                   "check" => { "type" => "did", "paths" => ["$.facilityId"] } }
     report = PassportLinter.new(catalogue: FakeCatalogue.new([criterion]), resolver: FakeResolver.new({}))