From cd14a191c1d20f13e788e6e057d438e0ee426375 Mon Sep 17 00:00:00 2001 From: Stephen A Pohl Date: Sat, 16 May 2026 23:22:29 -0400 Subject: [PATCH 1/2] Add crashes_after_fix rule to flag fixed crash bugs still crashing on Nightly The macOS and Windows Spotlight teams have repeatedly hit the same workflow gap: a crash gets a speculative fix, the patch lands, the bug is marked RESOLVED FIXED -- and the signature keeps firing on Nightly after the build containing the fix has shipped. With nothing prompting us to re-check crash-stats a few days post-landing, this verification step gets skipped, and we have ended up discovering only much later (in some cases weeks or months) that the speculative fix didn't actually move the crash numbers. This rule plugs that gap. Once a day it picks RESOLVED FIXED bugs where cf_status_firefox_nightly is "fixed" and cf_last_resolved falls between min_days_since_fix (default 4) and max_days_since_fix (default 10) ago, runs a faceted Socorro SuperSearch over Nightly for the bug's signature(s) starting the day after the fix landed, and -- if min_crash_count (default 5) or more crashes have been recorded in that window -- needinfos the assignee asking whether the fix was incomplete, whether the signature is shared with a different underlying crash, or whether a follow-up is needed. The four-day floor gives the Nightly build containing the fix time to roll out and accumulate user exposure before the bot will fire. The rule skips bugs that already have any open needinfo, and also skips bugs whose comment history contains the rule's marker phrase, so it only pings the assignee once per fix. --- bugbot/rules/crashes_after_fix.py | 198 +++++++++++++++++++++++ configs/rules.json | 5 + templates/crashes_after_fix.html | 23 +++ templates/crashes_after_fix_needinfo.txt | 4 + 4 files changed, 230 insertions(+) create mode 100644 bugbot/rules/crashes_after_fix.py create mode 100644 templates/crashes_after_fix.html create mode 100644 templates/crashes_after_fix_needinfo.txt diff --git a/bugbot/rules/crashes_after_fix.py b/bugbot/rules/crashes_after_fix.py new file mode 100644 index 000000000..ecba36064 --- /dev/null +++ b/bugbot/rules/crashes_after_fix.py @@ -0,0 +1,198 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from datetime import timedelta + +from libmozdata import utils as lmdutils +from libmozdata.socorro import SuperSearch + +from bugbot import utils +from bugbot.bzcleaner import BzCleaner + +# Marker phrase placed in the needinfo comment so subsequent runs can detect +# that this rule has already actioned a bug and skip it via the longdesc +# substring filter in get_bz_params(). Keep it stable across template edits. +COMMENT_MARKER = "crashes are still being reported against this signature" + + +class CrashesAfterFix(BzCleaner): + """Crash bugs whose signature is still crashing on Nightly some days + after the fix landed. Need-infos the assignee to ask whether the fix was + incomplete or whether a follow-up is needed.""" + + def __init__(self): + super().__init__() + self.min_days_since_fix = utils.get_config(self.name(), "min_days_since_fix", 4) + self.max_days_since_fix = utils.get_config( + self.name(), "max_days_since_fix", 10 + ) + self.min_crash_count = utils.get_config(self.name(), "min_crash_count", 5) + self.extra_ni = {} + # bug_id (str) -> per-bug context used to query Socorro and fill the + # needinfo template (see bughandler()). + self.bug_data = {} + + def description(self): + return ( + "Bugs whose crash signatures keep crashing on Nightly " + "{}-{} days after the fix landed" + ).format(self.min_days_since_fix, self.max_days_since_fix) + + def has_assignee(self): + return True + + def get_extra_for_template(self): + return { + "min_days": self.min_days_since_fix, + "max_days": self.max_days_since_fix, + "min_crashes": self.min_crash_count, + } + + def get_extra_for_needinfo_template(self): + return self.extra_ni + + def get_bz_params(self, date): + today = lmdutils.get_date_ymd(date) + # cf_last_resolved must fall in the window + # [today - max_days, today - min_days] + # so the fix has been on Nightly for at least min_days but not more + # than max_days. Bugzilla compares cf_last_resolved against a date + # string with greaterthan/lessthan. + oldest_fix = lmdutils.get_date_str( + today - timedelta(days=self.max_days_since_fix) + ) + newest_fix = lmdutils.get_date_str( + today - timedelta(days=self.min_days_since_fix) + ) + + fields = [ + "id", + "summary", + "assigned_to", + "assigned_to_detail", + "cf_crash_signature", + "cf_last_resolved", + "cf_status_firefox_nightly", + ] + + params = { + "include_fields": fields, + "resolution": "FIXED", + "bug_status": ["RESOLVED", "VERIFIED"], + # Has a non-empty crash signature. + "f1": "cf_crash_signature", + "o1": "isnotempty", + # The fix is in Nightly. + "f2": "cf_status_firefox_nightly", + "o2": "equals", + "v2": "fixed", + # cf_last_resolved > today - max_days (recent enough). + "f3": "cf_last_resolved", + "o3": "greaterthan", + "v3": oldest_fix, + # cf_last_resolved < today - min_days (old enough to be in nightly + # builds for the monitoring window). + "f4": "cf_last_resolved", + "o4": "lessthan", + "v4": newest_fix, + # Skip bugs that already have an open needinfo so we don't pile on. + "f5": "flagtypes.name", + "o5": "notsubstring", + "v5": "needinfo?", + # Skip bugs where we've already left a needinfo comment for this + # rule (idempotency across daily runs). + "n6": 1, + "f6": "longdesc", + "o6": "casesubstring", + "v6": COMMENT_MARKER, + } + + return params + + def bughandler(self, bug, data): + if not bug.get("cf_crash_signature"): + return + + sigs = sorted(utils.get_signatures(bug["cf_crash_signature"])) + if not sigs: + return + + assignee = bug.get("assigned_to") or "" + if utils.is_no_assignee(assignee): + return + + nickname = "" + if bug.get("assigned_to_detail"): + nickname = bug["assigned_to_detail"].get("nick", "") + + fix_date = bug.get("cf_last_resolved") + if not fix_date: + return + + bug_id = str(bug["id"]) + self.bug_data[bug_id] = { + "summary": self.get_summary(bug), + "signatures": sigs, + "fix_date": fix_date, + "assignee_email": assignee, + "assignee_nickname": nickname, + } + + def _query_socorro(self, info): + """Faceted SuperSearch over Nightly crashes since the day after the fix + landed. Returns (total_count, per_signature_counts, since_date_str).""" + fix_dt = lmdutils.get_date_ymd(info["fix_date"]) + since = lmdutils.get_date_str(fix_dt + timedelta(days=1)) + + counts = {} + + def handler(json, data): + if json.get("errors"): + return + for facet in json.get("facets", {}).get("signature", []): + data[facet["term"]] = int(facet["count"]) + + params = { + "product": "Firefox", + "release_channel": "nightly", + "date": ">=" + since, + "signature": ["=" + s for s in info["signatures"]], + "_results_number": 0, + "_facets": "signature", + "_facets_size": max(len(info["signatures"]), 1), + } + + SuperSearch(params=params, handler=handler, handlerdata=counts).wait() + return sum(counts.values()), counts, since + + def get_bugs(self, date="today", bug_ids=[]): + super().get_bugs(date=date, bug_ids=bug_ids) + + result = {} + for bug_id, info in self.bug_data.items(): + total, per_sig, since = self._query_socorro(info) + if total < self.min_crash_count: + continue + + self.extra_ni[bug_id] = { + "crash_count": total, + "since": since, + "fix_date": info["fix_date"][:10], + "signatures": info["signatures"], + "per_signature_counts": per_sig, + } + self.add_auto_ni( + bug_id, + { + "mail": info["assignee_email"], + "nickname": info["assignee_nickname"], + }, + ) + result[bug_id] = {"id": bug_id, "summary": info["summary"]} + + return result + + +if __name__ == "__main__": + CrashesAfterFix().run() diff --git a/configs/rules.json b/configs/rules.json index e2929bdfc..ba5849b80 100644 --- a/configs/rules.json +++ b/configs/rules.json @@ -112,6 +112,11 @@ "keyword_exception": ["testcase"], "sec": false }, + "crashes_after_fix": { + "min_days_since_fix": 4, + "max_days_since_fix": 10, + "min_crash_count": 5 + }, "newbie_with_ni": { "number_of_days": 7, "number_of_comments": 2 diff --git a/templates/crashes_after_fix.html b/templates/crashes_after_fix.html new file mode 100644 index 000000000..4e32a4f94 --- /dev/null +++ b/templates/crashes_after_fix.html @@ -0,0 +1,23 @@ +

+ The following {{ plural('bug', data) }} {{ plural('still has', data, pword='still have') }} reported crashes on Nightly between {{ extra['min_days'] }} and {{ extra['max_days'] }} days after the fix landed (>= {{ extra['min_crashes'] }} crashes since the day after landing): +

+ + + + + + + + + {% for i, (bugid, summary) in enumerate(data) -%} + + + + + {% endfor -%} + +
BugSummary
+ {{ bugid }} + {{ summary | e }}
diff --git a/templates/crashes_after_fix_needinfo.txt b/templates/crashes_after_fix_needinfo.txt new file mode 100644 index 000000000..ff59a4ed6 --- /dev/null +++ b/templates/crashes_after_fix_needinfo.txt @@ -0,0 +1,4 @@ +{% if nickname %}:{{ nickname }},{% endif %} +this bug was marked RESOLVED FIXED on {{ extra['fix_date'] }}, but crashes are still being reported against this signature on Nightly. Since the day after the fix landed ({{ extra['since'] }}), {{ extra['crash_count'] }} crash{{ 'es' if extra['crash_count'] != 1 else '' }} {{ 'have' if extra['crash_count'] != 1 else 'has' }} been recorded on Nightly across the {{ extra['signatures']|length }} signature{{ 's' if extra['signatures']|length != 1 else '' }} on this bug. +Could you take a look at the recent crash reports to determine whether the fix is incomplete, whether the signature is shared with a different underlying crash, or whether a follow-up is needed? +{{ documentation }} From c65145ad3d42ddb1c2ad774bf186a689c82f8f19 Mon Sep 17 00:00:00 2001 From: Stephen A Pohl Date: Mon, 18 May 2026 12:49:48 -0400 Subject: [PATCH 2/2] Address review: drop min_days_since_fix floor Per marco-c's review feedback: min_crash_count plus the "date >= fix_date + 1 day" Socorro filter already gate pings, so the 4-day floor was redundant. Removing it means the rule fires as soon as the threshold is crossed -- fast-burning regressions get caught earlier, slow-burning ones are still gated by min_crash_count. max_days_since_fix is kept as the upper bound on how long we keep polling a bug whose crash count is still below the threshold. --- bugbot/rules/crashes_after_fix.py | 48 +++++++++++++------------------ configs/rules.json | 1 - templates/crashes_after_fix.html | 2 +- 3 files changed, 21 insertions(+), 30 deletions(-) diff --git a/bugbot/rules/crashes_after_fix.py b/bugbot/rules/crashes_after_fix.py index ecba36064..7abdd75af 100644 --- a/bugbot/rules/crashes_after_fix.py +++ b/bugbot/rules/crashes_after_fix.py @@ -17,13 +17,12 @@ class CrashesAfterFix(BzCleaner): - """Crash bugs whose signature is still crashing on Nightly some days - after the fix landed. Need-infos the assignee to ask whether the fix was - incomplete or whether a follow-up is needed.""" + """Crash bugs whose signature is still crashing on Nightly after the fix + landed. Need-infos the assignee to ask whether the fix was incomplete or + whether a follow-up is needed.""" def __init__(self): super().__init__() - self.min_days_since_fix = utils.get_config(self.name(), "min_days_since_fix", 4) self.max_days_since_fix = utils.get_config( self.name(), "max_days_since_fix", 10 ) @@ -35,16 +34,15 @@ def __init__(self): def description(self): return ( - "Bugs whose crash signatures keep crashing on Nightly " - "{}-{} days after the fix landed" - ).format(self.min_days_since_fix, self.max_days_since_fix) + "Bugs whose crash signatures keep crashing on Nightly within " + "{} days after the fix landed" + ).format(self.max_days_since_fix) def has_assignee(self): return True def get_extra_for_template(self): return { - "min_days": self.min_days_since_fix, "max_days": self.max_days_since_fix, "min_crashes": self.min_crash_count, } @@ -54,17 +52,16 @@ def get_extra_for_needinfo_template(self): def get_bz_params(self, date): today = lmdutils.get_date_ymd(date) - # cf_last_resolved must fall in the window - # [today - max_days, today - min_days] - # so the fix has been on Nightly for at least min_days but not more - # than max_days. Bugzilla compares cf_last_resolved against a date - # string with greaterthan/lessthan. + # Only consider fixes resolved within the past max_days. Older fixes + # have aged out of the monitoring window: by then beta/release + # exposure and signature evolution carry more meaning than another + # post-landing ping. Bugs already actioned by this rule are filtered + # out below via the marker-substring check on longdesc, so we keep + # polling each candidate daily until it either crosses the crash + # threshold or ages past max_days. oldest_fix = lmdutils.get_date_str( today - timedelta(days=self.max_days_since_fix) ) - newest_fix = lmdutils.get_date_str( - today - timedelta(days=self.min_days_since_fix) - ) fields = [ "id", @@ -91,21 +88,16 @@ def get_bz_params(self, date): "f3": "cf_last_resolved", "o3": "greaterthan", "v3": oldest_fix, - # cf_last_resolved < today - min_days (old enough to be in nightly - # builds for the monitoring window). - "f4": "cf_last_resolved", - "o4": "lessthan", - "v4": newest_fix, # Skip bugs that already have an open needinfo so we don't pile on. - "f5": "flagtypes.name", - "o5": "notsubstring", - "v5": "needinfo?", + "f4": "flagtypes.name", + "o4": "notsubstring", + "v4": "needinfo?", # Skip bugs where we've already left a needinfo comment for this # rule (idempotency across daily runs). - "n6": 1, - "f6": "longdesc", - "o6": "casesubstring", - "v6": COMMENT_MARKER, + "n5": 1, + "f5": "longdesc", + "o5": "casesubstring", + "v5": COMMENT_MARKER, } return params diff --git a/configs/rules.json b/configs/rules.json index ba5849b80..830a40885 100644 --- a/configs/rules.json +++ b/configs/rules.json @@ -113,7 +113,6 @@ "sec": false }, "crashes_after_fix": { - "min_days_since_fix": 4, "max_days_since_fix": 10, "min_crash_count": 5 }, diff --git a/templates/crashes_after_fix.html b/templates/crashes_after_fix.html index 4e32a4f94..a46b3188b 100644 --- a/templates/crashes_after_fix.html +++ b/templates/crashes_after_fix.html @@ -1,5 +1,5 @@

- The following {{ plural('bug', data) }} {{ plural('still has', data, pword='still have') }} reported crashes on Nightly between {{ extra['min_days'] }} and {{ extra['max_days'] }} days after the fix landed (>= {{ extra['min_crashes'] }} crashes since the day after landing): + The following {{ plural('bug', data) }} {{ plural('still has', data, pword='still have') }} reported crashes on Nightly within {{ extra['max_days'] }} days after the fix landed (>= {{ extra['min_crashes'] }} crashes since the day after landing):