From 8450ac3679eff549235a2716a653aa5b8c334003 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Sj=C3=B6lund?= Date: Mon, 31 Aug 2026 20:35:25 +0200 Subject: [PATCH] Record reports after the upload, not before all-reports.py wrote a report's row into [history] as soon as it had written the html to the workspace, and mailed everyone about it in the same run. The files only reach the web server at the end of the stage, after all-plots.py and two dozen report.py invocations, so anything failing in between left a report that the database says exists, that nobody can read, and that the rerun skips because it is recorded. The mail linking to it had already gone out. Before the table existed the record was the published 00_history.html, which only existed if the upload had happened, so a lost upload healed itself on the next run; moving the record into the database made it permanent. The reports and the queued emails now go into pending-reports.json, and publish-reports.py records and sends them once the files are up. The stage uploads history/ right after all-plots.py and keeps a second transfer for overview*.html at the end. | scenario | now | | ----------------------------------- | ---------------------------- | | generate, publish, rerun | skipped | | stage fails before the upload | regenerated next run | | uploaded but not recorded | copied back from the index | | branch named twice in one run | one report | Assisted-by: Claude Opus 5 --- .CI/Jenkinsfile | 10 ++++-- all-reports.py | 71 +++++++++++----------------------------- publish-reports.py | 80 ++++++++++++++++++++++++++++++++++++++++++++++ resultsdb.py | 17 +++++++++- 4 files changed, 123 insertions(+), 55 deletions(-) create mode 100755 publish-reports.py diff --git a/.CI/Jenkinsfile b/.CI/Jenkinsfile index 8dbd48b..61aaaee 100644 --- a/.CI/Jenkinsfile +++ b/.CI/Jenkinsfile @@ -477,7 +477,7 @@ pipeline { PGPASSFILE = credentials('omdb-pgpass') } steps { - sh 'rm -rf *.html history' + sh 'rm -rf *.html history pending-reports.json' sh ''' if ! test -d OpenModelica; then git clone https://openmodelica.org/git-readonly/OpenModelica.git @@ -490,6 +490,12 @@ pipeline { sh "./all-reports.py --email --omcgitdir=OpenModelica ${env.GITBRANCHES} ${env.GITBRANCHES_FMI} ${env.GITBRANCHES_NEWINST} ${env.GITBRANCHES_DAE} ${env.GITBRANCHES_NEWBACKEND_DAE} ${env.GITBRANCHES_CPP} ${env.GITBRANCHES_WASM_JIT} ${env.GITBRANCHES_C_PLUS_RUST} conversion heavy_tests generateSymbolicJacobian gbode cvode ida" sh "./all-plots.py ${env.GITBRANCHES} ${env.GITBRANCHES_FMI} ${env.GITBRANCHES_NEWINST} ${env.GITBRANCHES_DAE} ${env.GITBRANCHES_NEWBACKEND_DAE} ${env.GITBRANCHES_CPP} ${env.GITBRANCHES_WASM_JIT} ${env.GITBRANCHES_C_PLUS_RUST} conversion heavy_tests generateSymbolicJacobian gbode cvode ida" + // Recorded and announced only once they are up: a report the database + // knows about is never generated again, so one that a later failure in + // this stage stops from being uploaded would be lost for good. + sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'history/**')])]) + sh './publish-reports.py' + sh "./report.py --branches='${env.GITBRANCHES} ${env.GITBRANCHES_WASM_JIT}' configs/conf.json configs/conf-old.json configs/conf-nonstandard.json" sh 'mv overview.html overview-combined.html' sh "./report.py --branches='${env.GITBRANCHES} ${env.GITBRANCHES_WASM_JIT}' configs/conf-old.json" @@ -569,7 +575,7 @@ pipeline { sh 'find overview*.html history -type f | wc -l' sh 'find overview*.html history' - sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'overview*.html,history/**')])]) + sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'overview*.html')])]) } } diff --git a/all-reports.py b/all-reports.py index 05a9ce8..4b203cf 100755 --- a/all-reports.py +++ b/all-reports.py @@ -17,6 +17,8 @@ parser.add_argument('--githuburltesting', default="https://github.com/OpenModelica/OpenModelicaLibraryTesting/commit") parser.add_argument('--omcgitdir', default="../OpenModelica/OpenModelica") parser.add_argument('--email', default=False, action='store_true') +parser.add_argument('--pending', default="pending-reports.json", + help="Where to leave this run's reports for publish-reports.py") resultsdb.addArgument(parser) args = parser.parse_args() @@ -30,6 +32,7 @@ githuburltesting = args.githuburltesting omcgitdir = args.omcgitdir doemail = args.email +pendingfile = args.pending if not os.path.exists(omcgitdir): raise Exception("Could not find OpenModelica.git directory, set it with --omcgitdir. Tried: %s" % omcgitdir) @@ -136,17 +139,16 @@ def renderEntry(branch, entry): def renderIndex(branch, entries, preamble): return "".join(line + "\n" for line in preamble + [renderEntry(branch, e) for e in entries]) +# The reports generated by this run. A row in [history] means the report is +# never generated again, so publish-reports.py writes them once they are up. +pending_entries = {} + def storedEntries(branch): - """The reports the database has for a branch, oldest first.""" - return [tuple(row) for row in cursor.execute( + """The reports there are for a branch, oldest first.""" + stored = [tuple(row) for row in cursor.execute( "SELECT date1,date2,fname,improved,regressions,perfimproved,perfregressions " "FROM history WHERE %s ORDER BY date1,date2" % db.likeNoCase("branch"), (branch,))] - -def storeEntries(branch, entries): - for entry in entries: - cursor.execute("INSERT INTO history (branch,date1,date2,fname,improved,regressions," - "perfimproved,perfregressions) VALUES (?,?,?,?,?,?,?,?)", (branch,) + entry) - db.commit() + return sorted(stored + pending_entries.get(branch, [])) historyRootServed = None @@ -214,7 +216,7 @@ def historyOf(branch, nruns): missing = [e for e in entries if (e[0], e[1]) not in known] if missing: print("Copying %d reports of %s from its index into the database" % (len(missing), branch)) - storeEntries(branch, missing) + db.insertHistory(branch, missing) stored = sorted(stored + missing) inindex = set((e[0], e[1]) for e in entries) onlyStored = [e for e in stored if (e[0], e[1]) not in inindex] @@ -412,7 +414,7 @@ def historyOf(branch, nruns): os.makedirs(historydir, exist_ok=True) with codecs.open(fname, "w", encoding="utf-8") as fout: fout.write(tpl) - storeEntries(branch, [entry]) + pending_entries.setdefault(branch, []).append(entry) reports = sorted(reports + [entry]) # The index is written whenever it does not already say what the database @@ -425,45 +427,10 @@ def historyOf(branch, nruns): with codecs.open(historyindex, "w", encoding="utf-8") as fout: fout.write(index) -if not doemail: - # We are done - sys.exit(0) - -# OK; send the emails :D -import smtplib - -from email.message import EmailMessage -from email.headerregistry import Address -from email.utils import make_msgid - -missing_plain = "" -missing_html = "" -if missing_branches: - missing_plain = ", ".join(missing_branches) - missing_plain = "Report asks for missing branches which we ignored: %s\n" % missing_plain - missing_html = ("%s %s %s" % ("

", missing_plain, " - - -%s -

The following reports contain regressions your account was involved with:

-%s - - -""" % (missing_html, "\n".join(reversed(emails_to_send[email]["html"]))), subtype='html') - with smtplib.SMTP('smtp.office365.com') as s: - s.starttls() - s.ehlo() - s.login(os.environ["IDA_EMAIL_USR"],os.environ["IDA_EMAIL_PSW"]) - s.send_message(msg) +pending = {"entries": {b: [list(e) for e in es] for (b, es) in pending_entries.items() if es}, + "emails": emails_to_send if doemail else {}, + "missing_branches": missing_branches} +with codecs.open(pendingfile, "w", encoding="utf-8") as fout: + json.dump(pending, fout, indent=1) +print("Generated %d reports, listed in %s for publish-reports.py to record after the upload" + % (sum(len(es) for es in pending["entries"].values()), pendingfile)) diff --git a/publish-reports.py b/publish-reports.py new file mode 100755 index 0000000..4016a50 --- /dev/null +++ b/publish-reports.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +"""Record the reports all-reports.py generated, and announce them. + +Run this after the upload, never before: a report the [history] table knows +about is never generated again, so a row written before the files are up turns +any later failure in the stage into a report that is lost for good. The emails +link to those files, so they are sent from here too - after the rows, since the +other order would mail the same regressions again on every rerun. +""" + +import argparse, os, sys +import simplejson as json +import resultsdb + +parser = argparse.ArgumentParser(description='Record and announce published library testing reports') +parser.add_argument('--pending', default="pending-reports.json", + help="The reports all-reports.py generated") +parser.add_argument('--keep', default=False, action='store_true', + help="Do not delete the pending file, so the run can be repeated") +resultsdb.addArgument(parser) +args = parser.parse_args() + +if not os.path.exists(args.pending): + print("%s does not exist; all-reports.py generated nothing to publish" % args.pending) + sys.exit(0) + +with open(args.pending) as fin: + pending = json.load(fin) + +entries = pending.get("entries", {}) +emails = pending.get("emails", {}) + +db = resultsdb.connect(args.db) +db.createHistoryTable() +for branch in sorted(entries.keys()): + print("Recording %d reports of %s" % (len(entries[branch]), branch)) + db.insertHistory(branch, entries[branch]) +db.close() + +if emails: + import smtplib + from email.message import EmailMessage + from email.headerregistry import Address + + missing_plain = "" + missing_html = "" + if pending.get("missing_branches"): + missing_plain = "Report asks for missing branches which we ignored: %s\n" % ", ".join(pending["missing_branches"]) + missing_html = ("%s %s %s" % ("

", missing_plain, " + + +%s +

The following reports contain regressions your account was involved with:

+%s + + +""" % (missing_html, "\n".join(reversed(emails[email]["html"]))), subtype='html') + with smtplib.SMTP('smtp.office365.com') as s: + s.starttls() + s.ehlo() + s.login(os.environ["IDA_EMAIL_USR"],os.environ["IDA_EMAIL_PSW"]) + s.send_message(msg) + print("Sent %d emails" % len(emails)) + +if not args.keep: + os.remove(args.pending) diff --git a/resultsdb.py b/resultsdb.py index 1c826a0..2d1a4fa 100644 --- a/resultsdb.py +++ b/resultsdb.py @@ -47,7 +47,8 @@ # reports, so it can be rebuilt from here when the published one is missing, # unreadable or out of date - and a run that cannot read it back from the web # server no longer has to choose between skipping the branch and publishing a -# history with only today's report in it. +# history with only today's report in it. A row says the report is published, +# so publish-reports.py writes it after the upload. HISTORY_COLUMNS = [ ("branch", "text"), ("date1", "bigint"), ("date2", "bigint"), ("fname", "text"), ("improved", "int"), ("regressions", "int"), @@ -194,6 +195,20 @@ def createHistoryTable(self): % (cols, ", ".join(HISTORY_KEY))) self.commit() + def insertHistory(self, branch, entries): + """Record reports that are on the web server. + + A report the table knows about is never generated again, so do not call + this until the files are up. + """ + cursor = self.cursor() + for entry in entries: + cursor.execute("INSERT INTO history (%s) VALUES (%s)%s" + % (",".join(c for (c, _) in HISTORY_COLUMNS), + ",".join(["?"] * len(HISTORY_COLUMNS)), self.insertIgnore()), + (branch,) + tuple(entry)) + self.commit() + def createDateIndex(self, branch): """The index the report queries need; test.py drops it before a run.""" self.execute("CREATE INDEX IF NOT EXISTS %s ON %s (date)"