Record reports after the upload, not before

.CI/Jenkinsfile

@@ -477,7 +477,7 @@ pipeline {
         PGPASSFILE = credentials('omdb-pgpass')
       }
       steps {
-        sh 'rm -rf *.html history'
+        sh 'rm -rf *.html history pending-reports.json'
         sh '''
         if ! test -d OpenModelica; then
           git clone https://openmodelica.org/git-readonly/OpenModelica.git
@@ -490,6 +490,12 @@ pipeline {
         sh "./all-reports.py --email --omcgitdir=OpenModelica ${env.GITBRANCHES} ${env.GITBRANCHES_FMI} ${env.GITBRANCHES_NEWINST} ${env.GITBRANCHES_DAE} ${env.GITBRANCHES_NEWBACKEND_DAE} ${env.GITBRANCHES_CPP} ${env.GITBRANCHES_WASM_JIT} ${env.GITBRANCHES_C_PLUS_RUST} conversion heavy_tests generateSymbolicJacobian gbode cvode ida"
         sh "./all-plots.py ${env.GITBRANCHES} ${env.GITBRANCHES_FMI} ${env.GITBRANCHES_NEWINST} ${env.GITBRANCHES_DAE} ${env.GITBRANCHES_NEWBACKEND_DAE} ${env.GITBRANCHES_CPP} ${env.GITBRANCHES_WASM_JIT} ${env.GITBRANCHES_C_PLUS_RUST} conversion heavy_tests generateSymbolicJacobian gbode cvode ida"
 
+        // Recorded and announced only once they are up: a report the database
+        // knows about is never generated again, so one that a later failure in
+        // this stage stops from being uploaded would be lost for good.
+        sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'history/**')])])
+        sh './publish-reports.py'
+
         sh "./report.py --branches='${env.GITBRANCHES} ${env.GITBRANCHES_WASM_JIT}' configs/conf.json configs/conf-old.json configs/conf-nonstandard.json"
         sh 'mv overview.html overview-combined.html'
         sh "./report.py --branches='${env.GITBRANCHES} ${env.GITBRANCHES_WASM_JIT}' configs/conf-old.json"
@@ -569,7 +575,7 @@ pipeline {
         sh 'find overview*.html history -type f | wc -l'
         sh 'find overview*.html history'
 
-        sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'overview*.html,history/**')])])
+        sshPublisher(publishers: [sshPublisherDesc(configName: 'LibraryTestingReports', transfers: [sshTransfer(sourceFiles: 'overview*.html')])])
       }
     }
 

all-reports.py

@@ -17,6 +17,8 @@
 parser.add_argument('--githuburltesting', default="https://github.com/OpenModelica/OpenModelicaLibraryTesting/commit")
 parser.add_argument('--omcgitdir', default="../OpenModelica/OpenModelica")
 parser.add_argument('--email', default=False, action='store_true')
+parser.add_argument('--pending', default="pending-reports.json",
+                    help="Where to leave this run's reports for publish-reports.py")
 resultsdb.addArgument(parser)
 args = parser.parse_args()
 
@@ -30,6 +32,7 @@
 githuburltesting = args.githuburltesting
 omcgitdir = args.omcgitdir
 doemail = args.email
+pendingfile = args.pending
 
 if not os.path.exists(omcgitdir):
   raise Exception("Could not find OpenModelica.git directory, set it with --omcgitdir. Tried: %s" % omcgitdir)
@@ -136,17 +139,16 @@ def renderEntry(branch, entry):
 def renderIndex(branch, entries, preamble):
   return "".join(line + "\n" for line in preamble + [renderEntry(branch, e) for e in entries])
 
+# The reports generated by this run. A row in [history] means the report is
+# never generated again, so publish-reports.py writes them once they are up.
+pending_entries = {}
+
 def storedEntries(branch):
-  """The reports the database has for a branch, oldest first."""
-  return [tuple(row) for row in cursor.execute(
+  """The reports there are for a branch, oldest first."""
+  stored = [tuple(row) for row in cursor.execute(
       "SELECT date1,date2,fname,improved,regressions,perfimproved,perfregressions "
       "FROM history WHERE %s ORDER BY date1,date2" % db.likeNoCase("branch"), (branch,))]
-
-def storeEntries(branch, entries):
-  for entry in entries:
-    cursor.execute("INSERT INTO history (branch,date1,date2,fname,improved,regressions,"
-                   "perfimproved,perfregressions) VALUES (?,?,?,?,?,?,?,?)", (branch,) + entry)
-  db.commit()
+  return sorted(stored + pending_entries.get(branch, []))
 
 historyRootServed = None
 
@@ -214,7 +216,7 @@ def historyOf(branch, nruns):
   missing = [e for e in entries if (e[0], e[1]) not in known]
   if missing:
     print("Copying %d reports of %s from its index into the database" % (len(missing), branch))
-    storeEntries(branch, missing)
+    db.insertHistory(branch, missing)
     stored = sorted(stored + missing)
   inindex = set((e[0], e[1]) for e in entries)
   onlyStored = [e for e in stored if (e[0], e[1]) not in inindex]
@@ -412,7 +414,7 @@ def historyOf(branch, nruns):
     os.makedirs(historydir, exist_ok=True)
     with codecs.open(fname, "w", encoding="utf-8") as fout:
       fout.write(tpl)
-    storeEntries(branch, [entry])
+    pending_entries.setdefault(branch, []).append(entry)
     reports = sorted(reports + [entry])
 
   # The index is written whenever it does not already say what the database
@@ -425,45 +427,10 @@ def historyOf(branch, nruns):
     with codecs.open(historyindex, "w", encoding="utf-8") as fout:
       fout.write(index)
 
-if not doemail:
-  # We are done
-  sys.exit(0)
-
-# OK; send the emails :D
-import smtplib
-
-from email.message import EmailMessage
-from email.headerregistry import Address
-from email.utils import make_msgid
-
-missing_plain = ""
-missing_html = ""
-if missing_branches:
-  missing_plain = ", ".join(missing_branches)
-  missing_plain = "Report asks for missing branches which we ignored: %s\n" % missing_plain
-  missing_html = ("%s %s %s" % ("<p style=\"color:red;\">", missing_plain, "</p"))
-
-for email in sorted(emails_to_send.keys()):
-  msg = EmailMessage()
-  msg['Subject'] = 'OpenModelica Library Testing Regressions'
-  msg['From'] = Address("OM Hudson", "openmodelicabuilds", "ida.liu.se")
-  msg['To'] = email
-  msg.set_content("""\
-%s
-The following reports contain regressions your account was involved with:
-""" % missing_plain + "\n".join(reversed(emails_to_send[email]["plain"])))
-  msg.add_alternative("""\
-<html lang="en">
-<head></head>
-<body>
-%s
-<p>The following reports contain regressions your account was involved with:</p>
-%s
-</body>
-</html>
-""" % (missing_html, "\n".join(reversed(emails_to_send[email]["html"]))), subtype='html')
-  with smtplib.SMTP('smtp.office365.com') as s:
-    s.starttls()
-    s.ehlo()
-    s.login(os.environ["IDA_EMAIL_USR"],os.environ["IDA_EMAIL_PSW"])
-    s.send_message(msg)
+pending = {"entries": {b: [list(e) for e in es] for (b, es) in pending_entries.items() if es},
+           "emails": emails_to_send if doemail else {},
+           "missing_branches": missing_branches}
+with codecs.open(pendingfile, "w", encoding="utf-8") as fout:
+  json.dump(pending, fout, indent=1)
+print("Generated %d reports, listed in %s for publish-reports.py to record after the upload"
+      % (sum(len(es) for es in pending["entries"].values()), pendingfile))

publish-reports.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""Record the reports all-reports.py generated, and announce them.
+
+Run this after the upload, never before: a report the [history] table knows
+about is never generated again, so a row written before the files are up turns
+any later failure in the stage into a report that is lost for good. The emails
+link to those files, so they are sent from here too - after the rows, since the
+other order would mail the same regressions again on every rerun.
+"""
+
+import argparse, os, sys
+import simplejson as json
+import resultsdb
+
+parser = argparse.ArgumentParser(description='Record and announce published library testing reports')
+parser.add_argument('--pending', default="pending-reports.json",
+                    help="The reports all-reports.py generated")
+parser.add_argument('--keep', default=False, action='store_true',
+                    help="Do not delete the pending file, so the run can be repeated")
+resultsdb.addArgument(parser)
+args = parser.parse_args()
+
+if not os.path.exists(args.pending):
+  print("%s does not exist; all-reports.py generated nothing to publish" % args.pending)
+  sys.exit(0)
+
+with open(args.pending) as fin:
+  pending = json.load(fin)
+
+entries = pending.get("entries", {})
+emails = pending.get("emails", {})
+
+db = resultsdb.connect(args.db)
+db.createHistoryTable()
+for branch in sorted(entries.keys()):
+  print("Recording %d reports of %s" % (len(entries[branch]), branch))
+  db.insertHistory(branch, entries[branch])
+db.close()
+
+if emails:
+  import smtplib
+  from email.message import EmailMessage
+  from email.headerregistry import Address
+
+  missing_plain = ""
+  missing_html = ""
+  if pending.get("missing_branches"):
+    missing_plain = "Report asks for missing branches which we ignored: %s\n" % ", ".join(pending["missing_branches"])
+    missing_html = ("%s %s %s" % ("<p style=\"color:red;\">", missing_plain, "</p"))
+
+  for email in sorted(emails.keys()):
+    msg = EmailMessage()
+    msg['Subject'] = 'OpenModelica Library Testing Regressions'
+    msg['From'] = Address("OM Hudson", "openmodelicabuilds", "ida.liu.se")
+    msg['To'] = email
+    msg.set_content("""\
+%s
+The following reports contain regressions your account was involved with:
+""" % missing_plain + "\n".join(reversed(emails[email]["plain"])))
+    msg.add_alternative("""\
+<html lang="en">
+<head></head>
+<body>
+%s
+<p>The following reports contain regressions your account was involved with:</p>
+%s
+</body>
+</html>
+""" % (missing_html, "\n".join(reversed(emails[email]["html"]))), subtype='html')
+    with smtplib.SMTP('smtp.office365.com') as s:
+      s.starttls()
+      s.ehlo()
+      s.login(os.environ["IDA_EMAIL_USR"],os.environ["IDA_EMAIL_PSW"])
+      s.send_message(msg)
+  print("Sent %d emails" % len(emails))
+
+if not args.keep:
+  os.remove(args.pending)

resultsdb.py

@@ -47,7 +47,8 @@
 # reports, so it can be rebuilt from here when the published one is missing,
 # unreadable or out of date - and a run that cannot read it back from the web
 # server no longer has to choose between skipping the branch and publishing a
-# history with only today's report in it.
+# history with only today's report in it.  A row says the report is published,
+# so publish-reports.py writes it after the upload.
 HISTORY_COLUMNS = [
     ("branch", "text"), ("date1", "bigint"), ("date2", "bigint"), ("fname", "text"),
     ("improved", "int"), ("regressions", "int"),
@@ -194,6 +195,20 @@ def createHistoryTable(self):
                  % (cols, ", ".join(HISTORY_KEY)))
     self.commit()
 
+  def insertHistory(self, branch, entries):
+    """Record reports that are on the web server.
+
+    A report the table knows about is never generated again, so do not call
+    this until the files are up.
+    """
+    cursor = self.cursor()
+    for entry in entries:
+      cursor.execute("INSERT INTO history (%s) VALUES (%s)%s"
+                     % (",".join(c for (c, _) in HISTORY_COLUMNS),
+                        ",".join(["?"] * len(HISTORY_COLUMNS)), self.insertIgnore()),
+                     (branch,) + tuple(entry))
+    self.commit()
+
   def createDateIndex(self, branch):
     """The index the report queries need; test.py drops it before a run."""
     self.execute("CREATE INDEX IF NOT EXISTS %s ON %s (date)"