write images in logs & use col("column") syntax for columns

frontend/src/components/ProgressView.tsx

@@ -4,16 +4,41 @@ import Box from "@mui/material/Box";
 import Button from "@mui/material/Button";
 import LinearProgress from "@mui/material/LinearProgress";
 import Paper from "@mui/material/Paper";
+import Slider from "@mui/material/Slider";
 import Typography from "@mui/material/Typography";
 import { cancelRun } from "../api";
 
 type StreamEvent =
   | { type: "progress"; percent: number }
   | { type: "log"; message: string }
+  | {
+      type: "image";
+      data_url: string;
+      caption: string | null;
+      timestamp: string;
+    }
   | { type: "error"; message: string }
   | { type: "done"; message: string }
   | { type: "cancelled"; message: string };
 
+type StreamImage = {
+  dataUrl: string;
+  caption: string | null;
+  timestamp: string;
+};
+
+function formatStreamTime(iso: string): string {
+  return new Date(iso).toLocaleTimeString(undefined, {
+    hour: "2-digit",
+    minute: "2-digit",
+    second: "2-digit",
+  });
+}
+
+function isNearBottom(el: HTMLElement, threshold = 48) {
+  return el.scrollHeight - el.scrollTop - el.clientHeight <= threshold;
+}
+
 export function ProgressView({
   runId,
   onReset,
@@ -23,22 +48,55 @@ export function ProgressView({
 }) {
   const [percent, setPercent] = useState(0);
   const [logs, setLogs] = useState<string[]>([]);
+  const [images, setImages] = useState<StreamImage[]>([]);
+  const [imageIndex, setImageIndex] = useState(0);
   const [done, setDone] = useState<string | null>(null);
   const [cancelled, setCancelled] = useState<string | null>(null);
   const [error, setError] = useState<string | null>(null);
   const [cancelPending, setCancelPending] = useState(false);
   const [cancelError, setCancelError] = useState<string | null>(null);
-  const bottomRef = useRef<HTMLDivElement>(null);
+  const logContainerRef = useRef<HTMLDivElement>(null);
+  const stickToBottomRef = useRef(true);
+  const stickToLatestImageRef = useRef(true);
+
+  function handleLogScroll() {
+    const el = logContainerRef.current;
+    if (!el) return;
+    stickToBottomRef.current = isNearBottom(el);
+  }
+
+  function handleImageSliderChange(_: Event, value: number | number[]) {
+    const index = value as number;
+    setImageIndex(index);
+    stickToLatestImageRef.current = index === images.length - 1;
+  }
 
   useEffect(() => {
+    stickToBottomRef.current = true;
+    stickToLatestImageRef.current = true;
     const es = new EventSource(`/api/runs/${runId}/stream`);
     es.onmessage = (e) => {
       try {
         const ev = JSON.parse(e.data) as StreamEvent;
         if (ev.type === "progress")
           setPercent(Math.min(100, Math.max(0, Math.ceil(ev.percent))));
         else if (ev.type === "log") setLogs((x) => [...x, ev.message]);
-        else if (ev.type === "error") {
+        else if (ev.type === "image") {
+          setImages((prev) => {
+            const next = [
+              ...prev,
+              {
+                dataUrl: ev.data_url,
+                caption: ev.caption,
+                timestamp: ev.timestamp,
+              },
+            ];
+            if (stickToLatestImageRef.current) {
+              setImageIndex(next.length - 1);
+            }
+            return next;
+          });
+        } else if (ev.type === "error") {
           setError(ev.message);
           es.close();
         } else if (ev.type === "cancelled") {
@@ -57,10 +115,13 @@ export function ProgressView({
   }, [runId]);
 
   useEffect(() => {
-    bottomRef.current?.scrollIntoView({ behavior: "smooth" });
+    const el = logContainerRef.current;
+    if (!el || !stickToBottomRef.current) return;
+    el.scrollTop = el.scrollHeight;
   }, [logs]);
 
   const canCancel = !done && !error && !cancelled && !cancelPending;
+  const currentImage = images[imageIndex];
 
   return (
     <Box sx={{ maxWidth: 720 }}>
@@ -80,14 +141,6 @@ export function ProgressView({
           {error}
         </Alert>
       )}
-      {done && (
-        <Alert
-          severity="success"
-          sx={{ mb: 2, whiteSpace: "pre-wrap", fontFamily: "monospace" }}
-        >
-          {done}
-        </Alert>
-      )}
       {cancelled && (
         <Alert
           severity="warning"
@@ -101,7 +154,49 @@ export function ProgressView({
           {cancelError}
         </Alert>
       )}
+      <Paper variant="outlined" sx={{ p: 2, mb: 2 }}>
+        {currentImage ? (
+          <>
+            <Box
+              component="img"
+              src={currentImage.dataUrl}
+              sx={{ maxWidth: "100%", display: "block" }}
+            />
+            {currentImage.caption && (
+              <Typography variant="caption" display="block" sx={{ mt: 1 }}>
+                {currentImage.caption}
+              </Typography>
+            )}
+            {images.length > 1 && (
+              <Slider
+                sx={{ mt: 2 }}
+                min={0}
+                max={images.length - 1}
+                step={1}
+                value={imageIndex}
+                onChange={handleImageSliderChange}
+                valueLabelDisplay="auto"
+                valueLabelFormat={(i) => formatStreamTime(images[i].timestamp)}
+              />
+            )}
+          </>
+        ) : (
+          <Typography variant="body2" color="text.secondary">
+            No charts yet
+          </Typography>
+        )}
+      </Paper>
+      {done && (
+        <Alert
+          severity="success"
+          sx={{ mb: 2, whiteSpace: "pre-wrap", fontFamily: "monospace" }}
+        >
+          {done}
+        </Alert>
+      )}
       <Paper
+        ref={logContainerRef}
+        onScroll={handleLogScroll}
         variant="outlined"
         sx={{
           p: 2,
@@ -115,7 +210,6 @@ export function ProgressView({
         {logs.map((line, i) => (
           <div key={i}>{line}</div>
         ))}
-        <div ref={bottomRef} />
       </Paper>
       <Box sx={{ mt: 2, display: "flex", gap: 1 }}>
         <Button

pyproject.toml

@@ -14,6 +14,7 @@ dependencies = [
     "openapi-python-client>=0.27.1",
     "pandas>=2.3.3",
     "numpy>=2.3.4",
+    "matplotlib>=3.9",
     "pyvo>=1.8",
     "psycopg[binary]>=3.2.0",
     "click~=8.3.1",

uploader/app/lib/expression.py

@@ -8,24 +8,25 @@
 import astropy.units as u
 import numpy as np
 
-CONST_PREFIX = "const_"
+COL_FUNCTION = "col"
 
 NAMED_CONSTANTS: dict[str, u.Quantity] = {
-    "const_pi": np.pi * u.dimensionless_unscaled,
-    "const_c": const.c,
-    "const_deg": 1 * u.deg,
-    "const_rad": 1 * u.rad,
-    "const_arcmin": 1 * u.arcmin,
-    "const_arcsec": 1 * u.arcsec,
-    "const_mag": 1 * u.mag,
+    "pi": np.pi * u.dimensionless_unscaled,
+    "c": const.c,
+    "deg": 1 * u.deg,
+    "rad": 1 * u.rad,
+    "arcmin": 1 * u.arcmin,
+    "arcsec": 1 * u.arcsec,
+    "mag": 1 * u.mag,
 }
 
 
 def expression_syntax_help() -> str:
     constants = ", ".join(sorted(NAMED_CONSTANTS))
     return (
-        "Bare identifiers refer to rawdata column names.\n"
-        "Identifiers starting with const_ refer to predefined constants.\n"
+        f'Use {COL_FUNCTION}("name") to refer to rawdata columns '
+        '(e.g. col("a"), col("SMASB22.5"), col("PA-LEDA")).\n'
+        "Bare identifiers refer to predefined constants.\n"
         "Operators: + - * /.\n"
         "Functions: sin(x), cos(x) (argument must be an angle).\n"
         "Numbers are dimensionless.\n"
@@ -55,6 +56,17 @@ def expression_syntax_help() -> str:
 }
 
 
+def _column_from_call(node: ast.Call) -> str | None:
+    if not isinstance(node.func, ast.Name) or node.func.id != COL_FUNCTION:
+        return None
+    if node.keywords or len(node.args) != 1:
+        raise ValueError(f"{COL_FUNCTION}() takes exactly one string argument")
+    arg = node.args[0]
+    if not isinstance(arg, ast.Constant) or not isinstance(arg.value, str):
+        raise ValueError(f"{COL_FUNCTION}() argument must be a string literal")
+    return arg.value
+
+
 @final
 @dataclass
 class Expression:
@@ -85,13 +97,13 @@ def collect(self, node: ast.AST) -> set[str]:
         return self.columns
 
     def visit_Call(self, node: ast.Call) -> None:
+        column = _column_from_call(node)
+        if column is not None:
+            self.columns.add(column)
+            return
         for arg in node.args:
             self.visit(arg)
 
-    def visit_Name(self, node: ast.Name) -> None:
-        if not node.id.startswith(CONST_PREFIX):
-            self.columns.add(node.id)
-
 
 @final
 class _Evaluator(ast.NodeVisitor):
@@ -105,10 +117,10 @@ def visit(self, node: ast.AST) -> u.Quantity:
                 return self._binop(left, op, right)
             case ast.UnaryOp(op=op, operand=operand):
                 return self._unaryop(op, operand)
-            case ast.Call(func=func, args=args, keywords=keywords):
-                return self._call(func, args, keywords)
+            case ast.Call() as call:
+                return self._call(call)
             case ast.Name(id=name):
-                return self._name(name)
+                return self._lookup_constant(name)
             case ast.Constant(value=value):
                 return self._constant(value)
             case _:
@@ -126,28 +138,32 @@ def _unaryop(self, op: ast.unaryop, operand: ast.AST) -> u.Quantity:
             raise ValueError(f"unsupported unary operator: {op_type.__name__}")
         return _UNARYOPS[op_type](self.visit(operand))
 
-    def _call(self, func: ast.AST, args: list[ast.AST], keywords: list[ast.keyword]) -> u.Quantity:
-        if keywords:
+    def _call(self, node: ast.Call) -> u.Quantity:
+        column = _column_from_call(node)
+        if column is not None:
+            return self._lookup_column(column)
+        if node.keywords:
             raise ValueError("keyword arguments are not allowed")
-        if not isinstance(func, ast.Name):
+        if not isinstance(node.func, ast.Name):
             raise ValueError("only simple function calls are allowed")
-        fn = _FUNCTIONS.get(func.id)
+        fn = _FUNCTIONS.get(node.func.id)
         if fn is None:
-            raise ValueError(f"unknown function: {func.id}")
-        if len(args) != 1:
-            raise ValueError(f"{func.id}() takes exactly one argument")
-        arg = self.visit(args[0]).to(u.rad)
+            raise ValueError(f"unknown function: {node.func.id}")
+        if len(node.args) != 1:
+            raise ValueError(f"{node.func.id}() takes exactly one argument")
+        arg = self.visit(node.args[0]).to(u.rad)
         result = fn(arg)
         if isinstance(result, u.Quantity):
             return result
         return float(result) * u.dimensionless_unscaled
 
-    def _name(self, name: str) -> u.Quantity:
-        if name.startswith(CONST_PREFIX):
-            constant = NAMED_CONSTANTS.get(name)
-            if constant is None:
-                raise ValueError(f"unknown constant {name!r}")
-            return constant
+    def _lookup_constant(self, name: str) -> u.Quantity:
+        constant = NAMED_CONSTANTS.get(name)
+        if constant is None:
+            raise ValueError(f"unknown constant {name!r}")
+        return constant
+
+    def _lookup_column(self, name: str) -> u.Quantity:
         if name not in self._values:
             raise ValueError(f"unknown column {name!r}")
         unit_str = self._units.get(name, "")

uploader/app/report.py

@@ -1,5 +1,13 @@
+import base64
+import io
 from dataclasses import dataclass
 
+import matplotlib
+import matplotlib.figure
+import matplotlib.pyplot as plt
+
+matplotlib.use("Agg")
+
 
 @dataclass(frozen=True)
 class LogEvent:
@@ -21,4 +29,24 @@ class ErrorEvent:
     message: str
 
 
-Event = LogEvent | ProgressEvent | DoneEvent | ErrorEvent
+@dataclass(frozen=True)
+class ImageEvent:
+    data_url: str
+    caption: str | None = None
+
+
+Event = LogEvent | ProgressEvent | DoneEvent | ErrorEvent | ImageEvent
+
+
+def image_event_from_figure(
+    fig: matplotlib.figure.Figure,
+    caption: str | None = None,
+    dpi: int = 120,
+) -> ImageEvent:
+    buf = io.BytesIO()
+    fig.savefig(buf, format="png", dpi=dpi, bbox_inches="tight")
+    plt.close(fig)
+    return ImageEvent(
+        data_url=f"data:image/png;base64,{base64.b64encode(buf.getvalue()).decode('ascii')}",
+        caption=caption,
+    )