OpenFOAM: a response row at the run's final time only; a utility refused as a utility

OpenFOAM/OutWit.Controller.OpenFOAM.Model/OutWit.Controller.OpenFOAM.Model.csproj

@@ -5,7 +5,7 @@
   <PropertyGroup>
     <ImplicitUsings>enable</ImplicitUsings>
     <ControllerName>OpenFOAM</ControllerName>
-    <Version>1.3.0</Version>
+    <Version>1.3.1</Version>
     <Description>Shared data types of the OpenFOAM case controller and the rules a case obeys: the case (base files, allow-listed recipe, response request, artifact policy), one variant's run (the case and its token values) and its outcome (per-step exit codes and times, convergence facts, the extracted response row, the artifact reference); the allow-list, recipe grammar, case-path, response, token-coverage and run-time-code rules, byte-exact substitution and the solver-class vocabulary, shared by the node, the Sweep host and client preflight. Consumed by the node controller, the Sweep orchestration controller and client applications.</Description>
     <PackageTags>openfoam;cfd;case;sweep;parameter-study;witengine;witcloud;controller;data-types</PackageTags>
   </PropertyGroup>

OpenFOAM/OutWit.Controller.OpenFOAM.Model/Rules/FoamRecipeRules.cs

@@ -57,6 +57,9 @@ public static IReadOnlyList<string> Validate(FoamRecipeData? recipe, Func<string
 
         if (string.IsNullOrEmpty(recipe.Application))
             findings.Add("The recipe names no application.");
+        else if (FoamAllowList.IsUtility(recipe.Application))
+            findings.Add($"'{recipe.Application}' is a utility on the allow-list, not a solver: the application is the solver the case runs " +
+                         $"(controlDict's 'application', such as simpleFoam); '{recipe.Application}' runs as a step of its own.");
         else if (!FoamAllowList.IsSolverName(recipe.Application))
             findings.Add($"'{recipe.Application}' is not a solver name (a solver's name ends in 'Foam').");
         else if (hasExecutable != null && !hasExecutable(recipe.Application))

OpenFOAM/OutWit.Controller.OpenFOAM.Tests/Extraction/FoamResponseExtractorTests.cs

@@ -72,6 +72,51 @@ public void ValuesAreNamedByResponseAndColumnAndTheTimeColumnIsSkippedTest()
         Assert.That(names.Count(name => name.StartsWith("absent", StringComparison.Ordinal)), Is.EqualTo(0), "a response whose function object wrote nothing contributes nothing");
     }
 
+    [Test]
+    public void ARowBeforeTheFinalTimeIsNotReportedTest()
+    {
+        // A function object that stopped writing before the run ended: its
+        // last row is not the final state, and a stale value is worse than none.
+        Write("postProcessing/coeffs/0/coefficient.dat", COEFFICIENT_DAT);
+        var request = new FoamExtractionRequestData { Responses = [new FoamResponseSpecData { Name = "coeffs", Kind = FoamResponseKind.ForceCoeffs }] };
+        var notes = new List<string>();
+
+        var row = FoamResponseExtractor.Extract(m_case, request, 250, notes);
+
+        Assert.That(row.Values, Is.Empty);
+        Assert.That(notes, Is.EqualTo(new[] { "coeffs: the last row of coefficient.dat is at time 200, the run ended at 250 - not reported." }));
+    }
+
+    [Test]
+    public void ARowAtTheFinalTimeIsReportedTest()
+    {
+        Write("postProcessing/coeffs/0/coefficient.dat", COEFFICIENT_DAT);
+        Write("postProcessing/probes/0/p", PROBES_P);
+        var request = new FoamExtractionRequestData
+        {
+            Responses =
+            [
+                new FoamResponseSpecData { Name = "coeffs", Kind = FoamResponseKind.ForceCoeffs },
+                new FoamResponseSpecData { Name = "probes", Kind = FoamResponseKind.Probe, Fields = ["p"] }
+            ]
+        };
+        var notes = new List<string>();
+
+        // Transient times carry the last digit's noise of the time loop.
+        Assert.That(FoamResponseExtractor.Extract(m_case, request, 200, notes).Values.Select(value => value.Name), Does.Contain("coeffs.Cd"));
+        Assert.That(FoamResponseExtractor.Extract(m_case, request, 0.20000000000000004, notes).Values.Select(value => value.Name), Does.Contain("probes.c1"));
+        Assert.That(notes, Has.Count.EqualTo(2), "each call judges its own final time: coeffs at 200 is stale against 0.2, probes at 0.2 against 200");
+    }
+
+    [Test]
+    public void TheNotesAreWrittenBesideTheStepLogsTest()
+    {
+        FoamResponseExtractor.WriteNotes(m_case, ["coeffs: the last row of coefficient.dat is at time 200, the run ended at 250 - not reported."]);
+        FoamResponseExtractor.WriteNotes(m_case, []);
+
+        Assert.That(File.ReadAllText(Path.Combine(m_case, "log.responses")), Does.Contain("coeffs: the last row"));
+    }
+
     [Test]
     public void AMultiFieldMinMaxRowIsReadWholeTest()
     {

OpenFOAM/OutWit.Controller.OpenFOAM.Tests/Model/Rules/FoamRecipeRulesTests.cs

@@ -120,13 +120,29 @@ public void AParallelStepOnASerialUtilityIsRefusedTest()
     public void TheApplicationMustBeASolverAndMustRunTest()
     {
         var recipe = MotorBike();
-        recipe.Application = "blockMesh";
-        Assert.That(FoamRecipeRules.Validate(recipe), Has.Some.Contains("not a solver name"));
+        recipe.Application = "myFlowSolver";
+        Assert.That(FoamRecipeRules.Validate(recipe), Has.Some.EqualTo("'myFlowSolver' is not a solver name (a solver's name ends in 'Foam')."));
 
         recipe.Application = "pisoFoam";
         Assert.That(FoamRecipeRules.Validate(recipe), Has.Some.Contains("No step runs the application 'pisoFoam'"));
     }
 
+    [Test]
+    public void AUtilityAsTheApplicationIsNamedAsAUtilityTest()
+    {
+        // potentialFoam ends in 'Foam' like a solver: the reason must be
+        // what it is, not the shape of its name.
+        var recipe = MotorBike();
+        recipe.Application = "potentialFoam";
+
+        Assert.That(FoamRecipeRules.Validate(recipe), Has.Some.EqualTo(
+            "'potentialFoam' is a utility on the allow-list, not a solver: the application is the solver the case runs " +
+            "(controlDict's 'application', such as simpleFoam); 'potentialFoam' runs as a step of its own."));
+
+        recipe.Application = "blockMesh";
+        Assert.That(FoamRecipeRules.Validate(recipe), Has.Some.StartsWith("'blockMesh' is a utility on the allow-list, not a solver"));
+    }
+
     [Test]
     public void NegativeNumbersAreValuesNotFlagsTest()
     {

OpenFOAM/OutWit.Controller.OpenFOAM/Extraction/FoamResponseExtractor.cs

@@ -9,7 +9,10 @@ namespace OutWit.Controller.OpenFOAM.Extraction;
 /// its latest time directory, each column a value named
 /// <c>&lt;response&gt;.&lt;column&gt;</c> (with the file's name in between when
 /// a function object writes more than one file). The time column is left out;
-/// it is the same for every column and the result carries it once.
+/// it is the same for every column and the result carries it once. Given the
+/// run's final time, a file whose last row is earlier - a function object
+/// that stopped writing before the run ended - is not reported: a stale value
+/// is worse than none, and a note names it.
 /// </summary>
 public static class FoamResponseExtractor
 {
@@ -19,6 +22,11 @@ public static class FoamResponseExtractor
 
     private const string TIME_COLUMN = "Time";
 
+    private const string NOTES_FILE = "log.responses";
+
+    /// <summary>Relative tolerance of "the same time": a transient time loop carries its last digit's noise.</summary>
+    private const double TIME_TOLERANCE = 1e-9;
+
     #endregion
 
     #region Functions
@@ -30,6 +38,19 @@ public static class FoamResponseExtractor
     /// <param name="request">The request; null yields an empty row.</param>
     /// <returns>The row; a response whose files are missing contributes nothing.</returns>
     public static FoamResponseRowData Extract(string caseDirectory, FoamExtractionRequestData? request)
+    {
+        return Extract(caseDirectory, request, null, null);
+    }
+
+    /// <summary>
+    /// Extracts the requested responses whose last row is at the run's final time.
+    /// </summary>
+    /// <param name="caseDirectory">The case root.</param>
+    /// <param name="request">The request; null yields an empty row.</param>
+    /// <param name="finalTime">The time the run ended at; null skips the check.</param>
+    /// <param name="notes">Receives one sentence per file left out; null discards them.</param>
+    /// <returns>The row; a response whose files are missing or stale contributes nothing.</returns>
+    public static FoamResponseRowData Extract(string caseDirectory, FoamExtractionRequestData? request, double? finalTime, ICollection<string>? notes)
     {
         var row = new FoamResponseRowData();
         if (request == null)
@@ -51,6 +72,13 @@ public static FoamResponseRowData Extract(string caseDirectory, FoamExtractionRe
                 if (table == null)
                     continue;
 
+                if (finalTime is { } final && IsStale(table.Value.Columns, table.Value.Values, final, out var time))
+                {
+                    notes?.Add(string.Create(CultureInfo.InvariantCulture,
+                        $"{response.Name}: the last row of {Path.GetFileName(file)} is at time {time}, the run ended at {final} - not reported."));
+                    continue;
+                }
+
                 var stem = Path.GetFileNameWithoutExtension(file);
                 var prefix = files.Count > 1 ? $"{response.Name}.{stem}" : response.Name;
 
@@ -72,6 +100,20 @@ public static FoamResponseRowData Extract(string caseDirectory, FoamExtractionRe
         return row;
     }
 
+    /// <summary>
+    /// Writes the notes of an extraction beside the step logs (<c>log.responses</c>),
+    /// where they travel with the logs; no notes, no file.
+    /// </summary>
+    /// <param name="caseDirectory">The case root.</param>
+    /// <param name="notes">The notes of <see cref="Extract(string, FoamExtractionRequestData?, double?, ICollection{string}?)"/>.</param>
+    public static void WriteNotes(string caseDirectory, IReadOnlyCollection<string> notes)
+    {
+        if (notes.Count == 0)
+            return;
+
+        File.WriteAllLines(Path.Combine(caseDirectory, NOTES_FILE), notes);
+    }
+
     /// <summary>
     /// The time directory with the largest numeric name under a function
     /// object's output directory.
@@ -103,4 +145,19 @@ public static FoamResponseRowData Extract(string caseDirectory, FoamExtractionRe
     }
 
     #endregion
+
+    #region Tools
+
+    // A last row whose time column is not the final time.
+    private static bool IsStale(IReadOnlyList<string> columns, IReadOnlyList<double> values, double finalTime, out double time)
+    {
+        time = double.NaN;
+        if (columns.Count == 0 || values.Count == 0 || !string.Equals(columns[0], TIME_COLUMN, StringComparison.OrdinalIgnoreCase))
+            return false;
+
+        time = values[0];
+        return System.Math.Abs(time - finalTime) > TIME_TOLERANCE * System.Math.Max(1.0, System.Math.Abs(finalTime));
+    }
+
+    #endregion
 }

OpenFOAM/OutWit.Controller.OpenFOAM/OutWit.Controller.OpenFOAM.csproj

@@ -4,7 +4,7 @@
 
   <PropertyGroup>
     <ControllerName>OpenFOAM</ControllerName>
-    <Version>1.0.5</Version>
+    <Version>1.0.6</Version>
     <Description>Runs complete OpenFOAM cases on compute nodes: each run is one whole case on one node - meshing, decomposition, the solver, the post step - through an allow-listed recipe, the unit of parameter-study throughput. Carries pinned OpenFOAM v2606 kits for every supported node platform as controller assets, so nodes need no preinstalled OpenFOAM (parallel steps on Windows additionally need Microsoft MPI on the node); responses are read from the finished case right on the node. OpenFOAM is a registered trademark of OpenCFD Limited; this offering is not approved or endorsed by OpenCFD Limited.</Description>
     <PackageTags>openfoam;cfd;solver;simulation;witengine;witcloud;controller</PackageTags>
     <ControllerFeatures>foam-run</ControllerFeatures>

OpenFOAM/OutWit.Controller.OpenFOAM/README.md

@@ -46,6 +46,12 @@ leftovers of an earlier run in the base case (`log.*`, `postProcessing/`,
 collides with a file the case ships under `system/` is refused rather than
 overwritten.
 
+A response is read from the last row of each file in its function object's
+latest `postProcessing/<name>/<time>/` directory. A file whose last row is
+not at the run's final time - a function object that stopped writing before
+the run ended - is not reported: the node names it in `log.responses`,
+which travels with the step logs.
+
 ## The controller's own step
 
 One step of a recipe is the controller's rather than the kit's:

OpenFOAM/OutWit.Controller.OpenFOAM/Runtime/FoamCaseSession.cs

@@ -110,7 +110,7 @@ public async Task<FoamResultData> RunAsync(FoamTaskData task, CancellationToken
 
             if (report.Succeeded)
             {
-                result.ResponseRow = ExtractResponses(caseDirectory, data.Extraction);
+                result.ResponseRow = ExtractResponses(caseDirectory, data.Extraction, result.FinalTime);
                 await UploadArtifactAsync(result, caseDirectory, scratch.UsablePath, data.ArtifactPolicy, cancellationToken);
             }
             else if (data.ArtifactPolicy?.Logs == true)
@@ -164,13 +164,21 @@ private void ReadFacts(FoamResultData result, string caseDirectory, FoamRecipeDa
         result.CellCount = cells;
     }
 
-    private FoamResponseRowData ExtractResponses(string caseDirectory, FoamExtractionRequestData? extraction)
+    private FoamResponseRowData ExtractResponses(string caseDirectory, FoamExtractionRequestData? extraction, double finalTime)
     {
         // A parsing surprise must not turn a finished run into a failure: the
         // row degrades to empty and the run's other facts stand.
         try
         {
-            return FoamResponseExtractor.Extract(caseDirectory, extraction);
+            // A response whose last row is not the run's final time is left
+            // out and named in log.responses, which travels with the logs;
+            // an unknown final time (no solver log) checks nothing.
+            var notes = new List<string>();
+            var row = FoamResponseExtractor.Extract(caseDirectory, extraction, finalTime > 0 ? finalTime : null, notes);
+            foreach (var note in notes)
+                Logger?.LogWarning("Foam.Run: {Note}", note);
+            FoamResponseExtractor.WriteNotes(caseDirectory, notes);
+            return row;
         }
         catch (Exception e)
         {

Sweep/OutWit.Controller.Sweep/OutWit.Controller.Sweep.csproj

@@ -4,7 +4,7 @@
 
   <PropertyGroup>
     <ControllerName>Sweep</ControllerName>
-    <Version>2.0.4</Version>
+    <Version>2.0.5</Version>
     <Description>Host-side orchestration of parameter studies for every solver family the host knows - CalculiX decks and OpenFOAM cases - with one activity vocabulary: validates the study and its family block up front, fans each chunk out with Grid.ForEach to the family's node activity, and harvests the results into a manifest blob after every chunk, each row the sweep's verdict and the node's own result, so partial results are first-class and progress is observable while the sweep runs. Progressive chunk sizing: the first chunk as wide as the fleet, geometric growth to a cap.</Description>
     <PackageTags>sweep;parameter-study;doe;calculix;ccx;openfoam;cfd;witengine;witcloud;controller</PackageTags>
     <ControllerFeatures>sweep-solve</ControllerFeatures>