Add matcher runner and initial smoke eval registry

registry/data/smoke/reply_ok/test.jsonl

@@ -0,0 +1,3 @@
+{"id":"case-1","input":"Reply with only OK.","ideal":"OK"}
+{"id":"case-2","input":"Say OK and nothing else.","ideal":"OK"}
+{"id":"case-3","input":"Your entire answer must be OK.","ideal":"OK"}

registry/evals/smoke/reply_ok.json

@@ -0,0 +1,13 @@
+{
+  "id": "smoke.reply_ok",
+  "group": "smoke",
+  "description": "Checks whether the model can follow a simple instruction and reply with OK only.",
+  "dataset_path": "registry/data/smoke/reply_ok/test.jsonl",
+  "split": "test",
+  "matcher": {
+    "kind": "exact_match",
+    "case_sensitive": false,
+    "trim_whitespace": true
+  },
+  "default_run_count": 1
+}

src/main.zig

@@ -1,8 +1,131 @@
 const std = @import("std");
+const registry = @import("zig_eval").registry;
 
 pub fn main() !void {
-    const args = try std.process.argsAlloc(std.heap.page_allocator);
-    defer std.process.argsFree(std.heap.page_allocator, args);
+    const allocator = std.heap.page_allocator;
 
-    std.debug.print("zig_eval: CLI not implemented yet\n", .{});
+    const args = try std.process.argsAlloc(allocator);
+    defer std.process.argsFree(allocator, args);
+
+    if (args.len < 2) {
+        printHelp();
+        return;
+    }
+
+    const command = args[1];
+
+    if (std.mem.eql(u8, command, "list")) {
+        try listEvals(allocator);
+    } else if (std.mem.eql(u8, command, "show")) {
+        if (args.len < 3) {
+            std.debug.print("Error: missing eval id\n\n", .{});
+            std.debug.print("Usage: zig build run -- show <eval_id>\n", .{});
+            return;
+        }
+
+        try showEval(allocator, args[2]);
+    } else {
+        std.debug.print("Unknown command: {s}\n\n", .{command});
+        printHelp();
+    }
+}
+
+fn printHelp() void {
+    std.debug.print(
+        \\zig_eval CLI
+        \\
+        \\Commands:
+        \\  list                 List all eval definitions
+        \\  show <eval_id>       Show one eval and its dataset cases
+        \\
+        \\Examples:
+        \\  zig build run -- list
+        \\  zig build run -- show smoke.reply_ok
+        \\
+    , .{});
+}
+
+fn listEvals(allocator: std.mem.Allocator) !void {
+    const cwd = std.fs.cwd();
+
+    var loaded = try registry.loadAllEvalDefinitions(
+        allocator,
+        cwd,
+        "registry/evals",
+    );
+    defer loaded.deinit();
+
+    if (loaded.items.len == 0) {
+        std.debug.print("No evals found.\n", .{});
+        return;
+    }
+
+    std.debug.print("Available evals:\n", .{});
+
+    for (loaded.items) |eval_def| {
+        std.debug.print(
+            "  - {s} [{s}] {s}\n",
+            .{
+                eval_def.id,
+                eval_def.group,
+                eval_def.description,
+            },
+        );
+    }
 }
+
+fn showEval(allocator: std.mem.Allocator, eval_id: []const u8) !void {
+    const cwd = std.fs.cwd();
+
+    var loaded = try registry.loadAllEvalDefinitions(
+        allocator,
+        cwd,
+        "registry/evals",
+    );
+    defer loaded.deinit();
+
+    const eval_def = findEvalById(loaded.items, eval_id) orelse {
+        std.debug.print("Eval not found: {s}\n", .{eval_id});
+        return;
+    };
+
+    std.debug.print("Eval ID: {s}\n", .{eval_def.id});
+    std.debug.print("Group: {s}\n", .{eval_def.group});
+    std.debug.print("Description: {s}\n", .{eval_def.description});
+    std.debug.print("Dataset: {s}\n", .{eval_def.dataset_path});
+    std.debug.print("Split: {s}\n", .{eval_def.split});
+    std.debug.print("Default run count: {}\n\n", .{eval_def.default_run_count});
+
+    var cases = try registry.loadEvalCases(
+        allocator,
+        cwd,
+        eval_def.dataset_path,
+    );
+    defer cases.deinit();
+
+    std.debug.print("Cases: {}\n", .{cases.items.len});
+
+    for (cases.items) |case| {
+        std.debug.print("\nCase ID: {s}\n", .{case.id});
+        std.debug.print("Input: {s}\n", .{case.input});
+
+        if (case.ideal) |ideal| {
+            std.debug.print("Ideal: {s}\n", .{ideal});
+        } else {
+            std.debug.print("Ideal: null\n", .{});
+        }
+    }
+}
+
+fn findEvalById(
+    items: []const registry.EvalDefinition,
+    eval_id: []const u8,
+) ?registry.EvalDefinition {
+    for (items) |item| {
+        if (std.mem.eql(u8, item.id, eval_id)) {
+            return item;
+        }
+    }
+
+    return null;
+}

src/matchers/root.zig

@@ -1,5 +1,7 @@
 const std = @import("std");
 
+pub const runner = @import("runner.zig");
+
 pub const MatcherKind = enum {
     exact_match,
     includes,

src/matchers/runner.zig

@@ -0,0 +1,200 @@
+const std = @import("std");
+const matchers = @import("root.zig");
+
+pub const MatchResult = struct {
+    passed: bool,
+    score: f64,
+    reason: ?[]const u8 = null,
+};
+
+pub fn evaluate(
+    allocator: std.mem.Allocator,
+    config: matchers.MatcherConfig,
+    output: []const u8,
+    ideal: ?[]const u8,
+) !MatchResult {
+    switch (config) {
+        .exact_match => |opts| {
+            const expected_raw = ideal orelse "";
+
+            const actual = if (opts.trim_whitespace)
+                std.mem.trim(u8, output, " \t\r\n")
+            else
+                output;
+
+            const expected = if (opts.trim_whitespace)
+                std.mem.trim(u8, expected_raw, " \t\r\n")
+            else
+                expected_raw;
+
+            const ok = if (opts.case_sensitive)
+                std.mem.eql(u8, actual, expected)
+            else
+                std.ascii.eqlIgnoreCase(actual, expected);
+
+            return .{
+                .passed = ok,
+                .score = if (ok) 1.0 else 0.0,
+                .reason = if (ok) null else "exact_match_failed",
+            };
+        },
+
+        .includes => |opts| {
+            const expected_raw = ideal orelse "";
+
+            const actual = if (opts.trim_whitespace)
+                std.mem.trim(u8, output, " \t\r\n")
+            else
+                output;
+
+            const expected = if (opts.trim_whitespace)
+                std.mem.trim(u8, expected_raw, " \t\r\n")
+            else
+                expected_raw;
+
+            const ok = if (opts.case_sensitive)
+                std.mem.indexOf(u8, actual, expected) != null
+            else
+                indexOfIgnoreCase(actual, expected) != null;
+
+            return .{
+                .passed = ok,
+                .score = if (ok) 1.0 else 0.0,
+                .reason = if (ok) null else "includes_failed",
+            };
+        },
+
+        .json_fields => |cfg| {
+            var parsed = std.json.parseFromSlice(
+                std.json.Value,
+                allocator,
+                output,
+                .{},
+            ) catch {
+                return .{
+                    .passed = false,
+                    .score = 0.0,
+                    .reason = "invalid_json",
+                };
+            };
+            defer parsed.deinit();
+
+            const obj = switch (parsed.value) {
+                .object => |object| object,
+                else => {
+                    return .{
+                        .passed = false,
+                        .score = 0.0,
+                        .reason = "json_not_object",
+                    };
+                },
+            };
+
+            for (cfg.required_fields) |field| {
+                if (!obj.contains(field)) {
+                    return .{
+                        .passed = false,
+                        .score = 0.0,
+                        .reason = "missing_required_json_field",
+                    };
+                }
+            }
+
+            return .{
+                .passed = true,
+                .score = 1.0,
+                .reason = null,
+            };
+        },
+    }
+}
+
+fn indexOfIgnoreCase(haystack: []const u8, needle: []const u8) ?usize {
+    if (needle.len == 0) return 0;
+    if (needle.len > haystack.len) return null;
+
+    var i: usize = 0;
+    while (i + needle.len <= haystack.len) : (i += 1) {
+        if (std.ascii.eqlIgnoreCase(haystack[i .. i + needle.len], needle)) {
+            return i;
+        }
+    }
+
+    return null;
+}
+
+test "exact_match passes when output equals ideal" {
+    const config = matchers.MatcherConfig{
+        .exact_match = .{
+            .case_sensitive = true,
+            .trim_whitespace = true,
+        },
+    };
+
+    const result = try evaluate(
+        std.testing.allocator,
+        config,
+        " OK ",
+        "OK",
+    );
+
+    try std.testing.expect(result.passed);
+    try std.testing.expectEqual(@as(f64, 1.0), result.score);
+}
+
+test "includes passes when output contains ideal" {
+    const config = matchers.MatcherConfig{
+        .includes = .{
+            .case_sensitive = false,
+            .trim_whitespace = true,
+        },
+    };
+
+    const result = try evaluate(
+        std.testing.allocator,
+        config,
+        "The answer is Paris.",
+        "paris",
+    );
+
+    try std.testing.expect(result.passed);
+}
+
+test "json_fields passes when all fields exist" {
+    const fields = [_][]const u8{ "answer", "score" };
+
+    const config = matchers.MatcherConfig{
+        .json_fields = .{
+            .required_fields = fields[0..],
+        },
+    };
+
+    const result = try evaluate(
+        std.testing.allocator,
+        config,
+        "{\"answer\":\"yes\",\"score\":1}",
+        null,
+    );
+
+    try std.testing.expect(result.passed);
+}
+
+test "json_fields fails on invalid json" {
+    const fields = [_][]const u8{ "answer" };
+
+    const config = matchers.MatcherConfig{
+        .json_fields = .{
+            .required_fields = fields[0..],
+        },
+    };
+
+    const result = try evaluate(
+        std.testing.allocator,
+        config,
+        "not json",
+        null,
+    );
+
+    try std.testing.expect(!result.passed);
+    try std.testing.expectEqualStrings("invalid_json", result.reason.?);
+}