diff --git a/registry/data/smoke/reply_ok/test.jsonl b/registry/data/smoke/reply_ok/test.jsonl new file mode 100644 index 0000000..c233bfd --- /dev/null +++ b/registry/data/smoke/reply_ok/test.jsonl @@ -0,0 +1,3 @@ +{"id":"case-1","input":"Reply with only OK.","ideal":"OK"} +{"id":"case-2","input":"Say OK and nothing else.","ideal":"OK"} +{"id":"case-3","input":"Your entire answer must be OK.","ideal":"OK"} \ No newline at end of file diff --git a/registry/evals/smoke/reply_ok.json b/registry/evals/smoke/reply_ok.json new file mode 100644 index 0000000..411e135 --- /dev/null +++ b/registry/evals/smoke/reply_ok.json @@ -0,0 +1,13 @@ +{ + "id": "smoke.reply_ok", + "group": "smoke", + "description": "Checks whether the model can follow a simple instruction and reply with OK only.", + "dataset_path": "registry/data/smoke/reply_ok/test.jsonl", + "split": "test", + "matcher": { + "kind": "exact_match", + "case_sensitive": false, + "trim_whitespace": true + }, + "default_run_count": 1 +} \ No newline at end of file diff --git a/src/main.zig b/src/main.zig index b1da54c..4a5c2f4 100644 --- a/src/main.zig +++ b/src/main.zig @@ -1,8 +1,131 @@ const std = @import("std"); +const registry = @import("zig_eval").registry; pub fn main() !void { - const args = try std.process.argsAlloc(std.heap.page_allocator); - defer std.process.argsFree(std.heap.page_allocator, args); + const allocator = std.heap.page_allocator; - std.debug.print("zig_eval: CLI not implemented yet\n", .{}); + const args = try std.process.argsAlloc(allocator); + defer std.process.argsFree(allocator, args); + + if (args.len < 2) { + printHelp(); + return; + } + + const command = args[1]; + + if (std.mem.eql(u8, command, "list")) { + try listEvals(allocator); + } else if (std.mem.eql(u8, command, "show")) { + if (args.len < 3) { + std.debug.print("Error: missing eval id\n\n", .{}); + std.debug.print("Usage: zig build run -- show \n", .{}); + return; + } + + try showEval(allocator, args[2]); + } else { + std.debug.print("Unknown command: {s}\n\n", .{command}); + printHelp(); + } +} + +fn printHelp() void { + std.debug.print( + \\zig_eval CLI + \\ + \\Commands: + \\ list List all eval definitions + \\ show Show one eval and its dataset cases + \\ + \\Examples: + \\ zig build run -- list + \\ zig build run -- show smoke.reply_ok + \\ + , .{}); +} + +fn listEvals(allocator: std.mem.Allocator) !void { + const cwd = std.fs.cwd(); + + var loaded = try registry.loadAllEvalDefinitions( + allocator, + cwd, + "registry/evals", + ); + defer loaded.deinit(); + + if (loaded.items.len == 0) { + std.debug.print("No evals found.\n", .{}); + return; + } + + std.debug.print("Available evals:\n", .{}); + + for (loaded.items) |eval_def| { + std.debug.print( + " - {s} [{s}] {s}\n", + .{ + eval_def.id, + eval_def.group, + eval_def.description, + }, + ); + } } + +fn showEval(allocator: std.mem.Allocator, eval_id: []const u8) !void { + const cwd = std.fs.cwd(); + + var loaded = try registry.loadAllEvalDefinitions( + allocator, + cwd, + "registry/evals", + ); + defer loaded.deinit(); + + const eval_def = findEvalById(loaded.items, eval_id) orelse { + std.debug.print("Eval not found: {s}\n", .{eval_id}); + return; + }; + + std.debug.print("Eval ID: {s}\n", .{eval_def.id}); + std.debug.print("Group: {s}\n", .{eval_def.group}); + std.debug.print("Description: {s}\n", .{eval_def.description}); + std.debug.print("Dataset: {s}\n", .{eval_def.dataset_path}); + std.debug.print("Split: {s}\n", .{eval_def.split}); + std.debug.print("Default run count: {}\n\n", .{eval_def.default_run_count}); + + var cases = try registry.loadEvalCases( + allocator, + cwd, + eval_def.dataset_path, + ); + defer cases.deinit(); + + std.debug.print("Cases: {}\n", .{cases.items.len}); + + for (cases.items) |case| { + std.debug.print("\nCase ID: {s}\n", .{case.id}); + std.debug.print("Input: {s}\n", .{case.input}); + + if (case.ideal) |ideal| { + std.debug.print("Ideal: {s}\n", .{ideal}); + } else { + std.debug.print("Ideal: null\n", .{}); + } + } +} + +fn findEvalById( + items: []const registry.EvalDefinition, + eval_id: []const u8, +) ?registry.EvalDefinition { + for (items) |item| { + if (std.mem.eql(u8, item.id, eval_id)) { + return item; + } + } + + return null; +} \ No newline at end of file diff --git a/src/matchers/root.zig b/src/matchers/root.zig index 40432aa..a86ecd6 100644 --- a/src/matchers/root.zig +++ b/src/matchers/root.zig @@ -1,5 +1,7 @@ const std = @import("std"); +pub const runner = @import("runner.zig"); + pub const MatcherKind = enum { exact_match, includes, diff --git a/src/matchers/runner.zig b/src/matchers/runner.zig new file mode 100644 index 0000000..b1f1d64 --- /dev/null +++ b/src/matchers/runner.zig @@ -0,0 +1,200 @@ +const std = @import("std"); +const matchers = @import("root.zig"); + +pub const MatchResult = struct { + passed: bool, + score: f64, + reason: ?[]const u8 = null, +}; + +pub fn evaluate( + allocator: std.mem.Allocator, + config: matchers.MatcherConfig, + output: []const u8, + ideal: ?[]const u8, +) !MatchResult { + switch (config) { + .exact_match => |opts| { + const expected_raw = ideal orelse ""; + + const actual = if (opts.trim_whitespace) + std.mem.trim(u8, output, " \t\r\n") + else + output; + + const expected = if (opts.trim_whitespace) + std.mem.trim(u8, expected_raw, " \t\r\n") + else + expected_raw; + + const ok = if (opts.case_sensitive) + std.mem.eql(u8, actual, expected) + else + std.ascii.eqlIgnoreCase(actual, expected); + + return .{ + .passed = ok, + .score = if (ok) 1.0 else 0.0, + .reason = if (ok) null else "exact_match_failed", + }; + }, + + .includes => |opts| { + const expected_raw = ideal orelse ""; + + const actual = if (opts.trim_whitespace) + std.mem.trim(u8, output, " \t\r\n") + else + output; + + const expected = if (opts.trim_whitespace) + std.mem.trim(u8, expected_raw, " \t\r\n") + else + expected_raw; + + const ok = if (opts.case_sensitive) + std.mem.indexOf(u8, actual, expected) != null + else + indexOfIgnoreCase(actual, expected) != null; + + return .{ + .passed = ok, + .score = if (ok) 1.0 else 0.0, + .reason = if (ok) null else "includes_failed", + }; + }, + + .json_fields => |cfg| { + var parsed = std.json.parseFromSlice( + std.json.Value, + allocator, + output, + .{}, + ) catch { + return .{ + .passed = false, + .score = 0.0, + .reason = "invalid_json", + }; + }; + defer parsed.deinit(); + + const obj = switch (parsed.value) { + .object => |object| object, + else => { + return .{ + .passed = false, + .score = 0.0, + .reason = "json_not_object", + }; + }, + }; + + for (cfg.required_fields) |field| { + if (!obj.contains(field)) { + return .{ + .passed = false, + .score = 0.0, + .reason = "missing_required_json_field", + }; + } + } + + return .{ + .passed = true, + .score = 1.0, + .reason = null, + }; + }, + } +} + +fn indexOfIgnoreCase(haystack: []const u8, needle: []const u8) ?usize { + if (needle.len == 0) return 0; + if (needle.len > haystack.len) return null; + + var i: usize = 0; + while (i + needle.len <= haystack.len) : (i += 1) { + if (std.ascii.eqlIgnoreCase(haystack[i .. i + needle.len], needle)) { + return i; + } + } + + return null; +} + +test "exact_match passes when output equals ideal" { + const config = matchers.MatcherConfig{ + .exact_match = .{ + .case_sensitive = true, + .trim_whitespace = true, + }, + }; + + const result = try evaluate( + std.testing.allocator, + config, + " OK ", + "OK", + ); + + try std.testing.expect(result.passed); + try std.testing.expectEqual(@as(f64, 1.0), result.score); +} + +test "includes passes when output contains ideal" { + const config = matchers.MatcherConfig{ + .includes = .{ + .case_sensitive = false, + .trim_whitespace = true, + }, + }; + + const result = try evaluate( + std.testing.allocator, + config, + "The answer is Paris.", + "paris", + ); + + try std.testing.expect(result.passed); +} + +test "json_fields passes when all fields exist" { + const fields = [_][]const u8{ "answer", "score" }; + + const config = matchers.MatcherConfig{ + .json_fields = .{ + .required_fields = fields[0..], + }, + }; + + const result = try evaluate( + std.testing.allocator, + config, + "{\"answer\":\"yes\",\"score\":1}", + null, + ); + + try std.testing.expect(result.passed); +} + +test "json_fields fails on invalid json" { + const fields = [_][]const u8{ "answer" }; + + const config = matchers.MatcherConfig{ + .json_fields = .{ + .required_fields = fields[0..], + }, + }; + + const result = try evaluate( + std.testing.allocator, + config, + "not json", + null, + ); + + try std.testing.expect(!result.passed); + try std.testing.expectEqualStrings("invalid_json", result.reason.?); +} \ No newline at end of file