From e9e0a29d73e4674d32dfeb6bb271647e0dc8aa24 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 22:23:07 +0000 Subject: [PATCH 01/14] learnercli: mark login and project-link errors, poll with Retry-After ErrLoginRequired and ErrProjectNotLinked let a caller tell a state a person must fix from other failures without matching messages; the messages are unchanged. AuthorizedPoll returns the status and Retry-After of a pending 202 answer, and of an error answer such as 429. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01K9oUWMHw3ZCHpfmoBGcgeF --- internal/learnercli/client.go | 66 ++++++++++++++++++++++++++++-- internal/learnercli/credentials.go | 8 ++-- internal/learnercli/errors.go | 30 ++++++++++++++ internal/learnercli/project.go | 4 +- 4 files changed, 99 insertions(+), 9 deletions(-) create mode 100644 internal/learnercli/errors.go diff --git a/internal/learnercli/client.go b/internal/learnercli/client.go index 382e918..ddbf6b6 100644 --- a/internal/learnercli/client.go +++ b/internal/learnercli/client.go @@ -176,6 +176,64 @@ func (c *Client) AuthorizedJSON( return c.jsonRequest(ctx, method, path, credentials.AccessToken, request, response) } +// PollResponse is the outcome of one authorized poll. Status is 200 for a +// ready resource and 202 while the server still works on it; RetryAfter is +// the server's minimum wait before the next poll, zero when it named none. +type PollResponse struct { + Status int + RetryAfter time.Duration +} + +// AuthorizedPoll performs an authorized GET for an endpoint that answers 202 +// with Retry-After while its result is pending. A non-2xx answer is an +// *HTTPError that carries Retry-After too, so a rate limit is also waited out +// at the interval the server asked for. +func (c *Client) AuthorizedPoll(ctx context.Context, path string, response any) (PollResponse, error) { + credentials, err := c.validCredentials(ctx) + if err != nil { + return PollResponse{}, err + } + result, err := c.pollOnce(ctx, path, credentials.AccessToken, response) + var statusError *HTTPError + if !errors.As(err, &statusError) || statusError.Status != http.StatusUnauthorized { + return result, err + } + credentials, err = c.refresh(ctx, credentials) + if err != nil { + return PollResponse{}, err + } + return c.pollOnce(ctx, path, credentials.AccessToken, response) +} + +func (c *Client) pollOnce(ctx context.Context, path, token string, response any) (PollResponse, error) { + status, body, headers, err := c.rawJSON(ctx, http.MethodGet, path, token, nil) + if err != nil { + return PollResponse{}, err + } + retryAfter := parseRetryAfter(headers.Get("Retry-After")) + if status < 200 || status >= 300 { + httpErr := parseHTTPError(status, body) + if typed, ok := httpErr.(*HTTPError); ok { + typed.RetryAfter = retryAfter + } + return PollResponse{}, httpErr + } + if err := decodeJSON(body, response); err != nil { + return PollResponse{}, err + } + return PollResponse{Status: status, RetryAfter: retryAfter}, nil +} + +// parseRetryAfter accepts the delta-seconds form the API sends. An HTTP date +// or a malformed value yields zero, and the caller falls back to its default. +func parseRetryAfter(value string) time.Duration { + seconds, err := strconv.Atoi(strings.TrimSpace(value)) + if err != nil || seconds < 1 { + return 0 + } + return time.Duration(seconds) * time.Second +} + func (c *Client) Submit( ctx context.Context, path string, @@ -251,7 +309,7 @@ func (c *Client) validCredentials(ctx context.Context) (Credentials, error) { credentials = loaded } if credentials.APIURL != c.BaseURL { - return Credentials{}, errors.New("stored credentials belong to a different API URL; login again") + return Credentials{}, loginRequired("stored credentials belong to a different API URL; login again") } if time.Now().Add(30 * time.Second).Before(credentials.AccessExpiresAt) { return credentials, nil @@ -259,11 +317,11 @@ func (c *Client) validCredentials(ctx context.Context) (Credentials, error) { now := time.Now() if !credentials.RefreshIdleExpiresAt.IsZero() && !now.Before(credentials.RefreshIdleExpiresAt) { - return Credentials{}, errors.New("CLI login expired; run `softpractice login`") + return Credentials{}, loginRequired("CLI login expired; run `softpractice login`") } if !credentials.RefreshAbsoluteExpiresAt.IsZero() && !now.Before(credentials.RefreshAbsoluteExpiresAt) { - return Credentials{}, errors.New("CLI login expired; run `softpractice login`") + return Credentials{}, loginRequired("CLI login expired; run `softpractice login`") } return c.refresh(ctx, credentials) } @@ -387,6 +445,8 @@ type HTTPError struct { Code string Message string SupportID string + // RetryAfter is set only by AuthorizedPoll, from the response header. + RetryAfter time.Duration } type StarterArchive struct { diff --git a/internal/learnercli/credentials.go b/internal/learnercli/credentials.go index a9311b7..6c0dc48 100644 --- a/internal/learnercli/credentials.go +++ b/internal/learnercli/credentials.go @@ -98,7 +98,7 @@ func (s CredentialStore) Load() (Credentials, error) { info, err := os.Lstat(s.Path) if err != nil { if errors.Is(err, os.ErrNotExist) { - return Credentials{}, errors.New("not logged in; run `softpractice login`") + return Credentials{}, loginRequired("not logged in; run `softpractice login`") } return Credentials{}, err } @@ -120,17 +120,17 @@ func (s CredentialStore) Load() (Credentials, error) { metadata.APIURL == "" || metadata.RefreshIdleExpiresAt.IsZero() || metadata.RefreshAbsoluteExpiresAt.IsZero() { - return Credentials{}, errors.New("credentials metadata is incomplete; login again") + return Credentials{}, loginRequired("credentials metadata is incomplete; login again") } refreshToken, err := s.Secrets.Get(keyringService, s.refreshTokenAccount(metadata.APIURL)) if err != nil { if errors.Is(err, keyring.ErrNotFound) { - return Credentials{}, errors.New("refresh credential is missing; run `softpractice login`") + return Credentials{}, loginRequired("refresh credential is missing; run `softpractice login`") } return Credentials{}, fmt.Errorf("read refresh credential from system keyring: %w", err) } if refreshToken == "" { - return Credentials{}, errors.New("refresh credential is empty; run `softpractice login`") + return Credentials{}, loginRequired("refresh credential is empty; run `softpractice login`") } return Credentials{ APIURL: metadata.APIURL, RefreshToken: refreshToken, diff --git a/internal/learnercli/errors.go b/internal/learnercli/errors.go new file mode 100644 index 0000000..d7a6f7e --- /dev/null +++ b/internal/learnercli/errors.go @@ -0,0 +1,30 @@ +package learnercli + +import "errors" + +// ErrLoginRequired marks every failure that only a new `softpractice login` +// resolves. Callers that must not start the device flow themselves, such as a +// command run by an agent, use it to report the state instead. +var ErrLoginRequired = errors.New("login required") + +// ErrProjectNotLinked marks a directory that holds no SoftPractice project +// link, so there is no workspace to read or submit to. +var ErrProjectNotLinked = errors.New("project not linked") + +// markedError keeps the original message and adds a sentinel for errors.Is. +type markedError struct { + message string + mark error +} + +func (e markedError) Error() string { return e.message } + +func (e markedError) Is(target error) bool { return target == e.mark } + +func loginRequired(message string) error { + return markedError{message: message, mark: ErrLoginRequired} +} + +func projectNotLinked(message string) error { + return markedError{message: message, mark: ErrProjectNotLinked} +} diff --git a/internal/learnercli/project.go b/internal/learnercli/project.go index c5c6397..1ade921 100644 --- a/internal/learnercli/project.go +++ b/internal/learnercli/project.go @@ -91,10 +91,10 @@ func LoadProjectLink(repositoryRoot string) (ProjectLink, error) { info, err := os.Lstat(path) if err != nil { if errors.Is(err, os.ErrNotExist) { - return ProjectLink{}, fmt.Errorf( + return ProjectLink{}, projectNotLinked(fmt.Sprintf( "%s is missing; download the project linked to your workspace", projectLinkRelativePath, - ) + )) } return ProjectLink{}, fmt.Errorf("inspect project link: %w", err) } From ba9a4b16decec3011c4fcddd4bdb6c9a9949c0c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 22:23:25 +0000 Subject: [PATCH 02/14] cli: share learner use cases between commands status, submit, submission show, and update now call learnerUseCases, which return results instead of printing them, so a later `softpractice mcp` can call the same operations. Command output is unchanged. status --format json gains additive fields: assignment.title, lesson_version_update while a newer lesson version is published, and latest_submission.evaluation_status once the evaluation is terminal. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01K9oUWMHw3ZCHpfmoBGcgeF --- cmd/softpractice/git.go | 4 +- cmd/softpractice/machine.go | 62 ++++--- cmd/softpractice/main.go | 51 ++---- cmd/softpractice/usecase.go | 339 ++++++++++++++++++++++++++++++++++++ 4 files changed, 386 insertions(+), 70 deletions(-) create mode 100644 cmd/softpractice/usecase.go diff --git a/cmd/softpractice/git.go b/cmd/softpractice/git.go index 6fdad10..7bc810e 100644 --- a/cmd/softpractice/git.go +++ b/cmd/softpractice/git.go @@ -12,6 +12,8 @@ import ( "unicode/utf8" ) +var errNotGitRepository = errors.New("current directory is not a Git repository") + type gitRepository struct { Root string CommitSHA string @@ -38,7 +40,7 @@ func inspectGitRepository( } rootOutput, err := exec.CommandContext(ctx, "git", rootArguments...).Output() if err != nil { - return gitRepository{}, errors.New("current directory is not a Git repository") + return gitRepository{}, errNotGitRepository } repository := gitRepository{Root: filepath.Clean(strings.TrimSpace(string(rootOutput)))} headOutput, err := gitOutput(ctx, repository.Root, "rev-parse", "HEAD") diff --git a/cmd/softpractice/machine.go b/cmd/softpractice/machine.go index 0d35206..e71d40c 100644 --- a/cmd/softpractice/machine.go +++ b/cmd/softpractice/machine.go @@ -50,6 +50,7 @@ type machineStatus struct { Assignment struct { ID string `json:"id"` Version int `json:"version"` + Title string `json:"title"` State string `json:"state"` } `json:"assignment"` Local struct { @@ -59,8 +60,17 @@ type machineStatus struct { // Transition is present only while the server has already opened the next // lesson and this folder still holds the previous one. It is additive: a // contract_version 1 reader that ignores unknown keys is unaffected. - Transition *machineTransition `json:"transition,omitempty"` - LatestSubmission *machineSubmissionSummary `json:"latest_submission"` + Transition *machineTransition `json:"transition,omitempty"` + // LessonVersionUpdate is present while the server publishes a newer + // version of the lesson this folder holds. Additive like Transition. + LessonVersionUpdate *machineLessonVersionUpdate `json:"lesson_version_update,omitempty"` + LatestSubmission *machineSubmissionSummary `json:"latest_submission"` +} + +type machineLessonVersionUpdate struct { + AssignmentID string `json:"assignment_id"` + FromVersion int `json:"from_version"` + ToVersion int `json:"to_version"` } type machineTransition struct { @@ -75,15 +85,13 @@ type machineSubmissionSummary struct { RevisionID string `json:"revision_id"` JobState string `json:"job_state"` SubmittedAt time.Time `json:"submitted_at"` + // EvaluationStatus is the evaluation result status (accepted, revise, + // ...) once the job is terminal; omitted while it is not. + EvaluationStatus string `json:"evaluation_status,omitempty"` } -func writeStatusJSON( - output io.Writer, - user currentUser, - link learnercli.ProjectLink, - workspace learnercli.WorkspaceStatus, - repository gitRepository, -) error { +func writeStatusJSON(output io.Writer, snapshot statusSnapshot) error { + user, link, workspace, repository := snapshot.User, snapshot.Link, snapshot.Workspace, snapshot.Repository payload := machineStatus{ContractVersion: 1, Kind: "softpractice.status"} payload.Account.ID = user.ID payload.Account.EmailVerified = user.EmailVerified @@ -92,6 +100,7 @@ func writeStatusJSON( payload.Workspace.State = workspace.Workspace.State payload.Assignment.ID = workspace.Assignment.ID payload.Assignment.Version = workspace.Assignment.Version + payload.Assignment.Title = workspace.Assignment.Title payload.Assignment.State = workspace.Assignment.State payload.Local.Head = repository.CommitSHA payload.Local.Clean = repository.Clean @@ -101,11 +110,18 @@ func writeStatusJSON( ToAssignmentID: workspace.Assignment.ID, ToAssignmentVersion: workspace.Assignment.Version, } } + if pendingLessonVersion(link, workspace) { + payload.LessonVersionUpdate = &machineLessonVersionUpdate{ + AssignmentID: link.AssignmentID, + FromVersion: link.AssignmentVersion, ToVersion: workspace.Assignment.Version, + } + } if workspace.LatestSubmission != nil { payload.LatestSubmission = &machineSubmissionSummary{ ID: workspace.LatestSubmission.ID, RevisionID: workspace.LatestSubmission.RevisionID, - JobState: workspace.LatestSubmission.JobState, - SubmittedAt: workspace.LatestSubmission.SubmittedAt, + JobState: workspace.LatestSubmission.JobState, + SubmittedAt: workspace.LatestSubmission.SubmittedAt, + EvaluationStatus: snapshot.LatestEvaluationStatus, } } return writeMachineJSON(output, payload) @@ -246,29 +262,19 @@ func showSubmission( } } submissionID := strings.TrimSpace(*id) + useCases := newLearnerUseCases(client, startDirectory) if submissionID == "" { - _, link, workspace, linkErr := linkedWorkspace(ctx, client, startDirectory, false) - if linkErr != nil { - return linkErr + latest, err := useCases.LatestSubmission(ctx) + if err != nil { + return err } - if workspace.LatestSubmission != nil { - submissionID = workspace.LatestSubmission.ID - } else { - // An accepted submission advances the workspace, so right after - // acceptance the newest submission belongs to the previous lesson. - assignmentID, latestID, found, err := client.LatestPracticumSubmission(ctx, link.ProjectID) - if err != nil { - return err - } - if !found { - return errors.New("the linked workspace has no submission") - } + if latest.PredecessorAssignmentID != "" { fmt.Fprintf(errorOutput, text(ctx, "По %s отправок ещё нет; показан принятый результат %s.\n", "%s has no submission yet; showing the accepted %s result.\n", - ), workspace.Assignment.ID, assignmentID) - submissionID = latestID + ), latest.CurrentAssignmentID, latest.PredecessorAssignmentID) } + submissionID = latest.SubmissionID } parsedID, err := uuid.Parse(submissionID) if err != nil || parsedID.String() != submissionID { diff --git a/cmd/softpractice/main.go b/cmd/softpractice/main.go index b57fdc7..9b077eb 100644 --- a/cmd/softpractice/main.go +++ b/cmd/softpractice/main.go @@ -628,7 +628,7 @@ func updateProject(ctx context.Context, client *learnercli.Client, args []string if flags.NArg() != 0 { return errors.New(text(ctx, "использование: softpractice update", "usage: softpractice update")) } - return updateLinkedProject(ctx, client, "", "", output) + return newLearnerUseCases(client, "").Update(ctx, output) } func updateLinkedProject( @@ -1394,17 +1394,14 @@ func status( if err != nil { return err } - repository, link, workspace, err := linkedWorkspace(ctx, client, startDirectory, false) + snapshot, err := newLearnerUseCases(client, startDirectory).Status(ctx, outputFormat == outputFormatJSON) if err != nil { return err } - var user currentUser - if err := client.AuthorizedJSON(ctx, "GET", "/v1/me", nil, &user); err != nil { - return err - } if outputFormat == outputFormatJSON { - return writeStatusJSON(output, user, link, workspace, repository) + return writeStatusJSON(output, snapshot) } + repository, link, workspace, user := snapshot.Repository, snapshot.Link, snapshot.Workspace, snapshot.User clean := text(ctx, "чистое", "clean") if !repository.Clean { clean = text(ctx, "есть незакоммиченные или неотслеживаемые изменения", "has uncommitted or untracked changes") @@ -1501,23 +1498,13 @@ func submit( if flags.NArg() != 0 { return errors.New(text(ctx, "использование: softpractice submit [--yes]", "usage: softpractice submit [--yes]")) } - repository, link, workspace, err := linkedWorkspace(ctx, client, startDirectory, true) + useCases := newLearnerUseCases(client, startDirectory) + prepared, err := useCases.PrepareSubmission(ctx) if err != nil { return err } - defer os.Remove(repository.ArchivePath) - if !repository.Clean { - return errors.New(text(ctx, "рабочее дерево не чистое; сделайте commit всех изменений перед отправкой", "working tree is not clean; commit all changes before submitting")) - } - if workspace.Workspace.State != "active" { - return fmt.Errorf(text(ctx, "workspace находится в состоянии %s и не принимает отправки", "workspace is %s and cannot accept a submission"), workspace.Workspace.State) - } - if workspace.Assignment.State != "available" && workspace.Assignment.State != "submitted" { - return fmt.Errorf( - text(ctx, "урок находится в состоянии %s и не принимает отправки", "assignment is %s and cannot accept a submission"), - workspace.Assignment.State, - ) - } + defer prepared.Close() + repository, link, workspace := prepared.Repository, prepared.Link, prepared.Workspace fmt.Fprintf( output, text(ctx, "Проект: %s\nУрок: %s v%d — %s\n\n", "Project: %s\nAssignment: %s v%d — %s\n\n"), @@ -1565,27 +1552,9 @@ func submit( return nil } } - requestPath, headers, idempotencyKey := submissionRequest( - link, - workspace, - repository.CommitSHA, - ) - var receipt submissionReceipt - err = client.Submit( - ctx, - requestPath, - repository.ArchivePath, - headers, - &receipt, - ) + receipt, err := useCases.SendSubmission(ctx, prepared) if err != nil { - return fmt.Errorf( - "submit %s at %s (idempotency key %s): %w", - repository.Root, - repository.CommitSHA, - idempotencyKey, - err, - ) + return err } replay := "" if receipt.Replayed { diff --git a/cmd/softpractice/usecase.go b/cmd/softpractice/usecase.go new file mode 100644 index 0000000..2779247 --- /dev/null +++ b/cmd/softpractice/usecase.go @@ -0,0 +1,339 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "time" + + "github.com/google/uuid" + + "github.com/arthur-s/softpractice-cli/internal/learnercli" +) + +// learnerUseCases holds the learner operations that CLI commands format for a +// terminal and that `softpractice mcp` will expose as tools. They write +// nothing to standard output: results are returned, and the few operations +// that report progress take the writer explicitly. +type learnerUseCases struct { + client *learnercli.Client + startDirectory string + now func() time.Time + sleep func(context.Context, time.Duration) error +} + +func newLearnerUseCases(client *learnercli.Client, startDirectory string) learnerUseCases { + return learnerUseCases{ + client: client, startDirectory: startDirectory, + now: time.Now, sleep: sleepContext, + } +} + +func sleepContext(ctx context.Context, delay time.Duration) error { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +// statusSnapshot is the linked project, its workspace, and the signed-in +// learner. LatestEvaluationStatus is set only when it was requested and the +// latest submission already has a terminal evaluation. +type statusSnapshot struct { + Repository gitRepository + Link learnercli.ProjectLink + Workspace learnercli.WorkspaceStatus + User currentUser + LatestEvaluationStatus string +} + +func (u learnerUseCases) Status(ctx context.Context, withEvaluation bool) (statusSnapshot, error) { + repository, link, workspace, err := linkedWorkspace(ctx, u.client, u.startDirectory, false) + if err != nil { + return statusSnapshot{}, err + } + var user currentUser + if err := u.client.AuthorizedJSON(ctx, "GET", "/v1/me", nil, &user); err != nil { + return statusSnapshot{}, err + } + snapshot := statusSnapshot{Repository: repository, Link: link, Workspace: workspace, User: user} + if withEvaluation && workspace.LatestSubmission != nil { + switch workspace.LatestSubmission.JobState { + case "completed", "failed": + // The status stays available when only this optional field fails. + result, err := u.Evaluation(ctx, workspace.LatestSubmission.ID, evaluationWait{}) + if err == nil && result.Outcome == evaluationReady && result.Submission.Evaluation != nil { + snapshot.LatestEvaluationStatus = result.Submission.Evaluation.Status + } + } + } + return snapshot, nil +} + +// assignmentSnapshot is the requirement the linked folder is working on: the +// lesson version its tree was made for, and the theory material linked to it +// when the lesson has one. +type assignmentSnapshot struct { + Link learnercli.ProjectLink + Workspace learnercli.WorkspaceStatus + Assignment assignmentDetail + Material *theoryMaterial +} + +type assignmentDetail struct { + ID string `json:"id"` + Version int `json:"version"` + Title string `json:"title"` + Kind string `json:"kind"` + ProjectSetup string `json:"project_setup"` + ExerciseID string `json:"exercise_id"` + EstimatedMinutes int `json:"estimated_minutes"` + InstructionsMarkdown string `json:"instructions_markdown"` + ContentSHA256 string `json:"content_sha256"` +} + +type theoryMaterial struct { + ID string `json:"id"` + Title string `json:"title"` + EstimatedMinutes int `json:"estimated_minutes"` + Markdown string `json:"markdown"` + Diagram json.RawMessage `json:"diagram"` +} + +func (u learnerUseCases) Assignment(ctx context.Context) (assignmentSnapshot, error) { + _, link, workspace, err := linkedWorkspace(ctx, u.client, u.startDirectory, false) + if err != nil { + return assignmentSnapshot{}, err + } + id := workspace.Assignment.ID + version := submittedLessonVersion(link, workspace) + query := "?version=" + strconv.Itoa(version) + var assignment assignmentDetail + if err := u.client.AuthorizedJSON( + ctx, "GET", "/v1/assignments/"+url.PathEscape(id)+query, nil, &assignment, + ); err != nil { + return assignmentSnapshot{}, err + } + if assignment.ID != id || assignment.Version != version { + return assignmentSnapshot{}, errors.New("assignment response does not match the requested lesson") + } + snapshot := assignmentSnapshot{Link: link, Workspace: workspace, Assignment: assignment} + var material theoryMaterial + err = u.client.AuthorizedJSON( + ctx, "GET", "/v1/assignments/"+url.PathEscape(id)+"/material"+query, nil, &material, + ) + var statusError *learnercli.HTTPError + switch { + case err == nil: + snapshot.Material = &material + case errors.As(err, &statusError) && statusError.Status == http.StatusNotFound: + // The lesson has no linked theory material. + default: + return assignmentSnapshot{}, err + } + return snapshot, nil +} + +// preparedSubmission is a linked, clean, submittable commit with its archive +// already built. Close removes the archive. +type preparedSubmission struct { + Repository gitRepository + Link learnercli.ProjectLink + Workspace learnercli.WorkspaceStatus +} + +func (p preparedSubmission) Close() { + if p.Repository.ArchivePath != "" { + _ = os.Remove(p.Repository.ArchivePath) + } +} + +// PrepareSubmission checks that HEAD can be submitted and packs it. It sends +// nothing: the caller shows the learner what will be submitted and asks for +// confirmation before SendSubmission. +func (u learnerUseCases) PrepareSubmission(ctx context.Context) (preparedSubmission, error) { + repository, link, workspace, err := linkedWorkspace(ctx, u.client, u.startDirectory, true) + if err != nil { + return preparedSubmission{}, err + } + prepared := preparedSubmission{Repository: repository, Link: link, Workspace: workspace} + if !repository.Clean { + prepared.Close() + return preparedSubmission{}, errors.New(text(ctx, "рабочее дерево не чистое; сделайте commit всех изменений перед отправкой", "working tree is not clean; commit all changes before submitting")) + } + if workspace.Workspace.State != "active" { + prepared.Close() + return preparedSubmission{}, fmt.Errorf(text(ctx, "workspace находится в состоянии %s и не принимает отправки", "workspace is %s and cannot accept a submission"), workspace.Workspace.State) + } + if workspace.Assignment.State != "available" && workspace.Assignment.State != "submitted" { + prepared.Close() + return preparedSubmission{}, fmt.Errorf( + text(ctx, "урок находится в состоянии %s и не принимает отправки", "assignment is %s and cannot accept a submission"), + workspace.Assignment.State, + ) + } + return prepared, nil +} + +func (u learnerUseCases) SendSubmission(ctx context.Context, prepared preparedSubmission) (submissionReceipt, error) { + requestPath, headers, idempotencyKey := submissionRequest( + prepared.Link, + prepared.Workspace, + prepared.Repository.CommitSHA, + ) + var receipt submissionReceipt + if err := u.client.Submit(ctx, requestPath, prepared.Repository.ArchivePath, headers, &receipt); err != nil { + return submissionReceipt{}, fmt.Errorf( + "submit %s at %s (idempotency key %s): %w", + prepared.Repository.Root, + prepared.Repository.CommitSHA, + idempotencyKey, + err, + ) + } + return receipt, nil +} + +// Update applies the pending lesson transition or lesson version update to +// the linked folder, with every check `softpractice update` makes. +func (u learnerUseCases) Update(ctx context.Context, output io.Writer) error { + return updateLinkedProject(ctx, u.client, u.startDirectory, "", output) +} + +// latestSubmission names the linked workspace's newest submission. Right +// after an acceptance the current lesson has none yet, and the newest one is +// the accepted submission of the previous lesson: predecessorAssignmentID +// then names that lesson. +type latestSubmission struct { + SubmissionID string + CurrentAssignmentID string + PredecessorAssignmentID string +} + +func (u learnerUseCases) LatestSubmission(ctx context.Context) (latestSubmission, error) { + _, link, workspace, err := linkedWorkspace(ctx, u.client, u.startDirectory, false) + if err != nil { + return latestSubmission{}, err + } + latest := latestSubmission{CurrentAssignmentID: workspace.Assignment.ID} + if workspace.LatestSubmission != nil { + latest.SubmissionID = workspace.LatestSubmission.ID + return latest, nil + } + assignmentID, submissionID, found, err := u.client.LatestPracticumSubmission(ctx, link.ProjectID) + if err != nil { + return latestSubmission{}, err + } + if !found { + return latestSubmission{}, errors.New("the linked workspace has no submission") + } + latest.SubmissionID = submissionID + latest.PredecessorAssignmentID = assignmentID + return latest, nil +} + +type evaluationOutcome string + +const ( + // evaluationReady: the evaluation finished, as a result or as a safe + // technical failure. + evaluationReady evaluationOutcome = "ready" + // evaluationPending: the evaluation is still queued or running. + evaluationPending evaluationOutcome = "pending" + // evaluationSuperseded: the server replaced this evaluation job; it will + // never have a result. + evaluationSuperseded evaluationOutcome = "superseded" +) + +// evaluationWait selects one poll (the zero value) or polling until the +// evaluation is ready, superseded, or Timeout would pass before the next poll +// the server allows. +type evaluationWait struct { + Wait bool + Timeout time.Duration +} + +// defaultEvaluationPoll is used when the server names no interval. +const defaultEvaluationPoll = 2 * time.Second + +type evaluationResult struct { + Outcome evaluationOutcome + SubmissionID string + // Submission and RawEvaluation are set for ready and pending outcomes. + Submission machineSubmission + RawEvaluation json.RawMessage +} + +func (u learnerUseCases) Evaluation( + ctx context.Context, + submissionID string, + wait evaluationWait, +) (evaluationResult, error) { + parsedID, err := uuid.Parse(submissionID) + if err != nil || parsedID.String() != submissionID { + return evaluationResult{}, errors.New("submission ID must be a canonical UUID") + } + deadline := u.now().Add(wait.Timeout) + path := "/v1/submissions/" + submissionID + "/evaluation" + // pending is the last pending answer; a rate limit does not replace it. + var pending evaluationResult + for { + var response evaluationAPIResponse + poll, err := u.client.AuthorizedPoll(ctx, path, &response) + var delay time.Duration + var statusError *learnercli.HTTPError + switch { + case err == nil: + payload, normalizeErr := normalizeSubmissionResponse(response) + if normalizeErr != nil { + return evaluationResult{}, normalizeErr + } + if payload.SubmissionID != submissionID { + return evaluationResult{}, errors.New("evaluation response does not match the requested submission") + } + if payload.Terminal { + return evaluationResult{ + Outcome: evaluationReady, SubmissionID: submissionID, + Submission: payload, RawEvaluation: response.Evaluation, + }, nil + } + pending = evaluationResult{Outcome: evaluationPending, SubmissionID: submissionID, Submission: payload} + delay = poll.RetryAfter + if delay == 0 { + delay = time.Duration(payload.NextPollSeconds) * time.Second + } + case errors.As(err, &statusError) && statusError.Status == http.StatusConflict && + statusError.Code == "evaluation_superseded": + return evaluationResult{Outcome: evaluationSuperseded, SubmissionID: submissionID}, nil + case wait.Wait && errors.As(err, &statusError) && statusError.Status == http.StatusTooManyRequests && + statusError.RetryAfter > 0: + delay = statusError.RetryAfter + default: + return evaluationResult{}, err + } + if delay < time.Second { + delay = defaultEvaluationPoll + } + if !wait.Wait || u.now().Add(delay).After(deadline) { + // Only a known pending state can be reported when the wait ends, + // so a rate limit before the first answer stays an error. + if pending.Outcome == "" { + return evaluationResult{}, err + } + return pending, nil + } + if err := u.sleep(ctx, delay); err != nil { + return evaluationResult{}, err + } + } +} From dda32a97f6faf51770546a4aa42a58aed8210e60 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 22:23:25 +0000 Subject: [PATCH 03/14] cli: add evaluation and assignment show for agents `softpractice evaluation [--wait] [--timeout]` reads /v1/submissions/{id}/evaluation, waits at the Retry-After interval the server sets, and reports a superseded evaluation and a wait that ends before the result as separate outcomes with exit codes 4 and 3. `softpractice assignment show` prints the assignment text of the lesson version the folder holds and its theory material. Both print JSON in the existing contract_version/kind format and, in JSON mode, a softpractice.error document that says when a login or a linked project is required. machineReviewFeedback keeps the reviewer's recommendations and other free text out of this output by default. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01K9oUWMHw3ZCHpfmoBGcgeF --- README.md | 16 + README_RU.md | 17 + cmd/softpractice/agent_commands.go | 360 +++++++++++++++ cmd/softpractice/agent_commands_test.go | 578 ++++++++++++++++++++++++ cmd/softpractice/feedback.go | 157 +++++++ cmd/softpractice/main.go | 8 + cmd/softpractice/ui.go | 8 +- packaging/npm/README.md | 16 + 8 files changed, 1158 insertions(+), 2 deletions(-) create mode 100644 cmd/softpractice/agent_commands.go create mode 100644 cmd/softpractice/agent_commands_test.go create mode 100644 cmd/softpractice/feedback.go diff --git a/README.md b/README.md index a5ad147..114109e 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,8 @@ softpractice update | `softpractice check` | Run public checks against the current working tree | | `softpractice submit` | Submit the current committed Git revision | | `softpractice submission show` | Show the latest submission and review result | +| `softpractice evaluation` | Show the evaluation result of the latest submission; `--wait` waits for it | +| `softpractice assignment show` | Show the assignment this project is working on | | `softpractice update` | Apply the transition to the next lesson, or move to a newer version of the current lesson (replaces only lesson files, never yours; optional until the lesson ends) | | `softpractice open` | Open the workspace or latest result in the browser | | `softpractice project restore` | Restore a project from the latest usable revision | @@ -108,6 +110,20 @@ softpractice set auto-checks true run. Unlike `softpractice check`, submission checks require a clean working tree and run in an isolated snapshot of `HEAD`. +## Machine-readable output + +`status`, `submission show`, `evaluation`, and `assignment show` accept +`--format json`. Each document has `contract_version` and `kind`. + +```bash +softpractice evaluation --wait --timeout 10m --format json +``` + +`evaluation` exit codes: `0` result ready, `1` error, `3` not ready yet, +`4` evaluation superseded, no result. Its output has the verdict, the +automated check results, and the number and priorities of reviewer findings, +without the reviewer's recommendations; the full review is at `result_url`. + ## Language SoftPractice CLI supports English and Russian and selects a language from the diff --git a/README_RU.md b/README_RU.md index 7301161..aa884ae 100644 --- a/README_RU.md +++ b/README_RU.md @@ -79,6 +79,8 @@ softpractice update | `softpractice check` | Запустить публичные проверки текущего рабочего дерева | | `softpractice submit` | Отправить текущую зафиксированную Git-ревизию | | `softpractice submission show` | Показать последнюю отправку и результат проверки | +| `softpractice evaluation` | Показать результат проверки последней отправки; `--wait` дожидается его | +| `softpractice assignment show` | Показать задание, над которым работает проект | | `softpractice update` | Применить переход к следующему уроку или перейти на новую версию текущего урока (заменяет только файлы урока, ваши файлы не трогает; до конца урока необязательно) | | `softpractice open` | Открыть workspace или последний результат в браузере | | `softpractice project restore` | Восстановить проект из последней подходящей ревизии | @@ -109,6 +111,21 @@ softpractice set auto-checks true отличие от `softpractice check`, проверка перед отправкой требует чистое рабочее дерево и запускается в изолированном снимке `HEAD`. + +## Машиночитаемый вывод + +`status`, `submission show`, `evaluation` и `assignment show` принимают +`--format json`. У каждого документа есть `contract_version` и `kind`. + +```bash +softpractice evaluation --wait --timeout 10m --format json +``` + +Коды выхода `evaluation`: `0` — результат готов, `1` — ошибка, `3` — проверка +ещё не готова, `4` — проверка заменена, результата не будет. В выводе есть +вердикт, результаты автоматических проверок, число и приоритеты замечаний +рецензента, но нет его рекомендаций; полный разбор — по ссылке `result_url`. + ## Язык SoftPractice CLI поддерживает английский и русский языки и по возможности diff --git a/cmd/softpractice/agent_commands.go b/cmd/softpractice/agent_commands.go new file mode 100644 index 0000000..6d62967 --- /dev/null +++ b/cmd/softpractice/agent_commands.go @@ -0,0 +1,360 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "net/http" + "sort" + "strings" + "time" + + "github.com/arthur-s/softpractice-cli/internal/learnercli" +) + +// exitStatus ends the process with its code after the command has already +// written its result, so main prints nothing more. +type exitStatus int + +func (e exitStatus) Error() string { return fmt.Sprintf("exit status %d", int(e)) } + +const ( + // exitNotReady: the evaluation is still queued or running. + exitNotReady exitStatus = 3 + // exitSuperseded: the evaluation was superseded and has no result. + exitSuperseded exitStatus = 4 +) + +const defaultEvaluationTimeout = 10 * time.Minute + +type machineEvaluationResult struct { + ContractVersion int `json:"contract_version"` + Kind string `json:"kind"` + Outcome evaluationOutcome `json:"outcome"` + SubmissionID string `json:"submission_id"` + EvaluationJobID string `json:"evaluation_job_id,omitempty"` + JobState string `json:"job_state"` + Terminal bool `json:"terminal"` + Attempt int `json:"attempt,omitempty"` + MaxAttempts int `json:"max_attempts,omitempty"` + NextPollSeconds int `json:"next_poll_seconds,omitempty"` + UpdatedAt *time.Time `json:"updated_at,omitempty"` + ResultPath string `json:"result_path"` + ResultURL string `json:"result_url"` + Evaluation *machineEvaluation `json:"evaluation,omitempty"` + Feedback *machineFeedback `json:"feedback,omitempty"` +} + +func buildMachineEvaluationResult( + result evaluationResult, + webURL string, + detail reviewFeedbackDetail, +) (machineEvaluationResult, error) { + payload := machineEvaluationResult{ + ContractVersion: 1, Kind: "softpractice.evaluation", + Outcome: result.Outcome, SubmissionID: result.SubmissionID, + ResultPath: "/submissions/" + result.SubmissionID + "/result", + } + payload.ResultURL = strings.TrimRight(webURL, "/") + payload.ResultPath + if result.Outcome == evaluationSuperseded { + payload.JobState = "superseded" + payload.Terminal = true + return payload, nil + } + submission := result.Submission + updatedAt := submission.UpdatedAt + payload.EvaluationJobID = submission.EvaluationJobID + payload.JobState = submission.JobState + payload.Terminal = submission.Terminal + payload.Attempt = submission.Attempt + payload.MaxAttempts = submission.MaxAttempts + payload.NextPollSeconds = submission.NextPollSeconds + payload.UpdatedAt = &updatedAt + payload.Evaluation = submission.Evaluation + if result.Outcome == evaluationReady && len(result.RawEvaluation) != 0 { + feedback, err := buildMachineFeedback(result.RawEvaluation, detail) + if err != nil { + return machineEvaluationResult{}, err + } + payload.Feedback = feedback + } + return payload, nil +} + +func evaluationCommand( + ctx context.Context, + useCases learnerUseCases, + args []string, + output, errorOutput io.Writer, +) error { + flags := flag.NewFlagSet("evaluation", flag.ContinueOnError) + flags.SetOutput(errorOutput) + id := flags.String("id", "", "submission ID; defaults to the latest linked submission") + wait := flags.Bool("wait", false, "poll until the evaluation is ready") + timeout := flags.Duration("timeout", defaultEvaluationTimeout, "maximum wait with --wait") + format := flags.String("format", "text", "output format: text or json") + if err := flags.Parse(args); err != nil { + return err + } + usage := text(ctx, + "использование: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--format text|json]", + "usage: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--format text|json]") + if flags.NArg() != 0 || *timeout < 0 { + return errors.New(usage) + } + outputFormat, err := parseOutputFormat(*format) + if err != nil { + return err + } + payload, err := evaluationPayload(ctx, useCases, strings.TrimSpace(*id), *wait, *timeout, errorOutput) + if err != nil { + if outputFormat == outputFormatJSON { + _ = writeMachineError(output, err) + } + return err + } + if outputFormat == outputFormatJSON { + if err := writeMachineJSON(output, payload); err != nil { + return err + } + } else { + writeEvaluationText(ctx, output, payload) + } + switch payload.Outcome { + case evaluationPending: + return exitNotReady + case evaluationSuperseded: + return exitSuperseded + } + return nil +} + +func evaluationPayload( + ctx context.Context, + useCases learnerUseCases, + submissionID string, + wait bool, + timeout time.Duration, + errorOutput io.Writer, +) (machineEvaluationResult, error) { + if submissionID == "" { + latest, err := useCases.LatestSubmission(ctx) + if err != nil { + return machineEvaluationResult{}, err + } + if latest.PredecessorAssignmentID != "" { + fmt.Fprintf(errorOutput, text(ctx, + "По %s отправок ещё нет; показан принятый результат %s.\n", + "%s has no submission yet; showing the accepted %s result.\n", + ), latest.CurrentAssignmentID, latest.PredecessorAssignmentID) + } + submissionID = latest.SubmissionID + } + result, err := useCases.Evaluation(ctx, submissionID, evaluationWait{Wait: wait, Timeout: timeout}) + if err != nil { + return machineEvaluationResult{}, err + } + return buildMachineEvaluationResult(result, settingsFromContext(ctx).WebURL, machineReviewFeedback) +} + +func writeEvaluationText(ctx context.Context, output io.Writer, payload machineEvaluationResult) { + fmt.Fprintf(output, text(ctx, "Отправка: %s\n", "Submission: %s\n"), payload.SubmissionID) + switch payload.Outcome { + case evaluationSuperseded: + fmt.Fprintln(output, text(ctx, + "Проверка этой отправки заменена; результата у неё не будет.", + "This evaluation was superseded; it will not have a result.")) + return + case evaluationPending: + fmt.Fprintf(output, text(ctx, + "Проверка ещё не готова (%s). Повторите: softpractice evaluation --wait\n", + "The evaluation is not ready yet (%s). Check again: softpractice evaluation --wait\n"), + payload.JobState) + return + } + if payload.Evaluation != nil { + fmt.Fprintf(output, text(ctx, "Результат: %s\n", "Result: %s\n"), payload.Evaluation.Status) + if technical := payload.Evaluation.TechnicalError; technical != nil { + fmt.Fprintf(output, text(ctx, "Техническая ошибка: %s (%s), support %s\n", "Technical error: %s (%s), support %s\n"), + technical.ErrorCode, technical.Stage, technical.SupportID) + } + } + if feedback := payload.Feedback; feedback != nil { + if deterministic := feedback.Deterministic; deterministic != nil { + fmt.Fprintf(output, text(ctx, "Автоматические проверки: %s\n", "Automated checks: %s\n"), deterministic.Status) + for _, check := range deterministic.Checks { + fmt.Fprintf(output, " %-4s %s: %s\n", check.Status, check.ID, check.Summary) + } + } + if review := feedback.Review; review != nil && review.Status != "skipped" { + verdict := "-" + if review.Verdict != nil { + verdict = *review.Verdict + } + fmt.Fprintf(output, text(ctx, "Ревью: %s (%s)\n", "Review: %s (%s)\n"), verdict, review.Mode) + fmt.Fprintf(output, text(ctx, "Замечаний рецензента: %d%s\n", "Reviewer findings: %d%s\n"), + review.FindingsCount, formatPriorityCounts(review.FindingsByPriority)) + for _, finding := range review.Findings { + fmt.Fprintf(output, " [%s] %s\n", finding.Priority, finding.Title) + } + } + } + fmt.Fprintf(output, text(ctx, "Полный разбор: %s\n", "Full review: %s\n"), payload.ResultURL) +} + +func formatPriorityCounts(counts map[string]int) string { + if len(counts) == 0 { + return "" + } + order := map[string]int{"high": 0, "medium": 1, "low": 2} + keys := make([]string, 0, len(counts)) + for key := range counts { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + left, leftKnown := order[keys[i]] + right, rightKnown := order[keys[j]] + if leftKnown != rightKnown { + return leftKnown + } + if left != right { + return left < right + } + return keys[i] < keys[j] + }) + parts := make([]string, 0, len(keys)) + for _, key := range keys { + parts = append(parts, fmt.Sprintf("%s %d", key, counts[key])) + } + return " (" + strings.Join(parts, ", ") + ")" +} + +type machineAssignment struct { + ContractVersion int `json:"contract_version"` + Kind string `json:"kind"` + WorkspaceID string `json:"workspace_id"` + ProjectID string `json:"project_id"` + Assignment assignmentDetail `json:"assignment"` + Material *theoryMaterial `json:"material"` + Transition *machineTransition `json:"transition,omitempty"` + LessonVersionUpdate *machineLessonVersionUpdate `json:"lesson_version_update,omitempty"` +} + +func assignmentCommand( + ctx context.Context, + useCases learnerUseCases, + args []string, + output, errorOutput io.Writer, +) error { + usage := text(ctx, + "использование: softpractice assignment show [--format text|json]", + "usage: softpractice assignment show [--format text|json]") + if len(args) == 0 || args[0] != "show" { + return errors.New(usage) + } + flags := flag.NewFlagSet("assignment show", flag.ContinueOnError) + flags.SetOutput(errorOutput) + format := flags.String("format", "text", "output format: text or json") + if err := flags.Parse(args[1:]); err != nil { + return err + } + if flags.NArg() != 0 { + return errors.New(usage) + } + outputFormat, err := parseOutputFormat(*format) + if err != nil { + return err + } + snapshot, err := useCases.Assignment(ctx) + if err != nil { + if outputFormat == outputFormatJSON { + _ = writeMachineError(output, err) + } + return err + } + payload := machineAssignment{ + ContractVersion: 1, Kind: "softpractice.assignment", + WorkspaceID: snapshot.Workspace.Workspace.ID, ProjectID: snapshot.Link.ProjectID, + Assignment: snapshot.Assignment, Material: snapshot.Material, + } + if pendingTransition(snapshot.Link, snapshot.Workspace) { + payload.Transition = &machineTransition{ + FromAssignmentID: snapshot.Link.AssignmentID, FromAssignmentVersion: snapshot.Link.AssignmentVersion, + ToAssignmentID: snapshot.Workspace.Assignment.ID, ToAssignmentVersion: snapshot.Workspace.Assignment.Version, + } + } + if pendingLessonVersion(snapshot.Link, snapshot.Workspace) { + payload.LessonVersionUpdate = &machineLessonVersionUpdate{ + AssignmentID: snapshot.Link.AssignmentID, + FromVersion: snapshot.Link.AssignmentVersion, ToVersion: snapshot.Workspace.Assignment.Version, + } + } + if outputFormat == outputFormatJSON { + return writeMachineJSON(output, payload) + } + fmt.Fprintf(output, "# %s (%s v%d)\n\n", payload.Assignment.Title, payload.Assignment.ID, payload.Assignment.Version) + if payload.Transition != nil { + fmt.Fprintf(output, text(ctx, + "Переход не применён: файлы урока %s ещё не в этой папке (выполните `softpractice update`).\n\n", + "Transition pending: the %s lesson files are not in this folder yet (run `softpractice update`).\n\n"), + payload.Transition.ToAssignmentID) + } + fmt.Fprintln(output, strings.TrimRight(payload.Assignment.InstructionsMarkdown, "\n")) + if material := payload.Material; material != nil { + fmt.Fprintf(output, "\n---\n\n%s\n", strings.TrimRight(material.Markdown, "\n")) + } + return nil +} + +// machineError is written to standard output by the agent-facing commands in +// JSON mode, so a caller that reads only stdout still learns why the command +// failed. The same message also goes to standard error. +type machineError struct { + ContractVersion int `json:"contract_version"` + Kind string `json:"kind"` + Error struct { + Code string `json:"code"` + Message string `json:"message"` + HTTPStatus int `json:"http_status,omitempty"` + APICode string `json:"api_code,omitempty"` + SupportID string `json:"support_id,omitempty"` + } `json:"error"` +} + +// errorCode classifies an error for machine output. login_required and +// project_not_linked tell the caller that a person has to act: sign in with +// `softpractice login`, or work inside the linked project. +func errorCode(err error) string { + var statusError *learnercli.HTTPError + switch { + case errors.Is(err, learnercli.ErrLoginRequired): + return "login_required" + case errors.As(err, &statusError) && statusError.Status == http.StatusUnauthorized: + return "login_required" + case errors.Is(err, learnercli.ErrProjectNotLinked): + return "project_not_linked" + case errors.Is(err, errNotGitRepository): + return "not_git_repository" + case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + return "cancelled" + case errors.As(err, &statusError): + return "api_error" + default: + return "error" + } +} + +func writeMachineError(output io.Writer, err error) error { + payload := machineError{ContractVersion: 1, Kind: "softpractice.error"} + payload.Error.Code = errorCode(err) + payload.Error.Message = err.Error() + var statusError *learnercli.HTTPError + if errors.As(err, &statusError) { + payload.Error.HTTPStatus = statusError.Status + payload.Error.APICode = statusError.Code + payload.Error.SupportID = statusError.SupportID + } + return writeMachineJSON(output, payload) +} diff --git a/cmd/softpractice/agent_commands_test.go b/cmd/softpractice/agent_commands_test.go new file mode 100644 index 0000000..b29a638 --- /dev/null +++ b/cmd/softpractice/agent_commands_test.go @@ -0,0 +1,578 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "os/exec" + "path/filepath" + "reflect" + "sort" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/arthur-s/softpractice-cli/internal/learnercli" +) + +const reviewerDirectionMarker = "DIRECTION-TEXT-MUST-NOT-LEAK" + +// fakeClock stands in for the wall clock and the sleep between polls. +type fakeClock struct { + mu sync.Mutex + now time.Time + sleeps []time.Duration +} + +func (c *fakeClock) useCases(client *learnercli.Client, startDirectory string) learnerUseCases { + useCases := newLearnerUseCases(client, startDirectory) + useCases.now = func() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now + } + useCases.sleep = func(_ context.Context, delay time.Duration) error { + c.mu.Lock() + defer c.mu.Unlock() + c.sleeps = append(c.sleeps, delay) + c.now = c.now.Add(delay) + return nil + } + return useCases +} + +func pendingEvaluationBody(submissionID string) map[string]any { + return map[string]any{ + "submission_id": submissionID, "evaluation_job_id": "018f47a2-8b54-7d52-8f6f-1b4f01c61a25", + "job_state": "leased", "attempt": 1, "max_attempts": 3, "next_poll_seconds": 2, + "updated_at": "2026-07-29T09:32:03Z", + } +} + +func reviseEvaluationBody(submissionID string) map[string]any { + return map[string]any{ + "submission_id": submissionID, "evaluation_job_id": "018f47a2-8b54-7d52-8f6f-1b4f01c61a25", + "job_state": "completed", "attempt": 1, "max_attempts": 3, + "updated_at": "2026-07-29T09:32:15Z", + "evaluation": map[string]any{ + "id": "018f47a3-551a-7d52-8f6f-1b4f01c61a26", "schema_version": 1, + "exercise_id": "pa-foundation-05", "evaluator_version": "0.7", + "profile_id": "local", "profile_version": "4", "profile_sha256": strings.Repeat("a", 64), + "status": "revise", "completed_at": "2026-07-29T09:32:15Z", + "evidence": map[string]any{"sha256": strings.Repeat("b", 64)}, + "deterministic": map[string]any{"status": "passed", "checks": []any{ + map[string]any{"id": "open-scenarios", "kind": "public_tests", "status": "pass", "summary": "Open scenarios passed."}, + map[string]any{ + "id": "required-scenarios", "kind": "required_behavior", "status": "pass", "summary": "Required behavior confirmed.", + "counterexample": nil, + }, + }}, + "review": map[string]any{ + "projection_version": 1, "status": "completed", "mode": "llm", + "authoritative": true, "verdict": "revise", + "defense_questions": []any{"DEFENSE-QUESTION-TEXT"}, + "uncertainty": map[string]any{"blocking": false, "reason": nil}, + "rubric": []any{ + map[string]any{"criterion": "responsibilities", "level": 1, "rationale": "RUBRIC-RATIONALE-TEXT", "evidence_refs": []any{"app.py"}}, + map[string]any{"criterion": "tests", "level": 2, "rationale": "RUBRIC-RATIONALE-TEXT", "evidence_refs": []any{"test_app.py"}}, + }, + "feedback": []any{ + map[string]any{ + "priority": "high", "title": "FINDING-TITLE-TEXT", "observation": "OBSERVATION-TEXT", + "risk": "RISK-TEXT", "direction": reviewerDirectionMarker, "evidence_refs": []any{"app.py"}, + }, + map[string]any{ + "priority": "medium", "title": "FINDING-TITLE-TEXT", "observation": "OBSERVATION-TEXT", + "risk": "RISK-TEXT", "direction": reviewerDirectionMarker, "evidence_refs": []any{"app.py"}, + }, + }, + }, + "technical_error": nil, + }, + } +} + +// evaluationServer answers the evaluation endpoint of submissionID with the +// scripted responses in order, repeating the last one. +func evaluationServer(t *testing.T, submissionID string, script ...func(http.ResponseWriter)) (*httptest.Server, *int) { + t.Helper() + calls := 0 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/v1/submissions/"+submissionID+"/evaluation" { + http.NotFound(writer, request) + return + } + step := script[min(calls, len(script)-1)] + calls++ + step(writer) + })) + t.Cleanup(server.Close) + return server, &calls +} + +func respondPending(retryAfter string) func(http.ResponseWriter) { + return func(writer http.ResponseWriter) { + writer.Header().Set("Retry-After", retryAfter) + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(writer).Encode(pendingEvaluationBody(currentSubmissionID)) + } +} + +func respondRevise(writer http.ResponseWriter) { + writeTestJSON(writer, reviseEvaluationBody(currentSubmissionID)) +} + +// currentSubmissionID keeps the scripted handlers short; tests do not run in +// parallel. +var currentSubmissionID = "018f47a2-165d-7d52-8f6f-1b4f01c61a23" + +func runEvaluation(t *testing.T, useCases learnerUseCases, args ...string) (string, string, error) { + t.Helper() + var output, errorOutput bytes.Buffer + ctx := withSettings(context.Background(), runtimeSettings{ + Language: languageEnglish, WebURL: "https://softpractice.example", + }) + err := evaluationCommand(ctx, useCases, args, &output, &errorOutput) + return output.String(), errorOutput.String(), err +} + +func TestEvaluationWaitHonoursRetryAfterAndHidesDirections(t *testing.T) { + server, calls := evaluationServer(t, currentSubmissionID, + respondPending("7"), respondPending("5"), respondRevise) + clock := &fakeClock{now: time.Unix(0, 0)} + useCases := clock.useCases(savedTestClient(t, server.URL), "") + + output, _, err := runEvaluation(t, useCases, "--id", currentSubmissionID, "--wait", "--timeout", "1m", "--format", "json") + if err != nil { + t.Fatal(err) + } + if *calls != 3 { + t.Fatalf("calls = %d, want 3", *calls) + } + if want := []time.Duration{7 * time.Second, 5 * time.Second}; !reflect.DeepEqual(clock.sleeps, want) { + t.Fatalf("sleeps = %v, want %v", clock.sleeps, want) + } + for _, secret := range []string{ + reviewerDirectionMarker, "OBSERVATION-TEXT", "RISK-TEXT", "FINDING-TITLE-TEXT", + "RUBRIC-RATIONALE-TEXT", "DEFENSE-QUESTION-TEXT", `"direction"`, + } { + if strings.Contains(output, secret) { + t.Fatalf("JSON output contains %q:\n%s", secret, output) + } + } + var payload machineEvaluationResult + if err := decodeTestJSON([]byte(output), &payload); err != nil { + t.Fatal(err) + } + review := payload.Feedback.Review + if payload.Outcome != evaluationReady || payload.Evaluation.Status != "revise" || + payload.Feedback.Detail != reviewFeedbackSummary || review.FindingsCount != 2 || + review.FindingsByPriority["high"] != 1 || review.FindingsByPriority["medium"] != 1 || + *review.Verdict != "revise" || len(review.Rubric) != 2 || review.Rubric[0].Level != 1 || + payload.ResultURL != "https://softpractice.example/submissions/"+currentSubmissionID+"/result" { + t.Fatalf("unexpected payload: %+v", payload) + } + + text, _, err := runEvaluation(t, useCases, "--id", currentSubmissionID) + if err != nil { + t.Fatal(err) + } + if strings.Contains(text, reviewerDirectionMarker) || strings.Contains(text, "FINDING-TITLE-TEXT") || + !strings.Contains(text, "Reviewer findings: 2 (high 1, medium 1)") { + t.Fatalf("unexpected text output:\n%s", text) + } +} + +func TestMachineReviewFeedbackDefaultsToSummary(t *testing.T) { + if machineReviewFeedback != reviewFeedbackSummary { + t.Fatalf("machineReviewFeedback = %q; reviewer directions must stay out of agent output by default", machineReviewFeedback) + } + raw, err := json.Marshal(reviseEvaluationBody(currentSubmissionID)["evaluation"]) + if err != nil { + t.Fatal(err) + } + full, err := buildMachineFeedback(raw, reviewFeedbackFull) + if err != nil { + t.Fatal(err) + } + if len(full.Review.Findings) != 2 || full.Review.Findings[0].Direction != reviewerDirectionMarker { + t.Fatalf("full detail lost the findings: %+v", full.Review) + } +} + +func TestEvaluationTimeoutReportsNotReady(t *testing.T) { + server, calls := evaluationServer(t, currentSubmissionID, respondPending("30")) + clock := &fakeClock{now: time.Unix(0, 0)} + useCases := clock.useCases(savedTestClient(t, server.URL), "") + + output, _, err := runEvaluation(t, useCases, "--id", currentSubmissionID, "--wait", "--timeout", "45s", "--format", "json") + if !errors.Is(err, exitNotReady) { + t.Fatalf("err = %v, want exit status 3", err) + } + // It never polls sooner than Retry-After and stops when the next allowed + // poll would fall after the timeout. + if *calls != 2 || !reflect.DeepEqual(clock.sleeps, []time.Duration{30 * time.Second}) { + t.Fatalf("calls = %d, sleeps = %v", *calls, clock.sleeps) + } + var payload machineEvaluationResult + if err := decodeTestJSON([]byte(output), &payload); err != nil { + t.Fatal(err) + } + if payload.Outcome != evaluationPending || payload.Terminal || payload.JobState != "leased" || + payload.Evaluation != nil || payload.Feedback != nil { + t.Fatalf("unexpected payload: %+v", payload) + } + + // Without --wait a pending evaluation is the same outcome after one poll. + _, _, err = runEvaluation(t, useCases, "--id", currentSubmissionID) + if !errors.Is(err, exitNotReady) || *calls != 3 { + t.Fatalf("err = %v, calls = %d", err, *calls) + } +} + +func TestEvaluationWaitsOutRateLimit(t *testing.T) { + server, _ := evaluationServer(t, currentSubmissionID, + respondPending("2"), + func(writer http.ResponseWriter) { + writer.Header().Set("Retry-After", "11") + writeTestAPIError(writer, http.StatusTooManyRequests, "rate_limited") + }, + respondRevise) + clock := &fakeClock{now: time.Unix(0, 0)} + useCases := clock.useCases(savedTestClient(t, server.URL), "") + if _, _, err := runEvaluation(t, useCases, "--id", currentSubmissionID, "--wait"); err != nil { + t.Fatal(err) + } + if want := []time.Duration{2 * time.Second, 11 * time.Second}; !reflect.DeepEqual(clock.sleeps, want) { + t.Fatalf("sleeps = %v, want %v", clock.sleeps, want) + } +} + +func TestEvaluationSupersededIsASeparateOutcome(t *testing.T) { + server, _ := evaluationServer(t, currentSubmissionID, func(writer http.ResponseWriter) { + writeTestAPIError(writer, http.StatusConflict, "evaluation_superseded") + }) + clock := &fakeClock{now: time.Unix(0, 0)} + output, _, err := runEvaluation(t, clock.useCases(savedTestClient(t, server.URL), ""), + "--id", currentSubmissionID, "--wait", "--format", "json") + if !errors.Is(err, exitSuperseded) { + t.Fatalf("err = %v, want exit status 4", err) + } + var payload machineEvaluationResult + if err := decodeTestJSON([]byte(output), &payload); err != nil { + t.Fatal(err) + } + if payload.Outcome != evaluationSuperseded || payload.JobState != "superseded" || len(clock.sleeps) != 0 { + t.Fatalf("unexpected payload: %+v", payload) + } +} + +func TestEvaluationUnauthorizedAsksForLogin(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writeTestAPIError(writer, http.StatusUnauthorized, "invalid_credentials") + })) + defer server.Close() + output, _, err := runEvaluation(t, newLearnerUseCases(savedTestClient(t, server.URL), ""), + "--id", currentSubmissionID, "--format", "json") + if err == nil { + t.Fatal("expected an error") + } + var payload machineError + if err := decodeTestJSON([]byte(output), &payload); err != nil { + t.Fatal(err) + } + if payload.Kind != "softpractice.error" || payload.Error.Code != "login_required" || payload.Error.HTTPStatus != 401 { + t.Fatalf("unexpected error payload: %+v", payload) + } + if strings.Contains(output, "access-token") || strings.Contains(output, "refresh-token") { + t.Fatalf("error output leaks a token:\n%s", output) + } + + // With no stored login at all the same code is reported and nothing + // reaches the API. + store := learnercli.CredentialStore{ + Path: filepath.Join(t.TempDir(), "credentials.json"), Secrets: &mainMemorySecretStore{values: map[string]string{}}, + } + client, err := learnercli.NewClient(server.URL, store) + if err != nil { + t.Fatal(err) + } + output, _, err = runEvaluation(t, newLearnerUseCases(client, ""), "--id", currentSubmissionID, "--format", "json") + if !errors.Is(err, learnercli.ErrLoginRequired) || !strings.Contains(output, `"code": "login_required"`) { + t.Fatalf("err = %v, output:\n%s", err, output) + } +} + +func TestAgentCommandsWithoutLinkedProject(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + t.Errorf("unexpected API request %s", request.URL.Path) + http.NotFound(writer, request) + })) + defer server.Close() + client := savedTestClient(t, server.URL) + + unlinked := t.TempDir() + if output, err := exec.Command("git", "-C", unlinked, "init", "--quiet").CombinedOutput(); err != nil { + t.Fatalf("git init: %v: %s", err, output) + } + if output, err := exec.Command("git", "-C", unlinked, + "-c", "user.name=Test", "-c", "user.email=test@example.com", + "commit", "--quiet", "--allow-empty", "-m", "initial").CombinedOutput(); err != nil { + t.Fatalf("git commit: %v: %s", err, output) + } + for _, tc := range []struct { + directory string + code string + }{ + {directory: unlinked, code: "project_not_linked"}, + {directory: t.TempDir(), code: "not_git_repository"}, + } { + output, _, err := runEvaluation(t, newLearnerUseCases(client, tc.directory), "--format", "json") + if err == nil || !strings.Contains(output, `"code": "`+tc.code+`"`) { + t.Fatalf("%s: err = %v, output:\n%s", tc.code, err, output) + } + var assignmentOutput, errorOutput bytes.Buffer + err = assignmentCommand(context.Background(), newLearnerUseCases(client, tc.directory), + []string{"show", "--format", "json"}, &assignmentOutput, &errorOutput) + if err == nil || !strings.Contains(assignmentOutput.String(), `"code": "`+tc.code+`"`) { + t.Fatalf("assignment %s: err = %v, output:\n%s", tc.code, err, assignmentOutput.String()) + } + } +} + +func assignmentServer(t *testing.T, workspaceID string, withMaterial bool) *httptest.Server { + t.Helper() + now := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch { + case request.URL.Path == "/v1/workspaces/"+workspaceID+"/current-assignment": + writeTestJSON(writer, map[string]any{ + "workspace": map[string]any{ + "id": workspaceID, "project_id": "equipment-rental-python", "state": "active", + "base_revision_id": nil, "support_mode": nil, "created_at": now, "updated_at": now, + }, + "assignment": map[string]any{ + "id": "pa-foundation-01", "version": 2, "title": "Price rule", + "state": "available", "local_checks": testLocalChecks(), + }, + "latest_submission": nil, + }) + case request.URL.Path == "/v1/assignments/pa-foundation-01" && request.URL.Query().Get("version") == "1": + writeTestJSON(writer, map[string]any{ + "id": "pa-foundation-01", "version": 1, "title": "Price rule", "kind": "foundation", + "project_setup": "download", "exercise_id": "pa-foundation-01", "estimated_minutes": 45, + "instructions_markdown": "Extract the price rule.", "starter_ref": "starter@1.0.0", + "content_sha256": strings.Repeat("c", 64), "published_at": now, + }) + case request.URL.Path == "/v1/assignments/pa-foundation-01/material" && request.URL.Query().Get("version") == "1": + if !withMaterial { + writeTestAPIError(writer, http.StatusNotFound, "resource_not_found") + return + } + writeTestJSON(writer, map[string]any{ + "id": "pa-theory-rules", "title": "Rules", "estimated_minutes": 6, + "markdown": "# Rules\n\nTheory text.\n", + "diagram": map[string]any{"title": "Flow", "text_alternative": "A calls B.", "nodes": []any{ + map[string]any{"title": "A", "text": "a"}, map[string]any{"title": "B", "text": "b"}, + }}, + }) + default: + t.Errorf("unexpected API request %s?%s", request.URL.Path, request.URL.RawQuery) + http.NotFound(writer, request) + } + })) + t.Cleanup(server.Close) + return server +} + +func TestAssignmentShowReadsTheFolderLessonVersion(t *testing.T) { + workspaceID := uuid.NewString() + // The folder is pinned to v1 while the server has published v2: the + // requirement the tree is being written for is v1. + root := createPinnedLinkedGitRepository(t, workspaceID, "pa-foundation-01", 1) + for _, withMaterial := range []bool{true, false} { + server := assignmentServer(t, workspaceID, withMaterial) + var output, errorOutput bytes.Buffer + if err := assignmentCommand(context.Background(), newLearnerUseCases(savedTestClient(t, server.URL), root), + []string{"show", "--format", "json"}, &output, &errorOutput); err != nil { + t.Fatal(err) + } + var payload machineAssignment + if err := decodeTestJSON(output.Bytes(), &payload); err != nil { + t.Fatal(err) + } + if payload.Kind != "softpractice.assignment" || payload.Assignment.Version != 1 || + payload.Assignment.InstructionsMarkdown != "Extract the price rule." || + payload.LessonVersionUpdate == nil || payload.LessonVersionUpdate.ToVersion != 2 { + t.Fatalf("unexpected payload: %+v", payload) + } + if withMaterial != (payload.Material != nil) { + t.Fatalf("material = %+v, want present=%t", payload.Material, withMaterial) + } + if !withMaterial && !strings.Contains(output.String(), `"material": null`) { + t.Fatalf("absent material must be null:\n%s", output.String()) + } + } +} + +// jsonKeyPaths lists every object key path in a JSON document, with array +// elements folded into "[]", so a test can pin the shape of a contract. +func jsonKeyPaths(t *testing.T, data []byte) []string { + t.Helper() + var document any + if err := json.Unmarshal(data, &document); err != nil { + t.Fatal(err) + } + seen := map[string]bool{} + var walk func(prefix string, value any) + walk = func(prefix string, value any) { + switch typed := value.(type) { + case map[string]any: + for key, child := range typed { + path := key + if prefix != "" { + path = prefix + "." + key + } + seen[path] = true + walk(path, child) + } + case []any: + for _, child := range typed { + walk(prefix+"[]", child) + } + } + } + walk("", document) + paths := make([]string, 0, len(seen)) + for path := range seen { + paths = append(paths, path) + } + sort.Strings(paths) + return paths +} + +// TestAgentJSONContractsAreStable pins the key sets of the agent-facing JSON +// documents. A change here is a contract change: add keys only, and bump +// contract_version when a key is removed, renamed, or changes meaning. +func TestAgentJSONContractsAreStable(t *testing.T) { + server, _ := evaluationServer(t, currentSubmissionID, respondRevise) + clock := &fakeClock{now: time.Unix(0, 0)} + output, _, err := runEvaluation(t, clock.useCases(savedTestClient(t, server.URL), ""), + "--id", currentSubmissionID, "--format", "json") + if err != nil { + t.Fatal(err) + } + want := []string{ + "attempt", "contract_version", "evaluation", "evaluation.deterministic_status", + "evaluation.exercise_id", "evaluation.review", "evaluation.review.authoritative", + "evaluation.review.mode", "evaluation.review.status", "evaluation.review.verdict", + "evaluation.schema_version", "evaluation.status", "evaluation_job_id", + "feedback", "feedback.detail", "feedback.deterministic", "feedback.deterministic.checks", + "feedback.deterministic.checks[].has_counterexample", "feedback.deterministic.checks[].id", + "feedback.deterministic.checks[].kind", "feedback.deterministic.checks[].status", + "feedback.deterministic.checks[].summary", "feedback.deterministic.status", + "feedback.review", "feedback.review.authoritative", "feedback.review.blocking_uncertainty", + "feedback.review.findings_by_priority", "feedback.review.findings_by_priority.high", + "feedback.review.findings_by_priority.medium", "feedback.review.findings_count", + "feedback.review.mode", "feedback.review.rubric", "feedback.review.rubric[].criterion", + "feedback.review.rubric[].level", "feedback.review.status", "feedback.review.verdict", + "job_state", "kind", "max_attempts", "outcome", "result_path", "result_url", + "submission_id", "terminal", "updated_at", + } + if got := jsonKeyPaths(t, []byte(output)); !reflect.DeepEqual(got, want) { + t.Fatalf("softpractice.evaluation keys changed:\ngot %q\nwant %q", got, want) + } + + var errorOutput bytes.Buffer + if err := writeMachineError(&errorOutput, &learnercli.HTTPError{ + Status: 503, Code: "service_unavailable", Message: "down", SupportID: uuid.NewString(), + }); err != nil { + t.Fatal(err) + } + wantError := []string{ + "contract_version", "error", "error.api_code", "error.code", "error.http_status", + "error.message", "error.support_id", "kind", + } + if got := jsonKeyPaths(t, errorOutput.Bytes()); !reflect.DeepEqual(got, wantError) { + t.Fatalf("softpractice.error keys changed:\ngot %q\nwant %q", got, wantError) + } + + workspaceID := uuid.NewString() + root := createPinnedLinkedGitRepository(t, workspaceID, "pa-foundation-01", 1) + var assignmentOutput bytes.Buffer + if err := assignmentCommand(context.Background(), + newLearnerUseCases(savedTestClient(t, assignmentServer(t, workspaceID, true).URL), root), + []string{"show", "--format", "json"}, &assignmentOutput, &bytes.Buffer{}); err != nil { + t.Fatal(err) + } + wantAssignment := []string{ + "assignment", "assignment.content_sha256", "assignment.estimated_minutes", + "assignment.exercise_id", "assignment.id", "assignment.instructions_markdown", + "assignment.kind", "assignment.project_setup", "assignment.title", "assignment.version", + "contract_version", "kind", "lesson_version_update", "lesson_version_update.assignment_id", + "lesson_version_update.from_version", "lesson_version_update.to_version", "material", + "material.diagram", "material.diagram.nodes", "material.diagram.nodes[].text", + "material.diagram.nodes[].title", "material.diagram.text_alternative", "material.diagram.title", + "material.estimated_minutes", "material.id", "material.markdown", "material.title", + "project_id", "workspace_id", + } + if got := jsonKeyPaths(t, assignmentOutput.Bytes()); !reflect.DeepEqual(got, wantAssignment) { + t.Fatalf("softpractice.assignment keys changed:\ngot %q\nwant %q", got, wantAssignment) + } +} + +func TestStatusJSONReportsTitleAndLatestEvaluationStatus(t *testing.T) { + workspaceID := uuid.NewString() + now := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/v1/me": + writeTestJSON(writer, map[string]any{ + "id": uuid.NewString(), "email": "ada@example.com", "display_name": "Ada", + "email_verified": true, "cli_last_session_at": nil, "created_at": now, + }) + case "/v1/workspaces/" + workspaceID + "/current-assignment": + writeTestJSON(writer, map[string]any{ + "workspace": map[string]any{ + "id": workspaceID, "project_id": "equipment-rental-python", "state": "active", + "base_revision_id": nil, "support_mode": nil, "created_at": now, "updated_at": now, + }, + "assignment": map[string]any{ + "id": "pa-foundation-05", "version": 1, "title": "Module boundaries", + "state": "submitted", "local_checks": testLocalChecks(), + }, + "latest_submission": map[string]any{ + "id": currentSubmissionID, "revision_id": uuid.NewString(), + "job_state": "completed", "submitted_at": now, + }, + }) + case "/v1/submissions/" + currentSubmissionID + "/evaluation": + respondRevise(writer) + default: + http.NotFound(writer, request) + } + })) + defer server.Close() + root := createPinnedLinkedGitRepository(t, workspaceID, "pa-foundation-05", 1) + var output, errorOutput bytes.Buffer + if err := status(context.Background(), savedTestClient(t, server.URL), root, + []string{"--format", "json"}, &output, &errorOutput); err != nil { + t.Fatal(err) + } + var payload machineStatus + if err := decodeTestJSON(output.Bytes(), &payload); err != nil { + t.Fatal(err) + } + if payload.Assignment.Title != "Module boundaries" || payload.LatestSubmission == nil || + payload.LatestSubmission.EvaluationStatus != "revise" || payload.Transition != nil || + payload.LessonVersionUpdate != nil { + t.Fatalf("unexpected status: %+v", payload) + } +} diff --git a/cmd/softpractice/feedback.go b/cmd/softpractice/feedback.go new file mode 100644 index 0000000..0f4930d --- /dev/null +++ b/cmd/softpractice/feedback.go @@ -0,0 +1,157 @@ +package main + +import ( + "encoding/json" + "fmt" +) + +// reviewFeedbackDetail says how much of the reviewer's feedback the `evaluation` +// command prints, in both text and JSON. +type reviewFeedbackDetail string + +const ( + // reviewFeedbackSummary prints the verdict, the deterministic check + // results, how many findings the reviewer made, their priorities, and the + // rubric levels. It leaves out every free-text part of the review: the + // findings' titles, observations, risks, and directions, the rubric + // rationales, the uncertainty reason, the defense questions, and the + // counterexample details. The full review is on the result page. + reviewFeedbackSummary reviewFeedbackDetail = "summary" + // reviewFeedbackFull also prints the reviewer's findings verbatim. + reviewFeedbackFull reviewFeedbackDetail = "full" +) + +// machineReviewFeedback is the one setting that decides whether the +// reviewer's recommendations reach command output. This output is read by +// coding agents; with the summary an agent can tell the learner what the +// verdict is and where to read it, but it does not get the recommendations to +// apply on the learner's behalf. +const machineReviewFeedback = reviewFeedbackSummary + +// evaluationFeedbackSource decodes only the parts of the public evaluation +// projection that any feedback detail level may print. +type evaluationFeedbackSource struct { + Deterministic *struct { + Status string `json:"status"` + Checks []struct { + ID string `json:"id"` + Kind string `json:"kind"` + Status string `json:"status"` + Summary string `json:"summary"` + Counterexample json.RawMessage `json:"counterexample"` + } `json:"checks"` + } `json:"deterministic"` + Review *struct { + Status string `json:"status"` + Mode string `json:"mode"` + Authoritative bool `json:"authoritative"` + Verdict *string `json:"verdict"` + Feedback []struct { + Priority string `json:"priority"` + Title string `json:"title"` + Observation string `json:"observation"` + Risk string `json:"risk"` + Direction string `json:"direction"` + } `json:"feedback"` + Rubric []struct { + Criterion string `json:"criterion"` + Level int `json:"level"` + } `json:"rubric"` + Uncertainty *struct { + Blocking bool `json:"blocking"` + } `json:"uncertainty"` + } `json:"review"` +} + +type machineFeedback struct { + Detail reviewFeedbackDetail `json:"detail"` + Deterministic *machineDeterministicPart `json:"deterministic,omitempty"` + Review *machineReviewPart `json:"review,omitempty"` +} + +type machineDeterministicPart struct { + Status string `json:"status"` + Checks []machineDeterministicCheck `json:"checks"` +} + +type machineDeterministicCheck struct { + ID string `json:"id"` + Kind string `json:"kind"` + Status string `json:"status"` + Summary string `json:"summary"` + HasCounterexample bool `json:"has_counterexample"` +} + +type machineReviewPart struct { + Status string `json:"status"` + Mode string `json:"mode"` + Authoritative bool `json:"authoritative"` + Verdict *string `json:"verdict"` + Blocking bool `json:"blocking_uncertainty"` + FindingsCount int `json:"findings_count"` + FindingsByPriority map[string]int `json:"findings_by_priority"` + Rubric []machineRubricLevel `json:"rubric"` + Findings []machineReviewFinding `json:"findings,omitempty"` +} + +type machineRubricLevel struct { + Criterion string `json:"criterion"` + Level int `json:"level"` +} + +// machineReviewFinding is printed only at reviewFeedbackFull. +type machineReviewFinding struct { + Priority string `json:"priority"` + Title string `json:"title"` + Observation string `json:"observation"` + Risk string `json:"risk"` + Direction string `json:"direction"` +} + +// buildMachineFeedback projects a terminal evaluation at the given detail. A +// technical failure has neither part and yields an empty summary. +func buildMachineFeedback(raw json.RawMessage, detail reviewFeedbackDetail) (*machineFeedback, error) { + var source evaluationFeedbackSource + if err := json.Unmarshal(raw, &source); err != nil { + return nil, fmt.Errorf("decode evaluation feedback: %w", err) + } + feedback := &machineFeedback{Detail: detail} + if source.Deterministic != nil { + part := &machineDeterministicPart{ + Status: source.Deterministic.Status, + Checks: make([]machineDeterministicCheck, 0, len(source.Deterministic.Checks)), + } + for _, check := range source.Deterministic.Checks { + part.Checks = append(part.Checks, machineDeterministicCheck{ + ID: check.ID, Kind: check.Kind, Status: check.Status, Summary: check.Summary, + HasCounterexample: len(check.Counterexample) != 0 && string(check.Counterexample) != "null", + }) + } + feedback.Deterministic = part + } + if review := source.Review; review != nil { + part := &machineReviewPart{ + Status: review.Status, Mode: review.Mode, Authoritative: review.Authoritative, + Verdict: review.Verdict, FindingsCount: len(review.Feedback), + FindingsByPriority: map[string]int{}, + Rubric: make([]machineRubricLevel, 0, len(review.Rubric)), + } + if review.Uncertainty != nil { + part.Blocking = review.Uncertainty.Blocking + } + for _, item := range review.Feedback { + part.FindingsByPriority[item.Priority]++ + if detail == reviewFeedbackFull { + part.Findings = append(part.Findings, machineReviewFinding{ + Priority: item.Priority, Title: item.Title, Observation: item.Observation, + Risk: item.Risk, Direction: item.Direction, + }) + } + } + for _, item := range review.Rubric { + part.Rubric = append(part.Rubric, machineRubricLevel{Criterion: item.Criterion, Level: item.Level}) + } + feedback.Review = part + } + return feedback, nil +} diff --git a/cmd/softpractice/main.go b/cmd/softpractice/main.go index 9b077eb..9c0af2b 100644 --- a/cmd/softpractice/main.go +++ b/cmd/softpractice/main.go @@ -93,6 +93,10 @@ func main() { if errors.Is(err, flag.ErrHelp) { return } + var status exitStatus + if errors.As(err, &status) { + os.Exit(int(status)) + } fmt.Fprintln(os.Stderr, "softpractice:", err) os.Exit(1) } @@ -216,6 +220,10 @@ func run( return status(ctx, client, "", remaining[1:], output, errorOutput) case "submission": return submissionCommand(ctx, client, "", remaining[1:], output, errorOutput) + case "evaluation": + return evaluationCommand(ctx, newLearnerUseCases(client, ""), remaining[1:], output, errorOutput) + case "assignment": + return assignmentCommand(ctx, newLearnerUseCases(client, ""), remaining[1:], output, errorOutput) case "project": return projectCommand(ctx, client, remaining[1:], output, errorOutput) case "starter": diff --git a/cmd/softpractice/ui.go b/cmd/softpractice/ui.go index 4990fb5..8575ad9 100644 --- a/cmd/softpractice/ui.go +++ b/cmd/softpractice/ui.go @@ -154,8 +154,8 @@ func text(ctx context.Context, russian, english string) string { func printHelp(ctx context.Context, output io.Writer) { fmt.Fprintln(output, text(ctx, - "Softpractice — тренажёр инженерной практики\n\nИспользование:\n softpractice [--api URL] [--lang ru|en] <команда>\n\nНачните здесь:\n login Войти через браузер\n logout Завершить CLI-сессию\n starter Скачать проект первого урока\n status Показать текущий урок и состояние проекта\n\nРабота в проекте:\n check Проверить текущие изменения\n submit Отправить зафиксированный Git-коммит\n submission show Показать состояние и результат отправки\n submission download Скачать ZIP сохранённой ревизии\n update Применить переход к следующему доступному уроку\n open Открыть workspace или результат в браузере\n\nНастройки:\n set api-url URL Сохранить адрес API\n set web-url URL Сохранить адрес web-приложения\n set lang ru|en Сохранить язык CLI\n set auto-checks true|false Автозапуск проверок при отправке\n config Показать действующие настройки\n\nВерсия:\n version Показать версию CLI\n\nПодробнее:\n softpractice help <команда>", - "Softpractice — engineering practice simulator\n\nUsage:\n softpractice [--api URL] [--lang ru|en] \n\nGet started:\n login Sign in in a browser\n logout Revoke the CLI session\n starter Download the first lesson project\n status Show the current lesson and project state\n\nWork in a project:\n check Check the current changes\n submit Submit the committed Git revision\n submission show Show submission state and result\n submission download Download a saved revision ZIP\n update Apply the next available lesson transition\n open Open the workspace or result in a browser\n\nSettings:\n set api-url URL Save the API address\n set web-url URL Save the web app address\n set lang ru|en Save the CLI language\n set auto-checks true|false Run checks automatically on submit\n config Show effective settings\n\nVersion:\n version Show the CLI version\n\nMore help:\n softpractice help ")) + "Softpractice — тренажёр инженерной практики\n\nИспользование:\n softpractice [--api URL] [--lang ru|en] <команда>\n\nНачните здесь:\n login Войти через браузер\n logout Завершить CLI-сессию\n starter Скачать проект первого урока\n status Показать текущий урок и состояние проекта\n\nРабота в проекте:\n assignment show Показать текст текущего задания\n check Проверить текущие изменения\n submit Отправить зафиксированный Git-коммит\n submission show Показать состояние и результат отправки\n submission download Скачать ZIP сохранённой ревизии\n evaluation Показать результат проверки, с --wait дождаться его\n update Применить переход к следующему доступному уроку\n open Открыть workspace или результат в браузере\n\nНастройки:\n set api-url URL Сохранить адрес API\n set web-url URL Сохранить адрес web-приложения\n set lang ru|en Сохранить язык CLI\n set auto-checks true|false Автозапуск проверок при отправке\n config Показать действующие настройки\n\nВерсия:\n version Показать версию CLI\n\nПодробнее:\n softpractice help <команда>", + "Softpractice — engineering practice simulator\n\nUsage:\n softpractice [--api URL] [--lang ru|en] \n\nGet started:\n login Sign in in a browser\n logout Revoke the CLI session\n starter Download the first lesson project\n status Show the current lesson and project state\n\nWork in a project:\n assignment show Show the current assignment text\n check Check the current changes\n submit Submit the committed Git revision\n submission show Show submission state and result\n submission download Download a saved revision ZIP\n evaluation Show the evaluation result; --wait waits for it\n update Apply the next available lesson transition\n open Open the workspace or result in a browser\n\nSettings:\n set api-url URL Save the API address\n set web-url URL Save the web app address\n set lang ru|en Save the CLI language\n set auto-checks true|false Run checks automatically on submit\n config Show effective settings\n\nVersion:\n version Show the CLI version\n\nMore help:\n softpractice help ")) fmt.Fprintln(output, text(ctx, "\nВосстановление:\n project restore Восстановить продолжимый связанный проект", "\nRecovery:\n project restore Restore a resumable linked project")) @@ -176,6 +176,10 @@ func printCommandHelp(ctx context.Context, output io.Writer, command string) err russian, english = "Использование: softpractice status [--format text|json]\n\nПоказывает состояние текущего связанного Git-проекта.", "Usage: softpractice status [--format text|json]\n\nShows the state of the current linked Git project." case "submission": russian, english = "Использование: softpractice submission show [--id ID] [--format text|json|json-v2]\n softpractice submission download --id ID [--output PATH]\n\nshow показывает безопасный статус и результат своей отправки. download сохраняет ZIP точной отправленной ревизии, как кнопка «Скачать проект» на странице результата.", "Usage: softpractice submission show [--id ID] [--format text|json|json-v2]\n softpractice submission download --id ID [--output PATH]\n\nshow displays the safe status and result of your submission. download saves the exact submitted revision ZIP, equivalent to the result-page Download project button." + case "evaluation": + russian, english = "Использование: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--format text|json]\n\nПоказывает результат проверки последней отправки или отправки --id. С --wait опрашивает сервер с интервалом, который он назначает, пока проверка не завершится или не истечёт --timeout (по умолчанию 10m). Рекомендации рецензента не выводятся; полный разбор — по ссылке.\n\nКоды выхода: 0 — результат готов, 1 — ошибка, 3 — проверка ещё не готова, 4 — проверка заменена и результата не будет.", "Usage: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--format text|json]\n\nShows the evaluation result of the latest submission or of --id. With --wait it polls at the interval the server sets until the evaluation finishes or --timeout (default 10m) passes. Reviewer recommendations are not printed; the full review is at the printed link.\n\nExit codes: 0 result ready, 1 error, 3 not ready yet, 4 superseded, no result." + case "assignment": + russian, english = "Использование: softpractice assignment show [--format text|json]\n\nПоказывает текст задания, над которым работает этот проект, и связанный с ним теоретический материал.", "Usage: softpractice assignment show [--format text|json]\n\nShows the assignment this project is working on and its linked theory material." case "submit": russian, english = "Использование: softpractice submit [--yes] [--checks=true|false]\n\nОтправляет чистый Git commit текущего урока на проверку.", "Usage: softpractice submit [--yes] [--checks=true|false]\n\nSubmits the clean Git commit for the current lesson." case "check": diff --git a/packaging/npm/README.md b/packaging/npm/README.md index a5ad147..114109e 100644 --- a/packaging/npm/README.md +++ b/packaging/npm/README.md @@ -79,6 +79,8 @@ softpractice update | `softpractice check` | Run public checks against the current working tree | | `softpractice submit` | Submit the current committed Git revision | | `softpractice submission show` | Show the latest submission and review result | +| `softpractice evaluation` | Show the evaluation result of the latest submission; `--wait` waits for it | +| `softpractice assignment show` | Show the assignment this project is working on | | `softpractice update` | Apply the transition to the next lesson, or move to a newer version of the current lesson (replaces only lesson files, never yours; optional until the lesson ends) | | `softpractice open` | Open the workspace or latest result in the browser | | `softpractice project restore` | Restore a project from the latest usable revision | @@ -108,6 +110,20 @@ softpractice set auto-checks true run. Unlike `softpractice check`, submission checks require a clean working tree and run in an isolated snapshot of `HEAD`. +## Machine-readable output + +`status`, `submission show`, `evaluation`, and `assignment show` accept +`--format json`. Each document has `contract_version` and `kind`. + +```bash +softpractice evaluation --wait --timeout 10m --format json +``` + +`evaluation` exit codes: `0` result ready, `1` error, `3` not ready yet, +`4` evaluation superseded, no result. Its output has the verdict, the +automated check results, and the number and priorities of reviewer findings, +without the reviewer's recommendations; the full review is at `result_url`. + ## Language SoftPractice CLI supports English and Russian and selects a language from the From 96484333858981b776ac6b88a761c26a329a103f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 07:00:36 +0000 Subject: [PATCH 04/14] cli: drop submission show --format json-v2 The CLI is not released yet, so there is no v2 consumer to keep. v2 passed the whole evaluation projection through, including the reviewer's recommendations and the defense questions; `--format json` is now the only machine format. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01K9oUWMHw3ZCHpfmoBGcgeF --- cmd/softpractice/machine.go | 63 ++++------------------------------- cmd/softpractice/main_test.go | 31 ----------------- cmd/softpractice/ui.go | 2 +- 3 files changed, 7 insertions(+), 89 deletions(-) diff --git a/cmd/softpractice/machine.go b/cmd/softpractice/machine.go index e71d40c..928a609 100644 --- a/cmd/softpractice/machine.go +++ b/cmd/softpractice/machine.go @@ -195,25 +195,6 @@ type machineSubmission struct { Evaluation *machineEvaluation `json:"evaluation,omitempty"` } -// machineSubmissionV2 preserves the entire learner-safe evaluation projection -// returned by the public API. Version 1 intentionally exposes a compact -// summary; v2 is for automation that must assert individual safe checks and -// review fields without parsing human-oriented CLI output. -type machineSubmissionV2 struct { - ContractVersion int `json:"contract_version"` - Kind string `json:"kind"` - SubmissionID string `json:"submission_id"` - EvaluationJobID string `json:"evaluation_job_id"` - JobState string `json:"job_state"` - Terminal bool `json:"terminal"` - Attempt int `json:"attempt"` - MaxAttempts int `json:"max_attempts"` - NextPollSeconds int `json:"next_poll_seconds,omitempty"` - UpdatedAt time.Time `json:"updated_at"` - ResultPath string `json:"result_path"` - Evaluation json.RawMessage `json:"evaluation,omitempty"` -} - func submissionCommand( ctx context.Context, client *learnercli.Client, @@ -244,22 +225,16 @@ func showSubmission( flags := flag.NewFlagSet("submission show", flag.ContinueOnError) flags.SetOutput(errorOutput) id := flags.String("id", "", "submission ID; defaults to the latest linked submission") - format := flags.String("format", "text", "output format: text, json, or json-v2") + format := flags.String("format", "text", "output format: text or json") if err := flags.Parse(args); err != nil { return err } if flags.NArg() != 0 { - return errors.New("usage: softpractice submission show [--id ID] [--format text|json|json-v2]") - } - formatValue := strings.ToLower(strings.TrimSpace(*format)) - outputFormat := outputFormatText - jsonV2 := formatValue == "json-v2" - if !jsonV2 { - var err error - outputFormat, err = parseOutputFormat(formatValue) - if err != nil { - return errors.New("format must be text, json, or json-v2") - } + return errors.New("usage: softpractice submission show [--id ID] [--format text|json]") + } + outputFormat, err := parseOutputFormat(*format) + if err != nil { + return err } submissionID := strings.TrimSpace(*id) useCases := newLearnerUseCases(client, startDirectory) @@ -293,9 +268,6 @@ func showSubmission( if payload.SubmissionID != submissionID { return errors.New("evaluation response does not match the requested submission") } - if jsonV2 { - return writeMachineJSON(output, normalizeSubmissionResponseV2(payload, response.Evaluation)) - } if outputFormat == outputFormatJSON { return writeMachineJSON(output, payload) } @@ -447,29 +419,6 @@ func normalizeSubmissionResponse(response evaluationAPIResponse) (machineSubmiss return payload, nil } -func normalizeSubmissionResponseV2( - payload machineSubmission, - evaluation json.RawMessage, -) machineSubmissionV2 { - result := machineSubmissionV2{ - ContractVersion: 2, - Kind: payload.Kind, - SubmissionID: payload.SubmissionID, - EvaluationJobID: payload.EvaluationJobID, - JobState: payload.JobState, - Terminal: payload.Terminal, - Attempt: payload.Attempt, - MaxAttempts: payload.MaxAttempts, - NextPollSeconds: payload.NextPollSeconds, - UpdatedAt: payload.UpdatedAt, - ResultPath: payload.ResultPath, - } - if payload.Terminal { - result.Evaluation = append(json.RawMessage(nil), evaluation...) - } - return result -} - func writeMachineJSON(output io.Writer, payload any) error { encoder := json.NewEncoder(output) encoder.SetEscapeHTML(false) diff --git a/cmd/softpractice/main_test.go b/cmd/softpractice/main_test.go index 43f44d8..0b09562 100644 --- a/cmd/softpractice/main_test.go +++ b/cmd/softpractice/main_test.go @@ -358,37 +358,6 @@ func TestLinkedStatusSubmitAndOpenCommandFlow(t *testing.T) { t.Fatalf("machine submission = %+v", submissionPayload) } - var submissionJSONV2 bytes.Buffer - if err := submissionCommand( - context.Background(), client, repositoryRoot, - []string{"show", "--id", submissionID, "--format", "json-v2"}, - &submissionJSONV2, &statusErrors, - ); err != nil { - t.Fatal(err) - } - var submissionPayloadV2 machineSubmissionV2 - if err := decodeTestJSON(submissionJSONV2.Bytes(), &submissionPayloadV2); err != nil { - t.Fatal(err) - } - var evaluationV2 struct { - Deterministic struct { - Checks []struct { - ID string `json:"id"` - } `json:"checks"` - } `json:"deterministic"` - Review struct { - Summary string `json:"summary"` - Feedback []any `json:"feedback"` - } `json:"review"` - } - if submissionPayloadV2.ContractVersion != 2 || !submissionPayloadV2.Terminal || - json.Unmarshal(submissionPayloadV2.Evaluation, &evaluationV2) != nil || - len(evaluationV2.Deterministic.Checks) != 1 || - evaluationV2.Deterministic.Checks[0].ID != "public-regression" || - evaluationV2.Review.Summary != "safe feedback" || len(evaluationV2.Review.Feedback) != 1 { - t.Fatalf("machine submission v2 = %+v, evaluation = %+v", submissionPayloadV2, evaluationV2) - } - for range 2 { var submitOutput, submitErrors bytes.Buffer if err := submit( diff --git a/cmd/softpractice/ui.go b/cmd/softpractice/ui.go index 8575ad9..9d8ccce 100644 --- a/cmd/softpractice/ui.go +++ b/cmd/softpractice/ui.go @@ -175,7 +175,7 @@ func printCommandHelp(ctx context.Context, output io.Writer, command string) err case "status": russian, english = "Использование: softpractice status [--format text|json]\n\nПоказывает состояние текущего связанного Git-проекта.", "Usage: softpractice status [--format text|json]\n\nShows the state of the current linked Git project." case "submission": - russian, english = "Использование: softpractice submission show [--id ID] [--format text|json|json-v2]\n softpractice submission download --id ID [--output PATH]\n\nshow показывает безопасный статус и результат своей отправки. download сохраняет ZIP точной отправленной ревизии, как кнопка «Скачать проект» на странице результата.", "Usage: softpractice submission show [--id ID] [--format text|json|json-v2]\n softpractice submission download --id ID [--output PATH]\n\nshow displays the safe status and result of your submission. download saves the exact submitted revision ZIP, equivalent to the result-page Download project button." + russian, english = "Использование: softpractice submission show [--id ID] [--format text|json]\n softpractice submission download --id ID [--output PATH]\n\nshow показывает безопасный статус и результат своей отправки. download сохраняет ZIP точной отправленной ревизии, как кнопка «Скачать проект» на странице результата.", "Usage: softpractice submission show [--id ID] [--format text|json]\n softpractice submission download --id ID [--output PATH]\n\nshow displays the safe status and result of your submission. download saves the exact submitted revision ZIP, equivalent to the result-page Download project button." case "evaluation": russian, english = "Использование: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--format text|json]\n\nПоказывает результат проверки последней отправки или отправки --id. С --wait опрашивает сервер с интервалом, который он назначает, пока проверка не завершится или не истечёт --timeout (по умолчанию 10m). Рекомендации рецензента не выводятся; полный разбор — по ссылке.\n\nКоды выхода: 0 — результат готов, 1 — ошибка, 3 — проверка ещё не готова, 4 — проверка заменена и результата не будет.", "Usage: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--format text|json]\n\nShows the evaluation result of the latest submission or of --id. With --wait it polls at the interval the server sets until the evaluation finishes or --timeout (default 10m) passes. Reviewer recommendations are not printed; the full review is at the printed link.\n\nExit codes: 0 result ready, 1 error, 3 not ready yet, 4 superseded, no result." case "assignment": From c21149700aece13eaa02869e3a2d408fbb77d182 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 07:31:52 +0000 Subject: [PATCH 05/14] cli: show the review as Markdown, directions on request, no contract_version `softpractice evaluation` now prints what the checks and the reviewer found and why: counterexamples, the review summary, the rubric with rationales, and each finding's observation and risk. The reviewer's directions and the counterexamples' next steps are printed only with --directions; defaultReviewFeedback keeps them out otherwise. The reviewer's questions are never printed, only their number. The text output is Markdown so it reads in a terminal and renders in an agent's chat. The JSON documents lose contract_version, since the CLI has no released consumers to version for, and the evaluation document carries one `result` object instead of two overlapping ones. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01K9oUWMHw3ZCHpfmoBGcgeF --- README.md | 9 +- README_RU.md | 7 +- cmd/softpractice/agent_commands.go | 258 ++++++++++++++-------- cmd/softpractice/agent_commands_test.go | 132 ++++++----- cmd/softpractice/feedback.go | 282 +++++++++++++++--------- cmd/softpractice/machine.go | 15 +- cmd/softpractice/main_test.go | 2 +- cmd/softpractice/ui.go | 2 +- packaging/npm/README.md | 9 +- 9 files changed, 453 insertions(+), 263 deletions(-) diff --git a/README.md b/README.md index 114109e..3c6a32a 100644 --- a/README.md +++ b/README.md @@ -113,16 +113,17 @@ tree and run in an isolated snapshot of `HEAD`. ## Machine-readable output `status`, `submission show`, `evaluation`, and `assignment show` accept -`--format json`. Each document has `contract_version` and `kind`. +`--format json`. Each document has a `kind`. ```bash softpractice evaluation --wait --timeout 10m --format json ``` `evaluation` exit codes: `0` result ready, `1` error, `3` not ready yet, -`4` evaluation superseded, no result. Its output has the verdict, the -automated check results, and the number and priorities of reviewer findings, -without the reviewer's recommendations; the full review is at `result_url`. +`4` evaluation superseded, no result. Its output has the automated check +results and the review without the reviewer's recommendations; +`--directions` adds them. The reviewer's questions are on the result page at +`result_url`. ## Language diff --git a/README_RU.md b/README_RU.md index aa884ae..70c8059 100644 --- a/README_RU.md +++ b/README_RU.md @@ -115,7 +115,7 @@ softpractice set auto-checks true ## Машиночитаемый вывод `status`, `submission show`, `evaluation` и `assignment show` принимают -`--format json`. У каждого документа есть `contract_version` и `kind`. +`--format json`. У каждого документа есть поле `kind`. ```bash softpractice evaluation --wait --timeout 10m --format json @@ -123,8 +123,9 @@ softpractice evaluation --wait --timeout 10m --format json Коды выхода `evaluation`: `0` — результат готов, `1` — ошибка, `3` — проверка ещё не готова, `4` — проверка заменена, результата не будет. В выводе есть -вердикт, результаты автоматических проверок, число и приоритеты замечаний -рецензента, но нет его рекомендаций; полный разбор — по ссылке `result_url`. +результаты автоматических проверок и ревью без рекомендаций рецензента; +`--directions` добавляет их. Вопросы рецензента — на странице результата по +ссылке `result_url`. ## Язык diff --git a/cmd/softpractice/agent_commands.go b/cmd/softpractice/agent_commands.go index 6d62967..6b664b0 100644 --- a/cmd/softpractice/agent_commands.go +++ b/cmd/softpractice/agent_commands.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "net/http" - "sort" "strings" "time" @@ -30,21 +29,18 @@ const ( const defaultEvaluationTimeout = 10 * time.Minute type machineEvaluationResult struct { - ContractVersion int `json:"contract_version"` - Kind string `json:"kind"` - Outcome evaluationOutcome `json:"outcome"` - SubmissionID string `json:"submission_id"` - EvaluationJobID string `json:"evaluation_job_id,omitempty"` - JobState string `json:"job_state"` - Terminal bool `json:"terminal"` - Attempt int `json:"attempt,omitempty"` - MaxAttempts int `json:"max_attempts,omitempty"` - NextPollSeconds int `json:"next_poll_seconds,omitempty"` - UpdatedAt *time.Time `json:"updated_at,omitempty"` - ResultPath string `json:"result_path"` - ResultURL string `json:"result_url"` - Evaluation *machineEvaluation `json:"evaluation,omitempty"` - Feedback *machineFeedback `json:"feedback,omitempty"` + Kind string `json:"kind"` + Outcome evaluationOutcome `json:"outcome"` + SubmissionID string `json:"submission_id"` + EvaluationJobID string `json:"evaluation_job_id,omitempty"` + JobState string `json:"job_state"` + Attempt int `json:"attempt,omitempty"` + MaxAttempts int `json:"max_attempts,omitempty"` + NextPollSeconds int `json:"next_poll_seconds,omitempty"` + UpdatedAt *time.Time `json:"updated_at,omitempty"` + ResultURL string `json:"result_url"` + // Result is present only for the ready outcome. + Result *machineResult `json:"result,omitempty"` } func buildMachineEvaluationResult( @@ -53,32 +49,27 @@ func buildMachineEvaluationResult( detail reviewFeedbackDetail, ) (machineEvaluationResult, error) { payload := machineEvaluationResult{ - ContractVersion: 1, Kind: "softpractice.evaluation", - Outcome: result.Outcome, SubmissionID: result.SubmissionID, - ResultPath: "/submissions/" + result.SubmissionID + "/result", + Kind: "softpractice.evaluation", Outcome: result.Outcome, SubmissionID: result.SubmissionID, + ResultURL: strings.TrimRight(webURL, "/") + "/submissions/" + result.SubmissionID + "/result", } - payload.ResultURL = strings.TrimRight(webURL, "/") + payload.ResultPath if result.Outcome == evaluationSuperseded { payload.JobState = "superseded" - payload.Terminal = true return payload, nil } submission := result.Submission updatedAt := submission.UpdatedAt payload.EvaluationJobID = submission.EvaluationJobID payload.JobState = submission.JobState - payload.Terminal = submission.Terminal payload.Attempt = submission.Attempt payload.MaxAttempts = submission.MaxAttempts payload.NextPollSeconds = submission.NextPollSeconds payload.UpdatedAt = &updatedAt - payload.Evaluation = submission.Evaluation - if result.Outcome == evaluationReady && len(result.RawEvaluation) != 0 { - feedback, err := buildMachineFeedback(result.RawEvaluation, detail) + if result.Outcome == evaluationReady { + built, err := buildMachineResult(result.RawEvaluation, detail) if err != nil { return machineEvaluationResult{}, err } - payload.Feedback = feedback + payload.Result = built } return payload, nil } @@ -94,13 +85,14 @@ func evaluationCommand( id := flags.String("id", "", "submission ID; defaults to the latest linked submission") wait := flags.Bool("wait", false, "poll until the evaluation is ready") timeout := flags.Duration("timeout", defaultEvaluationTimeout, "maximum wait with --wait") + directions := flags.Bool("directions", false, "also print the reviewer's directions and next steps") format := flags.String("format", "text", "output format: text or json") if err := flags.Parse(args); err != nil { return err } usage := text(ctx, - "использование: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--format text|json]", - "usage: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--format text|json]") + "использование: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--directions] [--format text|json]", + "usage: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--directions] [--format text|json]") if flags.NArg() != 0 || *timeout < 0 { return errors.New(usage) } @@ -108,7 +100,11 @@ func evaluationCommand( if err != nil { return err } - payload, err := evaluationPayload(ctx, useCases, strings.TrimSpace(*id), *wait, *timeout, errorOutput) + detail := defaultReviewFeedback + if *directions { + detail = feedbackWithDirections + } + payload, err := evaluationPayload(ctx, useCases, strings.TrimSpace(*id), *wait, *timeout, detail, errorOutput) if err != nil { if outputFormat == outputFormatJSON { _ = writeMachineError(output, err) @@ -120,7 +116,7 @@ func evaluationCommand( return err } } else { - writeEvaluationText(ctx, output, payload) + writeEvaluationMarkdown(ctx, output, payload) } switch payload.Outcome { case evaluationPending: @@ -137,6 +133,7 @@ func evaluationPayload( submissionID string, wait bool, timeout time.Duration, + detail reviewFeedbackDetail, errorOutput io.Writer, ) (machineEvaluationResult, error) { if submissionID == "" { @@ -156,83 +153,169 @@ func evaluationPayload( if err != nil { return machineEvaluationResult{}, err } - return buildMachineEvaluationResult(result, settingsFromContext(ctx).WebURL, machineReviewFeedback) + return buildMachineEvaluationResult(result, settingsFromContext(ctx).WebURL, detail) } -func writeEvaluationText(ctx context.Context, output io.Writer, payload machineEvaluationResult) { - fmt.Fprintf(output, text(ctx, "Отправка: %s\n", "Submission: %s\n"), payload.SubmissionID) +// writeEvaluationMarkdown prints the evaluation as Markdown, which reads as +// plain text in a terminal and renders in an agent's chat. +func writeEvaluationMarkdown(ctx context.Context, output io.Writer, payload machineEvaluationResult) { switch payload.Outcome { case evaluationSuperseded: - fmt.Fprintln(output, text(ctx, - "Проверка этой отправки заменена; результата у неё не будет.", - "This evaluation was superseded; it will not have a result.")) + fmt.Fprintf(output, text(ctx, + "Проверка отправки `%s` заменена; результата у неё не будет.\n", + "The evaluation of submission `%s` was superseded; it will not have a result.\n"), + payload.SubmissionID) return case evaluationPending: fmt.Fprintf(output, text(ctx, - "Проверка ещё не готова (%s). Повторите: softpractice evaluation --wait\n", - "The evaluation is not ready yet (%s). Check again: softpractice evaluation --wait\n"), - payload.JobState) + "Проверка отправки `%s` ещё не готова (%s). Дождаться: `softpractice evaluation --wait`\n", + "The evaluation of submission `%s` is not ready yet (%s). To wait: `softpractice evaluation --wait`\n"), + payload.SubmissionID, payload.JobState) return } - if payload.Evaluation != nil { - fmt.Fprintf(output, text(ctx, "Результат: %s\n", "Result: %s\n"), payload.Evaluation.Status) - if technical := payload.Evaluation.TechnicalError; technical != nil { - fmt.Fprintf(output, text(ctx, "Техническая ошибка: %s (%s), support %s\n", "Technical error: %s (%s), support %s\n"), - technical.ErrorCode, technical.Stage, technical.SupportID) - } + result := payload.Result + fmt.Fprintf(output, "# %s\n\n", evaluationStatusTitle(ctx, result.Status)) + fmt.Fprintf(output, text(ctx, "Отправка `%s`, попытка проверки %d из %d.\n", "Submission `%s`, evaluation attempt %d of %d.\n"), + payload.SubmissionID, payload.Attempt, payload.MaxAttempts) + if technical := result.TechnicalError; technical != nil { + fmt.Fprintf(output, text(ctx, "\nТехническая ошибка: `%s` на этапе `%s`, support ID `%s`.\n", + "\nTechnical error: `%s` at stage `%s`, support ID `%s`.\n"), + technical.ErrorCode, technical.Stage, technical.SupportID) } - if feedback := payload.Feedback; feedback != nil { - if deterministic := feedback.Deterministic; deterministic != nil { - fmt.Fprintf(output, text(ctx, "Автоматические проверки: %s\n", "Automated checks: %s\n"), deterministic.Status) - for _, check := range deterministic.Checks { - fmt.Fprintf(output, " %-4s %s: %s\n", check.Status, check.ID, check.Summary) + if deterministic := result.Deterministic; deterministic != nil { + fmt.Fprintf(output, text(ctx, "\n## Автоматические проверки: %s\n\n", "\n## Automated checks: %s\n\n"), + deterministicStatusLabel(ctx, deterministic.Status)) + for _, check := range deterministic.Checks { + fmt.Fprintf(output, "- [%s] %s: %s\n", check.Status, check.ID, check.Summary) + if example := check.Counterexample; example != nil { + fmt.Fprintf(output, text(ctx, " - Пример: %s\n", " - Example: %s\n"), example.Title) + fmt.Fprintf(output, text(ctx, " - Сценарий: %s\n", " - Scenario: %s\n"), example.Scenario) + fmt.Fprintf(output, text(ctx, " - Вход: %s\n", " - Input: %s\n"), example.Input) + fmt.Fprintf(output, text(ctx, " - Ожидалось: %s\n", " - Expected: %s\n"), example.Expected) + if example.NextStep != "" { + fmt.Fprintf(output, text(ctx, " - Следующий шаг: %s\n", " - Next step: %s\n"), example.NextStep) + } } } - if review := feedback.Review; review != nil && review.Status != "skipped" { - verdict := "-" - if review.Verdict != nil { - verdict = *review.Verdict + } + if review := result.Review; review != nil && review.Status != "skipped" { + fmt.Fprint(output, text(ctx, "\n## Ревью\n", "\n## Review\n")) + if !review.Authoritative { + fmt.Fprint(output, text(ctx, + "\nПредварительное ревью: оно не определяет результат.\n", + "\nPreliminary review: it does not decide the result.\n")) + } + if review.Summary != "" { + fmt.Fprintf(output, "\n%s\n", review.Summary) + } + if uncertainty := review.Uncertainty; uncertainty != nil && uncertainty.Reason != nil && *uncertainty.Reason != "" { + fmt.Fprintf(output, text(ctx, "\nЧто осталось неясным: %s\n", "\nWhat remains unclear: %s\n"), *uncertainty.Reason) + } + if len(review.Rubric) > 0 { + fmt.Fprint(output, text(ctx, "\n### Критерии\n\n", "\n### Criteria\n\n")) + for _, item := range review.Rubric { + fmt.Fprintf(output, "- **%s**: %s. %s%s\n", item.Criterion, rubricLevelLabel(ctx, item.Level), + item.Rationale, formatEvidence(ctx, item.EvidenceRefs)) } - fmt.Fprintf(output, text(ctx, "Ревью: %s (%s)\n", "Review: %s (%s)\n"), verdict, review.Mode) - fmt.Fprintf(output, text(ctx, "Замечаний рецензента: %d%s\n", "Reviewer findings: %d%s\n"), - review.FindingsCount, formatPriorityCounts(review.FindingsByPriority)) - for _, finding := range review.Findings { - fmt.Fprintf(output, " [%s] %s\n", finding.Priority, finding.Title) + } + if len(review.Findings) > 0 { + fmt.Fprint(output, text(ctx, "\n### Замечания\n", "\n### Findings\n")) + for index, finding := range review.Findings { + fmt.Fprintf(output, "\n%d. **%s** (%s)\n", index+1, finding.Title, priorityLabel(ctx, finding.Priority)) + fmt.Fprintf(output, text(ctx, " - Наблюдение: %s\n", " - Observation: %s\n"), finding.Observation) + fmt.Fprintf(output, text(ctx, " - Риск: %s\n", " - Risk: %s\n"), finding.Risk) + if finding.Direction != "" { + fmt.Fprintf(output, text(ctx, " - Направление: %s\n", " - Direction: %s\n"), finding.Direction) + } + if len(finding.EvidenceRefs) > 0 { + fmt.Fprintf(output, text(ctx, " - Где: %s\n", " - Where: %s\n"), strings.Join(finding.EvidenceRefs, ", ")) + } } } + if review.QuestionsCount > 0 { + fmt.Fprintf(output, text(ctx, + "\nВопросов рецензента к решению: %d. Они на странице результата.\n", + "\nReviewer questions about the solution: %d. They are on the result page.\n"), + review.QuestionsCount) + } } - fmt.Fprintf(output, text(ctx, "Полный разбор: %s\n", "Full review: %s\n"), payload.ResultURL) + if result.DirectionsHidden { + fmt.Fprint(output, text(ctx, + "\nРекомендации рецензента не показаны. Показать: `softpractice evaluation --directions`\n", + "\nThe reviewer's recommendations are not shown. To show them: `softpractice evaluation --directions`\n")) + } + fmt.Fprintf(output, text(ctx, "\nПолный разбор: %s\n", "\nFull review: %s\n"), payload.ResultURL) } -func formatPriorityCounts(counts map[string]int) string { - if len(counts) == 0 { - return "" +func evaluationStatusTitle(ctx context.Context, status string) string { + switch status { + case "accepted": + return text(ctx, "Решение принято", "Solution accepted") + case "revise": + return text(ctx, "Нужна доработка", "Revision needed") + case "uncertain": + return text(ctx, "Рецензенту нужно пояснение", "The reviewer needs an explanation") + case "deterministic_failed": + return text(ctx, "Автоматические проверки не пройдены", "Automated checks failed") + case "ready_for_review": + return text(ctx, "Автоматические проверки пройдены", "Automated checks passed") + case "technical_failure": + return text(ctx, "Проверка не завершилась из-за технической ошибки", "The evaluation failed for a technical reason") + default: + return status } - order := map[string]int{"high": 0, "medium": 1, "low": 2} - keys := make([]string, 0, len(counts)) - for key := range counts { - keys = append(keys, key) +} + +func deterministicStatusLabel(ctx context.Context, status string) string { + switch status { + case "passed": + return text(ctx, "пройдены", "passed") + case "failed": + return text(ctx, "не пройдены", "failed") + case "not_run": + return text(ctx, "не запускались", "not run") + case "technical_failure": + return text(ctx, "техническая ошибка", "technical failure") + default: + return status } - sort.Slice(keys, func(i, j int) bool { - left, leftKnown := order[keys[i]] - right, rightKnown := order[keys[j]] - if leftKnown != rightKnown { - return leftKnown - } - if left != right { - return left < right - } - return keys[i] < keys[j] - }) - parts := make([]string, 0, len(keys)) - for _, key := range keys { - parts = append(parts, fmt.Sprintf("%s %d", key, counts[key])) +} + +// rubricLevelLabel uses the level names of the result page. +func rubricLevelLabel(ctx context.Context, level int) string { + switch level { + case 1: + return text(ctx, "нужно доработать", "needs work") + case 2: + return text(ctx, "выполнено", "met") + case 3: + return text(ctx, "выполнено и закреплено", "met and reinforced") + default: + return text(ctx, "не подтверждено", "not confirmed") + } +} + +func priorityLabel(ctx context.Context, priority string) string { + switch priority { + case "high": + return text(ctx, "высокий приоритет", "high priority") + case "medium": + return text(ctx, "средний приоритет", "medium priority") + case "low": + return text(ctx, "низкий приоритет", "low priority") + default: + return priority + } +} + +func formatEvidence(ctx context.Context, refs []string) string { + if len(refs) == 0 { + return "" } - return " (" + strings.Join(parts, ", ") + ")" + return text(ctx, " Где: ", " Where: ") + strings.Join(refs, ", ") } type machineAssignment struct { - ContractVersion int `json:"contract_version"` Kind string `json:"kind"` WorkspaceID string `json:"workspace_id"` ProjectID string `json:"project_id"` @@ -275,7 +358,7 @@ func assignmentCommand( return err } payload := machineAssignment{ - ContractVersion: 1, Kind: "softpractice.assignment", + Kind: "softpractice.assignment", WorkspaceID: snapshot.Workspace.Workspace.ID, ProjectID: snapshot.Link.ProjectID, Assignment: snapshot.Assignment, Material: snapshot.Material, } @@ -312,9 +395,8 @@ func assignmentCommand( // JSON mode, so a caller that reads only stdout still learns why the command // failed. The same message also goes to standard error. type machineError struct { - ContractVersion int `json:"contract_version"` - Kind string `json:"kind"` - Error struct { + Kind string `json:"kind"` + Error struct { Code string `json:"code"` Message string `json:"message"` HTTPStatus int `json:"http_status,omitempty"` @@ -347,7 +429,7 @@ func errorCode(err error) string { } func writeMachineError(output io.Writer, err error) error { - payload := machineError{ContractVersion: 1, Kind: "softpractice.error"} + payload := machineError{Kind: "softpractice.error"} payload.Error.Code = errorCode(err) payload.Error.Message = err.Error() var statusError *learnercli.HTTPError diff --git a/cmd/softpractice/agent_commands_test.go b/cmd/softpractice/agent_commands_test.go index b29a638..4eceb45 100644 --- a/cmd/softpractice/agent_commands_test.go +++ b/cmd/softpractice/agent_commands_test.go @@ -70,12 +70,15 @@ func reviseEvaluationBody(submissionID string) map[string]any { map[string]any{"id": "open-scenarios", "kind": "public_tests", "status": "pass", "summary": "Open scenarios passed."}, map[string]any{ "id": "required-scenarios", "kind": "required_behavior", "status": "pass", "summary": "Required behavior confirmed.", - "counterexample": nil, + "counterexample": map[string]any{ + "title": "COUNTEREXAMPLE-TITLE", "scenario": "SCENARIO-TEXT", "input": "INPUT-TEXT", + "expected": "EXPECTED-TEXT", "next_step": "NEXT-STEP-TEXT", + }, }, }}, "review": map[string]any{ "projection_version": 1, "status": "completed", "mode": "llm", - "authoritative": true, "verdict": "revise", + "authoritative": true, "verdict": "revise", "summary": "REVIEW-SUMMARY-TEXT", "defense_questions": []any{"DEFENSE-QUESTION-TEXT"}, "uncertainty": map[string]any{"blocking": false, "reason": nil}, "rubric": []any{ @@ -159,23 +162,18 @@ func TestEvaluationWaitHonoursRetryAfterAndHidesDirections(t *testing.T) { if want := []time.Duration{7 * time.Second, 5 * time.Second}; !reflect.DeepEqual(clock.sleeps, want) { t.Fatalf("sleeps = %v, want %v", clock.sleeps, want) } - for _, secret := range []string{ - reviewerDirectionMarker, "OBSERVATION-TEXT", "RISK-TEXT", "FINDING-TITLE-TEXT", - "RUBRIC-RATIONALE-TEXT", "DEFENSE-QUESTION-TEXT", `"direction"`, - } { - if strings.Contains(output, secret) { - t.Fatalf("JSON output contains %q:\n%s", secret, output) - } - } + assertEvaluationOutput(t, output, false) var payload machineEvaluationResult if err := decodeTestJSON([]byte(output), &payload); err != nil { t.Fatal(err) } - review := payload.Feedback.Review - if payload.Outcome != evaluationReady || payload.Evaluation.Status != "revise" || - payload.Feedback.Detail != reviewFeedbackSummary || review.FindingsCount != 2 || - review.FindingsByPriority["high"] != 1 || review.FindingsByPriority["medium"] != 1 || - *review.Verdict != "revise" || len(review.Rubric) != 2 || review.Rubric[0].Level != 1 || + result := payload.Result + if payload.Outcome != evaluationReady || result.Status != "revise" || + result.Detail != feedbackWithoutDirections || !result.DirectionsHidden || + len(result.Review.Findings) != 2 || result.Review.Findings[0].Observation != "OBSERVATION-TEXT" || + result.Review.QuestionsCount != 1 || *result.Review.Verdict != "revise" || + len(result.Review.Rubric) != 2 || result.Review.Rubric[0].Rationale != "RUBRIC-RATIONALE-TEXT" || + result.Deterministic.Checks[1].Counterexample.Expected != "EXPECTED-TEXT" || payload.ResultURL != "https://softpractice.example/submissions/"+currentSubmissionID+"/result" { t.Fatalf("unexpected payload: %+v", payload) } @@ -184,26 +182,58 @@ func TestEvaluationWaitHonoursRetryAfterAndHidesDirections(t *testing.T) { if err != nil { t.Fatal(err) } - if strings.Contains(text, reviewerDirectionMarker) || strings.Contains(text, "FINDING-TITLE-TEXT") || - !strings.Contains(text, "Reviewer findings: 2 (high 1, medium 1)") { - t.Fatalf("unexpected text output:\n%s", text) + assertEvaluationOutput(t, text, false) + for _, want := range []string{ + "# Revision needed", "## Automated checks: passed", "## Review", "REVIEW-SUMMARY-TEXT", + "- **responsibilities**: needs work. RUBRIC-RATIONALE-TEXT Where: app.py", + "1. **FINDING-TITLE-TEXT** (high priority)", " - Risk: RISK-TEXT", + "Reviewer questions about the solution: 1.", "softpractice evaluation --directions", + } { + if !strings.Contains(text, want) { + t.Fatalf("text output lacks %q:\n%s", want, text) + } } } -func TestMachineReviewFeedbackDefaultsToSummary(t *testing.T) { - if machineReviewFeedback != reviewFeedbackSummary { - t.Fatalf("machineReviewFeedback = %q; reviewer directions must stay out of agent output by default", machineReviewFeedback) +// assertEvaluationOutput checks what the evaluation output shows at each +// detail: always what was found and why, the directions and next steps only +// on request, and the reviewer's questions never. +func assertEvaluationOutput(t *testing.T, output string, withDirections bool) { + t.Helper() + for _, shown := range []string{ + "OBSERVATION-TEXT", "RISK-TEXT", "FINDING-TITLE-TEXT", "RUBRIC-RATIONALE-TEXT", + "REVIEW-SUMMARY-TEXT", "SCENARIO-TEXT", "EXPECTED-TEXT", + } { + if !strings.Contains(output, shown) { + t.Fatalf("output lacks %q:\n%s", shown, output) + } } - raw, err := json.Marshal(reviseEvaluationBody(currentSubmissionID)["evaluation"]) - if err != nil { - t.Fatal(err) + if strings.Contains(output, "DEFENSE-QUESTION-TEXT") { + t.Fatalf("output contains a reviewer question:\n%s", output) } - full, err := buildMachineFeedback(raw, reviewFeedbackFull) - if err != nil { - t.Fatal(err) + for _, direction := range []string{reviewerDirectionMarker, "NEXT-STEP-TEXT"} { + if strings.Contains(output, direction) != withDirections { + t.Fatalf("output contains %q = %t, want %t:\n%s", direction, !withDirections, withDirections, output) + } + } +} + +func TestEvaluationDirectionsOnRequest(t *testing.T) { + if defaultReviewFeedback != feedbackWithoutDirections { + t.Fatalf("defaultReviewFeedback = %q; reviewer directions must stay out of output unless requested", defaultReviewFeedback) } - if len(full.Review.Findings) != 2 || full.Review.Findings[0].Direction != reviewerDirectionMarker { - t.Fatalf("full detail lost the findings: %+v", full.Review) + server, _ := evaluationServer(t, currentSubmissionID, respondRevise) + useCases := newLearnerUseCases(savedTestClient(t, server.URL), "") + for _, format := range []string{"json", "text"} { + output, _, err := runEvaluation(t, useCases, "--id", currentSubmissionID, "--directions", "--format", format) + if err != nil { + t.Fatal(err) + } + assertEvaluationOutput(t, output, true) + if format == "text" && (!strings.Contains(output, " - Direction: "+reviewerDirectionMarker) || + strings.Contains(output, "are not shown")) { + t.Fatalf("unexpected text output:\n%s", output) + } } } @@ -225,8 +255,7 @@ func TestEvaluationTimeoutReportsNotReady(t *testing.T) { if err := decodeTestJSON([]byte(output), &payload); err != nil { t.Fatal(err) } - if payload.Outcome != evaluationPending || payload.Terminal || payload.JobState != "leased" || - payload.Evaluation != nil || payload.Feedback != nil { + if payload.Outcome != evaluationPending || payload.JobState != "leased" || payload.Result != nil { t.Fatalf("unexpected payload: %+v", payload) } @@ -459,8 +488,7 @@ func jsonKeyPaths(t *testing.T, data []byte) []string { } // TestAgentJSONContractsAreStable pins the key sets of the agent-facing JSON -// documents. A change here is a contract change: add keys only, and bump -// contract_version when a key is removed, renamed, or changes meaning. +// documents, so that a change to what an agent reads is a deliberate one. func TestAgentJSONContractsAreStable(t *testing.T) { server, _ := evaluationServer(t, currentSubmissionID, respondRevise) clock := &fakeClock{now: time.Unix(0, 0)} @@ -470,21 +498,25 @@ func TestAgentJSONContractsAreStable(t *testing.T) { t.Fatal(err) } want := []string{ - "attempt", "contract_version", "evaluation", "evaluation.deterministic_status", - "evaluation.exercise_id", "evaluation.review", "evaluation.review.authoritative", - "evaluation.review.mode", "evaluation.review.status", "evaluation.review.verdict", - "evaluation.schema_version", "evaluation.status", "evaluation_job_id", - "feedback", "feedback.detail", "feedback.deterministic", "feedback.deterministic.checks", - "feedback.deterministic.checks[].has_counterexample", "feedback.deterministic.checks[].id", - "feedback.deterministic.checks[].kind", "feedback.deterministic.checks[].status", - "feedback.deterministic.checks[].summary", "feedback.deterministic.status", - "feedback.review", "feedback.review.authoritative", "feedback.review.blocking_uncertainty", - "feedback.review.findings_by_priority", "feedback.review.findings_by_priority.high", - "feedback.review.findings_by_priority.medium", "feedback.review.findings_count", - "feedback.review.mode", "feedback.review.rubric", "feedback.review.rubric[].criterion", - "feedback.review.rubric[].level", "feedback.review.status", "feedback.review.verdict", - "job_state", "kind", "max_attempts", "outcome", "result_path", "result_url", - "submission_id", "terminal", "updated_at", + "attempt", "evaluation_job_id", "job_state", "kind", "max_attempts", "outcome", "result", + "result.detail", "result.deterministic", "result.deterministic.checks", + "result.deterministic.checks[].counterexample", + "result.deterministic.checks[].counterexample.expected", + "result.deterministic.checks[].counterexample.input", + "result.deterministic.checks[].counterexample.scenario", + "result.deterministic.checks[].counterexample.title", "result.deterministic.checks[].id", + "result.deterministic.checks[].kind", "result.deterministic.checks[].status", + "result.deterministic.checks[].summary", "result.deterministic.status", + "result.directions_hidden", "result.exercise_id", "result.review", "result.review.authoritative", + "result.review.findings", "result.review.findings[].evidence_refs", + "result.review.findings[].observation", "result.review.findings[].priority", + "result.review.findings[].risk", "result.review.findings[].title", "result.review.mode", + "result.review.questions_count", "result.review.rubric", "result.review.rubric[].criterion", + "result.review.rubric[].evidence_refs", "result.review.rubric[].level", + "result.review.rubric[].rationale", "result.review.status", "result.review.summary", + "result.review.uncertainty", "result.review.uncertainty.blocking", + "result.review.uncertainty.reason", "result.review.verdict", "result.status", "result_url", + "submission_id", "updated_at", } if got := jsonKeyPaths(t, []byte(output)); !reflect.DeepEqual(got, want) { t.Fatalf("softpractice.evaluation keys changed:\ngot %q\nwant %q", got, want) @@ -497,7 +529,7 @@ func TestAgentJSONContractsAreStable(t *testing.T) { t.Fatal(err) } wantError := []string{ - "contract_version", "error", "error.api_code", "error.code", "error.http_status", + "error", "error.api_code", "error.code", "error.http_status", "error.message", "error.support_id", "kind", } if got := jsonKeyPaths(t, errorOutput.Bytes()); !reflect.DeepEqual(got, wantError) { @@ -516,7 +548,7 @@ func TestAgentJSONContractsAreStable(t *testing.T) { "assignment", "assignment.content_sha256", "assignment.estimated_minutes", "assignment.exercise_id", "assignment.id", "assignment.instructions_markdown", "assignment.kind", "assignment.project_setup", "assignment.title", "assignment.version", - "contract_version", "kind", "lesson_version_update", "lesson_version_update.assignment_id", + "kind", "lesson_version_update", "lesson_version_update.assignment_id", "lesson_version_update.from_version", "lesson_version_update.to_version", "material", "material.diagram", "material.diagram.nodes", "material.diagram.nodes[].text", "material.diagram.nodes[].title", "material.diagram.text_alternative", "material.diagram.title", diff --git a/cmd/softpractice/feedback.go b/cmd/softpractice/feedback.go index 0f4930d..7de29d2 100644 --- a/cmd/softpractice/feedback.go +++ b/cmd/softpractice/feedback.go @@ -5,153 +5,229 @@ import ( "fmt" ) -// reviewFeedbackDetail says how much of the reviewer's feedback the `evaluation` -// command prints, in both text and JSON. +// reviewFeedbackDetail says how much of an evaluation the `evaluation` command +// prints, in both text and JSON. type reviewFeedbackDetail string const ( - // reviewFeedbackSummary prints the verdict, the deterministic check - // results, how many findings the reviewer made, their priorities, and the - // rubric levels. It leaves out every free-text part of the review: the - // findings' titles, observations, risks, and directions, the rubric - // rationales, the uncertainty reason, the defense questions, and the - // counterexample details. The full review is on the result page. - reviewFeedbackSummary reviewFeedbackDetail = "summary" - // reviewFeedbackFull also prints the reviewer's findings verbatim. - reviewFeedbackFull reviewFeedbackDetail = "full" + // feedbackWithoutDirections prints what the checks and the reviewer found + // and why it matters: check results with their counterexamples, the + // review summary, the rubric with rationales, and each finding's title, + // observation, and risk. It leaves out what to do about it: the findings' + // directions and the counterexamples' next steps. + feedbackWithoutDirections reviewFeedbackDetail = "without_directions" + // feedbackWithDirections also prints the directions and next steps. + feedbackWithDirections reviewFeedbackDetail = "with_directions" ) -// machineReviewFeedback is the one setting that decides whether the -// reviewer's recommendations reach command output. This output is read by -// coding agents; with the summary an agent can tell the learner what the -// verdict is and where to read it, but it does not get the recommendations to -// apply on the learner's behalf. -const machineReviewFeedback = reviewFeedbackSummary +// defaultReviewFeedback is the one setting that decides whether the +// reviewer's recommendations reach command output when the learner did not +// ask for them with --directions. This output is also read by coding agents: +// by default an agent can show the learner the review, but it does not get a +// list of fixes to apply on the learner's behalf. +// +// The reviewer's questions to the learner are never printed at any detail; +// only their number is. They are answered by the learner, not by an agent +// that happens to read the output. +const defaultReviewFeedback = feedbackWithoutDirections -// evaluationFeedbackSource decodes only the parts of the public evaluation -// projection that any feedback detail level may print. -type evaluationFeedbackSource struct { +// evaluationSource decodes the public evaluation projection that the +// `evaluation` command renders. +type evaluationSource struct { + ExerciseID string `json:"exercise_id"` + Status string `json:"status"` + TechnicalError *struct { + ErrorCode string `json:"error_code"` + Stage string `json:"stage"` + SupportID string `json:"support_id"` + } `json:"technical_error"` Deterministic *struct { Status string `json:"status"` Checks []struct { - ID string `json:"id"` - Kind string `json:"kind"` - Status string `json:"status"` - Summary string `json:"summary"` - Counterexample json.RawMessage `json:"counterexample"` + ID string `json:"id"` + Kind string `json:"kind"` + Status string `json:"status"` + Summary string `json:"summary"` + Counterexample *struct { + Title string `json:"title"` + Scenario string `json:"scenario"` + Input string `json:"input"` + Expected string `json:"expected"` + NextStep string `json:"next_step"` + } `json:"counterexample"` } `json:"checks"` } `json:"deterministic"` Review *struct { - Status string `json:"status"` - Mode string `json:"mode"` - Authoritative bool `json:"authoritative"` - Verdict *string `json:"verdict"` - Feedback []struct { - Priority string `json:"priority"` - Title string `json:"title"` - Observation string `json:"observation"` - Risk string `json:"risk"` - Direction string `json:"direction"` - } `json:"feedback"` + Status string `json:"status"` + Mode string `json:"mode"` + Authoritative bool `json:"authoritative"` + Verdict *string `json:"verdict"` + Summary string `json:"summary"` + Questions []string `json:"defense_questions"` + Uncertainty *struct { + Blocking bool `json:"blocking"` + Reason *string `json:"reason"` + } `json:"uncertainty"` Rubric []struct { - Criterion string `json:"criterion"` - Level int `json:"level"` + Criterion string `json:"criterion"` + Level int `json:"level"` + Rationale string `json:"rationale"` + EvidenceRefs []string `json:"evidence_refs"` } `json:"rubric"` - Uncertainty *struct { - Blocking bool `json:"blocking"` - } `json:"uncertainty"` + Feedback []struct { + Priority string `json:"priority"` + Title string `json:"title"` + Observation string `json:"observation"` + Risk string `json:"risk"` + Direction string `json:"direction"` + EvidenceRefs []string `json:"evidence_refs"` + } `json:"feedback"` } `json:"review"` } -type machineFeedback struct { - Detail reviewFeedbackDetail `json:"detail"` - Deterministic *machineDeterministicPart `json:"deterministic,omitempty"` - Review *machineReviewPart `json:"review,omitempty"` +type machineResult struct { + Status string `json:"status"` + ExerciseID string `json:"exercise_id"` + Detail reviewFeedbackDetail `json:"detail"` + // DirectionsHidden is true when directions or next steps exist but this + // output leaves them out; --directions prints them. + DirectionsHidden bool `json:"directions_hidden"` + TechnicalError *machineTechnicalError `json:"technical_error,omitempty"` + Deterministic *machineDeterministic `json:"deterministic,omitempty"` + Review *machineReviewResult `json:"review,omitempty"` } -type machineDeterministicPart struct { +type machineDeterministic struct { Status string `json:"status"` Checks []machineDeterministicCheck `json:"checks"` } type machineDeterministicCheck struct { - ID string `json:"id"` - Kind string `json:"kind"` - Status string `json:"status"` - Summary string `json:"summary"` - HasCounterexample bool `json:"has_counterexample"` + ID string `json:"id"` + Kind string `json:"kind"` + Status string `json:"status"` + Summary string `json:"summary"` + Counterexample *machineCounterexample `json:"counterexample,omitempty"` +} + +type machineCounterexample struct { + Title string `json:"title"` + Scenario string `json:"scenario"` + Input string `json:"input"` + Expected string `json:"expected"` + // NextStep is printed only with directions. + NextStep string `json:"next_step,omitempty"` } -type machineReviewPart struct { - Status string `json:"status"` - Mode string `json:"mode"` - Authoritative bool `json:"authoritative"` - Verdict *string `json:"verdict"` - Blocking bool `json:"blocking_uncertainty"` - FindingsCount int `json:"findings_count"` - FindingsByPriority map[string]int `json:"findings_by_priority"` - Rubric []machineRubricLevel `json:"rubric"` - Findings []machineReviewFinding `json:"findings,omitempty"` +type machineReviewResult struct { + Status string `json:"status"` + Mode string `json:"mode"` + Authoritative bool `json:"authoritative"` + Verdict *string `json:"verdict"` + Summary string `json:"summary,omitempty"` + Uncertainty *machineUncertainty `json:"uncertainty,omitempty"` + Rubric []machineRubricItem `json:"rubric"` + Findings []machineReviewFinding `json:"findings"` + // QuestionsCount is how many questions the reviewer asked the learner. + // Their text is on the result page. + QuestionsCount int `json:"questions_count"` } -type machineRubricLevel struct { - Criterion string `json:"criterion"` - Level int `json:"level"` +type machineUncertainty struct { + Blocking bool `json:"blocking"` + Reason *string `json:"reason"` +} + +type machineRubricItem struct { + Criterion string `json:"criterion"` + Level int `json:"level"` + Rationale string `json:"rationale"` + EvidenceRefs []string `json:"evidence_refs"` } -// machineReviewFinding is printed only at reviewFeedbackFull. type machineReviewFinding struct { - Priority string `json:"priority"` - Title string `json:"title"` - Observation string `json:"observation"` - Risk string `json:"risk"` - Direction string `json:"direction"` + Priority string `json:"priority"` + Title string `json:"title"` + Observation string `json:"observation"` + Risk string `json:"risk"` + Direction string `json:"direction,omitempty"` + EvidenceRefs []string `json:"evidence_refs"` } -// buildMachineFeedback projects a terminal evaluation at the given detail. A -// technical failure has neither part and yields an empty summary. -func buildMachineFeedback(raw json.RawMessage, detail reviewFeedbackDetail) (*machineFeedback, error) { - var source evaluationFeedbackSource +// buildMachineResult projects a terminal evaluation at the given detail. +func buildMachineResult(raw json.RawMessage, detail reviewFeedbackDetail) (*machineResult, error) { + var source evaluationSource if err := json.Unmarshal(raw, &source); err != nil { - return nil, fmt.Errorf("decode evaluation feedback: %w", err) + return nil, fmt.Errorf("decode evaluation: %w", err) } - feedback := &machineFeedback{Detail: detail} - if source.Deterministic != nil { - part := &machineDeterministicPart{ - Status: source.Deterministic.Status, - Checks: make([]machineDeterministicCheck, 0, len(source.Deterministic.Checks)), + withDirections := detail == feedbackWithDirections + directionsHidden := false + result := &machineResult{Status: source.Status, ExerciseID: source.ExerciseID, Detail: detail} + if technical := source.TechnicalError; technical != nil { + result.TechnicalError = &machineTechnicalError{ + ErrorCode: technical.ErrorCode, Stage: technical.Stage, SupportID: technical.SupportID, } - for _, check := range source.Deterministic.Checks { - part.Checks = append(part.Checks, machineDeterministicCheck{ - ID: check.ID, Kind: check.Kind, Status: check.Status, Summary: check.Summary, - HasCounterexample: len(check.Counterexample) != 0 && string(check.Counterexample) != "null", - }) + } + if deterministic := source.Deterministic; deterministic != nil { + part := &machineDeterministic{ + Status: deterministic.Status, + Checks: make([]machineDeterministicCheck, 0, len(deterministic.Checks)), } - feedback.Deterministic = part + for _, check := range deterministic.Checks { + item := machineDeterministicCheck{ID: check.ID, Kind: check.Kind, Status: check.Status, Summary: check.Summary} + if example := check.Counterexample; example != nil { + item.Counterexample = &machineCounterexample{ + Title: example.Title, Scenario: example.Scenario, + Input: example.Input, Expected: example.Expected, + } + if withDirections { + item.Counterexample.NextStep = example.NextStep + } else if example.NextStep != "" { + directionsHidden = true + } + } + part.Checks = append(part.Checks, item) + } + result.Deterministic = part } if review := source.Review; review != nil { - part := &machineReviewPart{ + part := &machineReviewResult{ Status: review.Status, Mode: review.Mode, Authoritative: review.Authoritative, - Verdict: review.Verdict, FindingsCount: len(review.Feedback), - FindingsByPriority: map[string]int{}, - Rubric: make([]machineRubricLevel, 0, len(review.Rubric)), + Verdict: review.Verdict, Summary: review.Summary, + Rubric: make([]machineRubricItem, 0, len(review.Rubric)), + Findings: make([]machineReviewFinding, 0, len(review.Feedback)), + QuestionsCount: len(review.Questions), } - if review.Uncertainty != nil { - part.Blocking = review.Uncertainty.Blocking + if uncertainty := review.Uncertainty; uncertainty != nil { + part.Uncertainty = &machineUncertainty{Blocking: uncertainty.Blocking, Reason: uncertainty.Reason} + } + for _, item := range review.Rubric { + part.Rubric = append(part.Rubric, machineRubricItem{ + Criterion: item.Criterion, Level: item.Level, + Rationale: item.Rationale, EvidenceRefs: nonNilStrings(item.EvidenceRefs), + }) } for _, item := range review.Feedback { - part.FindingsByPriority[item.Priority]++ - if detail == reviewFeedbackFull { - part.Findings = append(part.Findings, machineReviewFinding{ - Priority: item.Priority, Title: item.Title, Observation: item.Observation, - Risk: item.Risk, Direction: item.Direction, - }) + finding := machineReviewFinding{ + Priority: item.Priority, Title: item.Title, Observation: item.Observation, + Risk: item.Risk, EvidenceRefs: nonNilStrings(item.EvidenceRefs), } + if withDirections { + finding.Direction = item.Direction + } else if item.Direction != "" { + directionsHidden = true + } + part.Findings = append(part.Findings, finding) } - for _, item := range review.Rubric { - part.Rubric = append(part.Rubric, machineRubricLevel{Criterion: item.Criterion, Level: item.Level}) - } - feedback.Review = part + result.Review = part + } + result.DirectionsHidden = directionsHidden + return result, nil +} + +func nonNilStrings(values []string) []string { + if values == nil { + return []string{} } - return feedback, nil + return values } diff --git a/cmd/softpractice/machine.go b/cmd/softpractice/machine.go index 928a609..7a5650c 100644 --- a/cmd/softpractice/machine.go +++ b/cmd/softpractice/machine.go @@ -36,9 +36,8 @@ func parseOutputFormat(value string) (outputFormat, error) { } type machineStatus struct { - ContractVersion int `json:"contract_version"` - Kind string `json:"kind"` - Account struct { + Kind string `json:"kind"` + Account struct { ID string `json:"id"` EmailVerified bool `json:"email_verified"` } `json:"account"` @@ -58,11 +57,10 @@ type machineStatus struct { Clean bool `json:"clean"` } `json:"local"` // Transition is present only while the server has already opened the next - // lesson and this folder still holds the previous one. It is additive: a - // contract_version 1 reader that ignores unknown keys is unaffected. + // lesson and this folder still holds the previous one. Transition *machineTransition `json:"transition,omitempty"` // LessonVersionUpdate is present while the server publishes a newer - // version of the lesson this folder holds. Additive like Transition. + // version of the lesson this folder holds. LessonVersionUpdate *machineLessonVersionUpdate `json:"lesson_version_update,omitempty"` LatestSubmission *machineSubmissionSummary `json:"latest_submission"` } @@ -92,7 +90,7 @@ type machineSubmissionSummary struct { func writeStatusJSON(output io.Writer, snapshot statusSnapshot) error { user, link, workspace, repository := snapshot.User, snapshot.Link, snapshot.Workspace, snapshot.Repository - payload := machineStatus{ContractVersion: 1, Kind: "softpractice.status"} + payload := machineStatus{Kind: "softpractice.status"} payload.Account.ID = user.ID payload.Account.EmailVerified = user.EmailVerified payload.Workspace.ID = workspace.Workspace.ID @@ -181,7 +179,6 @@ type machineTechnicalError struct { } type machineSubmission struct { - ContractVersion int `json:"contract_version"` Kind string `json:"kind"` SubmissionID string `json:"submission_id"` EvaluationJobID string `json:"evaluation_job_id"` @@ -352,7 +349,7 @@ func normalizeSubmissionResponse(response evaluationAPIResponse) (machineSubmiss return machineSubmission{}, errors.New("evaluation response is incomplete") } payload := machineSubmission{ - ContractVersion: 1, Kind: "softpractice.submission", + Kind: "softpractice.submission", SubmissionID: response.SubmissionID, EvaluationJobID: response.EvaluationJobID, JobState: response.JobState, Attempt: response.Attempt, MaxAttempts: response.MaxAttempts, NextPollSeconds: response.NextPollSeconds, UpdatedAt: response.UpdatedAt, diff --git a/cmd/softpractice/main_test.go b/cmd/softpractice/main_test.go index 0b09562..619a493 100644 --- a/cmd/softpractice/main_test.go +++ b/cmd/softpractice/main_test.go @@ -331,7 +331,7 @@ func TestLinkedStatusSubmitAndOpenCommandFlow(t *testing.T) { if err := decodeTestJSON(statusJSON.Bytes(), &statusPayload); err != nil { t.Fatal(err) } - if statusPayload.ContractVersion != 1 || statusPayload.Kind != "softpractice.status" || + if statusPayload.Kind != "softpractice.status" || statusPayload.Local.Head == "" || !statusPayload.Local.Clean || statusPayload.LatestSubmission == nil || statusPayload.LatestSubmission.ID != submissionID { t.Fatalf("machine status = %+v", statusPayload) diff --git a/cmd/softpractice/ui.go b/cmd/softpractice/ui.go index 9d8ccce..556213e 100644 --- a/cmd/softpractice/ui.go +++ b/cmd/softpractice/ui.go @@ -177,7 +177,7 @@ func printCommandHelp(ctx context.Context, output io.Writer, command string) err case "submission": russian, english = "Использование: softpractice submission show [--id ID] [--format text|json]\n softpractice submission download --id ID [--output PATH]\n\nshow показывает безопасный статус и результат своей отправки. download сохраняет ZIP точной отправленной ревизии, как кнопка «Скачать проект» на странице результата.", "Usage: softpractice submission show [--id ID] [--format text|json]\n softpractice submission download --id ID [--output PATH]\n\nshow displays the safe status and result of your submission. download saves the exact submitted revision ZIP, equivalent to the result-page Download project button." case "evaluation": - russian, english = "Использование: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--format text|json]\n\nПоказывает результат проверки последней отправки или отправки --id. С --wait опрашивает сервер с интервалом, который он назначает, пока проверка не завершится или не истечёт --timeout (по умолчанию 10m). Рекомендации рецензента не выводятся; полный разбор — по ссылке.\n\nКоды выхода: 0 — результат готов, 1 — ошибка, 3 — проверка ещё не готова, 4 — проверка заменена и результата не будет.", "Usage: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--format text|json]\n\nShows the evaluation result of the latest submission or of --id. With --wait it polls at the interval the server sets until the evaluation finishes or --timeout (default 10m) passes. Reviewer recommendations are not printed; the full review is at the printed link.\n\nExit codes: 0 result ready, 1 error, 3 not ready yet, 4 superseded, no result." + russian, english = "Использование: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--directions] [--format text|json]\n\nПоказывает результат проверки последней отправки или отправки --id в формате Markdown. С --wait опрашивает сервер с интервалом, который он назначает, пока проверка не завершится или не истечёт --timeout (по умолчанию 10m). Рекомендации рецензента и следующие шаги выводятся только с --directions. Вопросы рецензента — на странице результата.\n\nКоды выхода: 0 — результат готов, 1 — ошибка, 3 — проверка ещё не готова, 4 — проверка заменена и результата не будет.", "Usage: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--directions] [--format text|json]\n\nShows the evaluation result of the latest submission or of --id as Markdown. With --wait it polls at the interval the server sets until the evaluation finishes or --timeout (default 10m) passes. The reviewer's recommendations and next steps are printed only with --directions. The reviewer's questions are on the result page.\n\nExit codes: 0 result ready, 1 error, 3 not ready yet, 4 superseded, no result." case "assignment": russian, english = "Использование: softpractice assignment show [--format text|json]\n\nПоказывает текст задания, над которым работает этот проект, и связанный с ним теоретический материал.", "Usage: softpractice assignment show [--format text|json]\n\nShows the assignment this project is working on and its linked theory material." case "submit": diff --git a/packaging/npm/README.md b/packaging/npm/README.md index 114109e..3c6a32a 100644 --- a/packaging/npm/README.md +++ b/packaging/npm/README.md @@ -113,16 +113,17 @@ tree and run in an isolated snapshot of `HEAD`. ## Machine-readable output `status`, `submission show`, `evaluation`, and `assignment show` accept -`--format json`. Each document has `contract_version` and `kind`. +`--format json`. Each document has a `kind`. ```bash softpractice evaluation --wait --timeout 10m --format json ``` `evaluation` exit codes: `0` result ready, `1` error, `3` not ready yet, -`4` evaluation superseded, no result. Its output has the verdict, the -automated check results, and the number and priorities of reviewer findings, -without the reviewer's recommendations; the full review is at `result_url`. +`4` evaluation superseded, no result. Its output has the automated check +results and the review without the reviewer's recommendations; +`--directions` adds them. The reviewer's questions are on the result page at +`result_url`. ## Language From 4e2b4bda57d9cfca949d0490760b46268adc6bd6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 07:48:40 +0000 Subject: [PATCH 06/14] cli: organize commands around the lesson loop Commands follow what a learner does: task, material (--lesson for an earlier lesson), hint, submit --wait, result, submissions, and open task|material|result. They replace assignment show, evaluation, submission show, and submission download (now submissions download). status ends with the next step, and lists next_actions in JSON: apply the update, commit, submit, wait, read the result, or answer the reviewer's questions on the result page. A new commit after a submission is detected from the submission's commit_sha. The reviewer's questions and opening the next hint stay on the site: result, status, and hint give the page link. The CLI never prints the questions' text. Every JSON-capable command takes --json and prints a softpractice.error document on failure. Invalid arguments exit with code 2. open no longer links to /workspaces/{id}, which the web app does not serve. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01K9oUWMHw3ZCHpfmoBGcgeF --- README.md | 50 +- README_RU.md | 49 +- cmd/softpractice/agent_commands.go | 442 -------------- cmd/softpractice/coursebase_test.go | 4 +- cmd/softpractice/feedback.go | 10 +- cmd/softpractice/learning_commands.go | 545 ++++++++++++++++++ ...ands_test.go => learning_commands_test.go} | 293 ++++++++-- cmd/softpractice/machine.go | 213 +++---- cmd/softpractice/main.go | 201 +++++-- cmd/softpractice/main_test.go | 67 ++- cmd/softpractice/output.go | 126 ++++ cmd/softpractice/project_checks.go | 2 +- cmd/softpractice/ui.go | 113 +++- cmd/softpractice/usecase.go | 257 +++++++-- packaging/npm/README.md | 50 +- 15 files changed, 1592 insertions(+), 830 deletions(-) delete mode 100644 cmd/softpractice/agent_commands.go create mode 100644 cmd/softpractice/learning_commands.go rename cmd/softpractice/{agent_commands_test.go => learning_commands_test.go} (65%) create mode 100644 cmd/softpractice/output.go diff --git a/README.md b/README.md index 3c6a32a..7a7b774 100644 --- a/README.md +++ b/README.md @@ -42,25 +42,27 @@ Download the starter project for your current practicum: softpractice starter ``` -Open the created directory and check the current lesson: +Open the created directory, check the project state, and read the assignment: ```bash cd softpractice status +softpractice task +softpractice material ``` Work on the project and commit your changes with Git. When the solution is -ready, submit the current commit: +ready, submit the current commit and wait for the result: ```bash -softpractice submit +softpractice submit --wait ``` -Check the submission and open its result: +Read the result again at any time, or open it in the browser: ```bash -softpractice submission show -softpractice open +softpractice result +softpractice open result ``` After an accepted solution, apply the transition to the next available lesson: @@ -69,20 +71,24 @@ After an accepted solution, apply the transition to the next available lesson: softpractice update ``` +`softpractice status` always ends with the next step. + ## Common commands | Command | Description | | --- | --- | | `softpractice login` | Sign in through the browser | | `softpractice starter` | Download the first lesson project | -| `softpractice status` | Show the current lesson and local project state | +| `softpractice status` | Show the project state and the next step | +| `softpractice task` | Show the assignment | +| `softpractice material` | Show the theory material; `--lesson ID` for another lesson | +| `softpractice hint` | Show the hints you have opened; the next one is opened on the assignment page | | `softpractice check` | Run public checks against the current working tree | -| `softpractice submit` | Submit the current committed Git revision | -| `softpractice submission show` | Show the latest submission and review result | -| `softpractice evaluation` | Show the evaluation result of the latest submission; `--wait` waits for it | -| `softpractice assignment show` | Show the assignment this project is working on | +| `softpractice submit` | Submit the current committed Git revision; `--wait` waits for the result | +| `softpractice result` | Show the evaluation result; `--wait` waits for it, `--directions` adds the reviewer's recommendations | +| `softpractice submissions` | Show the recent submissions of the lesson | | `softpractice update` | Apply the transition to the next lesson, or move to a newer version of the current lesson (replaces only lesson files, never yours; optional until the lesson ends) | -| `softpractice open` | Open the workspace or latest result in the browser | +| `softpractice open` | Open the task, material, or latest result in the browser | | `softpractice project restore` | Restore a project from the latest usable revision | Run `softpractice help ` for detailed help. @@ -110,20 +116,24 @@ softpractice set auto-checks true run. Unlike `softpractice check`, submission checks require a clean working tree and run in an isolated snapshot of `HEAD`. +## Reviewer questions + +When the reviewer asks questions about your solution, `softpractice result` +and `softpractice status` say so and give the link to the result page, where +you answer them. + ## Machine-readable output -`status`, `submission show`, `evaluation`, and `assignment show` accept -`--format json`. Each document has a `kind`. +`status`, `task`, `material`, `hint`, `result`, `submissions`, and `submit` +accept `--json`. Each document has a `kind`; on an error the command prints a +`softpractice.error` document. ```bash -softpractice evaluation --wait --timeout 10m --format json +softpractice result --wait --timeout 10m --json ``` -`evaluation` exit codes: `0` result ready, `1` error, `3` not ready yet, -`4` evaluation superseded, no result. Its output has the automated check -results and the review without the reviewer's recommendations; -`--directions` adds them. The reviewer's questions are on the result page at -`result_url`. +Exit codes: `0` done, `1` error, `2` invalid arguments, `3` the evaluation is +not ready yet, `4` the evaluation was superseded and will have no result. ## Language diff --git a/README_RU.md b/README_RU.md index 70c8059..f53a75b 100644 --- a/README_RU.md +++ b/README_RU.md @@ -42,25 +42,27 @@ softpractice login softpractice starter ``` -Перейди в созданный каталог и проверь текущий урок: +Перейди в созданный каталог, проверь состояние проекта и прочитай задание: ```bash cd <каталог-проекта> softpractice status +softpractice task +softpractice material ``` Работай над проектом и фиксируй изменения в Git. Когда решение будет готово, -отправь текущий коммит: +отправь текущий коммит и дождись результата: ```bash -softpractice submit +softpractice submit --wait ``` -Проверь состояние отправки и открой результат: +Результат можно посмотреть снова в любой момент или открыть в браузере: ```bash -softpractice submission show -softpractice open +softpractice result +softpractice open result ``` После принятия решения примени переход к следующему доступному уроку: @@ -69,20 +71,24 @@ softpractice open softpractice update ``` +`softpractice status` всегда заканчивается следующим шагом. + ## Основные команды | Команда | Описание | | --- | --- | | `softpractice login` | Войти через браузер | | `softpractice starter` | Скачать проект первого урока | -| `softpractice status` | Показать текущий урок и состояние локального проекта | +| `softpractice status` | Показать состояние проекта и следующий шаг | +| `softpractice task` | Показать задание | +| `softpractice material` | Показать теоретический материал; `--lesson ID` — другого урока | +| `softpractice hint` | Показать открытые подсказки; следующая открывается на странице задания | | `softpractice check` | Запустить публичные проверки текущего рабочего дерева | -| `softpractice submit` | Отправить текущую зафиксированную Git-ревизию | -| `softpractice submission show` | Показать последнюю отправку и результат проверки | -| `softpractice evaluation` | Показать результат проверки последней отправки; `--wait` дожидается его | -| `softpractice assignment show` | Показать задание, над которым работает проект | +| `softpractice submit` | Отправить текущую зафиксированную Git-ревизию; `--wait` дожидается результата | +| `softpractice result` | Показать результат проверки; `--wait` дожидается его, `--directions` добавляет рекомендации рецензента | +| `softpractice submissions` | Показать последние отправки урока | | `softpractice update` | Применить переход к следующему уроку или перейти на новую версию текущего урока (заменяет только файлы урока, ваши файлы не трогает; до конца урока необязательно) | -| `softpractice open` | Открыть workspace или последний результат в браузере | +| `softpractice open` | Открыть задание, материал или последний результат в браузере | | `softpractice project restore` | Восстановить проект из последней подходящей ревизии | Подробную справку можно получить командой `softpractice help <команда>`. @@ -111,21 +117,24 @@ softpractice set auto-checks true отличие от `softpractice check`, проверка перед отправкой требует чистое рабочее дерево и запускается в изолированном снимке `HEAD`. +## Вопросы рецензента + +Если у рецензента есть вопросы к решению, `softpractice result` и +`softpractice status` сообщают об этом и дают ссылку на страницу результата: +отвечать нужно там. ## Машиночитаемый вывод -`status`, `submission show`, `evaluation` и `assignment show` принимают -`--format json`. У каждого документа есть поле `kind`. +`status`, `task`, `material`, `hint`, `result`, `submissions` и `submit` +принимают `--json`. У каждого документа есть поле `kind`; при ошибке команда +выводит документ `softpractice.error`. ```bash -softpractice evaluation --wait --timeout 10m --format json +softpractice result --wait --timeout 10m --json ``` -Коды выхода `evaluation`: `0` — результат готов, `1` — ошибка, `3` — проверка -ещё не готова, `4` — проверка заменена, результата не будет. В выводе есть -результаты автоматических проверок и ревью без рекомендаций рецензента; -`--directions` добавляет их. Вопросы рецензента — на странице результата по -ссылке `result_url`. +Коды выхода: `0` — готово, `1` — ошибка, `2` — неверные аргументы, `3` — +проверка ещё не готова, `4` — проверка заменена, результата не будет. ## Язык diff --git a/cmd/softpractice/agent_commands.go b/cmd/softpractice/agent_commands.go deleted file mode 100644 index 6b664b0..0000000 --- a/cmd/softpractice/agent_commands.go +++ /dev/null @@ -1,442 +0,0 @@ -package main - -import ( - "context" - "errors" - "flag" - "fmt" - "io" - "net/http" - "strings" - "time" - - "github.com/arthur-s/softpractice-cli/internal/learnercli" -) - -// exitStatus ends the process with its code after the command has already -// written its result, so main prints nothing more. -type exitStatus int - -func (e exitStatus) Error() string { return fmt.Sprintf("exit status %d", int(e)) } - -const ( - // exitNotReady: the evaluation is still queued or running. - exitNotReady exitStatus = 3 - // exitSuperseded: the evaluation was superseded and has no result. - exitSuperseded exitStatus = 4 -) - -const defaultEvaluationTimeout = 10 * time.Minute - -type machineEvaluationResult struct { - Kind string `json:"kind"` - Outcome evaluationOutcome `json:"outcome"` - SubmissionID string `json:"submission_id"` - EvaluationJobID string `json:"evaluation_job_id,omitempty"` - JobState string `json:"job_state"` - Attempt int `json:"attempt,omitempty"` - MaxAttempts int `json:"max_attempts,omitempty"` - NextPollSeconds int `json:"next_poll_seconds,omitempty"` - UpdatedAt *time.Time `json:"updated_at,omitempty"` - ResultURL string `json:"result_url"` - // Result is present only for the ready outcome. - Result *machineResult `json:"result,omitempty"` -} - -func buildMachineEvaluationResult( - result evaluationResult, - webURL string, - detail reviewFeedbackDetail, -) (machineEvaluationResult, error) { - payload := machineEvaluationResult{ - Kind: "softpractice.evaluation", Outcome: result.Outcome, SubmissionID: result.SubmissionID, - ResultURL: strings.TrimRight(webURL, "/") + "/submissions/" + result.SubmissionID + "/result", - } - if result.Outcome == evaluationSuperseded { - payload.JobState = "superseded" - return payload, nil - } - submission := result.Submission - updatedAt := submission.UpdatedAt - payload.EvaluationJobID = submission.EvaluationJobID - payload.JobState = submission.JobState - payload.Attempt = submission.Attempt - payload.MaxAttempts = submission.MaxAttempts - payload.NextPollSeconds = submission.NextPollSeconds - payload.UpdatedAt = &updatedAt - if result.Outcome == evaluationReady { - built, err := buildMachineResult(result.RawEvaluation, detail) - if err != nil { - return machineEvaluationResult{}, err - } - payload.Result = built - } - return payload, nil -} - -func evaluationCommand( - ctx context.Context, - useCases learnerUseCases, - args []string, - output, errorOutput io.Writer, -) error { - flags := flag.NewFlagSet("evaluation", flag.ContinueOnError) - flags.SetOutput(errorOutput) - id := flags.String("id", "", "submission ID; defaults to the latest linked submission") - wait := flags.Bool("wait", false, "poll until the evaluation is ready") - timeout := flags.Duration("timeout", defaultEvaluationTimeout, "maximum wait with --wait") - directions := flags.Bool("directions", false, "also print the reviewer's directions and next steps") - format := flags.String("format", "text", "output format: text or json") - if err := flags.Parse(args); err != nil { - return err - } - usage := text(ctx, - "использование: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--directions] [--format text|json]", - "usage: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--directions] [--format text|json]") - if flags.NArg() != 0 || *timeout < 0 { - return errors.New(usage) - } - outputFormat, err := parseOutputFormat(*format) - if err != nil { - return err - } - detail := defaultReviewFeedback - if *directions { - detail = feedbackWithDirections - } - payload, err := evaluationPayload(ctx, useCases, strings.TrimSpace(*id), *wait, *timeout, detail, errorOutput) - if err != nil { - if outputFormat == outputFormatJSON { - _ = writeMachineError(output, err) - } - return err - } - if outputFormat == outputFormatJSON { - if err := writeMachineJSON(output, payload); err != nil { - return err - } - } else { - writeEvaluationMarkdown(ctx, output, payload) - } - switch payload.Outcome { - case evaluationPending: - return exitNotReady - case evaluationSuperseded: - return exitSuperseded - } - return nil -} - -func evaluationPayload( - ctx context.Context, - useCases learnerUseCases, - submissionID string, - wait bool, - timeout time.Duration, - detail reviewFeedbackDetail, - errorOutput io.Writer, -) (machineEvaluationResult, error) { - if submissionID == "" { - latest, err := useCases.LatestSubmission(ctx) - if err != nil { - return machineEvaluationResult{}, err - } - if latest.PredecessorAssignmentID != "" { - fmt.Fprintf(errorOutput, text(ctx, - "По %s отправок ещё нет; показан принятый результат %s.\n", - "%s has no submission yet; showing the accepted %s result.\n", - ), latest.CurrentAssignmentID, latest.PredecessorAssignmentID) - } - submissionID = latest.SubmissionID - } - result, err := useCases.Evaluation(ctx, submissionID, evaluationWait{Wait: wait, Timeout: timeout}) - if err != nil { - return machineEvaluationResult{}, err - } - return buildMachineEvaluationResult(result, settingsFromContext(ctx).WebURL, detail) -} - -// writeEvaluationMarkdown prints the evaluation as Markdown, which reads as -// plain text in a terminal and renders in an agent's chat. -func writeEvaluationMarkdown(ctx context.Context, output io.Writer, payload machineEvaluationResult) { - switch payload.Outcome { - case evaluationSuperseded: - fmt.Fprintf(output, text(ctx, - "Проверка отправки `%s` заменена; результата у неё не будет.\n", - "The evaluation of submission `%s` was superseded; it will not have a result.\n"), - payload.SubmissionID) - return - case evaluationPending: - fmt.Fprintf(output, text(ctx, - "Проверка отправки `%s` ещё не готова (%s). Дождаться: `softpractice evaluation --wait`\n", - "The evaluation of submission `%s` is not ready yet (%s). To wait: `softpractice evaluation --wait`\n"), - payload.SubmissionID, payload.JobState) - return - } - result := payload.Result - fmt.Fprintf(output, "# %s\n\n", evaluationStatusTitle(ctx, result.Status)) - fmt.Fprintf(output, text(ctx, "Отправка `%s`, попытка проверки %d из %d.\n", "Submission `%s`, evaluation attempt %d of %d.\n"), - payload.SubmissionID, payload.Attempt, payload.MaxAttempts) - if technical := result.TechnicalError; technical != nil { - fmt.Fprintf(output, text(ctx, "\nТехническая ошибка: `%s` на этапе `%s`, support ID `%s`.\n", - "\nTechnical error: `%s` at stage `%s`, support ID `%s`.\n"), - technical.ErrorCode, technical.Stage, technical.SupportID) - } - if deterministic := result.Deterministic; deterministic != nil { - fmt.Fprintf(output, text(ctx, "\n## Автоматические проверки: %s\n\n", "\n## Automated checks: %s\n\n"), - deterministicStatusLabel(ctx, deterministic.Status)) - for _, check := range deterministic.Checks { - fmt.Fprintf(output, "- [%s] %s: %s\n", check.Status, check.ID, check.Summary) - if example := check.Counterexample; example != nil { - fmt.Fprintf(output, text(ctx, " - Пример: %s\n", " - Example: %s\n"), example.Title) - fmt.Fprintf(output, text(ctx, " - Сценарий: %s\n", " - Scenario: %s\n"), example.Scenario) - fmt.Fprintf(output, text(ctx, " - Вход: %s\n", " - Input: %s\n"), example.Input) - fmt.Fprintf(output, text(ctx, " - Ожидалось: %s\n", " - Expected: %s\n"), example.Expected) - if example.NextStep != "" { - fmt.Fprintf(output, text(ctx, " - Следующий шаг: %s\n", " - Next step: %s\n"), example.NextStep) - } - } - } - } - if review := result.Review; review != nil && review.Status != "skipped" { - fmt.Fprint(output, text(ctx, "\n## Ревью\n", "\n## Review\n")) - if !review.Authoritative { - fmt.Fprint(output, text(ctx, - "\nПредварительное ревью: оно не определяет результат.\n", - "\nPreliminary review: it does not decide the result.\n")) - } - if review.Summary != "" { - fmt.Fprintf(output, "\n%s\n", review.Summary) - } - if uncertainty := review.Uncertainty; uncertainty != nil && uncertainty.Reason != nil && *uncertainty.Reason != "" { - fmt.Fprintf(output, text(ctx, "\nЧто осталось неясным: %s\n", "\nWhat remains unclear: %s\n"), *uncertainty.Reason) - } - if len(review.Rubric) > 0 { - fmt.Fprint(output, text(ctx, "\n### Критерии\n\n", "\n### Criteria\n\n")) - for _, item := range review.Rubric { - fmt.Fprintf(output, "- **%s**: %s. %s%s\n", item.Criterion, rubricLevelLabel(ctx, item.Level), - item.Rationale, formatEvidence(ctx, item.EvidenceRefs)) - } - } - if len(review.Findings) > 0 { - fmt.Fprint(output, text(ctx, "\n### Замечания\n", "\n### Findings\n")) - for index, finding := range review.Findings { - fmt.Fprintf(output, "\n%d. **%s** (%s)\n", index+1, finding.Title, priorityLabel(ctx, finding.Priority)) - fmt.Fprintf(output, text(ctx, " - Наблюдение: %s\n", " - Observation: %s\n"), finding.Observation) - fmt.Fprintf(output, text(ctx, " - Риск: %s\n", " - Risk: %s\n"), finding.Risk) - if finding.Direction != "" { - fmt.Fprintf(output, text(ctx, " - Направление: %s\n", " - Direction: %s\n"), finding.Direction) - } - if len(finding.EvidenceRefs) > 0 { - fmt.Fprintf(output, text(ctx, " - Где: %s\n", " - Where: %s\n"), strings.Join(finding.EvidenceRefs, ", ")) - } - } - } - if review.QuestionsCount > 0 { - fmt.Fprintf(output, text(ctx, - "\nВопросов рецензента к решению: %d. Они на странице результата.\n", - "\nReviewer questions about the solution: %d. They are on the result page.\n"), - review.QuestionsCount) - } - } - if result.DirectionsHidden { - fmt.Fprint(output, text(ctx, - "\nРекомендации рецензента не показаны. Показать: `softpractice evaluation --directions`\n", - "\nThe reviewer's recommendations are not shown. To show them: `softpractice evaluation --directions`\n")) - } - fmt.Fprintf(output, text(ctx, "\nПолный разбор: %s\n", "\nFull review: %s\n"), payload.ResultURL) -} - -func evaluationStatusTitle(ctx context.Context, status string) string { - switch status { - case "accepted": - return text(ctx, "Решение принято", "Solution accepted") - case "revise": - return text(ctx, "Нужна доработка", "Revision needed") - case "uncertain": - return text(ctx, "Рецензенту нужно пояснение", "The reviewer needs an explanation") - case "deterministic_failed": - return text(ctx, "Автоматические проверки не пройдены", "Automated checks failed") - case "ready_for_review": - return text(ctx, "Автоматические проверки пройдены", "Automated checks passed") - case "technical_failure": - return text(ctx, "Проверка не завершилась из-за технической ошибки", "The evaluation failed for a technical reason") - default: - return status - } -} - -func deterministicStatusLabel(ctx context.Context, status string) string { - switch status { - case "passed": - return text(ctx, "пройдены", "passed") - case "failed": - return text(ctx, "не пройдены", "failed") - case "not_run": - return text(ctx, "не запускались", "not run") - case "technical_failure": - return text(ctx, "техническая ошибка", "technical failure") - default: - return status - } -} - -// rubricLevelLabel uses the level names of the result page. -func rubricLevelLabel(ctx context.Context, level int) string { - switch level { - case 1: - return text(ctx, "нужно доработать", "needs work") - case 2: - return text(ctx, "выполнено", "met") - case 3: - return text(ctx, "выполнено и закреплено", "met and reinforced") - default: - return text(ctx, "не подтверждено", "not confirmed") - } -} - -func priorityLabel(ctx context.Context, priority string) string { - switch priority { - case "high": - return text(ctx, "высокий приоритет", "high priority") - case "medium": - return text(ctx, "средний приоритет", "medium priority") - case "low": - return text(ctx, "низкий приоритет", "low priority") - default: - return priority - } -} - -func formatEvidence(ctx context.Context, refs []string) string { - if len(refs) == 0 { - return "" - } - return text(ctx, " Где: ", " Where: ") + strings.Join(refs, ", ") -} - -type machineAssignment struct { - Kind string `json:"kind"` - WorkspaceID string `json:"workspace_id"` - ProjectID string `json:"project_id"` - Assignment assignmentDetail `json:"assignment"` - Material *theoryMaterial `json:"material"` - Transition *machineTransition `json:"transition,omitempty"` - LessonVersionUpdate *machineLessonVersionUpdate `json:"lesson_version_update,omitempty"` -} - -func assignmentCommand( - ctx context.Context, - useCases learnerUseCases, - args []string, - output, errorOutput io.Writer, -) error { - usage := text(ctx, - "использование: softpractice assignment show [--format text|json]", - "usage: softpractice assignment show [--format text|json]") - if len(args) == 0 || args[0] != "show" { - return errors.New(usage) - } - flags := flag.NewFlagSet("assignment show", flag.ContinueOnError) - flags.SetOutput(errorOutput) - format := flags.String("format", "text", "output format: text or json") - if err := flags.Parse(args[1:]); err != nil { - return err - } - if flags.NArg() != 0 { - return errors.New(usage) - } - outputFormat, err := parseOutputFormat(*format) - if err != nil { - return err - } - snapshot, err := useCases.Assignment(ctx) - if err != nil { - if outputFormat == outputFormatJSON { - _ = writeMachineError(output, err) - } - return err - } - payload := machineAssignment{ - Kind: "softpractice.assignment", - WorkspaceID: snapshot.Workspace.Workspace.ID, ProjectID: snapshot.Link.ProjectID, - Assignment: snapshot.Assignment, Material: snapshot.Material, - } - if pendingTransition(snapshot.Link, snapshot.Workspace) { - payload.Transition = &machineTransition{ - FromAssignmentID: snapshot.Link.AssignmentID, FromAssignmentVersion: snapshot.Link.AssignmentVersion, - ToAssignmentID: snapshot.Workspace.Assignment.ID, ToAssignmentVersion: snapshot.Workspace.Assignment.Version, - } - } - if pendingLessonVersion(snapshot.Link, snapshot.Workspace) { - payload.LessonVersionUpdate = &machineLessonVersionUpdate{ - AssignmentID: snapshot.Link.AssignmentID, - FromVersion: snapshot.Link.AssignmentVersion, ToVersion: snapshot.Workspace.Assignment.Version, - } - } - if outputFormat == outputFormatJSON { - return writeMachineJSON(output, payload) - } - fmt.Fprintf(output, "# %s (%s v%d)\n\n", payload.Assignment.Title, payload.Assignment.ID, payload.Assignment.Version) - if payload.Transition != nil { - fmt.Fprintf(output, text(ctx, - "Переход не применён: файлы урока %s ещё не в этой папке (выполните `softpractice update`).\n\n", - "Transition pending: the %s lesson files are not in this folder yet (run `softpractice update`).\n\n"), - payload.Transition.ToAssignmentID) - } - fmt.Fprintln(output, strings.TrimRight(payload.Assignment.InstructionsMarkdown, "\n")) - if material := payload.Material; material != nil { - fmt.Fprintf(output, "\n---\n\n%s\n", strings.TrimRight(material.Markdown, "\n")) - } - return nil -} - -// machineError is written to standard output by the agent-facing commands in -// JSON mode, so a caller that reads only stdout still learns why the command -// failed. The same message also goes to standard error. -type machineError struct { - Kind string `json:"kind"` - Error struct { - Code string `json:"code"` - Message string `json:"message"` - HTTPStatus int `json:"http_status,omitempty"` - APICode string `json:"api_code,omitempty"` - SupportID string `json:"support_id,omitempty"` - } `json:"error"` -} - -// errorCode classifies an error for machine output. login_required and -// project_not_linked tell the caller that a person has to act: sign in with -// `softpractice login`, or work inside the linked project. -func errorCode(err error) string { - var statusError *learnercli.HTTPError - switch { - case errors.Is(err, learnercli.ErrLoginRequired): - return "login_required" - case errors.As(err, &statusError) && statusError.Status == http.StatusUnauthorized: - return "login_required" - case errors.Is(err, learnercli.ErrProjectNotLinked): - return "project_not_linked" - case errors.Is(err, errNotGitRepository): - return "not_git_repository" - case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): - return "cancelled" - case errors.As(err, &statusError): - return "api_error" - default: - return "error" - } -} - -func writeMachineError(output io.Writer, err error) error { - payload := machineError{Kind: "softpractice.error"} - payload.Error.Code = errorCode(err) - payload.Error.Message = err.Error() - var statusError *learnercli.HTTPError - if errors.As(err, &statusError) { - payload.Error.HTTPStatus = statusError.Status - payload.Error.APICode = statusError.Code - payload.Error.SupportID = statusError.SupportID - } - return writeMachineJSON(output, payload) -} diff --git a/cmd/softpractice/coursebase_test.go b/cmd/softpractice/coursebase_test.go index 5725f56..c9c1732 100644 --- a/cmd/softpractice/coursebase_test.go +++ b/cmd/softpractice/coursebase_test.go @@ -123,7 +123,7 @@ func TestStatusReportsAPendingTransitionAndScopesTheSubmissionLine(t *testing.T) var machineOutput bytes.Buffer if err := status( - context.Background(), client, root, []string{"--format", "json"}, &machineOutput, &errorOutput, + context.Background(), client, root, []string{"--json"}, &machineOutput, &errorOutput, ); err != nil { t.Fatal(err) } @@ -161,7 +161,7 @@ func TestMachineStatusOmitsTransitionWhenTheProjectIsOnTheCurrentLesson(t *testi var output, errorOutput bytes.Buffer if err := status( - context.Background(), client, root, []string{"--format", "json"}, &output, &errorOutput, + context.Background(), client, root, []string{"--json"}, &output, &errorOutput, ); err != nil { t.Fatal(err) } diff --git a/cmd/softpractice/feedback.go b/cmd/softpractice/feedback.go index 7de29d2..2b77d57 100644 --- a/cmd/softpractice/feedback.go +++ b/cmd/softpractice/feedback.go @@ -5,7 +5,7 @@ import ( "fmt" ) -// reviewFeedbackDetail says how much of an evaluation the `evaluation` command +// reviewFeedbackDetail says how much of an evaluation the `result` command // prints, in both text and JSON. type reviewFeedbackDetail string @@ -32,7 +32,7 @@ const ( const defaultReviewFeedback = feedbackWithoutDirections // evaluationSource decodes the public evaluation projection that the -// `evaluation` command renders. +// `result` command renders. type evaluationSource struct { ExerciseID string `json:"exercise_id"` Status string `json:"status"` @@ -131,6 +131,10 @@ type machineReviewResult struct { // QuestionsCount is how many questions the reviewer asked the learner. // Their text is on the result page. QuestionsCount int `json:"questions_count"` + // QuestionsAnswerable is true when the learner is expected to answer the + // questions on the result page: an authoritative uncertain review. Other + // questions are for self-check. + QuestionsAnswerable bool `json:"questions_answerable"` } type machineUncertainty struct { @@ -197,6 +201,8 @@ func buildMachineResult(raw json.RawMessage, detail reviewFeedbackDetail) (*mach Rubric: make([]machineRubricItem, 0, len(review.Rubric)), Findings: make([]machineReviewFinding, 0, len(review.Feedback)), QuestionsCount: len(review.Questions), + QuestionsAnswerable: review.Authoritative && review.Verdict != nil && *review.Verdict == "uncertain" && + len(review.Questions) > 0, } if uncertainty := review.Uncertainty; uncertainty != nil { part.Uncertainty = &machineUncertainty{Blocking: uncertainty.Blocking, Reason: uncertainty.Reason} diff --git a/cmd/softpractice/learning_commands.go b/cmd/softpractice/learning_commands.go new file mode 100644 index 0000000..b128393 --- /dev/null +++ b/cmd/softpractice/learning_commands.go @@ -0,0 +1,545 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io" + "strconv" + "strings" + "time" +) + +const defaultEvaluationTimeout = 10 * time.Minute + +type machineEvaluationResult struct { + Kind string `json:"kind"` + Outcome evaluationOutcome `json:"outcome"` + SubmissionID string `json:"submission_id"` + EvaluationJobID string `json:"evaluation_job_id,omitempty"` + JobState string `json:"job_state"` + Attempt int `json:"attempt,omitempty"` + MaxAttempts int `json:"max_attempts,omitempty"` + NextPollSeconds int `json:"next_poll_seconds,omitempty"` + UpdatedAt *time.Time `json:"updated_at,omitempty"` + ResultURL string `json:"result_url"` + // Result is present only for the ready outcome. + Result *machineResult `json:"result,omitempty"` +} + +// buildMachineEvaluationResult projects one evaluation; resultURL is its +// result page. +func buildMachineEvaluationResult( + result evaluationResult, + resultURL string, + detail reviewFeedbackDetail, +) (machineEvaluationResult, error) { + payload := machineEvaluationResult{ + Kind: "softpractice.evaluation", Outcome: result.Outcome, SubmissionID: result.SubmissionID, + ResultURL: resultURL, + } + if result.Outcome == evaluationSuperseded { + payload.JobState = "superseded" + return payload, nil + } + submission := result.Submission + updatedAt := submission.UpdatedAt + payload.EvaluationJobID = submission.EvaluationJobID + payload.JobState = submission.JobState + payload.Attempt = submission.Attempt + payload.MaxAttempts = submission.MaxAttempts + payload.NextPollSeconds = submission.NextPollSeconds + payload.UpdatedAt = &updatedAt + if result.Outcome == evaluationReady { + built, err := buildMachineResult(result.RawEvaluation, detail) + if err != nil { + return machineEvaluationResult{}, err + } + payload.Result = built + } + return payload, nil +} + +func resultCommand( + ctx context.Context, + useCases learnerUseCases, + args []string, + output, errorOutput io.Writer, +) error { + flags := flag.NewFlagSet("result", flag.ContinueOnError) + flags.SetOutput(errorOutput) + id := flags.String("id", "", "submission ID; defaults to the latest submission") + wait := flags.Bool("wait", false, "wait until the evaluation is ready") + timeout := flags.Duration("timeout", defaultEvaluationTimeout, "maximum wait with --wait") + directions := flags.Bool("directions", false, "also print the reviewer's directions and next steps") + jsonOutput := jsonFlag(flags) + if err := parseFlags(flags, args); err != nil { + return err + } + if flags.NArg() != 0 || *timeout < 0 { + return usage(ctx, + "использование: softpractice result [--id ID] [--wait] [--timeout DURATION] [--directions] [--json]", + "usage: softpractice result [--id ID] [--wait] [--timeout DURATION] [--directions] [--json]") + } + submissionID := strings.TrimSpace(*id) + if submissionID == "" { + latest, err := useCases.LatestSubmission(ctx) + if err != nil { + return reportJSONError(*jsonOutput, output, err) + } + if latest.PredecessorAssignmentID != "" { + fmt.Fprintf(errorOutput, text(ctx, + "По %s отправок ещё нет; показан принятый результат %s.\n", + "%s has no submission yet; showing the accepted %s result.\n", + ), latest.CurrentAssignmentID, latest.PredecessorAssignmentID) + } + submissionID = latest.SubmissionID + } + return reportJSONError(*jsonOutput, output, + printEvaluation(ctx, useCases, submissionID, evaluationWait{Wait: *wait, Timeout: *timeout}, + *directions, *jsonOutput, output)) +} + +// printEvaluation reads or waits for one evaluation, prints it, and returns +// the exit status of a pending or superseded outcome. +func printEvaluation( + ctx context.Context, + useCases learnerUseCases, + submissionID string, + wait evaluationWait, + directions, jsonOutput bool, + output io.Writer, +) error { + detail := defaultReviewFeedback + if directions { + detail = feedbackWithDirections + } + result, err := useCases.Evaluation(ctx, submissionID, wait) + if err != nil { + return err + } + payload, err := buildMachineEvaluationResult(result, webResultURL(ctx, submissionID), detail) + if err != nil { + return err + } + if jsonOutput { + if err := writeMachineJSON(output, payload); err != nil { + return err + } + } else { + writeEvaluationMarkdown(ctx, output, payload) + } + switch payload.Outcome { + case evaluationPending: + return exitNotReady + case evaluationSuperseded: + return exitSuperseded + } + return nil +} + +// writeEvaluationMarkdown prints the evaluation as Markdown, which reads as +// plain text in a terminal and renders in an agent's chat. +func writeEvaluationMarkdown(ctx context.Context, output io.Writer, payload machineEvaluationResult) { + switch payload.Outcome { + case evaluationSuperseded: + fmt.Fprintf(output, text(ctx, + "Проверка отправки `%s` заменена; результата у неё не будет.\n", + "The evaluation of submission `%s` was superseded; it will not have a result.\n"), + payload.SubmissionID) + return + case evaluationPending: + fmt.Fprintf(output, text(ctx, + "Проверка отправки `%s` ещё не готова (%s). Дождаться: `softpractice result --wait`\n", + "The evaluation of submission `%s` is not ready yet (%s). To wait: `softpractice result --wait`\n"), + payload.SubmissionID, payload.JobState) + return + } + result := payload.Result + fmt.Fprintf(output, "# %s\n\n", evaluationStatusTitle(ctx, result.Status)) + fmt.Fprintf(output, text(ctx, "Отправка `%s`, попытка проверки %d из %d.\n", "Submission `%s`, evaluation attempt %d of %d.\n"), + payload.SubmissionID, payload.Attempt, payload.MaxAttempts) + if technical := result.TechnicalError; technical != nil { + fmt.Fprintf(output, text(ctx, "\nТехническая ошибка: `%s` на этапе `%s`, support ID `%s`.\n", + "\nTechnical error: `%s` at stage `%s`, support ID `%s`.\n"), + technical.ErrorCode, technical.Stage, technical.SupportID) + } + if deterministic := result.Deterministic; deterministic != nil { + fmt.Fprintf(output, text(ctx, "\n## Автоматические проверки: %s\n\n", "\n## Automated checks: %s\n\n"), + deterministicStatusLabel(ctx, deterministic.Status)) + for _, check := range deterministic.Checks { + fmt.Fprintf(output, "- [%s] %s: %s\n", check.Status, check.ID, check.Summary) + if example := check.Counterexample; example != nil { + fmt.Fprintf(output, text(ctx, " - Пример: %s\n", " - Example: %s\n"), example.Title) + fmt.Fprintf(output, text(ctx, " - Сценарий: %s\n", " - Scenario: %s\n"), example.Scenario) + fmt.Fprintf(output, text(ctx, " - Вход: %s\n", " - Input: %s\n"), example.Input) + fmt.Fprintf(output, text(ctx, " - Ожидалось: %s\n", " - Expected: %s\n"), example.Expected) + if example.NextStep != "" { + fmt.Fprintf(output, text(ctx, " - Следующий шаг: %s\n", " - Next step: %s\n"), example.NextStep) + } + } + } + } + if review := result.Review; review != nil && review.Status != "skipped" { + fmt.Fprint(output, text(ctx, "\n## Ревью\n", "\n## Review\n")) + if !review.Authoritative { + fmt.Fprint(output, text(ctx, + "\nПредварительное ревью: оно не определяет результат.\n", + "\nPreliminary review: it does not decide the result.\n")) + } + if review.Summary != "" { + fmt.Fprintf(output, "\n%s\n", review.Summary) + } + if uncertainty := review.Uncertainty; uncertainty != nil && uncertainty.Reason != nil && *uncertainty.Reason != "" { + fmt.Fprintf(output, text(ctx, "\nЧто осталось неясным: %s\n", "\nWhat remains unclear: %s\n"), *uncertainty.Reason) + } + if len(review.Rubric) > 0 { + fmt.Fprint(output, text(ctx, "\n### Критерии\n\n", "\n### Criteria\n\n")) + for _, item := range review.Rubric { + fmt.Fprintf(output, "- **%s**: %s. %s%s\n", item.Criterion, rubricLevelLabel(ctx, item.Level), + item.Rationale, formatEvidence(ctx, item.EvidenceRefs)) + } + } + if len(review.Findings) > 0 { + fmt.Fprint(output, text(ctx, "\n### Замечания\n", "\n### Findings\n")) + for index, finding := range review.Findings { + fmt.Fprintf(output, "\n%d. **%s** (%s)\n", index+1, finding.Title, priorityLabel(ctx, finding.Priority)) + fmt.Fprintf(output, text(ctx, " - Наблюдение: %s\n", " - Observation: %s\n"), finding.Observation) + fmt.Fprintf(output, text(ctx, " - Риск: %s\n", " - Risk: %s\n"), finding.Risk) + if finding.Direction != "" { + fmt.Fprintf(output, text(ctx, " - Направление: %s\n", " - Direction: %s\n"), finding.Direction) + } + if len(finding.EvidenceRefs) > 0 { + fmt.Fprintf(output, text(ctx, " - Где: %s\n", " - Where: %s\n"), strings.Join(finding.EvidenceRefs, ", ")) + } + } + } + switch { + case review.QuestionsAnswerable: + fmt.Fprintf(output, text(ctx, + "\nРецензенту нужны ваши ответы на вопросы о решении (%d). Ответьте на странице результата: %s\n", + "\nThe reviewer needs your answers to questions about the solution (%d). Answer them on the result page: %s\n"), + review.QuestionsCount, payload.ResultURL) + case review.QuestionsCount > 0: + fmt.Fprintf(output, text(ctx, + "\nВопросы рецензента для самопроверки (%d) — на странице результата.\n", + "\nThe reviewer's self-check questions (%d) are on the result page.\n"), + review.QuestionsCount) + } + } + if result.DirectionsHidden { + fmt.Fprint(output, text(ctx, + "\nРекомендации рецензента не показаны. Показать: `softpractice result --directions`\n", + "\nThe reviewer's recommendations are not shown. To show them: `softpractice result --directions`\n")) + } + fmt.Fprintf(output, text(ctx, "\nПолный разбор: %s\n", "\nFull review: %s\n"), payload.ResultURL) +} + +func evaluationStatusTitle(ctx context.Context, status string) string { + switch status { + case "accepted": + return text(ctx, "Решение принято", "Solution accepted") + case "revise": + return text(ctx, "Нужна доработка", "Revision needed") + case "uncertain": + return text(ctx, "Рецензенту нужно пояснение", "The reviewer needs an explanation") + case "deterministic_failed": + return text(ctx, "Автоматические проверки не пройдены", "Automated checks failed") + case "ready_for_review": + return text(ctx, "Автоматические проверки пройдены", "Automated checks passed") + case "technical_failure": + return text(ctx, "Проверка не завершилась из-за технической ошибки", "The evaluation failed for a technical reason") + default: + return status + } +} + +func deterministicStatusLabel(ctx context.Context, status string) string { + switch status { + case "passed": + return text(ctx, "пройдены", "passed") + case "failed": + return text(ctx, "не пройдены", "failed") + case "not_run": + return text(ctx, "не запускались", "not run") + case "technical_failure": + return text(ctx, "техническая ошибка", "technical failure") + default: + return status + } +} + +// rubricLevelLabel uses the level names of the result page. +func rubricLevelLabel(ctx context.Context, level int) string { + switch level { + case 1: + return text(ctx, "нужно доработать", "needs work") + case 2: + return text(ctx, "выполнено", "met") + case 3: + return text(ctx, "выполнено и закреплено", "met and reinforced") + default: + return text(ctx, "не подтверждено", "not confirmed") + } +} + +func priorityLabel(ctx context.Context, priority string) string { + switch priority { + case "high": + return text(ctx, "высокий приоритет", "high priority") + case "medium": + return text(ctx, "средний приоритет", "medium priority") + case "low": + return text(ctx, "низкий приоритет", "low priority") + default: + return priority + } +} + +func formatEvidence(ctx context.Context, refs []string) string { + if len(refs) == 0 { + return "" + } + return text(ctx, " Где: ", " Where: ") + strings.Join(refs, ", ") +} + +// webLessonURL is the assignment page of a lesson; suffix selects a page +// under it, such as "/material". +func webLessonURL(ctx context.Context, assignmentID string, version int, suffix string) string { + target := strings.TrimRight(settingsFromContext(ctx).WebURL, "/") + "/assignments/" + assignmentID + suffix + if version > 0 { + target += "?version=" + strconv.Itoa(version) + } + return target +} + +func webResultURL(ctx context.Context, submissionID string) string { + return strings.TrimRight(settingsFromContext(ctx).WebURL, "/") + "/submissions/" + submissionID + "/result" +} + +type machineTask struct { + Kind string `json:"kind"` + WorkspaceID string `json:"workspace_id"` + ProjectID string `json:"project_id"` + Assignment assignmentDetail `json:"assignment"` + URL string `json:"url"` + Transition *machineTransition `json:"transition,omitempty"` + LessonVersionUpdate *machineLessonVersionUpdate `json:"lesson_version_update,omitempty"` +} + +func taskCommand(ctx context.Context, useCases learnerUseCases, args []string, output, errorOutput io.Writer) error { + flags := flag.NewFlagSet("task", flag.ContinueOnError) + flags.SetOutput(errorOutput) + jsonOutput := jsonFlag(flags) + if err := parseFlags(flags, args); err != nil { + return err + } + if flags.NArg() != 0 { + return usage(ctx, "использование: softpractice task [--json]", "usage: softpractice task [--json]") + } + lesson, assignment, err := useCases.Task(ctx) + if err != nil { + return reportJSONError(*jsonOutput, output, err) + } + payload := machineTask{ + Kind: "softpractice.task", WorkspaceID: lesson.Workspace.Workspace.ID, ProjectID: lesson.Link.ProjectID, + Assignment: assignment, URL: webLessonURL(ctx, assignment.ID, assignment.Version, ""), + } + if pendingTransition(lesson.Link, lesson.Workspace) { + payload.Transition = &machineTransition{ + FromAssignmentID: lesson.Link.AssignmentID, FromAssignmentVersion: lesson.Link.AssignmentVersion, + ToAssignmentID: lesson.Workspace.Assignment.ID, ToAssignmentVersion: lesson.Workspace.Assignment.Version, + } + } + if pendingLessonVersion(lesson.Link, lesson.Workspace) { + payload.LessonVersionUpdate = &machineLessonVersionUpdate{ + AssignmentID: lesson.Link.AssignmentID, + FromVersion: lesson.Link.AssignmentVersion, ToVersion: lesson.Workspace.Assignment.Version, + } + } + if *jsonOutput { + return writeMachineJSON(output, payload) + } + fmt.Fprintf(output, "# %s\n\n", assignment.Title) + fmt.Fprintf(output, text(ctx, "Урок `%s` v%d, около %d мин.\n\n", "Lesson `%s` v%d, about %d min.\n\n"), + assignment.ID, assignment.Version, assignment.EstimatedMinutes) + if payload.Transition != nil { + fmt.Fprintf(output, text(ctx, + "Файлов урока `%s` ещё нет в этой папке: выполните `softpractice update`.\n\n", + "The `%s` lesson files are not in this folder yet: run `softpractice update`.\n\n"), + payload.Transition.ToAssignmentID) + } + if payload.LessonVersionUpdate != nil { + fmt.Fprintf(output, text(ctx, + "Опубликована версия v%d этого урока; перейти на неё: `softpractice update`.\n\n", + "Version v%d of this lesson is published; to move to it: `softpractice update`.\n\n"), + payload.LessonVersionUpdate.ToVersion) + } + fmt.Fprintln(output, strings.TrimRight(assignment.InstructionsMarkdown, "\n")) + fmt.Fprintf(output, text(ctx, "\nМатериал: `softpractice material`. Страница задания: %s\n", + "\nMaterial: `softpractice material`. Assignment page: %s\n"), payload.URL) + return nil +} + +type machineMaterial struct { + Kind string `json:"kind"` + AssignmentID string `json:"assignment_id"` + Version int `json:"version,omitempty"` + Material *theoryMaterial `json:"material"` + URL string `json:"url"` +} + +func materialCommand(ctx context.Context, useCases learnerUseCases, args []string, output, errorOutput io.Writer) error { + flags := flag.NewFlagSet("material", flag.ContinueOnError) + flags.SetOutput(errorOutput) + lessonID := flags.String("lesson", "", "lesson ID; defaults to the lesson of this project") + jsonOutput := jsonFlag(flags) + if err := parseFlags(flags, args); err != nil { + return err + } + if flags.NArg() != 0 { + return usage(ctx, "использование: softpractice material [--lesson ID] [--json]", + "usage: softpractice material [--lesson ID] [--json]") + } + snapshot, err := useCases.Material(ctx, strings.TrimSpace(*lessonID)) + if err != nil { + return reportJSONError(*jsonOutput, output, err) + } + payload := machineMaterial{ + Kind: "softpractice.material", AssignmentID: snapshot.AssignmentID, Version: snapshot.Version, + Material: snapshot.Material, URL: webLessonURL(ctx, snapshot.AssignmentID, snapshot.Version, "/material"), + } + if *jsonOutput { + return writeMachineJSON(output, payload) + } + material := snapshot.Material + if material == nil { + fmt.Fprintf(output, text(ctx, "У урока `%s` нет теоретического материала.\n", + "Lesson `%s` has no theory material.\n"), snapshot.AssignmentID) + return nil + } + fmt.Fprintln(output, strings.TrimRight(material.Markdown, "\n")) + if diagram := material.Diagram; diagram.Title != "" { + fmt.Fprintf(output, text(ctx, "\n## Схема: %s\n\n", "\n## Diagram: %s\n\n"), diagram.Title) + for _, node := range diagram.Nodes { + fmt.Fprintf(output, "- **%s**: %s\n", node.Title, node.Text) + } + if diagram.TextAlternative != "" { + fmt.Fprintf(output, "\n%s\n", diagram.TextAlternative) + } + } + fmt.Fprintf(output, text(ctx, "\nМатериал на сайте: %s\n", "\nMaterial on the site: %s\n"), payload.URL) + return nil +} + +type machineHints struct { + Kind string `json:"kind"` + AssignmentID string `json:"assignment_id"` + Version int `json:"version"` + Status string `json:"status"` + Revealed []preparedHint `json:"revealed"` + // URL is the assignment page, where the learner opens the next hint. + URL string `json:"url"` +} + +func hintCommand(ctx context.Context, useCases learnerUseCases, args []string, output, errorOutput io.Writer) error { + flags := flag.NewFlagSet("hint", flag.ContinueOnError) + flags.SetOutput(errorOutput) + jsonOutput := jsonFlag(flags) + if err := parseFlags(flags, args); err != nil { + return err + } + if flags.NArg() != 0 { + return usage(ctx, "использование: softpractice hint [--json]", "usage: softpractice hint [--json]") + } + snapshot, err := useCases.Hints(ctx) + if err != nil { + return reportJSONError(*jsonOutput, output, err) + } + payload := machineHints{ + Kind: "softpractice.hints", AssignmentID: snapshot.Lesson.AssignmentID, Version: snapshot.Lesson.Version, + Status: snapshot.Status, Revealed: snapshot.Revealed, + URL: webLessonURL(ctx, snapshot.Lesson.AssignmentID, snapshot.Lesson.Version, ""), + } + if *jsonOutput { + return writeMachineJSON(output, payload) + } + if snapshot.Status == "unavailable" { + fmt.Fprintf(output, text(ctx, "У урока `%s` нет подготовленных подсказок.\n", + "Lesson `%s` has no prepared hints.\n"), snapshot.Lesson.AssignmentID) + return nil + } + for _, hint := range snapshot.Revealed { + fmt.Fprintf(output, text(ctx, "## Подсказка %d\n\n%s\n\n", "## Hint %d\n\n%s\n\n"), + hint.Order, strings.TrimRight(hint.Markdown, "\n")) + } + switch { + case snapshot.Status == "available" && len(snapshot.Revealed) == 0: + fmt.Fprintf(output, text(ctx, "Открытых подсказок пока нет. Открыть подсказку можно на странице задания: %s\n", + "No hints are open yet. You can open a hint on the assignment page: %s\n"), payload.URL) + case snapshot.Status == "available": + fmt.Fprintf(output, text(ctx, "Следующую подсказку можно открыть на странице задания: %s\n", + "You can open the next hint on the assignment page: %s\n"), payload.URL) + default: + fmt.Fprintln(output, text(ctx, "Все подсказки урока открыты.", "All hints of this lesson are open.")) + } + return nil +} + +type machineSubmissions struct { + Kind string `json:"kind"` + AssignmentID string `json:"assignment_id"` + Version int `json:"version"` + History []submissionAttempt `json:"history"` +} + +func submissionsCommand( + ctx context.Context, + useCases learnerUseCases, + args []string, + output, errorOutput io.Writer, +) error { + if len(args) > 0 && args[0] == "download" { + return downloadSubmissionRevision(ctx, useCases.client, args[1:], output, errorOutput) + } + flags := flag.NewFlagSet("submissions", flag.ContinueOnError) + flags.SetOutput(errorOutput) + jsonOutput := jsonFlag(flags) + if err := parseFlags(flags, args); err != nil { + return err + } + if flags.NArg() != 0 { + return usage(ctx, + "использование: softpractice submissions [--json]\n softpractice submissions download --id ID [--output PATH]", + "usage: softpractice submissions [--json]\n softpractice submissions download --id ID [--output PATH]") + } + snapshot, err := useCases.Submissions(ctx) + if err != nil { + return reportJSONError(*jsonOutput, output, err) + } + payload := machineSubmissions{ + Kind: "softpractice.submissions", AssignmentID: snapshot.Lesson.AssignmentID, + Version: snapshot.Lesson.Version, History: snapshot.History, + } + if *jsonOutput { + return writeMachineJSON(output, payload) + } + if len(snapshot.History) == 0 { + fmt.Fprintf(output, text(ctx, "По уроку `%s` отправок пока нет.\n", "Lesson `%s` has no submissions yet.\n"), + snapshot.Lesson.AssignmentID) + return nil + } + fmt.Fprintf(output, text(ctx, "Отправки урока `%s`, сначала новые:\n\n", "Submissions of lesson `%s`, newest first:\n\n"), + snapshot.Lesson.AssignmentID) + for _, attempt := range snapshot.History { + status := attempt.JobState + if attempt.EvaluationStatus != nil { + status = evaluationStatusTitle(ctx, *attempt.EvaluationStatus) + } + fmt.Fprintf(output, text(ctx, "%d. %s — %s — `%s`\n", "%d. %s — %s — `%s`\n"), + attempt.LearningAttempt, attempt.SubmittedAt.Local().Format("2006-01-02 15:04"), status, attempt.SubmissionID) + } + fmt.Fprintln(output, text(ctx, "\nРезультат отправки: `softpractice result --id ID`", + "\nResult of a submission: `softpractice result --id ID`")) + return nil +} diff --git a/cmd/softpractice/agent_commands_test.go b/cmd/softpractice/learning_commands_test.go similarity index 65% rename from cmd/softpractice/agent_commands_test.go rename to cmd/softpractice/learning_commands_test.go index 4eceb45..f856977 100644 --- a/cmd/softpractice/agent_commands_test.go +++ b/cmd/softpractice/learning_commands_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "io" "net/http" "net/http/httptest" "os/exec" @@ -142,7 +143,7 @@ func runEvaluation(t *testing.T, useCases learnerUseCases, args ...string) (stri ctx := withSettings(context.Background(), runtimeSettings{ Language: languageEnglish, WebURL: "https://softpractice.example", }) - err := evaluationCommand(ctx, useCases, args, &output, &errorOutput) + err := resultCommand(ctx, useCases, args, &output, &errorOutput) return output.String(), errorOutput.String(), err } @@ -152,7 +153,7 @@ func TestEvaluationWaitHonoursRetryAfterAndHidesDirections(t *testing.T) { clock := &fakeClock{now: time.Unix(0, 0)} useCases := clock.useCases(savedTestClient(t, server.URL), "") - output, _, err := runEvaluation(t, useCases, "--id", currentSubmissionID, "--wait", "--timeout", "1m", "--format", "json") + output, _, err := runEvaluation(t, useCases, "--id", currentSubmissionID, "--wait", "--timeout", "1m", "--json") if err != nil { t.Fatal(err) } @@ -187,7 +188,7 @@ func TestEvaluationWaitHonoursRetryAfterAndHidesDirections(t *testing.T) { "# Revision needed", "## Automated checks: passed", "## Review", "REVIEW-SUMMARY-TEXT", "- **responsibilities**: needs work. RUBRIC-RATIONALE-TEXT Where: app.py", "1. **FINDING-TITLE-TEXT** (high priority)", " - Risk: RISK-TEXT", - "Reviewer questions about the solution: 1.", "softpractice evaluation --directions", + "The reviewer's self-check questions (1) are on the result page.", "softpractice result --directions", } { if !strings.Contains(text, want) { t.Fatalf("text output lacks %q:\n%s", want, text) @@ -225,7 +226,11 @@ func TestEvaluationDirectionsOnRequest(t *testing.T) { server, _ := evaluationServer(t, currentSubmissionID, respondRevise) useCases := newLearnerUseCases(savedTestClient(t, server.URL), "") for _, format := range []string{"json", "text"} { - output, _, err := runEvaluation(t, useCases, "--id", currentSubmissionID, "--directions", "--format", format) + args := []string{"--id", currentSubmissionID, "--directions"} + if format == "json" { + args = append(args, "--json") + } + output, _, err := runEvaluation(t, useCases, args...) if err != nil { t.Fatal(err) } @@ -242,7 +247,7 @@ func TestEvaluationTimeoutReportsNotReady(t *testing.T) { clock := &fakeClock{now: time.Unix(0, 0)} useCases := clock.useCases(savedTestClient(t, server.URL), "") - output, _, err := runEvaluation(t, useCases, "--id", currentSubmissionID, "--wait", "--timeout", "45s", "--format", "json") + output, _, err := runEvaluation(t, useCases, "--id", currentSubmissionID, "--wait", "--timeout", "45s", "--json") if !errors.Is(err, exitNotReady) { t.Fatalf("err = %v, want exit status 3", err) } @@ -290,7 +295,7 @@ func TestEvaluationSupersededIsASeparateOutcome(t *testing.T) { }) clock := &fakeClock{now: time.Unix(0, 0)} output, _, err := runEvaluation(t, clock.useCases(savedTestClient(t, server.URL), ""), - "--id", currentSubmissionID, "--wait", "--format", "json") + "--id", currentSubmissionID, "--wait", "--json") if !errors.Is(err, exitSuperseded) { t.Fatalf("err = %v, want exit status 4", err) } @@ -309,7 +314,7 @@ func TestEvaluationUnauthorizedAsksForLogin(t *testing.T) { })) defer server.Close() output, _, err := runEvaluation(t, newLearnerUseCases(savedTestClient(t, server.URL), ""), - "--id", currentSubmissionID, "--format", "json") + "--id", currentSubmissionID, "--json") if err == nil { t.Fatal("expected an error") } @@ -333,7 +338,7 @@ func TestEvaluationUnauthorizedAsksForLogin(t *testing.T) { if err != nil { t.Fatal(err) } - output, _, err = runEvaluation(t, newLearnerUseCases(client, ""), "--id", currentSubmissionID, "--format", "json") + output, _, err = runEvaluation(t, newLearnerUseCases(client, ""), "--id", currentSubmissionID, "--json") if !errors.Is(err, learnercli.ErrLoginRequired) || !strings.Contains(output, `"code": "login_required"`) { t.Fatalf("err = %v, output:\n%s", err, output) } @@ -363,15 +368,19 @@ func TestAgentCommandsWithoutLinkedProject(t *testing.T) { {directory: unlinked, code: "project_not_linked"}, {directory: t.TempDir(), code: "not_git_repository"}, } { - output, _, err := runEvaluation(t, newLearnerUseCases(client, tc.directory), "--format", "json") + output, _, err := runEvaluation(t, newLearnerUseCases(client, tc.directory), "--json") if err == nil || !strings.Contains(output, `"code": "`+tc.code+`"`) { t.Fatalf("%s: err = %v, output:\n%s", tc.code, err, output) } - var assignmentOutput, errorOutput bytes.Buffer - err = assignmentCommand(context.Background(), newLearnerUseCases(client, tc.directory), - []string{"show", "--format", "json"}, &assignmentOutput, &errorOutput) - if err == nil || !strings.Contains(assignmentOutput.String(), `"code": "`+tc.code+`"`) { - t.Fatalf("assignment %s: err = %v, output:\n%s", tc.code, err, assignmentOutput.String()) + for name, command := range map[string]func(context.Context, learnerUseCases, []string, io.Writer, io.Writer) error{ + "task": taskCommand, "material": materialCommand, "hint": hintCommand, "submissions": submissionsCommand, + } { + var commandOutput, errorOutput bytes.Buffer + err = command(context.Background(), newLearnerUseCases(client, tc.directory), + []string{"--json"}, &commandOutput, &errorOutput) + if err == nil || !strings.Contains(commandOutput.String(), `"code": "`+tc.code+`"`) { + t.Fatalf("%s %s: err = %v, output:\n%s", name, tc.code, err, commandOutput.String()) + } } } } @@ -412,6 +421,19 @@ func assignmentServer(t *testing.T, workspaceID string, withMaterial bool) *http map[string]any{"title": "A", "text": "a"}, map[string]any{"title": "B", "text": "b"}, }}, }) + case request.URL.Path == "/v1/assignments/pa-foundation-01/prepared-hints" && request.URL.Query().Get("version") == "1": + writeTestJSON(writer, map[string]any{ + "status": "available", + "revealed": []any{map[string]any{"id": "rule-first", "order": 1, "markdown": "HINT-ONE-TEXT"}}, + }) + case request.URL.Path == "/v1/assignments/pa-foundation-00/material" && request.URL.RawQuery == "": + writeTestJSON(writer, map[string]any{ + "id": "pa-theory-intro", "title": "Intro", "estimated_minutes": 5, + "markdown": "EARLIER-MATERIAL-TEXT", + "diagram": map[string]any{"title": "Map", "text_alternative": "X then Y.", "nodes": []any{ + map[string]any{"title": "X", "text": "x"}, map[string]any{"title": "Y", "text": "y"}, + }}, + }) default: t.Errorf("unexpected API request %s?%s", request.URL.Path, request.URL.RawQuery) http.NotFound(writer, request) @@ -421,32 +443,100 @@ func assignmentServer(t *testing.T, workspaceID string, withMaterial bool) *http return server } -func TestAssignmentShowReadsTheFolderLessonVersion(t *testing.T) { +func runLessonCommand( + t *testing.T, + command func(context.Context, learnerUseCases, []string, io.Writer, io.Writer) error, + useCases learnerUseCases, + args ...string, +) string { + t.Helper() + var output, errorOutput bytes.Buffer + ctx := withSettings(context.Background(), runtimeSettings{ + Language: languageEnglish, WebURL: "https://softpractice.example", + }) + if err := command(ctx, useCases, args, &output, &errorOutput); err != nil { + t.Fatalf("%v\n%s", err, errorOutput.String()) + } + return output.String() +} + +func TestTaskAndMaterialReadTheFolderLessonVersion(t *testing.T) { workspaceID := uuid.NewString() // The folder is pinned to v1 while the server has published v2: the // requirement the tree is being written for is v1. root := createPinnedLinkedGitRepository(t, workspaceID, "pa-foundation-01", 1) for _, withMaterial := range []bool{true, false} { - server := assignmentServer(t, workspaceID, withMaterial) - var output, errorOutput bytes.Buffer - if err := assignmentCommand(context.Background(), newLearnerUseCases(savedTestClient(t, server.URL), root), - []string{"show", "--format", "json"}, &output, &errorOutput); err != nil { + useCases := newLearnerUseCases(savedTestClient(t, assignmentServer(t, workspaceID, withMaterial).URL), root) + var task machineTask + if err := decodeTestJSON([]byte(runLessonCommand(t, taskCommand, useCases, "--json")), &task); err != nil { t.Fatal(err) } - var payload machineAssignment - if err := decodeTestJSON(output.Bytes(), &payload); err != nil { + if task.Kind != "softpractice.task" || task.Assignment.Version != 1 || + task.Assignment.InstructionsMarkdown != "Extract the price rule." || + task.URL != "https://softpractice.example/assignments/pa-foundation-01?version=1" || + task.LessonVersionUpdate == nil || task.LessonVersionUpdate.ToVersion != 2 { + t.Fatalf("unexpected task: %+v", task) + } + output := runLessonCommand(t, materialCommand, useCases, "--json") + var material machineMaterial + if err := decodeTestJSON([]byte(output), &material); err != nil { t.Fatal(err) } - if payload.Kind != "softpractice.assignment" || payload.Assignment.Version != 1 || - payload.Assignment.InstructionsMarkdown != "Extract the price rule." || - payload.LessonVersionUpdate == nil || payload.LessonVersionUpdate.ToVersion != 2 { - t.Fatalf("unexpected payload: %+v", payload) + if withMaterial != (material.Material != nil) || material.Version != 1 || + material.URL != "https://softpractice.example/assignments/pa-foundation-01/material?version=1" { + t.Fatalf("material = %+v, want present=%t", material, withMaterial) + } + if !withMaterial && !strings.Contains(output, `"material": null`) { + t.Fatalf("absent material must be null:\n%s", output) } - if withMaterial != (payload.Material != nil) { - t.Fatalf("material = %+v, want present=%t", payload.Material, withMaterial) + text := runLessonCommand(t, materialCommand, useCases) + if withMaterial && (!strings.Contains(text, "# Rules") || !strings.Contains(text, "- **A**: a")) { + t.Fatalf("unexpected material text:\n%s", text) } - if !withMaterial && !strings.Contains(output.String(), `"material": null`) { - t.Fatalf("absent material must be null:\n%s", output.String()) + if !withMaterial && !strings.Contains(text, "has no theory material") { + t.Fatalf("unexpected material text:\n%s", text) + } + } +} + +func TestMaterialOfAnEarlierLesson(t *testing.T) { + workspaceID := uuid.NewString() + root := createPinnedLinkedGitRepository(t, workspaceID, "pa-foundation-01", 1) + useCases := newLearnerUseCases(savedTestClient(t, assignmentServer(t, workspaceID, true).URL), root) + output := runLessonCommand(t, materialCommand, useCases, "--lesson", "pa-foundation-00") + if !strings.Contains(output, "EARLIER-MATERIAL-TEXT") || + !strings.Contains(output, "https://softpractice.example/assignments/pa-foundation-00/material\n") { + t.Fatalf("unexpected output:\n%s", output) + } +} + +func TestHintListsOpenedHintsAndLinksToTheSite(t *testing.T) { + workspaceID := uuid.NewString() + root := createPinnedLinkedGitRepository(t, workspaceID, "pa-foundation-01", 1) + useCases := newLearnerUseCases(savedTestClient(t, assignmentServer(t, workspaceID, true).URL), root) + output := runLessonCommand(t, hintCommand, useCases) + if !strings.Contains(output, "## Hint 1\n\nHINT-ONE-TEXT") || + !strings.Contains(output, "open the next hint on the assignment page: https://softpractice.example/assignments/pa-foundation-01?version=1") { + t.Fatalf("unexpected output:\n%s", output) + } + var hints machineHints + if err := decodeTestJSON([]byte(runLessonCommand(t, hintCommand, useCases, "--json")), &hints); err != nil { + t.Fatal(err) + } + if hints.Status != "available" || len(hints.Revealed) != 1 || hints.Version != 1 { + t.Fatalf("unexpected hints: %+v", hints) + } +} + +func TestInvalidArgumentsAreUsageErrors(t *testing.T) { + useCases := newLearnerUseCases(savedTestClient(t, "http://127.0.0.1:1"), t.TempDir()) + for name, args := range map[string][]string{ + "unknown flag": {"--no-such-flag"}, "extra argument": {"extra"}, "negative timeout": {"--timeout", "-1s"}, + } { + _, _, err := runEvaluation(t, useCases, args...) + var usageErr usageError + if !errors.As(err, &usageErr) { + t.Fatalf("%s: err = %v, want a usage error", name, err) } } } @@ -493,7 +583,7 @@ func TestAgentJSONContractsAreStable(t *testing.T) { server, _ := evaluationServer(t, currentSubmissionID, respondRevise) clock := &fakeClock{now: time.Unix(0, 0)} output, _, err := runEvaluation(t, clock.useCases(savedTestClient(t, server.URL), ""), - "--id", currentSubmissionID, "--format", "json") + "--id", currentSubmissionID, "--json") if err != nil { t.Fatal(err) } @@ -511,7 +601,7 @@ func TestAgentJSONContractsAreStable(t *testing.T) { "result.review.findings", "result.review.findings[].evidence_refs", "result.review.findings[].observation", "result.review.findings[].priority", "result.review.findings[].risk", "result.review.findings[].title", "result.review.mode", - "result.review.questions_count", "result.review.rubric", "result.review.rubric[].criterion", + "result.review.questions_answerable", "result.review.questions_count", "result.review.rubric", "result.review.rubric[].criterion", "result.review.rubric[].evidence_refs", "result.review.rubric[].level", "result.review.rubric[].rationale", "result.review.status", "result.review.summary", "result.review.uncertainty", "result.review.uncertainty.blocking", @@ -538,25 +628,35 @@ func TestAgentJSONContractsAreStable(t *testing.T) { workspaceID := uuid.NewString() root := createPinnedLinkedGitRepository(t, workspaceID, "pa-foundation-01", 1) - var assignmentOutput bytes.Buffer - if err := assignmentCommand(context.Background(), - newLearnerUseCases(savedTestClient(t, assignmentServer(t, workspaceID, true).URL), root), - []string{"show", "--format", "json"}, &assignmentOutput, &bytes.Buffer{}); err != nil { - t.Fatal(err) - } - wantAssignment := []string{ - "assignment", "assignment.content_sha256", "assignment.estimated_minutes", - "assignment.exercise_id", "assignment.id", "assignment.instructions_markdown", - "assignment.kind", "assignment.project_setup", "assignment.title", "assignment.version", - "kind", "lesson_version_update", "lesson_version_update.assignment_id", - "lesson_version_update.from_version", "lesson_version_update.to_version", "material", - "material.diagram", "material.diagram.nodes", "material.diagram.nodes[].text", - "material.diagram.nodes[].title", "material.diagram.text_alternative", "material.diagram.title", - "material.estimated_minutes", "material.id", "material.markdown", "material.title", - "project_id", "workspace_id", - } - if got := jsonKeyPaths(t, assignmentOutput.Bytes()); !reflect.DeepEqual(got, wantAssignment) { - t.Fatalf("softpractice.assignment keys changed:\ngot %q\nwant %q", got, wantAssignment) + useCases := newLearnerUseCases(savedTestClient(t, assignmentServer(t, workspaceID, true).URL), root) + for _, tc := range []struct { + kind string + command func(context.Context, learnerUseCases, []string, io.Writer, io.Writer) error + want []string + }{ + {kind: "softpractice.task", command: taskCommand, want: []string{ + "assignment", "assignment.content_sha256", "assignment.estimated_minutes", + "assignment.exercise_id", "assignment.id", "assignment.instructions_markdown", + "assignment.kind", "assignment.project_setup", "assignment.title", "assignment.version", + "kind", "lesson_version_update", "lesson_version_update.assignment_id", + "lesson_version_update.from_version", "lesson_version_update.to_version", + "project_id", "url", "workspace_id", + }}, + {kind: "softpractice.material", command: materialCommand, want: []string{ + "assignment_id", "kind", "material", + "material.diagram", "material.diagram.nodes", "material.diagram.nodes[].text", + "material.diagram.nodes[].title", "material.diagram.text_alternative", "material.diagram.title", + "material.estimated_minutes", "material.id", "material.markdown", "material.title", + "url", "version", + }}, + {kind: "softpractice.hints", command: hintCommand, want: []string{ + "assignment_id", "kind", "revealed", "revealed[].id", "revealed[].markdown", "revealed[].order", + "status", "url", "version", + }}, + } { + if got := jsonKeyPaths(t, []byte(runLessonCommand(t, tc.command, useCases, "--json"))); !reflect.DeepEqual(got, tc.want) { + t.Fatalf("%s keys changed:\ngot %q\nwant %q", tc.kind, got, tc.want) + } } } @@ -595,7 +695,7 @@ func TestStatusJSONReportsTitleAndLatestEvaluationStatus(t *testing.T) { root := createPinnedLinkedGitRepository(t, workspaceID, "pa-foundation-05", 1) var output, errorOutput bytes.Buffer if err := status(context.Background(), savedTestClient(t, server.URL), root, - []string{"--format", "json"}, &output, &errorOutput); err != nil { + []string{"--json"}, &output, &errorOutput); err != nil { t.Fatal(err) } var payload machineStatus @@ -604,7 +704,98 @@ func TestStatusJSONReportsTitleAndLatestEvaluationStatus(t *testing.T) { } if payload.Assignment.Title != "Module boundaries" || payload.LatestSubmission == nil || payload.LatestSubmission.EvaluationStatus != "revise" || payload.Transition != nil || + len(payload.NextActions) != 1 || payload.NextActions[0].Command != "softpractice result" || payload.LessonVersionUpdate != nil { t.Fatalf("unexpected status: %+v", payload) } } + +func TestNextActions(t *testing.T) { + ctx := withSettings(context.Background(), runtimeSettings{WebURL: "https://softpractice.example"}) + submission := func(jobState string) *struct { + ID string `json:"id"` + RevisionID string `json:"revision_id"` + JobState string `json:"job_state"` + SubmittedAt time.Time `json:"submitted_at"` + } { + return &struct { + ID string `json:"id"` + RevisionID string `json:"revision_id"` + JobState string `json:"job_state"` + SubmittedAt time.Time `json:"submitted_at"` + }{ID: currentSubmissionID, JobState: jobState} + } + snapshot := func(pinned string, pinnedVersion int, clean bool, head string) statusSnapshot { + var result statusSnapshot + result.Link = learnercli.ProjectLink{SchemaVersion: 2, AssignmentID: pinned, AssignmentVersion: pinnedVersion} + result.Workspace.Assignment.ID = "pa-foundation-02" + result.Workspace.Assignment.Version = 2 + result.Repository = gitRepository{Clean: clean, CommitSHA: head} + return result + } + codes := func(actions []nextAction) string { + parts := make([]string, 0, len(actions)) + for _, action := range actions { + parts = append(parts, action.Code) + } + return strings.Join(parts, ",") + } + + transition := snapshot("pa-foundation-01", 1, true, "abc") + fresh := snapshot("pa-foundation-02", 2, true, "abc") + dirty := snapshot("pa-foundation-02", 2, false, "abc") + queued := snapshot("pa-foundation-02", 2, true, "abc") + queued.Workspace.LatestSubmission = submission("leased") + questions := snapshot("pa-foundation-02", 2, true, "abc") + questions.Workspace.LatestSubmission = submission("completed") + questions.Latest = &latestResult{Status: "uncertain", QuestionsAnswerable: true, CommitSHA: "abc"} + reviewed := snapshot("pa-foundation-02", 2, true, "abc") + reviewed.Workspace.LatestSubmission = submission("completed") + reviewed.Latest = &latestResult{Status: "revise", CommitSHA: "abc"} + reworked := reviewed + reworked.Repository.CommitSHA = "def" + olderVersion := snapshot("pa-foundation-02", 1, true, "abc") + + for name, tc := range map[string]struct { + snapshot statusSnapshot + want string + }{ + "transition": {transition, "apply_update"}, + "no submission": {fresh, "submit"}, + "dirty tree": {dirty, "commit_changes"}, + "evaluating": {queued, "wait_result"}, + "questions": {questions, "answer_questions,read_result"}, + "reviewed": {reviewed, "read_result"}, + "new commit": {reworked, "submit"}, + "older version": {olderVersion, "submit,update_lesson_version"}, + } { + actions := nextActions(ctx, tc.snapshot) + if got := codes(actions); got != tc.want { + t.Fatalf("%s: next actions = %s, want %s", name, got, tc.want) + } + if name == "questions" && actions[0].URL != "https://softpractice.example/submissions/"+currentSubmissionID+"/result" { + t.Fatalf("questions URL = %q", actions[0].URL) + } + } +} + +func TestResultAsksToAnswerQuestionsOnTheSite(t *testing.T) { + server, _ := evaluationServer(t, currentSubmissionID, func(writer http.ResponseWriter) { + body := reviseEvaluationBody(currentSubmissionID) + evaluation := body["evaluation"].(map[string]any) + evaluation["status"] = "uncertain" + review := evaluation["review"].(map[string]any) + review["verdict"] = "uncertain" + review["uncertainty"] = map[string]any{"blocking": true, "reason": "UNCERTAINTY-REASON"} + writeTestJSON(writer, body) + }) + output, _, err := runEvaluation(t, newLearnerUseCases(savedTestClient(t, server.URL), ""), "--id", currentSubmissionID) + if err != nil { + t.Fatal(err) + } + want := "Answer them on the result page: https://softpractice.example/submissions/" + currentSubmissionID + "/result" + if !strings.Contains(output, want) || !strings.Contains(output, "UNCERTAINTY-REASON") || + strings.Contains(output, "DEFENSE-QUESTION-TEXT") { + t.Fatalf("unexpected output:\n%s", output) + } +} diff --git a/cmd/softpractice/machine.go b/cmd/softpractice/machine.go index 7a5650c..9f3b277 100644 --- a/cmd/softpractice/machine.go +++ b/cmd/softpractice/machine.go @@ -17,24 +17,6 @@ import ( "github.com/arthur-s/softpractice-cli/internal/learnercli" ) -type outputFormat string - -const ( - outputFormatText outputFormat = "text" - outputFormatJSON outputFormat = "json" -) - -func parseOutputFormat(value string) (outputFormat, error) { - switch outputFormat(strings.ToLower(strings.TrimSpace(value))) { - case outputFormatText: - return outputFormatText, nil - case outputFormatJSON: - return outputFormatJSON, nil - default: - return "", errors.New("format must be text or json") - } -} - type machineStatus struct { Kind string `json:"kind"` Account struct { @@ -63,6 +45,81 @@ type machineStatus struct { // version of the lesson this folder holds. LessonVersionUpdate *machineLessonVersionUpdate `json:"lesson_version_update,omitempty"` LatestSubmission *machineSubmissionSummary `json:"latest_submission"` + // NextActions lists what to do next, most important first. + NextActions []nextAction `json:"next_actions"` +} + +// nextAction is one step the learner can take now. Command is a CLI command; +// URL is a page for an action that happens on the site. +type nextAction struct { + Code string `json:"code"` + Command string `json:"command,omitempty"` + URL string `json:"url,omitempty"` +} + +// nextActions derives the learner's next steps from the status, most +// important first. The list is never empty. +func nextActions(ctx context.Context, snapshot statusSnapshot) []nextAction { + link, workspace, repository := snapshot.Link, snapshot.Workspace, snapshot.Repository + var actions []nextAction + submit := nextAction{Code: "submit", Command: "softpractice submit --wait"} + if !repository.Clean { + submit = nextAction{Code: "commit_changes"} + } + latest := workspace.LatestSubmission + switch { + case pendingTransition(link, workspace): + actions = append(actions, nextAction{Code: "apply_update", Command: "softpractice update"}) + case latest == nil: + actions = append(actions, submit) + case latest.JobState == "queued" || latest.JobState == "leased": + actions = append(actions, nextAction{Code: "wait_result", Command: "softpractice result --wait"}) + default: + result := snapshot.Latest + readResult := nextAction{Code: "read_result", Command: "softpractice result"} + switch { + case result != nil && result.QuestionsAnswerable: + actions = append(actions, + nextAction{Code: "answer_questions", URL: webResultURL(ctx, latest.ID)}, readResult) + case !repository.Clean || (result != nil && result.CommitSHA != "" && result.CommitSHA != repository.CommitSHA): + // Work continued after this submission. + actions = append(actions, submit) + default: + actions = append(actions, readResult) + } + } + if pendingLessonVersion(link, workspace) { + actions = append(actions, nextAction{Code: "update_lesson_version", Command: "softpractice update"}) + } + return actions +} + +func nextActionText(ctx context.Context, action nextAction) string { + switch action.Code { + case "apply_update": + return text(ctx, "примените переход к следующему уроку: `softpractice update`", + "apply the transition to the next lesson: `softpractice update`") + case "commit_changes": + return text(ctx, "сделайте commit изменений и отправьте решение: `softpractice submit --wait`", + "commit your changes and submit: `softpractice submit --wait`") + case "submit": + return text(ctx, "когда решение готово и закоммичено, отправьте его: `softpractice submit --wait`", + "when the solution is committed, submit it: `softpractice submit --wait`") + case "wait_result": + return text(ctx, "дождитесь результата проверки: `softpractice result --wait`", + "wait for the evaluation: `softpractice result --wait`") + case "read_result": + return text(ctx, "прочитайте результат проверки: `softpractice result`", + "read the evaluation result: `softpractice result`") + case "answer_questions": + return text(ctx, "ответьте на вопросы рецензента на странице результата: ", + "answer the reviewer's questions on the result page: ") + action.URL + case "update_lesson_version": + return text(ctx, "можно перейти на новую версию урока: `softpractice update`", + "you can move to the new lesson version: `softpractice update`") + default: + return action.Code + } } type machineLessonVersionUpdate struct { @@ -88,7 +145,7 @@ type machineSubmissionSummary struct { EvaluationStatus string `json:"evaluation_status,omitempty"` } -func writeStatusJSON(output io.Writer, snapshot statusSnapshot) error { +func writeStatusJSON(ctx context.Context, output io.Writer, snapshot statusSnapshot) error { user, link, workspace, repository := snapshot.User, snapshot.Link, snapshot.Workspace, snapshot.Repository payload := machineStatus{Kind: "softpractice.status"} payload.Account.ID = user.ID @@ -117,11 +174,14 @@ func writeStatusJSON(output io.Writer, snapshot statusSnapshot) error { if workspace.LatestSubmission != nil { payload.LatestSubmission = &machineSubmissionSummary{ ID: workspace.LatestSubmission.ID, RevisionID: workspace.LatestSubmission.RevisionID, - JobState: workspace.LatestSubmission.JobState, - SubmittedAt: workspace.LatestSubmission.SubmittedAt, - EvaluationStatus: snapshot.LatestEvaluationStatus, + JobState: workspace.LatestSubmission.JobState, + SubmittedAt: workspace.LatestSubmission.SubmittedAt, + } + if snapshot.Latest != nil { + payload.LatestSubmission.EvaluationStatus = snapshot.Latest.Status } } + payload.NextActions = nextActions(ctx, snapshot) return writeMachineJSON(output, payload) } @@ -179,7 +239,6 @@ type machineTechnicalError struct { } type machineSubmission struct { - Kind string `json:"kind"` SubmissionID string `json:"submission_id"` EvaluationJobID string `json:"evaluation_job_id"` JobState string `json:"job_state"` @@ -188,115 +247,26 @@ type machineSubmission struct { MaxAttempts int `json:"max_attempts"` NextPollSeconds int `json:"next_poll_seconds,omitempty"` UpdatedAt time.Time `json:"updated_at"` - ResultPath string `json:"result_path"` Evaluation *machineEvaluation `json:"evaluation,omitempty"` } -func submissionCommand( - ctx context.Context, - client *learnercli.Client, - startDirectory string, - args []string, - output, errorOutput io.Writer, -) error { - if len(args) == 0 { - return errors.New("usage: softpractice submission show|download ...") - } - switch args[0] { - case "show": - return showSubmission(ctx, client, startDirectory, args[1:], output, errorOutput) - case "download": - return downloadSubmissionRevision(ctx, client, args[1:], output, errorOutput) - default: - return errors.New("usage: softpractice submission show|download ...") - } -} - -func showSubmission( - ctx context.Context, - client *learnercli.Client, - startDirectory string, - args []string, - output, errorOutput io.Writer, -) error { - flags := flag.NewFlagSet("submission show", flag.ContinueOnError) - flags.SetOutput(errorOutput) - id := flags.String("id", "", "submission ID; defaults to the latest linked submission") - format := flags.String("format", "text", "output format: text or json") - if err := flags.Parse(args); err != nil { - return err - } - if flags.NArg() != 0 { - return errors.New("usage: softpractice submission show [--id ID] [--format text|json]") - } - outputFormat, err := parseOutputFormat(*format) - if err != nil { - return err - } - submissionID := strings.TrimSpace(*id) - useCases := newLearnerUseCases(client, startDirectory) - if submissionID == "" { - latest, err := useCases.LatestSubmission(ctx) - if err != nil { - return err - } - if latest.PredecessorAssignmentID != "" { - fmt.Fprintf(errorOutput, text(ctx, - "По %s отправок ещё нет; показан принятый результат %s.\n", - "%s has no submission yet; showing the accepted %s result.\n", - ), latest.CurrentAssignmentID, latest.PredecessorAssignmentID) - } - submissionID = latest.SubmissionID - } - parsedID, err := uuid.Parse(submissionID) - if err != nil || parsedID.String() != submissionID { - return errors.New("submission ID must be a canonical UUID") - } - var response evaluationAPIResponse - if err := client.AuthorizedJSON( - ctx, "GET", "/v1/submissions/"+submissionID+"/evaluation", nil, &response, - ); err != nil { - return err - } - payload, err := normalizeSubmissionResponse(response) - if err != nil { - return err - } - if payload.SubmissionID != submissionID { - return errors.New("evaluation response does not match the requested submission") - } - if outputFormat == outputFormatJSON { - return writeMachineJSON(output, payload) - } - fmt.Fprintf( - output, - text(ctx, "Отправка: %s\nСостояние проверки: %s\n", "Submission: %s\nEvaluation state: %s\n"), - payload.SubmissionID, - payload.JobState, - ) - if payload.Evaluation != nil { - fmt.Fprintf(output, text(ctx, "Результат: %s\n", "Result: %s\n"), payload.Evaluation.Status) - } - return nil -} - func downloadSubmissionRevision( ctx context.Context, client *learnercli.Client, args []string, output, errorOutput io.Writer, ) error { - flags := flag.NewFlagSet("submission download", flag.ContinueOnError) + flags := flag.NewFlagSet("submissions download", flag.ContinueOnError) flags.SetOutput(errorOutput) id := flags.String("id", "", "accepted submission ID from the result page") destination := flags.String("output", "", "destination ZIP path") - if err := flags.Parse(args); err != nil { + if err := parseFlags(flags, args); err != nil { return err } if flags.NArg() != 0 || strings.TrimSpace(*id) == "" { - return errors.New(text(ctx, - "использование: softpractice submission download --id ID [--output PATH]", - "usage: softpractice submission download --id ID [--output PATH]")) + return usage(ctx, + "использование: softpractice submissions download --id ID [--output PATH]", + "usage: softpractice submissions download --id ID [--output PATH]") } submissionID := strings.TrimSpace(*id) parsedID, err := uuid.Parse(submissionID) @@ -349,11 +319,9 @@ func normalizeSubmissionResponse(response evaluationAPIResponse) (machineSubmiss return machineSubmission{}, errors.New("evaluation response is incomplete") } payload := machineSubmission{ - Kind: "softpractice.submission", SubmissionID: response.SubmissionID, EvaluationJobID: response.EvaluationJobID, JobState: response.JobState, Attempt: response.Attempt, MaxAttempts: response.MaxAttempts, NextPollSeconds: response.NextPollSeconds, UpdatedAt: response.UpdatedAt, - ResultPath: "/submissions/" + response.SubmissionID + "/result", } switch response.JobState { case "queued", "leased": @@ -415,10 +383,3 @@ func normalizeSubmissionResponse(response evaluationAPIResponse) (machineSubmiss payload.Evaluation = &machine return payload, nil } - -func writeMachineJSON(output io.Writer, payload any) error { - encoder := json.NewEncoder(output) - encoder.SetEscapeHTML(false) - encoder.SetIndent("", " ") - return encoder.Encode(payload) -} diff --git a/cmd/softpractice/main.go b/cmd/softpractice/main.go index 9c0af2b..ace39d0 100644 --- a/cmd/softpractice/main.go +++ b/cmd/softpractice/main.go @@ -98,6 +98,10 @@ func main() { os.Exit(int(status)) } fmt.Fprintln(os.Stderr, "softpractice:", err) + var usageErr usageError + if errors.As(err, &usageErr) { + os.Exit(int(exitUsage)) + } os.Exit(1) } } @@ -122,7 +126,7 @@ func run( apiURL := root.String("api", "", "API base URL") languageValue := root.String("lang", "", "CLI language: ru or en") showVersion := root.Bool("version", false, "print the CLI version and exit") - if err := root.Parse(args); err != nil { + if err := parseFlags(root, args); err != nil { return err } var languageOverride language @@ -148,7 +152,7 @@ func run( } if remaining[0] == "help" || remaining[0] == "--help" || remaining[0] == "-h" { if len(remaining) > 2 { - return errors.New(text(ctx, "использование: softpractice help [команда]", "usage: softpractice help [command]")) + return usage(ctx, "использование: softpractice help [команда]", "usage: softpractice help [command]") } if len(remaining) == 2 { return printCommandHelp(ctx, output, remaining[1]) @@ -158,7 +162,7 @@ func run( } if remaining[0] == "config" { if len(remaining) != 1 { - return errors.New(text(ctx, "использование: softpractice config", "usage: softpractice config")) + return usage(ctx, "использование: softpractice config", "usage: softpractice config") } settings, err = resolveSettings(config) if err != nil { @@ -209,7 +213,7 @@ func run( return login(ctx, client, remaining[1:], output, errorOutput) case "logout": if len(remaining) != 1 { - return errors.New("usage: softpractice logout") + return usageError{message: "usage: softpractice logout"} } if err := client.Logout(ctx); err != nil { return err @@ -218,12 +222,16 @@ func run( return nil case "status": return status(ctx, client, "", remaining[1:], output, errorOutput) - case "submission": - return submissionCommand(ctx, client, "", remaining[1:], output, errorOutput) - case "evaluation": - return evaluationCommand(ctx, newLearnerUseCases(client, ""), remaining[1:], output, errorOutput) - case "assignment": - return assignmentCommand(ctx, newLearnerUseCases(client, ""), remaining[1:], output, errorOutput) + case "task": + return taskCommand(ctx, newLearnerUseCases(client, ""), remaining[1:], output, errorOutput) + case "material": + return materialCommand(ctx, newLearnerUseCases(client, ""), remaining[1:], output, errorOutput) + case "hint": + return hintCommand(ctx, newLearnerUseCases(client, ""), remaining[1:], output, errorOutput) + case "result": + return resultCommand(ctx, newLearnerUseCases(client, ""), remaining[1:], output, errorOutput) + case "submissions": + return submissionsCommand(ctx, newLearnerUseCases(client, ""), remaining[1:], output, errorOutput) case "project": return projectCommand(ctx, client, remaining[1:], output, errorOutput) case "starter": @@ -235,7 +243,7 @@ func run( case "open": return openCurrent(ctx, client, "", remaining[1:], output, errorOutput) default: - return fmt.Errorf(text(ctx, "неизвестная команда %q", "unknown command %q"), remaining[0]) + return usageError{message: fmt.Sprintf(text(ctx, "неизвестная команда %q", "unknown command %q"), remaining[0])} } } @@ -246,21 +254,17 @@ func projectCommand( output, errorOutput io.Writer, ) error { if len(args) == 0 || args[0] != "restore" { - return errors.New(text(ctx, - "использование: softpractice project restore [--practicum ID] [--directory PATH]", - "usage: softpractice project restore [--practicum ID] [--directory PATH]")) + return usage(ctx, "использование: softpractice project restore [--practicum ID] [--directory PATH]", "usage: softpractice project restore [--practicum ID] [--directory PATH]") } flags := flag.NewFlagSet("project restore", flag.ContinueOnError) flags.SetOutput(errorOutput) practicumID := flags.String("practicum", "", "started practicum ID") directory := flags.String("directory", "", "new destination directory") - if err := flags.Parse(args[1:]); err != nil { + if err := parseFlags(flags, args[1:]); err != nil { return err } if flags.NArg() != 0 { - return errors.New(text(ctx, - "использование: softpractice project restore [--practicum ID] [--directory PATH]", - "usage: softpractice project restore [--practicum ID] [--directory PATH]")) + return usage(ctx, "использование: softpractice project restore [--practicum ID] [--directory PATH]", "usage: softpractice project restore [--practicum ID] [--directory PATH]") } source, err := client.ResolveProjectRestoreSource(ctx, strings.TrimSpace(*practicumID)) if err != nil { @@ -416,9 +420,7 @@ func showConfig(ctx context.Context, output io.Writer) error { func setConfig(ctx context.Context, args []string, output io.Writer) error { if len(args) != 2 { - return errors.New(text(ctx, - "использование: softpractice set api-url|web-url|lang|auto-checks VALUE", - "usage: softpractice set api-url|web-url|lang|auto-checks VALUE")) + return usage(ctx, "использование: softpractice set api-url|web-url|lang|auto-checks VALUE", "usage: softpractice set api-url|web-url|lang|auto-checks VALUE") } if args[0] == "auto-checks" { return setProjectAutoChecks(ctx, args[1], output) @@ -446,7 +448,7 @@ func setConfig(ctx context.Context, args []string, output io.Writer) error { } config.Language = string(value) default: - return fmt.Errorf(text(ctx, "неизвестная настройка %q", "unknown setting %q"), args[0]) + return usageError{message: fmt.Sprintf(text(ctx, "неизвестная настройка %q", "unknown setting %q"), args[0])} } if err := settings.Config.Save(config); err != nil { return err @@ -486,11 +488,11 @@ func downloadStarter(ctx context.Context, client *learnercli.Client, args []stri flags.SetOutput(errorOutput) directory := flags.String("directory", "", "new destination directory") practicumID := flags.String("practicum", "", "started practicum ID") - if err := flags.Parse(args); err != nil { + if err := parseFlags(flags, args); err != nil { return err } if flags.NArg() != 0 { - return errors.New(text(ctx, "использование: softpractice starter [--practicum PRACTICUM_ID] [--directory DIRECTORY]", "usage: softpractice starter [--practicum PRACTICUM_ID] [--directory DIRECTORY]")) + return usage(ctx, "использование: softpractice starter [--practicum PRACTICUM_ID] [--directory DIRECTORY]", "usage: softpractice starter [--practicum PRACTICUM_ID] [--directory DIRECTORY]") } practicum, err := client.StartedPracticum(ctx, *practicumID) if err != nil { @@ -630,11 +632,11 @@ func initializeLinkedGit( func updateProject(ctx context.Context, client *learnercli.Client, args []string, output, errorOutput io.Writer) error { flags := flag.NewFlagSet("update", flag.ContinueOnError) flags.SetOutput(errorOutput) - if err := flags.Parse(args); err != nil { + if err := parseFlags(flags, args); err != nil { return err } if flags.NArg() != 0 { - return errors.New(text(ctx, "использование: softpractice update", "usage: softpractice update")) + return usage(ctx, "использование: softpractice update", "usage: softpractice update") } return newLearnerUseCases(client, "").Update(ctx, output) } @@ -1343,11 +1345,11 @@ func login( flags := flag.NewFlagSet("login", flag.ContinueOnError) flags.SetOutput(errorOutput) noBrowser := flags.Bool("no-browser", false, "print the verification URL without opening it") - if err := flags.Parse(args); err != nil { + if err := parseFlags(flags, args); err != nil { return err } if flags.NArg() != 0 { - return errors.New(text(ctx, "использование: softpractice login [--no-browser]", "usage: softpractice login [--no-browser]")) + return usage(ctx, "использование: softpractice login [--no-browser]", "usage: softpractice login [--no-browser]") } authorization, err := client.StartLogin(ctx, "softpractice-cli/"+cliVersion) if err != nil { @@ -1391,23 +1393,19 @@ func status( ) error { flags := flag.NewFlagSet("status", flag.ContinueOnError) flags.SetOutput(errorOutput) - format := flags.String("format", "text", "output format: text or json") - if err := flags.Parse(args); err != nil { + jsonOutput := jsonFlag(flags) + if err := parseFlags(flags, args); err != nil { return err } if flags.NArg() != 0 { - return errors.New("usage: softpractice status [--format text|json]") + return usage(ctx, "использование: softpractice status [--json]", "usage: softpractice status [--json]") } - outputFormat, err := parseOutputFormat(*format) + snapshot, err := newLearnerUseCases(client, startDirectory).Status(ctx) if err != nil { - return err - } - snapshot, err := newLearnerUseCases(client, startDirectory).Status(ctx, outputFormat == outputFormatJSON) - if err != nil { - return err + return reportJSONError(*jsonOutput, output, err) } - if outputFormat == outputFormatJSON { - return writeStatusJSON(output, snapshot) + if *jsonOutput { + return writeStatusJSON(ctx, output, snapshot) } repository, link, workspace, user := snapshot.Repository, snapshot.Link, snapshot.Workspace, snapshot.User clean := text(ctx, "чистое", "clean") @@ -1467,6 +1465,17 @@ func status( workspace.LatestSubmission.JobState, workspace.LatestSubmission.SubmittedAt.Format(time.RFC3339), ) + if snapshot.Latest != nil { + fmt.Fprintf(output, text(ctx, "Результат проверки: %s\n", "Evaluation result: %s\n"), + evaluationStatusTitle(ctx, snapshot.Latest.Status)) + } + } + for index, action := range nextActions(ctx, snapshot) { + label := text(ctx, "Следующий шаг: ", "Next step: ") + if index > 0 { + label = text(ctx, "Также: ", "Also: ") + } + fmt.Fprintln(output, label+nextActionText(ctx, action)) } return nil } @@ -1500,21 +1509,32 @@ func submit( flags.SetOutput(errorOutput) yes := flags.Bool("yes", false, "submit without an interactive confirmation") checksFlag := flags.Bool("checks", false, "run local public checks before submitting") - if err := flags.Parse(args); err != nil { + wait := flags.Bool("wait", false, "wait for the evaluation result after submitting") + timeout := flags.Duration("timeout", defaultEvaluationTimeout, "maximum wait with --wait") + jsonOutput := jsonFlag(flags) + if err := parseFlags(flags, args); err != nil { return err } - if flags.NArg() != 0 { - return errors.New(text(ctx, "использование: softpractice submit [--yes]", "usage: softpractice submit [--yes]")) + if flags.NArg() != 0 || *timeout < 0 { + return usage(ctx, + "использование: softpractice submit [--yes] [--checks=true|false] [--wait] [--timeout DURATION] [--json]", + "usage: softpractice submit [--yes] [--checks=true|false] [--wait] [--timeout DURATION] [--json]") + } + // With --json, stdout carries only the JSON document; everything meant + // for a person goes to stderr. + human := output + if *jsonOutput { + human = errorOutput } useCases := newLearnerUseCases(client, startDirectory) prepared, err := useCases.PrepareSubmission(ctx) if err != nil { - return err + return reportJSONError(*jsonOutput, output, err) } defer prepared.Close() repository, link, workspace := prepared.Repository, prepared.Link, prepared.Workspace fmt.Fprintf( - output, + human, text(ctx, "Проект: %s\nУрок: %s v%d — %s\n\n", "Project: %s\nAssignment: %s v%d — %s\n\n"), link.ProjectID, workspace.Assignment.ID, @@ -1522,7 +1542,7 @@ func submit( workspace.Assignment.Title, ) fmt.Fprintf( - output, + human, text(ctx, "Commit: %s\nФайлы: %d\nСжатый размер: %s\n", "Commit: %s\nFiles: %d\nCompressed size: %s\n"), repository.CommitSHA, len(repository.Files), @@ -1540,36 +1560,36 @@ func submit( if !explicit { enabled, err = projectAutoChecks(ctx, repository.Root) if err != nil { - return err + return reportJSONError(*jsonOutput, output, err) } } if enabled { - if err := runSubmissionChecks(ctx, repository, output, errorOutput); err != nil { - return err + if err := runSubmissionChecks(ctx, repository, human, errorOutput); err != nil { + return reportJSONError(*jsonOutput, output, err) } } else { - fmt.Fprintln(output, text(ctx, "Локальные проверки не запускались. Решение проверит сервер. Для отдельного локального запуска: softpractice check.", "Local checks were not run. The server will check your solution. To run them separately: softpractice check.")) + fmt.Fprintln(human, text(ctx, "Локальные проверки не запускались. Решение проверит сервер. Для отдельного локального запуска: softpractice check.", "Local checks were not run. The server will check your solution. To run them separately: softpractice check.")) } if !*yes { - confirmed, err := confirmSubmission(ctx, input, output) + confirmed, err := confirmSubmission(ctx, input, human) if err != nil { - return err + return reportJSONError(*jsonOutput, output, err) } if !confirmed { - fmt.Fprintln(output, text(ctx, "Отправка отменена.", "Submission cancelled.")) + fmt.Fprintln(human, text(ctx, "Отправка отменена.", "Submission cancelled.")) return nil } } receipt, err := useCases.SendSubmission(ctx, prepared) if err != nil { - return err + return reportJSONError(*jsonOutput, output, err) } replay := "" if receipt.Replayed { - replay = " (already submitted)" + replay = text(ctx, " (уже была отправлена)", " (already submitted)") } fmt.Fprintf( - output, + human, text(ctx, "Отправка: %s%s\nСостояние проверки: %s\n", "Submission: %s%s\nEvaluation state: %s\n"), receipt.SubmissionID, replay, @@ -1579,13 +1599,40 @@ func submit( // The submission is accepted and evaluated as usual. The notice only // says a newer version exists; updating stays the learner's choice. fmt.Fprintf( - output, + human, text(ctx, "\nУрок обновлён: %s v%d → v%d. Решение принято и проверяется как обычно. Чтобы перейти на новую версию, выполните `softpractice update`.\n", "\nLesson updated: %s v%d → v%d. Your submission is accepted and evaluated as usual. To move to the new version, run `softpractice update`.\n"), update.AssignmentID, update.SubmittedVersion, update.CurrentVersion, ) } - return nil + if !*wait { + if *jsonOutput { + return writeMachineJSON(output, machineSubmissionReceipt{ + Kind: "softpractice.submission", SubmissionID: receipt.SubmissionID, RevisionID: receipt.RevisionID, + EvaluationJobID: receipt.EvaluationJobID, JobState: receipt.JobState, Replayed: receipt.Replayed, + SubmittedAt: receipt.SubmittedAt, ResultURL: webResultURL(ctx, receipt.SubmissionID), + LessonUpdate: receipt.LessonUpdate, + }) + } + fmt.Fprintln(human, text(ctx, "Результат: `softpractice result --wait`", "Result: `softpractice result --wait`")) + return nil + } + fmt.Fprintln(human, text(ctx, "Ждём результат проверки…\n", "Waiting for the evaluation…\n")) + return reportJSONError(*jsonOutput, output, printEvaluation(ctx, useCases, receipt.SubmissionID, + evaluationWait{Wait: true, Timeout: *timeout}, false, *jsonOutput, output)) +} + +// machineSubmissionReceipt is what `submit --json` prints without --wait. +type machineSubmissionReceipt struct { + Kind string `json:"kind"` + SubmissionID string `json:"submission_id"` + RevisionID string `json:"revision_id"` + EvaluationJobID string `json:"evaluation_job_id"` + JobState string `json:"job_state"` + Replayed bool `json:"replayed"` + SubmittedAt time.Time `json:"submitted_at"` + ResultURL string `json:"result_url"` + LessonUpdate *lessonUpdateNotice `json:"lesson_update,omitempty"` } // submittedLessonVersion is the lesson version this folder's tree was made @@ -1649,11 +1696,18 @@ func openCurrent( "web app base URL", ) noBrowser := flags.Bool("no-browser", false, "print without opening a browser") - if err := flags.Parse(args); err != nil { + usageErr := usage(ctx, + "использование: softpractice open [task|material|result] [--web URL] [--no-browser]", + "usage: softpractice open [task|material|result] [--web URL] [--no-browser]") + page := "" + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + page, args = args[0], args[1:] + } + if err := parseFlags(flags, args); err != nil { return err } - if flags.NArg() != 0 { - return errors.New(text(ctx, "использование: softpractice open [--web URL] [--no-browser]", "usage: softpractice open [--web URL] [--no-browser]")) + if flags.NArg() != 0 || (page != "" && page != "task" && page != "material" && page != "result") { + return usageErr } if err := validateWebURL(*webURL); err != nil { return err @@ -1662,10 +1716,29 @@ func openCurrent( if err != nil { return err } - target := strings.TrimRight(*webURL, "/") + "/workspaces/" + link.WorkspaceID - if workspace.LatestSubmission != nil { - target = strings.TrimRight(*webURL, "/") + - "/submissions/" + workspace.LatestSubmission.ID + "/result" + settings := settingsFromContext(ctx) + settings.WebURL = *webURL + ctx = withSettings(ctx, settings) + version := submittedLessonVersion(link, workspace) + if page == "" { + // Without a page: the latest result, or the task before the first + // submission of this lesson. + page = "task" + if workspace.LatestSubmission != nil { + page = "result" + } + } + var target string + switch page { + case "task": + target = webLessonURL(ctx, workspace.Assignment.ID, version, "") + case "material": + target = webLessonURL(ctx, workspace.Assignment.ID, version, "/material") + case "result": + if workspace.LatestSubmission == nil { + return fmt.Errorf(text(ctx, "по уроку %s отправок пока нет", "lesson %s has no submissions yet"), workspace.Assignment.ID) + } + target = webResultURL(ctx, workspace.LatestSubmission.ID) } fmt.Fprintln(output, target) if *noBrowser { diff --git a/cmd/softpractice/main_test.go b/cmd/softpractice/main_test.go index 619a493..f441fc5 100644 --- a/cmd/softpractice/main_test.go +++ b/cmd/softpractice/main_test.go @@ -322,7 +322,7 @@ func TestLinkedStatusSubmitAndOpenCommandFlow(t *testing.T) { } var statusJSON bytes.Buffer if err := status( - context.Background(), client, repositoryRoot, []string{"--format", "json"}, + context.Background(), client, repositoryRoot, []string{"--json"}, &statusJSON, &statusErrors, ); err != nil { t.Fatal(err) @@ -338,24 +338,24 @@ func TestLinkedStatusSubmitAndOpenCommandFlow(t *testing.T) { } var submissionJSON bytes.Buffer - if err := submissionCommand( - context.Background(), client, repositoryRoot, - []string{"show", "--id", submissionID, "--format", "json"}, + if err := resultCommand( + context.Background(), newLearnerUseCases(client, repositoryRoot), + []string{"--id", submissionID, "--json"}, &submissionJSON, &statusErrors, ); err != nil { t.Fatal(err) } - var submissionPayload machineSubmission + var submissionPayload machineEvaluationResult if err := decodeTestJSON(submissionJSON.Bytes(), &submissionPayload); err != nil { t.Fatal(err) } - if !submissionPayload.Terminal || submissionPayload.Evaluation == nil || - submissionPayload.Evaluation.Status != "revise" || - submissionPayload.Evaluation.DeterministicStatus != "passed" || - submissionPayload.Evaluation.Review == nil || - submissionPayload.Evaluation.Review.Verdict == nil || - *submissionPayload.Evaluation.Review.Verdict != "revise" { - t.Fatalf("machine submission = %+v", submissionPayload) + if submissionPayload.Outcome != evaluationReady || submissionPayload.Result == nil || + submissionPayload.Result.Status != "revise" || + submissionPayload.Result.Deterministic.Status != "passed" || + submissionPayload.Result.Review == nil || + submissionPayload.Result.Review.Verdict == nil || + *submissionPayload.Result.Review.Verdict != "revise" { + t.Fatalf("machine result = %+v", submissionPayload) } for range 2 { @@ -381,6 +381,35 @@ func TestLinkedStatusSubmitAndOpenCommandFlow(t *testing.T) { t.Fatalf("idempotency keys = %v", idempotencyKeys) } + // submit --wait --json prints only the evaluation on stdout; what is meant + // for a person goes to stderr. + var waitOutput, waitErrors bytes.Buffer + if err := submit( + context.Background(), client, repositoryRoot, + []string{"--yes", "--wait", "--json"}, strings.NewReader(""), &waitOutput, &waitErrors, + ); err != nil { + t.Fatal(err) + } + var waited machineEvaluationResult + if err := decodeTestJSON(waitOutput.Bytes(), &waited); err != nil { + t.Fatalf("%v\n%s", err, waitOutput.String()) + } + if waited.Outcome != evaluationReady || waited.SubmissionID != submissionID || waited.Result.Status != "revise" || + !strings.Contains(waitErrors.String(), "Submission: "+submissionID) { + t.Fatalf("submit --wait = %+v\nstderr: %s", waited, waitErrors.String()) + } + + var taskOutput bytes.Buffer + if err := openCurrent( + context.Background(), client, repositoryRoot, + []string{"task", "--no-browser", "--web", "https://app.example"}, &taskOutput, &bytes.Buffer{}, + ); err != nil { + t.Fatal(err) + } + if strings.TrimSpace(taskOutput.String()) != "https://app.example/assignments/pa-foundation-05?version=1" { + t.Fatalf("open task output = %q", taskOutput.String()) + } + var openOutput, openErrors bytes.Buffer if err := openCurrent( context.Background(), @@ -440,7 +469,7 @@ func TestSubmissionDownloadCommandSavesOwnedRevisionArchive(t *testing.T) { } } -func TestSubmissionShowFallsBackToAcceptedPredecessor(t *testing.T) { +func TestResultFallsBackToAcceptedPredecessor(t *testing.T) { workspaceID := uuid.NewString() baseRevisionID := uuid.NewString() acceptedSubmissionID := uuid.NewString() @@ -504,18 +533,18 @@ func TestSubmissionShowFallsBackToAcceptedPredecessor(t *testing.T) { repositoryRoot := createLinkedGitRepository(t, workspaceID) var output, errorOutput bytes.Buffer - if err := submissionCommand( - context.Background(), client, repositoryRoot, - []string{"show", "--format", "json"}, &output, &errorOutput, + if err := resultCommand( + context.Background(), newLearnerUseCases(client, repositoryRoot), + []string{"--json"}, &output, &errorOutput, ); err != nil { t.Fatal(err) } - var payload machineSubmission + var payload machineEvaluationResult if err := decodeTestJSON(output.Bytes(), &payload); err != nil { t.Fatal(err) } - if payload.SubmissionID != acceptedSubmissionID || payload.Evaluation == nil || - payload.Evaluation.Status != "accepted" { + if payload.SubmissionID != acceptedSubmissionID || payload.Result == nil || + payload.Result.Status != "accepted" { t.Fatalf("machine submission = %+v", payload) } if !strings.Contains(errorOutput.String(), "pa-foundation-02 has no submission yet") || diff --git a/cmd/softpractice/output.go b/cmd/softpractice/output.go new file mode 100644 index 0000000..0fdf9b1 --- /dev/null +++ b/cmd/softpractice/output.go @@ -0,0 +1,126 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net/http" + + "github.com/arthur-s/softpractice-cli/internal/learnercli" +) + +// exitStatus ends the process with its code after the command has already +// written its result, so main prints nothing more. +type exitStatus int + +func (e exitStatus) Error() string { return fmt.Sprintf("exit status %d", int(e)) } + +// Exit codes shared by every command. 0 is success and 1 is any other error. +const ( + // exitUsage: the arguments are invalid. main uses it for usageError. + exitUsage exitStatus = 2 + // exitNotReady: the evaluation is still queued or running. + exitNotReady exitStatus = 3 + // exitSuperseded: the evaluation was superseded and has no result. + exitSuperseded exitStatus = 4 +) + +// usageError is an invalid command line. main prints it and exits with +// exitUsage. +type usageError struct{ message string } + +func (e usageError) Error() string { return e.message } + +func usage(ctx context.Context, russian, english string) error { + return usageError{message: text(ctx, russian, english)} +} + +// parseFlags parses args and reports a malformed flag as a usageError. The +// flag package has already printed the flag's own help by then. +func parseFlags(flags *flag.FlagSet, args []string) error { + if err := flags.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return err + } + return usageError{message: err.Error()} + } + return nil +} + +func jsonFlag(flags *flag.FlagSet) *bool { + return flags.Bool("json", false, "print JSON instead of text") +} + +func writeMachineJSON(output io.Writer, payload any) error { + encoder := json.NewEncoder(output) + encoder.SetEscapeHTML(false) + encoder.SetIndent("", " ") + return encoder.Encode(payload) +} + +// machineError is written to standard output in JSON mode, so a caller that +// reads only stdout still learns why the command failed. The same message +// also goes to standard error. +type machineError struct { + Kind string `json:"kind"` + Error struct { + Code string `json:"code"` + Message string `json:"message"` + HTTPStatus int `json:"http_status,omitempty"` + APICode string `json:"api_code,omitempty"` + SupportID string `json:"support_id,omitempty"` + } `json:"error"` +} + +// errorCode classifies an error for machine output. login_required and +// project_not_linked tell the caller that a person has to act: sign in with +// `softpractice login`, or work inside the linked project. +func errorCode(err error) string { + var statusError *learnercli.HTTPError + var usageErr usageError + switch { + case errors.As(err, &usageErr): + return "usage" + case errors.Is(err, learnercli.ErrLoginRequired): + return "login_required" + case errors.As(err, &statusError) && statusError.Status == http.StatusUnauthorized: + return "login_required" + case errors.Is(err, learnercli.ErrProjectNotLinked): + return "project_not_linked" + case errors.Is(err, errNotGitRepository): + return "not_git_repository" + case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + return "cancelled" + case errors.As(err, &statusError): + return "api_error" + default: + return "error" + } +} + +func writeMachineError(output io.Writer, err error) error { + payload := machineError{Kind: "softpractice.error"} + payload.Error.Code = errorCode(err) + payload.Error.Message = err.Error() + var statusError *learnercli.HTTPError + if errors.As(err, &statusError) { + payload.Error.HTTPStatus = statusError.Status + payload.Error.APICode = statusError.Code + payload.Error.SupportID = statusError.SupportID + } + return writeMachineJSON(output, payload) +} + +// reportJSONError writes err as a softpractice.error document when the +// command runs with --json. An exit status is a result, not an error, and a +// request for help is neither. +func reportJSONError(jsonOutput bool, output io.Writer, err error) error { + var status exitStatus + if jsonOutput && err != nil && !errors.As(err, &status) && !errors.Is(err, flag.ErrHelp) { + _ = writeMachineError(output, err) + } + return err +} diff --git a/cmd/softpractice/project_checks.go b/cmd/softpractice/project_checks.go index 3c355df..b4db130 100644 --- a/cmd/softpractice/project_checks.go +++ b/cmd/softpractice/project_checks.go @@ -46,7 +46,7 @@ func setProjectAutoChecks(ctx context.Context, value string, output io.Writer) e func checkProject(ctx context.Context, startDirectory string, args []string, output, errorOutput io.Writer) error { if len(args) != 0 { - return errors.New(text(ctx, "использование: softpractice check", "usage: softpractice check")) + return usage(ctx, "использование: softpractice check", "usage: softpractice check") } repository, err := inspectGitRepository(ctx, startDirectory, false) if err != nil { diff --git a/cmd/softpractice/ui.go b/cmd/softpractice/ui.go index 556213e..515046e 100644 --- a/cmd/softpractice/ui.go +++ b/cmd/softpractice/ui.go @@ -154,11 +154,90 @@ func text(ctx context.Context, russian, english string) string { func printHelp(ctx context.Context, output io.Writer) { fmt.Fprintln(output, text(ctx, - "Softpractice — тренажёр инженерной практики\n\nИспользование:\n softpractice [--api URL] [--lang ru|en] <команда>\n\nНачните здесь:\n login Войти через браузер\n logout Завершить CLI-сессию\n starter Скачать проект первого урока\n status Показать текущий урок и состояние проекта\n\nРабота в проекте:\n assignment show Показать текст текущего задания\n check Проверить текущие изменения\n submit Отправить зафиксированный Git-коммит\n submission show Показать состояние и результат отправки\n submission download Скачать ZIP сохранённой ревизии\n evaluation Показать результат проверки, с --wait дождаться его\n update Применить переход к следующему доступному уроку\n open Открыть workspace или результат в браузере\n\nНастройки:\n set api-url URL Сохранить адрес API\n set web-url URL Сохранить адрес web-приложения\n set lang ru|en Сохранить язык CLI\n set auto-checks true|false Автозапуск проверок при отправке\n config Показать действующие настройки\n\nВерсия:\n version Показать версию CLI\n\nПодробнее:\n softpractice help <команда>", - "Softpractice — engineering practice simulator\n\nUsage:\n softpractice [--api URL] [--lang ru|en] \n\nGet started:\n login Sign in in a browser\n logout Revoke the CLI session\n starter Download the first lesson project\n status Show the current lesson and project state\n\nWork in a project:\n assignment show Show the current assignment text\n check Check the current changes\n submit Submit the committed Git revision\n submission show Show submission state and result\n submission download Download a saved revision ZIP\n evaluation Show the evaluation result; --wait waits for it\n update Apply the next available lesson transition\n open Open the workspace or result in a browser\n\nSettings:\n set api-url URL Save the API address\n set web-url URL Save the web app address\n set lang ru|en Save the CLI language\n set auto-checks true|false Run checks automatically on submit\n config Show effective settings\n\nVersion:\n version Show the CLI version\n\nMore help:\n softpractice help ")) - fmt.Fprintln(output, text(ctx, - "\nВосстановление:\n project restore Восстановить продолжимый связанный проект", - "\nRecovery:\n project restore Restore a resumable linked project")) + `Softpractice — тренажёр инженерной практики + +Использование: + softpractice [--api URL] [--lang ru|en] <команда> + +Начните здесь: + login Войти через браузер + starter Скачать проект первого урока + status Показать состояние проекта и следующий шаг + +Урок: + task Показать задание + material Показать теоретический материал + hint Показать открытые подсказки + +Решение: + check Проверить текущие изменения + submit Отправить зафиксированный Git-коммит + result Показать результат проверки + submissions Показать отправки урока + update Применить переход к следующему уроку + open Открыть задание, материал или результат в браузере + +Настройки: + set api-url URL Сохранить адрес API + set web-url URL Сохранить адрес web-приложения + set lang ru|en Сохранить язык CLI + set auto-checks true|false Автозапуск проверок при отправке + config Показать действующие настройки + +Прочее: + logout Завершить CLI-сессию + project restore Восстановить продолжимый связанный проект + +Версия: + version Показать версию CLI + +Команды status, task, material, hint, result, submissions и submit принимают --json. +Коды выхода: 0 — готово, 1 — ошибка, 2 — неверные аргументы, 3 — проверка ещё не готова, 4 — проверка заменена. + +Подробнее: + softpractice help <команда>`, + `Softpractice — engineering practice simulator + +Usage: + softpractice [--api URL] [--lang ru|en] + +Get started: + login Sign in in a browser + starter Download the first lesson project + status Show the project state and the next step + +Lesson: + task Show the assignment + material Show the theory material + hint Show opened hints + +Solution: + check Check the current changes + submit Submit the committed Git revision + result Show the evaluation result + submissions Show the lesson's submissions + update Apply the next lesson transition + open Open the task, material, or result in a browser + +Settings: + set api-url URL Save the API address + set web-url URL Save the web app address + set lang ru|en Save the CLI language + set auto-checks true|false Run checks automatically on submit + config Show effective settings + +Other: + logout Revoke the CLI session + project restore Restore a resumable linked project + +Version: + version Show the CLI version + +status, task, material, hint, result, submissions, and submit accept --json. +Exit codes: 0 done, 1 error, 2 invalid arguments, 3 evaluation not ready yet, 4 evaluation superseded. + +More help: + softpractice help `)) } func printCommandHelp(ctx context.Context, output io.Writer, command string) error { @@ -173,21 +252,25 @@ func printCommandHelp(ctx context.Context, output io.Writer, command string) err case "project": russian, english = "Использование: softpractice project restore [--practicum ID] [--directory PATH]\n\nВосстанавливает последнюю отправленную ревизию в новый связанный Git-проект. Если текущий урок ещё не имеет отправок, восстанавливает принятую предыдущую ревизию и применяет авторизованный update. Без истории отправок использует starter текущего урока.", "Usage: softpractice project restore [--practicum ID] [--directory PATH]\n\nRestores the latest submitted revision into a new linked Git project. If the current lesson has no submissions, it restores the accepted predecessor and applies the authorized update. With no submission history it uses the current lesson starter." case "status": - russian, english = "Использование: softpractice status [--format text|json]\n\nПоказывает состояние текущего связанного Git-проекта.", "Usage: softpractice status [--format text|json]\n\nShows the state of the current linked Git project." - case "submission": - russian, english = "Использование: softpractice submission show [--id ID] [--format text|json]\n softpractice submission download --id ID [--output PATH]\n\nshow показывает безопасный статус и результат своей отправки. download сохраняет ZIP точной отправленной ревизии, как кнопка «Скачать проект» на странице результата.", "Usage: softpractice submission show [--id ID] [--format text|json]\n softpractice submission download --id ID [--output PATH]\n\nshow displays the safe status and result of your submission. download saves the exact submitted revision ZIP, equivalent to the result-page Download project button." - case "evaluation": - russian, english = "Использование: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--directions] [--format text|json]\n\nПоказывает результат проверки последней отправки или отправки --id в формате Markdown. С --wait опрашивает сервер с интервалом, который он назначает, пока проверка не завершится или не истечёт --timeout (по умолчанию 10m). Рекомендации рецензента и следующие шаги выводятся только с --directions. Вопросы рецензента — на странице результата.\n\nКоды выхода: 0 — результат готов, 1 — ошибка, 3 — проверка ещё не готова, 4 — проверка заменена и результата не будет.", "Usage: softpractice evaluation [--id ID] [--wait] [--timeout DURATION] [--directions] [--format text|json]\n\nShows the evaluation result of the latest submission or of --id as Markdown. With --wait it polls at the interval the server sets until the evaluation finishes or --timeout (default 10m) passes. The reviewer's recommendations and next steps are printed only with --directions. The reviewer's questions are on the result page.\n\nExit codes: 0 result ready, 1 error, 3 not ready yet, 4 superseded, no result." - case "assignment": - russian, english = "Использование: softpractice assignment show [--format text|json]\n\nПоказывает текст задания, над которым работает этот проект, и связанный с ним теоретический материал.", "Usage: softpractice assignment show [--format text|json]\n\nShows the assignment this project is working on and its linked theory material." + russian, english = "Использование: softpractice status [--json]\n\nПоказывает состояние связанного проекта, последнюю отправку и следующий шаг. В JSON следующие шаги — в next_actions.", "Usage: softpractice status [--json]\n\nShows the linked project, its latest submission, and the next step. In JSON the next steps are in next_actions." + case "submissions": + russian, english = "Использование: softpractice submissions [--json]\n softpractice submissions download --id ID [--output PATH]\n\nПоказывает последние отправки текущего урока с результатами. download сохраняет ZIP точной отправленной ревизии, как кнопка «Скачать проект» на странице результата.", "Usage: softpractice submissions [--json]\n softpractice submissions download --id ID [--output PATH]\n\nShows the recent submissions of the current lesson with their results. download saves the exact submitted revision ZIP, equivalent to the result-page Download project button." + case "result": + russian, english = "Использование: softpractice result [--id ID] [--wait] [--timeout DURATION] [--directions] [--json]\n\nПоказывает результат проверки последней отправки или отправки --id в формате Markdown. С --wait ждёт завершения проверки, опрашивая сервер с назначенным им интервалом, не дольше --timeout (по умолчанию 10m). Рекомендации рецензента и следующие шаги выводятся только с --directions. На вопросы рецензента отвечают на странице результата.\n\nКоды выхода: 0 — результат готов, 1 — ошибка, 3 — проверка ещё не готова, 4 — проверка заменена и результата не будет.", "Usage: softpractice result [--id ID] [--wait] [--timeout DURATION] [--directions] [--json]\n\nShows the evaluation result of the latest submission or of --id as Markdown. With --wait it waits for the evaluation, polling at the interval the server sets, for at most --timeout (default 10m). The reviewer's recommendations and next steps are printed only with --directions. The reviewer's questions are answered on the result page.\n\nExit codes: 0 result ready, 1 error, 3 not ready yet, 4 superseded, no result." + case "task": + russian, english = "Использование: softpractice task [--json]\n\nПоказывает задание той версии урока, над которой работает этот проект.", "Usage: softpractice task [--json]\n\nShows the assignment of the lesson version this project is working on." + case "material": + russian, english = "Использование: softpractice material [--lesson ID] [--json]\n\nПоказывает теоретический материал урока этого проекта или урока --lesson.", "Usage: softpractice material [--lesson ID] [--json]\n\nShows the theory material of this project's lesson or of --lesson." + case "hint": + russian, english = "Использование: softpractice hint [--json]\n\nПоказывает подсказки урока, которые вы уже открыли. Следующая подсказка открывается на странице задания.", "Usage: softpractice hint [--json]\n\nShows the lesson hints you have already opened. The next hint is opened on the assignment page." case "submit": - russian, english = "Использование: softpractice submit [--yes] [--checks=true|false]\n\nОтправляет чистый Git commit текущего урока на проверку.", "Usage: softpractice submit [--yes] [--checks=true|false]\n\nSubmits the clean Git commit for the current lesson." + russian, english = "Использование: softpractice submit [--yes] [--checks=true|false] [--wait] [--timeout DURATION] [--json]\n\nОтправляет чистый Git commit текущего урока на проверку. С --wait затем ждёт результат, как softpractice result --wait.", "Usage: softpractice submit [--yes] [--checks=true|false] [--wait] [--timeout DURATION] [--json]\n\nSubmits the clean Git commit for the current lesson. With --wait it then waits for the result, like softpractice result --wait." case "check": russian, english = "Использование: softpractice check\n\nЗапускает публичные проверки для текущего рабочего дерева, включая незакоммиченные изменения. Ничего не отправляет на сервер.", "Usage: softpractice check\n\nRuns public checks against the current working tree, including uncommitted changes. Does not submit anything to the server." case "update": russian, english = "Использование: softpractice update\n\nПосле принятия решения применяет в этом же проекте переход к следующему уроку: добавляет, заменяет или удаляет только явно объявленные файлы.\n\nПереход применяется к принятому решению прошлого урока. Если текущий коммит — другой, команда ничего не меняет и показывает оба коммита и способ продолжить.", "Usage: softpractice update\n\nAfter acceptance, applies the next-lesson transition in the same project, adding, replacing, or removing only explicitly declared files.\n\nThe transition applies to the accepted solution of the previous lesson. When the current commit is a different one, the command changes nothing and shows both commits and how to continue." case "open": - russian, english = "Использование: softpractice open [--web URL] [--no-browser]\n\nОткрывает workspace или последний результат в браузере.", "Usage: softpractice open [--web URL] [--no-browser]\n\nOpens the workspace or the latest result in a browser." + russian, english = "Использование: softpractice open [task|material|result] [--web URL] [--no-browser]\n\nОткрывает в браузере задание, материал или последний результат. Без аргумента — результат, а до первой отправки — задание.", "Usage: softpractice open [task|material|result] [--web URL] [--no-browser]\n\nOpens the task, the material, or the latest result in a browser. Without an argument: the result, or the task before the first submission." case "set": russian, english = "Использование: softpractice set api-url|web-url|lang|auto-checks VALUE\n\nСохраняет пользовательскую настройку CLI. auto-checks true|false действует только в текущем проекте.", "Usage: softpractice set api-url|web-url|lang|auto-checks VALUE\n\nSaves a user-level CLI setting. auto-checks true|false applies only to the current project." case "config": @@ -195,7 +278,7 @@ func printCommandHelp(ctx context.Context, output io.Writer, command string) err case "version": russian, english = "Использование: softpractice version, или softpractice --version\n\nПоказывает версию CLI. Сборка из исходников без версии отправку решения не пройдёт: сервер принимает только семантическую версию.", "Usage: softpractice version, or softpractice --version\n\nShows the CLI version. A source build without one cannot submit: the server accepts only a semantic version." default: - return fmt.Errorf(text(ctx, "неизвестная команда %q", "unknown command %q"), command) + return usageError{message: fmt.Sprintf(text(ctx, "неизвестная команда %q", "unknown command %q"), command)} } fmt.Fprintln(output, text(ctx, russian, english)) return nil diff --git a/cmd/softpractice/usecase.go b/cmd/softpractice/usecase.go index 2779247..d5a5064 100644 --- a/cmd/softpractice/usecase.go +++ b/cmd/softpractice/usecase.go @@ -47,17 +47,29 @@ func sleepContext(ctx context.Context, delay time.Duration) error { } // statusSnapshot is the linked project, its workspace, and the signed-in -// learner. LatestEvaluationStatus is set only when it was requested and the -// latest submission already has a terminal evaluation. +// learner. Latest is set once the latest submission has a terminal +// evaluation, and only when reading it succeeded. type statusSnapshot struct { - Repository gitRepository - Link learnercli.ProjectLink - Workspace learnercli.WorkspaceStatus - User currentUser - LatestEvaluationStatus string + Repository gitRepository + Link learnercli.ProjectLink + Workspace learnercli.WorkspaceStatus + User currentUser + Latest *latestResult +} + +// latestResult is what the next step depends on in a terminal evaluation of +// the latest submission. +type latestResult struct { + Status string + // QuestionsAnswerable is true when the reviewer's questions can be + // answered on the result page: an authoritative uncertain review. + QuestionsAnswerable bool + // CommitSHA is the commit the submission was made from; empty when the + // API did not return it. + CommitSHA string } -func (u learnerUseCases) Status(ctx context.Context, withEvaluation bool) (statusSnapshot, error) { +func (u learnerUseCases) Status(ctx context.Context) (statusSnapshot, error) { repository, link, workspace, err := linkedWorkspace(ctx, u.client, u.startDirectory, false) if err != nil { return statusSnapshot{}, err @@ -67,27 +79,63 @@ func (u learnerUseCases) Status(ctx context.Context, withEvaluation bool) (statu return statusSnapshot{}, err } snapshot := statusSnapshot{Repository: repository, Link: link, Workspace: workspace, User: user} - if withEvaluation && workspace.LatestSubmission != nil { - switch workspace.LatestSubmission.JobState { - case "completed", "failed": - // The status stays available when only this optional field fails. - result, err := u.Evaluation(ctx, workspace.LatestSubmission.ID, evaluationWait{}) - if err == nil && result.Outcome == evaluationReady && result.Submission.Evaluation != nil { - snapshot.LatestEvaluationStatus = result.Submission.Evaluation.Status + if latest := workspace.LatestSubmission; latest != nil && (latest.JobState == "completed" || latest.JobState == "failed") { + // The status stays available when only these optional reads fail. + if result, err := u.Evaluation(ctx, latest.ID, evaluationWait{}); err == nil && result.Outcome == evaluationReady { + if built, err := buildMachineResult(result.RawEvaluation, defaultReviewFeedback); err == nil { + snapshot.Latest = &latestResult{ + Status: built.Status, + QuestionsAnswerable: built.Review != nil && built.Review.QuestionsAnswerable, + } + snapshot.Latest.CommitSHA, _ = u.SubmittedCommit(ctx, latest.ID) } } } return snapshot, nil } -// assignmentSnapshot is the requirement the linked folder is working on: the -// lesson version its tree was made for, and the theory material linked to it -// when the lesson has one. -type assignmentSnapshot struct { - Link learnercli.ProjectLink - Workspace learnercli.WorkspaceStatus - Assignment assignmentDetail - Material *theoryMaterial +// SubmittedCommit returns the Git commit a submission was made from. +func (u learnerUseCases) SubmittedCommit(ctx context.Context, submissionID string) (string, error) { + if err := validateSubmissionID(submissionID); err != nil { + return "", err + } + var submission struct { + ID string `json:"id"` + Revision struct { + CommitSHA string `json:"commit_sha"` + } `json:"revision"` + } + if err := u.client.AuthorizedJSON(ctx, "GET", "/v1/submissions/"+submissionID, nil, &submission); err != nil { + return "", err + } + if submission.ID != submissionID { + return "", errors.New("submission response does not match the requested submission") + } + return submission.Revision.CommitSHA, nil +} + +// lessonContext names the lesson the linked folder is working on: the +// current lesson, at the version its tree was made for. +type lessonContext struct { + Link learnercli.ProjectLink + Workspace learnercli.WorkspaceStatus + AssignmentID string + Version int +} + +func (u learnerUseCases) Lesson(ctx context.Context) (lessonContext, error) { + _, link, workspace, err := linkedWorkspace(ctx, u.client, u.startDirectory, false) + if err != nil { + return lessonContext{}, err + } + return lessonContext{ + Link: link, Workspace: workspace, + AssignmentID: workspace.Assignment.ID, Version: submittedLessonVersion(link, workspace), + }, nil +} + +func (l lessonContext) query() string { + return "?version=" + strconv.Itoa(l.Version) } type assignmentDetail struct { @@ -102,35 +150,65 @@ type assignmentDetail struct { ContentSHA256 string `json:"content_sha256"` } -type theoryMaterial struct { - ID string `json:"id"` - Title string `json:"title"` - EstimatedMinutes int `json:"estimated_minutes"` - Markdown string `json:"markdown"` - Diagram json.RawMessage `json:"diagram"` -} - -func (u learnerUseCases) Assignment(ctx context.Context) (assignmentSnapshot, error) { - _, link, workspace, err := linkedWorkspace(ctx, u.client, u.startDirectory, false) +// Task returns the assignment text of the folder's lesson. +func (u learnerUseCases) Task(ctx context.Context) (lessonContext, assignmentDetail, error) { + lesson, err := u.Lesson(ctx) if err != nil { - return assignmentSnapshot{}, err + return lessonContext{}, assignmentDetail{}, err } - id := workspace.Assignment.ID - version := submittedLessonVersion(link, workspace) - query := "?version=" + strconv.Itoa(version) var assignment assignmentDetail if err := u.client.AuthorizedJSON( - ctx, "GET", "/v1/assignments/"+url.PathEscape(id)+query, nil, &assignment, + ctx, "GET", "/v1/assignments/"+url.PathEscape(lesson.AssignmentID)+lesson.query(), nil, &assignment, ); err != nil { - return assignmentSnapshot{}, err + return lessonContext{}, assignmentDetail{}, err } - if assignment.ID != id || assignment.Version != version { - return assignmentSnapshot{}, errors.New("assignment response does not match the requested lesson") + if assignment.ID != lesson.AssignmentID || assignment.Version != lesson.Version { + return lessonContext{}, assignmentDetail{}, errors.New("assignment response does not match the requested lesson") + } + return lesson, assignment, nil +} + +type theoryMaterial struct { + ID string `json:"id"` + Title string `json:"title"` + EstimatedMinutes int `json:"estimated_minutes"` + Markdown string `json:"markdown"` + Diagram theoryDiagram `json:"diagram"` +} + +type theoryDiagram struct { + Title string `json:"title"` + TextAlternative string `json:"text_alternative"` + Nodes []struct { + Title string `json:"title"` + Text string `json:"text"` + } `json:"nodes"` +} + +// materialSnapshot is the theory material of one lesson. Material is nil when +// the lesson has none. Version is zero for a lesson named explicitly: the API +// then serves its current published version. +type materialSnapshot struct { + AssignmentID string + Version int + Material *theoryMaterial +} + +// Material returns the theory material of the folder's lesson, or of +// lessonID when it is set, such as an earlier lesson of the practicum. +func (u learnerUseCases) Material(ctx context.Context, lessonID string) (materialSnapshot, error) { + snapshot := materialSnapshot{AssignmentID: lessonID} + query := "" + if lessonID == "" { + lesson, err := u.Lesson(ctx) + if err != nil { + return materialSnapshot{}, err + } + snapshot.AssignmentID, snapshot.Version, query = lesson.AssignmentID, lesson.Version, lesson.query() } - snapshot := assignmentSnapshot{Link: link, Workspace: workspace, Assignment: assignment} var material theoryMaterial - err = u.client.AuthorizedJSON( - ctx, "GET", "/v1/assignments/"+url.PathEscape(id)+"/material"+query, nil, &material, + err := u.client.AuthorizedJSON( + ctx, "GET", "/v1/assignments/"+url.PathEscape(snapshot.AssignmentID)+"/material"+query, nil, &material, ) var statusError *learnercli.HTTPError switch { @@ -139,11 +217,95 @@ func (u learnerUseCases) Assignment(ctx context.Context) (assignmentSnapshot, er case errors.As(err, &statusError) && statusError.Status == http.StatusNotFound: // The lesson has no linked theory material. default: - return assignmentSnapshot{}, err + return materialSnapshot{}, err + } + return snapshot, nil +} + +type preparedHint struct { + ID string `json:"id"` + Order int `json:"order"` + Markdown string `json:"markdown"` +} + +// hintsSnapshot lists the prepared hints the learner has already opened for +// the folder's lesson. Status is available while a closed hint remains, +// exhausted when all are open, and unavailable when the lesson has none. +// Opening the next hint is a learner action on the assignment page. +type hintsSnapshot struct { + Lesson lessonContext + Status string + Revealed []preparedHint +} + +func (u learnerUseCases) Hints(ctx context.Context) (hintsSnapshot, error) { + lesson, err := u.Lesson(ctx) + if err != nil { + return hintsSnapshot{}, err + } + var state struct { + Status string `json:"status"` + Revealed []preparedHint `json:"revealed"` + } + if err := u.client.AuthorizedJSON( + ctx, "GET", "/v1/assignments/"+url.PathEscape(lesson.AssignmentID)+"/prepared-hints"+lesson.query(), nil, &state, + ); err != nil { + return hintsSnapshot{}, err + } + if state.Revealed == nil { + state.Revealed = []preparedHint{} + } + return hintsSnapshot{Lesson: lesson, Status: state.Status, Revealed: state.Revealed}, nil +} + +type submissionAttempt struct { + SubmissionID string `json:"submission_id"` + LearningAttempt int `json:"learning_attempt"` + JobState string `json:"job_state"` + EvaluationStatus *string `json:"evaluation_status"` + SubmittedAt time.Time `json:"submitted_at"` +} + +// submissionsSnapshot is the recent submission history of the folder's +// lesson, newest first. The API returns at most five entries. +type submissionsSnapshot struct { + Lesson lessonContext + History []submissionAttempt +} + +func (u learnerUseCases) Submissions(ctx context.Context) (submissionsSnapshot, error) { + lesson, err := u.Lesson(ctx) + if err != nil { + return submissionsSnapshot{}, err + } + snapshot := submissionsSnapshot{Lesson: lesson, History: []submissionAttempt{}} + latest := lesson.Workspace.LatestSubmission + if latest == nil { + return snapshot, nil + } + if err := validateSubmissionID(latest.ID); err != nil { + return submissionsSnapshot{}, err + } + var iteration struct { + History []submissionAttempt `json:"history"` + } + if err := u.client.AuthorizedJSON(ctx, "GET", "/v1/submissions/"+latest.ID+"/iteration", nil, &iteration); err != nil { + return submissionsSnapshot{}, err + } + if iteration.History != nil { + snapshot.History = iteration.History } return snapshot, nil } +func validateSubmissionID(submissionID string) error { + parsed, err := uuid.Parse(submissionID) + if err != nil || parsed.String() != submissionID { + return errors.New("submission ID must be a canonical UUID") + } + return nil +} + // preparedSubmission is a linked, clean, submittable commit with its archive // already built. Close removes the archive. type preparedSubmission struct { @@ -279,9 +441,8 @@ func (u learnerUseCases) Evaluation( submissionID string, wait evaluationWait, ) (evaluationResult, error) { - parsedID, err := uuid.Parse(submissionID) - if err != nil || parsedID.String() != submissionID { - return evaluationResult{}, errors.New("submission ID must be a canonical UUID") + if err := validateSubmissionID(submissionID); err != nil { + return evaluationResult{}, err } deadline := u.now().Add(wait.Timeout) path := "/v1/submissions/" + submissionID + "/evaluation" diff --git a/packaging/npm/README.md b/packaging/npm/README.md index 3c6a32a..7a7b774 100644 --- a/packaging/npm/README.md +++ b/packaging/npm/README.md @@ -42,25 +42,27 @@ Download the starter project for your current practicum: softpractice starter ``` -Open the created directory and check the current lesson: +Open the created directory, check the project state, and read the assignment: ```bash cd softpractice status +softpractice task +softpractice material ``` Work on the project and commit your changes with Git. When the solution is -ready, submit the current commit: +ready, submit the current commit and wait for the result: ```bash -softpractice submit +softpractice submit --wait ``` -Check the submission and open its result: +Read the result again at any time, or open it in the browser: ```bash -softpractice submission show -softpractice open +softpractice result +softpractice open result ``` After an accepted solution, apply the transition to the next available lesson: @@ -69,20 +71,24 @@ After an accepted solution, apply the transition to the next available lesson: softpractice update ``` +`softpractice status` always ends with the next step. + ## Common commands | Command | Description | | --- | --- | | `softpractice login` | Sign in through the browser | | `softpractice starter` | Download the first lesson project | -| `softpractice status` | Show the current lesson and local project state | +| `softpractice status` | Show the project state and the next step | +| `softpractice task` | Show the assignment | +| `softpractice material` | Show the theory material; `--lesson ID` for another lesson | +| `softpractice hint` | Show the hints you have opened; the next one is opened on the assignment page | | `softpractice check` | Run public checks against the current working tree | -| `softpractice submit` | Submit the current committed Git revision | -| `softpractice submission show` | Show the latest submission and review result | -| `softpractice evaluation` | Show the evaluation result of the latest submission; `--wait` waits for it | -| `softpractice assignment show` | Show the assignment this project is working on | +| `softpractice submit` | Submit the current committed Git revision; `--wait` waits for the result | +| `softpractice result` | Show the evaluation result; `--wait` waits for it, `--directions` adds the reviewer's recommendations | +| `softpractice submissions` | Show the recent submissions of the lesson | | `softpractice update` | Apply the transition to the next lesson, or move to a newer version of the current lesson (replaces only lesson files, never yours; optional until the lesson ends) | -| `softpractice open` | Open the workspace or latest result in the browser | +| `softpractice open` | Open the task, material, or latest result in the browser | | `softpractice project restore` | Restore a project from the latest usable revision | Run `softpractice help ` for detailed help. @@ -110,20 +116,24 @@ softpractice set auto-checks true run. Unlike `softpractice check`, submission checks require a clean working tree and run in an isolated snapshot of `HEAD`. +## Reviewer questions + +When the reviewer asks questions about your solution, `softpractice result` +and `softpractice status` say so and give the link to the result page, where +you answer them. + ## Machine-readable output -`status`, `submission show`, `evaluation`, and `assignment show` accept -`--format json`. Each document has a `kind`. +`status`, `task`, `material`, `hint`, `result`, `submissions`, and `submit` +accept `--json`. Each document has a `kind`; on an error the command prints a +`softpractice.error` document. ```bash -softpractice evaluation --wait --timeout 10m --format json +softpractice result --wait --timeout 10m --json ``` -`evaluation` exit codes: `0` result ready, `1` error, `3` not ready yet, -`4` evaluation superseded, no result. Its output has the automated check -results and the review without the reviewer's recommendations; -`--directions` adds them. The reviewer's questions are on the result page at -`result_url`. +Exit codes: `0` done, `1` error, `2` invalid arguments, `3` the evaluation is +not ready yet, `4` the evaluation was superseded and will have no result. ## Language From 6feb0e2285b64f035495f41ea1bd010d1f87fb53 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 08:27:03 +0000 Subject: [PATCH 07/14] cli: fix review findings in --json errors, next steps, and open - With --json, every error is a softpractice.error document, including an invalid command line (code usage): the commands now report errors in one deferred place and read --json from the raw arguments. - Declining the submit confirmation is error code declined (exit 1) instead of a silent success with empty stdout. - status next steps: practicum_completed with the completion page for a completed workspace, workspace_inactive for an archived one, retry_evaluation with the result page after a technical failure, resubmit after a superseded evaluation, and stash_changes before apply_update on a dirty tree. - A malformed --id is a usage error (exit 2) for result and submissions download. - open accepts the page after the flags, and open result falls back to the accepted predecessor like result does. - submit --wait accepts --directions. - Drop the unused evaluation projection built by normalizeSubmissionResponse. - Tests for JSON usage errors, declined submit, submit --json receipt, rate limits without an answer, cancellation while waiting, token refresh during polling, and the new next steps. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019Xxe94ZEJnVQm1pzvTR1aK --- README.md | 3 +- README_RU.md | 3 +- cmd/softpractice/learning_commands.go | 37 ++-- cmd/softpractice/learning_commands_test.go | 186 +++++++++++++++++++-- cmd/softpractice/machine.go | 118 ++++++------- cmd/softpractice/main.go | 51 ++++-- cmd/softpractice/main_test.go | 44 +++++ cmd/softpractice/output.go | 35 ++++ cmd/softpractice/ui.go | 2 +- packaging/npm/README.md | 3 +- 10 files changed, 373 insertions(+), 109 deletions(-) diff --git a/README.md b/README.md index 7a7b774..463037e 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,8 @@ you answer them. `status`, `task`, `material`, `hint`, `result`, `submissions`, and `submit` accept `--json`. Each document has a `kind`; on an error the command prints a -`softpractice.error` document. +`softpractice.error` document, also for an invalid command line (code +`usage`) and for a declined `submit` confirmation (code `declined`). ```bash softpractice result --wait --timeout 10m --json diff --git a/README_RU.md b/README_RU.md index f53a75b..17a0e5a 100644 --- a/README_RU.md +++ b/README_RU.md @@ -127,7 +127,8 @@ softpractice set auto-checks true `status`, `task`, `material`, `hint`, `result`, `submissions` и `submit` принимают `--json`. У каждого документа есть поле `kind`; при ошибке команда -выводит документ `softpractice.error`. +выводит документ `softpractice.error`, в том числе при неверных аргументах +(код `usage`) и при отказе в подтверждении `submit` (код `declined`). ```bash softpractice result --wait --timeout 10m --json diff --git a/cmd/softpractice/learning_commands.go b/cmd/softpractice/learning_commands.go index b128393..2b6c64e 100644 --- a/cmd/softpractice/learning_commands.go +++ b/cmd/softpractice/learning_commands.go @@ -65,7 +65,8 @@ func resultCommand( useCases learnerUseCases, args []string, output, errorOutput io.Writer, -) error { +) (err error) { + defer func() { err = reportJSONError(requestsJSON(args), output, err) }() flags := flag.NewFlagSet("result", flag.ContinueOnError) flags.SetOutput(errorOutput) id := flags.String("id", "", "submission ID; defaults to the latest submission") @@ -82,10 +83,13 @@ func resultCommand( "usage: softpractice result [--id ID] [--wait] [--timeout DURATION] [--directions] [--json]") } submissionID := strings.TrimSpace(*id) + if submissionID != "" && validateSubmissionID(submissionID) != nil { + return usage(ctx, "ID отправки должен быть каноническим UUID", "submission ID must be a canonical UUID") + } if submissionID == "" { latest, err := useCases.LatestSubmission(ctx) if err != nil { - return reportJSONError(*jsonOutput, output, err) + return err } if latest.PredecessorAssignmentID != "" { fmt.Fprintf(errorOutput, text(ctx, @@ -95,9 +99,8 @@ func resultCommand( } submissionID = latest.SubmissionID } - return reportJSONError(*jsonOutput, output, - printEvaluation(ctx, useCases, submissionID, evaluationWait{Wait: *wait, Timeout: *timeout}, - *directions, *jsonOutput, output)) + return printEvaluation(ctx, useCases, submissionID, evaluationWait{Wait: *wait, Timeout: *timeout}, + *directions, *jsonOutput, output) } // printEvaluation reads or waits for one evaluation, prints it, and returns @@ -313,6 +316,10 @@ func webLessonURL(ctx context.Context, assignmentID string, version int, suffix return target } +func webPracticumCompletionURL(ctx context.Context, practicumID string) string { + return strings.TrimRight(settingsFromContext(ctx).WebURL, "/") + "/practicums/" + practicumID + "/completion" +} + func webResultURL(ctx context.Context, submissionID string) string { return strings.TrimRight(settingsFromContext(ctx).WebURL, "/") + "/submissions/" + submissionID + "/result" } @@ -327,7 +334,8 @@ type machineTask struct { LessonVersionUpdate *machineLessonVersionUpdate `json:"lesson_version_update,omitempty"` } -func taskCommand(ctx context.Context, useCases learnerUseCases, args []string, output, errorOutput io.Writer) error { +func taskCommand(ctx context.Context, useCases learnerUseCases, args []string, output, errorOutput io.Writer) (err error) { + defer func() { err = reportJSONError(requestsJSON(args), output, err) }() flags := flag.NewFlagSet("task", flag.ContinueOnError) flags.SetOutput(errorOutput) jsonOutput := jsonFlag(flags) @@ -339,7 +347,7 @@ func taskCommand(ctx context.Context, useCases learnerUseCases, args []string, o } lesson, assignment, err := useCases.Task(ctx) if err != nil { - return reportJSONError(*jsonOutput, output, err) + return err } payload := machineTask{ Kind: "softpractice.task", WorkspaceID: lesson.Workspace.Workspace.ID, ProjectID: lesson.Link.ProjectID, @@ -389,7 +397,8 @@ type machineMaterial struct { URL string `json:"url"` } -func materialCommand(ctx context.Context, useCases learnerUseCases, args []string, output, errorOutput io.Writer) error { +func materialCommand(ctx context.Context, useCases learnerUseCases, args []string, output, errorOutput io.Writer) (err error) { + defer func() { err = reportJSONError(requestsJSON(args), output, err) }() flags := flag.NewFlagSet("material", flag.ContinueOnError) flags.SetOutput(errorOutput) lessonID := flags.String("lesson", "", "lesson ID; defaults to the lesson of this project") @@ -403,7 +412,7 @@ func materialCommand(ctx context.Context, useCases learnerUseCases, args []strin } snapshot, err := useCases.Material(ctx, strings.TrimSpace(*lessonID)) if err != nil { - return reportJSONError(*jsonOutput, output, err) + return err } payload := machineMaterial{ Kind: "softpractice.material", AssignmentID: snapshot.AssignmentID, Version: snapshot.Version, @@ -442,7 +451,8 @@ type machineHints struct { URL string `json:"url"` } -func hintCommand(ctx context.Context, useCases learnerUseCases, args []string, output, errorOutput io.Writer) error { +func hintCommand(ctx context.Context, useCases learnerUseCases, args []string, output, errorOutput io.Writer) (err error) { + defer func() { err = reportJSONError(requestsJSON(args), output, err) }() flags := flag.NewFlagSet("hint", flag.ContinueOnError) flags.SetOutput(errorOutput) jsonOutput := jsonFlag(flags) @@ -454,7 +464,7 @@ func hintCommand(ctx context.Context, useCases learnerUseCases, args []string, o } snapshot, err := useCases.Hints(ctx) if err != nil { - return reportJSONError(*jsonOutput, output, err) + return err } payload := machineHints{ Kind: "softpractice.hints", AssignmentID: snapshot.Lesson.AssignmentID, Version: snapshot.Lesson.Version, @@ -498,7 +508,8 @@ func submissionsCommand( useCases learnerUseCases, args []string, output, errorOutput io.Writer, -) error { +) (err error) { + defer func() { err = reportJSONError(requestsJSON(args), output, err) }() if len(args) > 0 && args[0] == "download" { return downloadSubmissionRevision(ctx, useCases.client, args[1:], output, errorOutput) } @@ -515,7 +526,7 @@ func submissionsCommand( } snapshot, err := useCases.Submissions(ctx) if err != nil { - return reportJSONError(*jsonOutput, output, err) + return err } payload := machineSubmissions{ Kind: "softpractice.submissions", AssignmentID: snapshot.Lesson.AssignmentID, diff --git a/cmd/softpractice/learning_commands_test.go b/cmd/softpractice/learning_commands_test.go index f856977..0b0713b 100644 --- a/cmd/softpractice/learning_commands_test.go +++ b/cmd/softpractice/learning_commands_test.go @@ -289,6 +289,81 @@ func TestEvaluationWaitsOutRateLimit(t *testing.T) { } } +func TestEvaluationRateLimitWithoutAnAnswer(t *testing.T) { + rateLimited := func(retryAfter string) func(http.ResponseWriter) { + return func(writer http.ResponseWriter) { + if retryAfter != "" { + writer.Header().Set("Retry-After", retryAfter) + } + writeTestAPIError(writer, http.StatusTooManyRequests, "rate_limited") + } + } + for name, tc := range map[string]struct { + retryAfter string + timeout string + calls int + }{ + // Without Retry-After there is no interval to honour, so it is not + // retried at all. + "no Retry-After": {retryAfter: "", timeout: "1m", calls: 1}, + // A rate limit before the first answer leaves no pending state to + // report when the next allowed poll falls after the timeout. + "past the timeout": {retryAfter: "90", timeout: "1m", calls: 1}, + } { + server, calls := evaluationServer(t, currentSubmissionID, rateLimited(tc.retryAfter)) + clock := &fakeClock{now: time.Unix(0, 0)} + output, _, err := runEvaluation(t, clock.useCases(savedTestClient(t, server.URL), ""), + "--id", currentSubmissionID, "--wait", "--timeout", tc.timeout, "--json") + var statusError *learnercli.HTTPError + if !errors.As(err, &statusError) || statusError.Status != http.StatusTooManyRequests || + *calls != tc.calls || len(clock.sleeps) != 0 || !strings.Contains(output, `"code": "api_error"`) { + t.Fatalf("%s: err = %v, calls = %d, sleeps = %v, output:\n%s", name, err, *calls, clock.sleeps, output) + } + } +} + +func TestEvaluationWaitStopsWhenCancelled(t *testing.T) { + server, calls := evaluationServer(t, currentSubmissionID, respondPending("5")) + useCases := newLearnerUseCases(savedTestClient(t, server.URL), "") + ctx, cancel := context.WithCancel(context.Background()) + useCases.sleep = func(sleepCtx context.Context, delay time.Duration) error { + cancel() + return sleepContext(sleepCtx, delay) + } + _, err := useCases.Evaluation(ctx, currentSubmissionID, evaluationWait{Wait: true, Timeout: time.Hour}) + if !errors.Is(err, context.Canceled) || *calls != 1 || errorCode(err) != "cancelled" { + t.Fatalf("err = %v, calls = %d", err, *calls) + } +} + +func TestEvaluationPollRefreshesAnExpiredAccessToken(t *testing.T) { + refreshes := 0 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + switch { + case request.URL.Path == "/v1/auth/tokens/refresh": + refreshes++ + writeTestJSON(writer, map[string]any{ + "token_type": "Bearer", "access_token": "new-access-token", "expires_in_seconds": 900, + "refresh_token": "new-refresh-token", "refresh_idle_expires_at": time.Now().Add(time.Hour), + "refresh_absolute_expires_at": time.Now().Add(2 * time.Hour), "scopes": []string{"workspace:read"}, + }) + case request.Header.Get("Authorization") != "Bearer new-access-token": + writeTestAPIError(writer, http.StatusUnauthorized, "invalid_credentials") + default: + respondPending("3")(writer) + } + })) + defer server.Close() + clock := &fakeClock{now: time.Unix(0, 0)} + output, _, err := runEvaluation(t, clock.useCases(savedTestClient(t, server.URL), ""), + "--id", currentSubmissionID, "--json") + var payload machineEvaluationResult + if !errors.Is(err, exitNotReady) || refreshes != 1 || decodeTestJSON([]byte(output), &payload) != nil || + payload.Outcome != evaluationPending { + t.Fatalf("err = %v, refreshes = %d, output:\n%s", err, refreshes, output) + } +} + func TestEvaluationSupersededIsASeparateOutcome(t *testing.T) { server, _ := evaluationServer(t, currentSubmissionID, func(writer http.ResponseWriter) { writeTestAPIError(writer, http.StatusConflict, "evaluation_superseded") @@ -539,6 +614,52 @@ func TestInvalidArgumentsAreUsageErrors(t *testing.T) { t.Fatalf("%s: err = %v, want a usage error", name, err) } } + for _, args := range [][]string{{"--id", "not-a-uuid"}, {"--id", strings.ToUpper(currentSubmissionID)}} { + if _, _, err := runEvaluation(t, useCases, args...); !errors.As(err, new(usageError)) { + t.Fatalf("%v: err = %v, want a usage error", args, err) + } + } + + // With --json even a command line that does not parse is reported as + // one softpractice.error document on stdout. + commands := map[string]func(context.Context, learnerUseCases, []string, io.Writer, io.Writer) error{ + "result": resultCommand, "task": taskCommand, "material": materialCommand, + "hint": hintCommand, "submissions": submissionsCommand, + } + for name, command := range commands { + for _, args := range [][]string{{"--json", "--no-such-flag"}, {"--json", "extra"}, {"--no-such-flag", "--json=true"}} { + var output bytes.Buffer + err := command(context.Background(), useCases, args, &output, io.Discard) + var payload machineError + if decodeErr := decodeTestJSON(output.Bytes(), &payload); decodeErr != nil || !errors.As(err, new(usageError)) || + payload.Kind != "softpractice.error" || payload.Error.Code != "usage" { + t.Fatalf("%s %v: err = %v, output:\n%s", name, args, err, output.String()) + } + } + } + client := savedTestClient(t, "http://127.0.0.1:1") + for name, command := range map[string]func(io.Writer) error{ + "status": func(output io.Writer) error { + return status(context.Background(), client, t.TempDir(), []string{"--json", "extra"}, output, io.Discard) + }, + "submit": func(output io.Writer) error { + return submit(context.Background(), client, t.TempDir(), []string{"--json", "--timeout", "-1s"}, + strings.NewReader(""), output, io.Discard) + }, + } { + var output bytes.Buffer + err := command(&output) + if !errors.As(err, new(usageError)) || !strings.Contains(output.String(), `"code": "usage"`) { + t.Fatalf("%s: err = %v, output:\n%s", name, err, output.String()) + } + } + + // Without --json, or with --json=false, stdout stays empty. + var output bytes.Buffer + if err := taskCommand(context.Background(), useCases, []string{"--json=false", "extra"}, &output, io.Discard); err == nil || + output.Len() != 0 { + t.Fatalf("err = %v, output:\n%s", err, output.String()) + } } // jsonKeyPaths lists every object key path in a JSON document, with array @@ -727,7 +848,8 @@ func TestNextActions(t *testing.T) { } snapshot := func(pinned string, pinnedVersion int, clean bool, head string) statusSnapshot { var result statusSnapshot - result.Link = learnercli.ProjectLink{SchemaVersion: 2, AssignmentID: pinned, AssignmentVersion: pinnedVersion} + result.Link = learnercli.ProjectLink{SchemaVersion: 2, ProjectID: "pa-foundation", AssignmentID: pinned, AssignmentVersion: pinnedVersion} + result.Workspace.Workspace.State = "active" result.Workspace.Assignment.ID = "pa-foundation-02" result.Workspace.Assignment.Version = 2 result.Repository = gitRepository{Clean: clean, CommitSHA: head} @@ -755,26 +877,68 @@ func TestNextActions(t *testing.T) { reworked := reviewed reworked.Repository.CommitSHA = "def" olderVersion := snapshot("pa-foundation-02", 1, true, "abc") + dirtyTransition := snapshot("pa-foundation-01", 1, false, "abc") + dirtyReviewed := reviewed + dirtyReviewed.Repository.Clean = false + unreadable := snapshot("pa-foundation-02", 2, true, "abc") + unreadable.Workspace.LatestSubmission = submission("completed") + superseded := snapshot("pa-foundation-02", 2, true, "abc") + superseded.Workspace.LatestSubmission = submission("superseded") + dirtySuperseded := superseded + dirtySuperseded.Repository.Clean = false + technical := snapshot("pa-foundation-02", 2, true, "abc") + technical.Workspace.LatestSubmission = submission("failed") + technical.Latest = &latestResult{Status: "technical_failure", CommitSHA: "abc"} + technicalReworked := technical + technicalReworked.Repository.CommitSHA = "def" + completed := reviewed + completed.Workspace.Workspace.State = "completed" + completed.Workspace.Assignment.State = "accepted" + completed.Latest = &latestResult{Status: "accepted", CommitSHA: "abc"} + completedReworked := completed + completedReworked.Repository = gitRepository{Clean: false, CommitSHA: "def"} + archived := fresh + archived.Workspace.Workspace.State = "archived" for name, tc := range map[string]struct { snapshot statusSnapshot want string }{ - "transition": {transition, "apply_update"}, - "no submission": {fresh, "submit"}, - "dirty tree": {dirty, "commit_changes"}, - "evaluating": {queued, "wait_result"}, - "questions": {questions, "answer_questions,read_result"}, - "reviewed": {reviewed, "read_result"}, - "new commit": {reworked, "submit"}, - "older version": {olderVersion, "submit,update_lesson_version"}, + "transition": {transition, "apply_update"}, + "no submission": {fresh, "submit"}, + "dirty tree": {dirty, "commit_changes"}, + "evaluating": {queued, "wait_result"}, + "questions": {questions, "answer_questions,read_result"}, + "reviewed": {reviewed, "read_result"}, + "new commit": {reworked, "submit"}, + "older version": {olderVersion, "submit,update_lesson_version"}, + "dirty transition": {dirtyTransition, "stash_changes,apply_update"}, + "dirty after review": {dirtyReviewed, "commit_changes"}, + "result unreadable": {unreadable, "read_result"}, + "superseded": {superseded, "resubmit"}, + "superseded, dirty": {dirtySuperseded, "commit_changes"}, + "technical failure": {technical, "retry_evaluation,read_result"}, + "technical failure, rework": {technicalReworked, "submit"}, + "practicum completed": {completed, "practicum_completed"}, + "completed, rework": {completedReworked, "practicum_completed"}, + "archived": {archived, "workspace_inactive"}, } { actions := nextActions(ctx, tc.snapshot) if got := codes(actions); got != tc.want { t.Fatalf("%s: next actions = %s, want %s", name, got, tc.want) } - if name == "questions" && actions[0].URL != "https://softpractice.example/submissions/"+currentSubmissionID+"/result" { - t.Fatalf("questions URL = %q", actions[0].URL) + if (name == "questions" || name == "technical failure") && + actions[0].URL != "https://softpractice.example/submissions/"+currentSubmissionID+"/result" { + t.Fatalf("%s: URL = %q", name, actions[0].URL) + } + if name == "practicum completed" && actions[0].URL != "https://softpractice.example/practicums/"+ + tc.snapshot.Link.ProjectID+"/completion" { + t.Fatalf("completion URL = %q", actions[0].URL) + } + for _, action := range actions { + if nextActionText(ctx, action) == action.Code { + t.Fatalf("%s: action %s has no text", name, action.Code) + } } } } diff --git a/cmd/softpractice/machine.go b/cmd/softpractice/machine.go index 9f3b277..b2a83a1 100644 --- a/cmd/softpractice/machine.go +++ b/cmd/softpractice/machine.go @@ -61,6 +61,14 @@ type nextAction struct { // important first. The list is never empty. func nextActions(ctx context.Context, snapshot statusSnapshot) []nextAction { link, workspace, repository := snapshot.Link, snapshot.Workspace, snapshot.Repository + switch workspace.Workspace.State { + case "active": + case "completed": + // The practicum is finished: nothing is left to submit or update. + return []nextAction{{Code: "practicum_completed", URL: webPracticumCompletionURL(ctx, link.ProjectID)}} + default: + return []nextAction{{Code: "workspace_inactive"}} + } var actions []nextAction submit := nextAction{Code: "submit", Command: "softpractice submit --wait"} if !repository.Clean { @@ -69,11 +77,24 @@ func nextActions(ctx context.Context, snapshot statusSnapshot) []nextAction { latest := workspace.LatestSubmission switch { case pendingTransition(link, workspace): + if !repository.Clean { + // update refuses a dirty tree, and committing the changes would + // turn the accepted tree into a continued project it does not + // overlay. + actions = append(actions, nextAction{Code: "stash_changes", Command: "git stash"}) + } actions = append(actions, nextAction{Code: "apply_update", Command: "softpractice update"}) case latest == nil: actions = append(actions, submit) case latest.JobState == "queued" || latest.JobState == "leased": actions = append(actions, nextAction{Code: "wait_result", Command: "softpractice result --wait"}) + case latest.JobState == "superseded": + // This evaluation will have no result, and submitting the same commit + // again replays this submission: only a new commit is evaluated. + if repository.Clean { + submit.Code = "resubmit" + } + actions = append(actions, submit) default: result := snapshot.Latest readResult := nextAction{Code: "read_result", Command: "softpractice result"} @@ -84,6 +105,11 @@ func nextActions(ctx context.Context, snapshot statusSnapshot) []nextAction { case !repository.Clean || (result != nil && result.CommitSHA != "" && result.CommitSHA != repository.CommitSHA): // Work continued after this submission. actions = append(actions, submit) + case result != nil && result.Status == "technical_failure": + // The same revision can be evaluated again only from the result + // page; submitting the same commit replays this submission. + actions = append(actions, + nextAction{Code: "retry_evaluation", URL: webResultURL(ctx, latest.ID)}, readResult) default: actions = append(actions, readResult) } @@ -96,6 +122,14 @@ func nextActions(ctx context.Context, snapshot statusSnapshot) []nextAction { func nextActionText(ctx context.Context, action nextAction) string { switch action.Code { + case "practicum_completed": + return text(ctx, "практикум пройден: ", "the practicum is completed: ") + action.URL + case "workspace_inactive": + return text(ctx, "workspace не активен и не принимает отправки", + "the workspace is not active and accepts no submissions") + case "stash_changes": + return text(ctx, "уберите незакоммиченные изменения, например `git stash`: `softpractice update` применяет переход только к чистому принятому решению", + "set uncommitted changes aside, for example with `git stash`: `softpractice update` applies the transition only to the clean accepted solution") case "apply_update": return text(ctx, "примените переход к следующему уроку: `softpractice update`", "apply the transition to the next lesson: `softpractice update`") @@ -105,6 +139,9 @@ func nextActionText(ctx context.Context, action nextAction) string { case "submit": return text(ctx, "когда решение готово и закоммичено, отправьте его: `softpractice submit --wait`", "when the solution is committed, submit it: `softpractice submit --wait`") + case "resubmit": + return text(ctx, "проверка последней отправки заменена, результата у неё не будет; сделайте новый commit и отправьте его: `softpractice submit --wait`", + "the evaluation of the latest submission was superseded and will have no result; make a new commit and submit it: `softpractice submit --wait`") case "wait_result": return text(ctx, "дождитесь результата проверки: `softpractice result --wait`", "wait for the evaluation: `softpractice result --wait`") @@ -114,6 +151,9 @@ func nextActionText(ctx context.Context, action nextAction) string { case "answer_questions": return text(ctx, "ответьте на вопросы рецензента на странице результата: ", "answer the reviewer's questions on the result page: ") + action.URL + case "retry_evaluation": + return text(ctx, "проверка не завершилась из-за технической ошибки; повторите её на странице результата: ", + "the evaluation failed for a technical reason; retry it on the result page: ") + action.URL case "update_lesson_version": return text(ctx, "можно перейти на новую версию урока: `softpractice update`", "you can move to the new lesson version: `softpractice update`") @@ -196,40 +236,12 @@ type evaluationAPIResponse struct { UpdatedAt time.Time `json:"updated_at"` } +// evaluationProjection is the part of a terminal evaluation that +// normalizeSubmissionResponse validates; `result` renders the rest. type evaluationProjection struct { SchemaVersion int `json:"schema_version"` ExerciseID string `json:"exercise_id"` Status string `json:"status"` - Deterministic *struct { - Status string `json:"status"` - } `json:"deterministic"` - Review *struct { - Status string `json:"status"` - Mode string `json:"mode"` - Authoritative bool `json:"authoritative"` - Verdict *string `json:"verdict"` - } `json:"review"` - TechnicalError *struct { - ErrorCode string `json:"error_code"` - Stage string `json:"stage"` - SupportID string `json:"support_id"` - } `json:"technical_error"` -} - -type machineEvaluation struct { - SchemaVersion int `json:"schema_version"` - ExerciseID string `json:"exercise_id"` - Status string `json:"status"` - DeterministicStatus string `json:"deterministic_status,omitempty"` - Review *machineReview `json:"review,omitempty"` - TechnicalError *machineTechnicalError `json:"technical_error,omitempty"` -} - -type machineReview struct { - Status string `json:"status"` - Mode string `json:"mode"` - Authoritative bool `json:"authoritative"` - Verdict *string `json:"verdict"` } type machineTechnicalError struct { @@ -239,15 +251,14 @@ type machineTechnicalError struct { } type machineSubmission struct { - SubmissionID string `json:"submission_id"` - EvaluationJobID string `json:"evaluation_job_id"` - JobState string `json:"job_state"` - Terminal bool `json:"terminal"` - Attempt int `json:"attempt"` - MaxAttempts int `json:"max_attempts"` - NextPollSeconds int `json:"next_poll_seconds,omitempty"` - UpdatedAt time.Time `json:"updated_at"` - Evaluation *machineEvaluation `json:"evaluation,omitempty"` + SubmissionID string `json:"submission_id"` + EvaluationJobID string `json:"evaluation_job_id"` + JobState string `json:"job_state"` + Terminal bool `json:"terminal"` + Attempt int `json:"attempt"` + MaxAttempts int `json:"max_attempts"` + NextPollSeconds int `json:"next_poll_seconds,omitempty"` + UpdatedAt time.Time `json:"updated_at"` } func downloadSubmissionRevision( @@ -269,15 +280,14 @@ func downloadSubmissionRevision( "usage: softpractice submissions download --id ID [--output PATH]") } submissionID := strings.TrimSpace(*id) - parsedID, err := uuid.Parse(submissionID) - if err != nil || parsedID.String() != submissionID { - return errors.New(text(ctx, "ID отправки должен быть каноническим UUID", "submission ID must be a canonical UUID")) + if validateSubmissionID(submissionID) != nil { + return usage(ctx, "ID отправки должен быть каноническим UUID", "submission ID must be a canonical UUID") } target := strings.TrimSpace(*destination) if target == "" { target = "softpractice-" + submissionID + ".zip" } - target, err = filepath.Abs(target) + target, err := filepath.Abs(target) if err != nil { return err } @@ -359,27 +369,5 @@ func normalizeSubmissionResponse(response evaluationAPIResponse) (machineSubmiss (response.JobState == "failed") != (evaluation.Status == "technical_failure") { return machineSubmission{}, errors.New("evaluation projection status is inconsistent") } - machine := machineEvaluation{ - SchemaVersion: evaluation.SchemaVersion, - ExerciseID: evaluation.ExerciseID, - Status: evaluation.Status, - } - if evaluation.Deterministic != nil { - machine.DeterministicStatus = evaluation.Deterministic.Status - } - if evaluation.Review != nil { - machine.Review = &machineReview{ - Status: evaluation.Review.Status, Mode: evaluation.Review.Mode, - Authoritative: evaluation.Review.Authoritative, Verdict: evaluation.Review.Verdict, - } - } - if evaluation.TechnicalError != nil { - machine.TechnicalError = &machineTechnicalError{ - ErrorCode: evaluation.TechnicalError.ErrorCode, - Stage: evaluation.TechnicalError.Stage, - SupportID: evaluation.TechnicalError.SupportID, - } - } - payload.Evaluation = &machine return payload, nil } diff --git a/cmd/softpractice/main.go b/cmd/softpractice/main.go index ace39d0..68a76ef 100644 --- a/cmd/softpractice/main.go +++ b/cmd/softpractice/main.go @@ -1390,7 +1390,8 @@ func status( startDirectory string, args []string, output, errorOutput io.Writer, -) error { +) (err error) { + defer func() { err = reportJSONError(requestsJSON(args), output, err) }() flags := flag.NewFlagSet("status", flag.ContinueOnError) flags.SetOutput(errorOutput) jsonOutput := jsonFlag(flags) @@ -1402,7 +1403,7 @@ func status( } snapshot, err := newLearnerUseCases(client, startDirectory).Status(ctx) if err != nil { - return reportJSONError(*jsonOutput, output, err) + return err } if *jsonOutput { return writeStatusJSON(ctx, output, snapshot) @@ -1504,21 +1505,23 @@ func submit( args []string, input io.Reader, output, errorOutput io.Writer, -) error { +) (err error) { + defer func() { err = reportJSONError(requestsJSON(args), output, err) }() flags := flag.NewFlagSet("submit", flag.ContinueOnError) flags.SetOutput(errorOutput) yes := flags.Bool("yes", false, "submit without an interactive confirmation") checksFlag := flags.Bool("checks", false, "run local public checks before submitting") wait := flags.Bool("wait", false, "wait for the evaluation result after submitting") timeout := flags.Duration("timeout", defaultEvaluationTimeout, "maximum wait with --wait") + directions := flags.Bool("directions", false, "with --wait, also print the reviewer's directions and next steps") jsonOutput := jsonFlag(flags) if err := parseFlags(flags, args); err != nil { return err } if flags.NArg() != 0 || *timeout < 0 { return usage(ctx, - "использование: softpractice submit [--yes] [--checks=true|false] [--wait] [--timeout DURATION] [--json]", - "usage: softpractice submit [--yes] [--checks=true|false] [--wait] [--timeout DURATION] [--json]") + "использование: softpractice submit [--yes] [--checks=true|false] [--wait] [--timeout DURATION] [--directions] [--json]", + "usage: softpractice submit [--yes] [--checks=true|false] [--wait] [--timeout DURATION] [--directions] [--json]") } // With --json, stdout carries only the JSON document; everything meant // for a person goes to stderr. @@ -1529,7 +1532,7 @@ func submit( useCases := newLearnerUseCases(client, startDirectory) prepared, err := useCases.PrepareSubmission(ctx) if err != nil { - return reportJSONError(*jsonOutput, output, err) + return err } defer prepared.Close() repository, link, workspace := prepared.Repository, prepared.Link, prepared.Workspace @@ -1560,12 +1563,12 @@ func submit( if !explicit { enabled, err = projectAutoChecks(ctx, repository.Root) if err != nil { - return reportJSONError(*jsonOutput, output, err) + return err } } if enabled { if err := runSubmissionChecks(ctx, repository, human, errorOutput); err != nil { - return reportJSONError(*jsonOutput, output, err) + return err } } else { fmt.Fprintln(human, text(ctx, "Локальные проверки не запускались. Решение проверит сервер. Для отдельного локального запуска: softpractice check.", "Local checks were not run. The server will check your solution. To run them separately: softpractice check.")) @@ -1573,16 +1576,15 @@ func submit( if !*yes { confirmed, err := confirmSubmission(ctx, input, human) if err != nil { - return reportJSONError(*jsonOutput, output, err) + return err } if !confirmed { - fmt.Fprintln(human, text(ctx, "Отправка отменена.", "Submission cancelled.")) - return nil + return errSubmissionDeclined{message: text(ctx, "отправка отменена", "submission cancelled")} } } receipt, err := useCases.SendSubmission(ctx, prepared) if err != nil { - return reportJSONError(*jsonOutput, output, err) + return err } replay := "" if receipt.Replayed { @@ -1618,8 +1620,8 @@ func submit( return nil } fmt.Fprintln(human, text(ctx, "Ждём результат проверки…\n", "Waiting for the evaluation…\n")) - return reportJSONError(*jsonOutput, output, printEvaluation(ctx, useCases, receipt.SubmissionID, - evaluationWait{Wait: true, Timeout: *timeout}, false, *jsonOutput, output)) + return printEvaluation(ctx, useCases, receipt.SubmissionID, + evaluationWait{Wait: true, Timeout: *timeout}, *directions, *jsonOutput, output) } // machineSubmissionReceipt is what `submit --json` prints without --wait. @@ -1706,6 +1708,13 @@ func openCurrent( if err := parseFlags(flags, args); err != nil { return err } + // The page may also follow the flags: flag parsing stops at it. + if page == "" && flags.NArg() > 0 { + page = flags.Arg(0) + if err := parseFlags(flags, flags.Args()[1:]); err != nil { + return err + } + } if flags.NArg() != 0 || (page != "" && page != "task" && page != "material" && page != "result") { return usageErr } @@ -1735,10 +1744,20 @@ func openCurrent( case "material": target = webLessonURL(ctx, workspace.Assignment.ID, version, "/material") case "result": - if workspace.LatestSubmission == nil { + // Like `softpractice result`: right after an acceptance, the accepted + // result of the previous lesson. + if workspace.LatestSubmission != nil { + target = webResultURL(ctx, workspace.LatestSubmission.ID) + break + } + _, submissionID, found, err := client.LatestPracticumSubmission(ctx, link.ProjectID) + if err != nil { + return err + } + if !found { return fmt.Errorf(text(ctx, "по уроку %s отправок пока нет", "lesson %s has no submissions yet"), workspace.Assignment.ID) } - target = webResultURL(ctx, workspace.LatestSubmission.ID) + target = webResultURL(ctx, submissionID) } fmt.Fprintln(output, target) if *noBrowser { diff --git a/cmd/softpractice/main_test.go b/cmd/softpractice/main_test.go index f441fc5..85e118b 100644 --- a/cmd/softpractice/main_test.go +++ b/cmd/softpractice/main_test.go @@ -399,6 +399,38 @@ func TestLinkedStatusSubmitAndOpenCommandFlow(t *testing.T) { t.Fatalf("submit --wait = %+v\nstderr: %s", waited, waitErrors.String()) } + // Without --wait, submit --json prints the receipt as its one document. + var receiptOutput bytes.Buffer + if err := submit( + context.Background(), client, repositoryRoot, + []string{"--yes", "--json"}, strings.NewReader(""), &receiptOutput, &bytes.Buffer{}, + ); err != nil { + t.Fatal(err) + } + var receipt machineSubmissionReceipt + if err := decodeTestJSON(receiptOutput.Bytes(), &receipt); err != nil { + t.Fatalf("%v\n%s", err, receiptOutput.String()) + } + if receipt.Kind != "softpractice.submission" || receipt.SubmissionID != submissionID { + t.Fatalf("submit --json = %+v", receipt) + } + + // Declining the confirmation is not a success: nothing is sent, and + // stdout still carries one document that says so. + uploadsBefore := len(idempotencyKeys) + var declinedOutput bytes.Buffer + err = submit( + context.Background(), client, repositoryRoot, + []string{"--json"}, strings.NewReader("n\n"), &declinedOutput, &bytes.Buffer{}, + ) + var declined machineError + if decodeErr := decodeTestJSON(declinedOutput.Bytes(), &declined); decodeErr != nil { + t.Fatalf("%v\n%s", decodeErr, declinedOutput.String()) + } + if err == nil || declined.Error.Code != "declined" || len(idempotencyKeys) != uploadsBefore { + t.Fatalf("declined submit: err = %v, payload = %+v, uploads = %d", err, declined, len(idempotencyKeys)) + } + var taskOutput bytes.Buffer if err := openCurrent( context.Background(), client, repositoryRoot, @@ -410,6 +442,18 @@ func TestLinkedStatusSubmitAndOpenCommandFlow(t *testing.T) { t.Fatalf("open task output = %q", taskOutput.String()) } + // The page may also follow the flags. + var materialOutput bytes.Buffer + if err := openCurrent( + context.Background(), client, repositoryRoot, + []string{"--no-browser", "material", "--web", "https://app.example"}, &materialOutput, &bytes.Buffer{}, + ); err != nil { + t.Fatal(err) + } + if strings.TrimSpace(materialOutput.String()) != "https://app.example/assignments/pa-foundation-05/material?version=1" { + t.Fatalf("open material output = %q", materialOutput.String()) + } + var openOutput, openErrors bytes.Buffer if err := openCurrent( context.Background(), diff --git a/cmd/softpractice/output.go b/cmd/softpractice/output.go index 0fdf9b1..9a97b53 100644 --- a/cmd/softpractice/output.go +++ b/cmd/softpractice/output.go @@ -8,6 +8,8 @@ import ( "fmt" "io" "net/http" + "strconv" + "strings" "github.com/arthur-s/softpractice-cli/internal/learnercli" ) @@ -34,6 +36,12 @@ type usageError struct{ message string } func (e usageError) Error() string { return e.message } +// errSubmissionDeclined is the learner's "no" to the submit confirmation: +// nothing was sent, so the command does not report success. +type errSubmissionDeclined struct{ message string } + +func (e errSubmissionDeclined) Error() string { return e.message } + func usage(ctx context.Context, russian, english string) error { return usageError{message: text(ctx, russian, english)} } @@ -50,6 +58,30 @@ func parseFlags(flags *flag.FlagSet, args []string) error { return nil } +// requestsJSON reports whether args ask for --json. It reads the raw +// arguments, so a command line that fails to parse is still reported as a +// softpractice.error document when it asked for JSON. +func requestsJSON(args []string) bool { + for _, arg := range args { + if arg == "--" { + break + } + if !strings.HasPrefix(arg, "-") { + continue + } + name, value, hasValue := strings.Cut(strings.TrimLeft(arg, "-"), "=") + if name != "json" { + continue + } + if !hasValue { + return true + } + enabled, err := strconv.ParseBool(value) + return err == nil && enabled + } + return false +} + func jsonFlag(flags *flag.FlagSet) *bool { return flags.Bool("json", false, "print JSON instead of text") } @@ -81,9 +113,12 @@ type machineError struct { func errorCode(err error) string { var statusError *learnercli.HTTPError var usageErr usageError + var declined errSubmissionDeclined switch { case errors.As(err, &usageErr): return "usage" + case errors.As(err, &declined): + return "declined" case errors.Is(err, learnercli.ErrLoginRequired): return "login_required" case errors.As(err, &statusError) && statusError.Status == http.StatusUnauthorized: diff --git a/cmd/softpractice/ui.go b/cmd/softpractice/ui.go index 515046e..1fe2551 100644 --- a/cmd/softpractice/ui.go +++ b/cmd/softpractice/ui.go @@ -264,7 +264,7 @@ func printCommandHelp(ctx context.Context, output io.Writer, command string) err case "hint": russian, english = "Использование: softpractice hint [--json]\n\nПоказывает подсказки урока, которые вы уже открыли. Следующая подсказка открывается на странице задания.", "Usage: softpractice hint [--json]\n\nShows the lesson hints you have already opened. The next hint is opened on the assignment page." case "submit": - russian, english = "Использование: softpractice submit [--yes] [--checks=true|false] [--wait] [--timeout DURATION] [--json]\n\nОтправляет чистый Git commit текущего урока на проверку. С --wait затем ждёт результат, как softpractice result --wait.", "Usage: softpractice submit [--yes] [--checks=true|false] [--wait] [--timeout DURATION] [--json]\n\nSubmits the clean Git commit for the current lesson. With --wait it then waits for the result, like softpractice result --wait." + russian, english = "Использование: softpractice submit [--yes] [--checks=true|false] [--wait] [--timeout DURATION] [--directions] [--json]\n\nОтправляет чистый Git commit текущего урока на проверку. С --wait затем ждёт результат, как softpractice result --wait; --directions действует так же, как у result. Отказ в подтверждении завершает команду с кодом 1: ничего не отправлено.", "Usage: softpractice submit [--yes] [--checks=true|false] [--wait] [--timeout DURATION] [--directions] [--json]\n\nSubmits the clean Git commit for the current lesson. With --wait it then waits for the result, like softpractice result --wait; --directions works as it does for result. Declining the confirmation exits with code 1: nothing was submitted." case "check": russian, english = "Использование: softpractice check\n\nЗапускает публичные проверки для текущего рабочего дерева, включая незакоммиченные изменения. Ничего не отправляет на сервер.", "Usage: softpractice check\n\nRuns public checks against the current working tree, including uncommitted changes. Does not submit anything to the server." case "update": diff --git a/packaging/npm/README.md b/packaging/npm/README.md index 7a7b774..463037e 100644 --- a/packaging/npm/README.md +++ b/packaging/npm/README.md @@ -126,7 +126,8 @@ you answer them. `status`, `task`, `material`, `hint`, `result`, `submissions`, and `submit` accept `--json`. Each document has a `kind`; on an error the command prints a -`softpractice.error` document. +`softpractice.error` document, also for an invalid command line (code +`usage`) and for a declined `submit` confirmation (code `declined`). ```bash softpractice result --wait --timeout 10m --json From 64f6a1ff3b98d8404461da321519ea9928136e99 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 09:10:47 +0000 Subject: [PATCH 08/14] cli: prepare use cases for the MCP server - Evaluation projects the evaluation at a requested detail itself; the raw API projection, with directions and reviewer questions, no longer leaves the use case. - update is split into PrepareUpdate, which checks everything and lists the file operations without touching the folder, and ApplyUpdate, which refuses when HEAD or the working tree moved since the plan was made. - CheckPublished runs the public checks only when .softpractice/checks.json matches the configuration the server publishes for the folder's lesson. The CLI behaves as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XKQjpw1FmuSnc8k4EtFspp --- cmd/softpractice/learning_commands.go | 25 +-- cmd/softpractice/learning_commands_test.go | 2 +- cmd/softpractice/main.go | 129 +---------- cmd/softpractice/main_test.go | 38 +--- cmd/softpractice/output.go | 2 + cmd/softpractice/project_checks.go | 44 +++- cmd/softpractice/project_checks_test.go | 48 +++++ cmd/softpractice/update_plan.go | 235 +++++++++++++++++++++ cmd/softpractice/update_plan_test.go | 115 ++++++++++ cmd/softpractice/usecase.go | 48 +++-- 10 files changed, 489 insertions(+), 197 deletions(-) create mode 100644 cmd/softpractice/project_checks_test.go create mode 100644 cmd/softpractice/update_plan.go create mode 100644 cmd/softpractice/update_plan_test.go diff --git a/cmd/softpractice/learning_commands.go b/cmd/softpractice/learning_commands.go index 2b6c64e..92cb32b 100644 --- a/cmd/softpractice/learning_commands.go +++ b/cmd/softpractice/learning_commands.go @@ -29,18 +29,14 @@ type machineEvaluationResult struct { // buildMachineEvaluationResult projects one evaluation; resultURL is its // result page. -func buildMachineEvaluationResult( - result evaluationResult, - resultURL string, - detail reviewFeedbackDetail, -) (machineEvaluationResult, error) { +func buildMachineEvaluationResult(result evaluationResult, resultURL string) machineEvaluationResult { payload := machineEvaluationResult{ Kind: "softpractice.evaluation", Outcome: result.Outcome, SubmissionID: result.SubmissionID, ResultURL: resultURL, } if result.Outcome == evaluationSuperseded { payload.JobState = "superseded" - return payload, nil + return payload } submission := result.Submission updatedAt := submission.UpdatedAt @@ -50,14 +46,8 @@ func buildMachineEvaluationResult( payload.MaxAttempts = submission.MaxAttempts payload.NextPollSeconds = submission.NextPollSeconds payload.UpdatedAt = &updatedAt - if result.Outcome == evaluationReady { - built, err := buildMachineResult(result.RawEvaluation, detail) - if err != nil { - return machineEvaluationResult{}, err - } - payload.Result = built - } - return payload, nil + payload.Result = result.Result + return payload } func resultCommand( @@ -117,14 +107,11 @@ func printEvaluation( if directions { detail = feedbackWithDirections } - result, err := useCases.Evaluation(ctx, submissionID, wait) - if err != nil { - return err - } - payload, err := buildMachineEvaluationResult(result, webResultURL(ctx, submissionID), detail) + result, err := useCases.Evaluation(ctx, submissionID, wait, detail) if err != nil { return err } + payload := buildMachineEvaluationResult(result, webResultURL(ctx, submissionID)) if jsonOutput { if err := writeMachineJSON(output, payload); err != nil { return err diff --git a/cmd/softpractice/learning_commands_test.go b/cmd/softpractice/learning_commands_test.go index 0b0713b..07beb7a 100644 --- a/cmd/softpractice/learning_commands_test.go +++ b/cmd/softpractice/learning_commands_test.go @@ -330,7 +330,7 @@ func TestEvaluationWaitStopsWhenCancelled(t *testing.T) { cancel() return sleepContext(sleepCtx, delay) } - _, err := useCases.Evaluation(ctx, currentSubmissionID, evaluationWait{Wait: true, Timeout: time.Hour}) + _, err := useCases.Evaluation(ctx, currentSubmissionID, evaluationWait{Wait: true, Timeout: time.Hour}, defaultReviewFeedback) if !errors.Is(err, context.Canceled) || *calls != 1 || errorCode(err) != "cancelled" { t.Fatalf("err = %v, calls = %d", err, *calls) } diff --git a/cmd/softpractice/main.go b/cmd/softpractice/main.go index 68a76ef..bca994d 100644 --- a/cmd/softpractice/main.go +++ b/cmd/softpractice/main.go @@ -638,83 +638,12 @@ func updateProject(ctx context.Context, client *learnercli.Client, args []string if flags.NArg() != 0 { return usage(ctx, "использование: softpractice update", "usage: softpractice update") } - return newLearnerUseCases(client, "").Update(ctx, output) -} - -func updateLinkedProject( - ctx context.Context, - client *learnercli.Client, - startDirectory string, - expectedBaseRevisionID string, - output io.Writer, -) error { - repository, link, workspace, err := linkedWorkspace(ctx, client, startDirectory, true) - if err != nil { - return err - } - defer os.Remove(repository.ArchivePath) - if !repository.Clean { - return errors.New(text(ctx, "рабочее дерево не чистое; сделайте commit или stash перед обновлением проекта", "working tree is not clean; commit or stash changes before updating the course project")) - } - if workspace.Workspace.State != "active" { - return fmt.Errorf(text(ctx, "workspace находится в состоянии %s и не имеет обновления курса", "workspace is %s and has no course update"), workspace.Workspace.State) - } - if pendingLessonVersion(link, workspace) { - return updateLessonVersion(ctx, client, repository.Root, link, workspace, output) - } - if link.SchemaVersion == 2 && workspace.Assignment.ID == link.AssignmentID && workspace.Assignment.Version == link.AssignmentVersion { - checksPath := filepath.Join(repository.Root, filepath.FromSlash(localchecks.RelativePath)) - if _, statErr := os.Stat(checksPath); errors.Is(statErr, os.ErrNotExist) { - if err := writeLocalChecks(repository.Root, workspace.Assignment.LocalChecks); err != nil { - return err - } - if err := commitLocalChecks(ctx, repository.Root, "Add Softpractice public checks"); err != nil { - return err - } - fmt.Fprintln(output, text(ctx, - "Конфигурация публичных проверок добавлена в проект. Теперь повторите `softpractice submit`.", - "Public checks configuration was added to the project. Now run `softpractice submit` again.")) - return nil - } else if statErr != nil { - return fmt.Errorf("inspect local checks: %w", statErr) - } - return fmt.Errorf(text(ctx, "урок %s v%d всё ещё текущий; отправьте решение и дождитесь принятого результата перед обновлением", "lesson %s v%d is still current; submit and receive an accepted result before updating"), link.AssignmentID, link.AssignmentVersion) - } - update, err := client.PrepareCourseUpdate(ctx, link.WorkspaceID) - if err != nil { - return fmt.Errorf("prepare course update: %w", err) - } - // The transition may start from a newer version of the lesson this folder - // holds: the lesson was republished after this tree was made, and the tree - // was accepted on its own version. The server offers that only when the - // transition itself replaces every lesson file the republication changed; - // the base content check below still pins the exact accepted tree. - if (link.SchemaVersion == 2 && (update.FromAssignmentID != link.AssignmentID || update.FromAssignmentVersion < link.AssignmentVersion)) || - update.ToAssignmentID != workspace.Assignment.ID || update.ToAssignmentVersion != workspace.Assignment.Version { - return errors.New(text(ctx, "сервер вернул обновление для другого урока; проект не изменён", "server returned a course update for a different lesson; project was left unchanged")) - } - if expectedBaseRevisionID != "" && update.BaseRevisionID != expectedBaseRevisionID { - return errors.New("workspace base revision changed while restoring the project; retry") - } - normalized, err := submission.NormalizeTarGz(repository.ArchivePath, os.TempDir(), submission.DefaultArchiveLimits()) + useCases := newLearnerUseCases(client, "") + plan, err := useCases.PrepareUpdate(ctx) if err != nil { - return fmt.Errorf("verify local project content: %w", err) - } - defer os.Remove(normalized.Path) - if normalized.ContentSHA256 != update.BaseContentSHA256 { - // A non-empty startDirectory means this is the staging copy driven by - // `project restore`, not a folder the learner is working in. - return courseUpdateMismatchError(ctx, repository, update, startDirectory != "") - } - archivePath := "/v1/workspaces/" + link.WorkspaceID + "/current-assignment/course-update/archive?format=tar.gz" - if err := downloadAndApplyCourseUpdate(ctx, client, repository.Root, link, update, workspace.Assignment.LocalChecks, archivePath); err != nil { return err } - fmt.Fprintf(output, text(ctx, "Проект курса обновлён в этой папке: %s\n", "Course project updated in place: %s\n"), repository.Root) - fmt.Fprintln(output, text(ctx, - "Принятая предыдущая ревизия осталась в истории Git. Просмотрите изменения урока и продолжайте с `softpractice submit`.", - "The accepted previous revision remains in Git history. Review the lesson changes, then continue with `softpractice submit`.")) - return nil + return useCases.ApplyUpdate(ctx, plan, output) } func downloadAndApplyCourseUpdate( @@ -1143,51 +1072,6 @@ func safeProjectRelativePath(value string) bool { // to the version that replaced it. The update replaces only author-owned files // the bump changed and never the learner's work. It is voluntary: a folder on // the retired version can still submit until the lesson ends. -func updateLessonVersion( - ctx context.Context, - client *learnercli.Client, - root string, - link learnercli.ProjectLink, - workspace learnercli.WorkspaceStatus, - output io.Writer, -) error { - update, err := client.LessonVersionUpdate(ctx, link.WorkspaceID, link.AssignmentVersion) - if err != nil { - return fmt.Errorf("prepare lesson version update: %w", err) - } - if update.AssignmentID != link.AssignmentID || update.FromVersion != link.AssignmentVersion || - update.ToVersion != workspace.Assignment.Version { - return errors.New(text(ctx, "сервер вернул обновление для другого урока; проект не изменён", "server returned a course update for a different lesson; project was left unchanged")) - } - if len(update.Operations) == 0 { - return refreshLessonVersion(ctx, root, link, workspace, output) - } - replaced := make([]string, 0, len(update.Operations)) - for _, operation := range update.Operations { - if operation.Kind != "replace" { - return errors.New(text(ctx, "обновление урока может только заменять файлы урока; проект не изменён", "a lesson update may only replace lesson files; project was left unchanged")) - } - replaced = append(replaced, operation.Path) - } - courseUpdate := learnercli.CourseUpdate{ - Ref: update.Ref, - FromAssignmentID: update.AssignmentID, FromAssignmentVersion: update.FromVersion, - ToAssignmentID: update.AssignmentID, ToAssignmentVersion: update.ToVersion, - ArchiveSHA256: update.ArchiveSHA256, ArchiveSize: update.ArchiveSize, - Files: update.Files, Operations: update.Operations, - } - archivePath := fmt.Sprintf("/v1/workspaces/%s/current-assignment/lesson-version-update/archive?from_version=%d&format=tar.gz", - link.WorkspaceID, link.AssignmentVersion) - if err := downloadAndApplyCourseUpdate(ctx, client, root, link, courseUpdate, workspace.Assignment.LocalChecks, archivePath); err != nil { - return err - } - fmt.Fprintf(output, - text(ctx, "Урок %s обновлён: v%d → v%d. Заменены файлы урока: %s. Ваши файлы не изменены; посмотрите обновлённое задание и продолжайте.\n", - "Lesson %s updated: v%d → v%d. Lesson files replaced: %s. Your files are unchanged; review the updated assignment and continue.\n"), - link.AssignmentID, update.FromVersion, update.ToVersion, strings.Join(replaced, ", ")) - return nil -} - // refreshLessonVersion moves this folder onto the republished version of the // lesson it already holds when the bump changed no project file. There is no // archive to apply: the learner's work stays exactly as it is and only the pin @@ -1196,12 +1080,13 @@ func refreshLessonVersion( ctx context.Context, root string, link learnercli.ProjectLink, - workspace learnercli.WorkspaceStatus, + toVersion int, + localChecks []byte, output io.Writer, ) error { previousVersion := link.AssignmentVersion - link.AssignmentVersion = workspace.Assignment.Version - if err := writeLocalChecks(root, workspace.Assignment.LocalChecks); err != nil { + link.AssignmentVersion = toVersion + if err := writeLocalChecks(root, localChecks); err != nil { return err } if err := writeProjectLink(root, link); err != nil { diff --git a/cmd/softpractice/main_test.go b/cmd/softpractice/main_test.go index 85e118b..7d9ed85 100644 --- a/cmd/softpractice/main_test.go +++ b/cmd/softpractice/main_test.go @@ -1691,43 +1691,7 @@ func TestUpdateRefreshesTheLessonVersionWithoutTouchingLearnerFiles(t *testing.T // update. `update` replaces exactly those files and nothing the learner wrote. func TestUpdateReplacesRefactoredAuthorFilesAndKeepsLearnerWork(t *testing.T) { workspaceID := uuid.NewString() - payloadRoot := t.TempDir() - refactored := []byte("# refactored author module\n") - if err := os.WriteFile(filepath.Join(payloadRoot, "main.py"), refactored, 0o600); err != nil { - t.Fatal(err) - } - digest := sha256.Sum256(refactored) - var archive bytes.Buffer - metadata, err := starterbundle.Build(payloadRoot, []starterbundle.File{ - {Path: "main.py", SHA256: hex.EncodeToString(digest[:])}, - }, &archive) - if err != nil { - t.Fatal(err) - } - const ref = "pa-foundation-01-v1-to-pa-foundation-01-v2@1.0.0" - server := lessonBumpServer(t, workspaceID, func(writer http.ResponseWriter, request *http.Request) { - switch request.URL.Path { - case "/v1/workspaces/" + workspaceID + "/current-assignment/lesson-version-update": - writeTestJSON(writer, map[string]any{ - "ref": ref, "assignment_id": "pa-foundation-01", "from_version": 1, "to_version": 2, - "archive_sha256": metadata.SHA256, "archive_size": metadata.Size, - "files": []any{map[string]any{"path": "main.py", "sha256": hex.EncodeToString(digest[:])}}, - "operations": []any{map[string]any{"kind": "replace", "path": "main.py"}}, - }) - case "/v1/workspaces/" + workspaceID + "/current-assignment/lesson-version-update/archive": - if request.URL.Query().Get("from_version") != "1" { - http.NotFound(writer, request) - return - } - writer.Header().Set("Content-Type", "application/gzip") - writer.Header().Set("Content-Length", fmt.Sprint(metadata.Size)) - writer.Header().Set("X-Softpractice-Course-Update-SHA256", metadata.SHA256) - writer.Header().Set("X-Softpractice-Course-Update-Ref", ref) - _, _ = writer.Write(archive.Bytes()) - default: - http.NotFound(writer, request) - } - }) + server, refactored := refactoredLessonServer(t, workspaceID) defer server.Close() client := savedTestClient(t, server.URL) root := createPinnedLinkedGitRepository(t, workspaceID, "pa-foundation-01", 1) diff --git a/cmd/softpractice/output.go b/cmd/softpractice/output.go index 9a97b53..9a22efe 100644 --- a/cmd/softpractice/output.go +++ b/cmd/softpractice/output.go @@ -127,6 +127,8 @@ func errorCode(err error) string { return "project_not_linked" case errors.Is(err, errNotGitRepository): return "not_git_repository" + case errors.Is(err, errChecksNotPublished): + return "checks_not_published" case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): return "cancelled" case errors.As(err, &statusError): diff --git a/cmd/softpractice/project_checks.go b/cmd/softpractice/project_checks.go index b4db130..b3cef5c 100644 --- a/cmd/softpractice/project_checks.go +++ b/cmd/softpractice/project_checks.go @@ -4,12 +4,14 @@ import ( "context" "errors" "fmt" - "github.com/arthur-s/softpractice-cli/internal/learnercli" - "github.com/arthur-s/softpractice-cli/internal/localchecks" "io" "os" "os/exec" + "reflect" "strings" + + "github.com/arthur-s/softpractice-cli/internal/learnercli" + "github.com/arthur-s/softpractice-cli/internal/localchecks" ) const autoChecksKey = "softpractice.autoChecks" @@ -98,3 +100,41 @@ func runSubmissionChecks(ctx context.Context, repository gitRepository, output, fmt.Fprintln(output, text(ctx, "Все локальные публичные проверки прошли.", "All local public checks passed.")) return nil } + +// errChecksNotPublished means the folder's public checks configuration is not +// the one the server publishes for its lesson: it was edited, or the folder +// holds another lesson version. Running it would run commands nobody but the +// folder vouches for. +var errChecksNotPublished = errors.New("local checks configuration differs from the one published for this lesson") + +// CheckPublished runs the public checks on the working tree of the linked +// folder, like `softpractice check`, but only when .softpractice/checks.json +// is exactly the configuration the server publishes for the folder's lesson. +// Callers that cannot see what a check runs, such as an MCP client, use it. +func (u learnerUseCases) CheckPublished(ctx context.Context, output, errorOutput io.Writer) error { + repository, link, workspace, err := linkedWorkspace(ctx, u.client, u.startDirectory, false) + if err != nil { + return err + } + checks, err := localchecks.Load(repository.Root) + if err != nil { + return err + } + published, err := localchecks.Parse(workspace.Assignment.LocalChecks) + if err != nil || pendingTransition(link, workspace) || pendingLessonVersion(link, workspace) || + !reflect.DeepEqual(checks, published) { + return fmt.Errorf("%w: %s", errChecksNotPublished, text(ctx, + "запустите `softpractice check` в терминале, посмотрев, какие команды он выполнит, или примените обновление урока", + "run `softpractice check` in a terminal after reviewing the commands it runs, or apply the lesson update")) + } + fmt.Fprintln(output, text(ctx, + "Локальные публичные проверки рабочего дерева:", + "Local public checks for the working tree:")) + if err := localchecks.Run(ctx, repository.Root, checks, output, errorOutput); err != nil { + return err + } + fmt.Fprintln(output, text(ctx, + "Все локальные публичные проверки прошли.", + "All local public checks passed.")) + return nil +} diff --git a/cmd/softpractice/project_checks_test.go b/cmd/softpractice/project_checks_test.go new file mode 100644 index 0000000..2e55c37 --- /dev/null +++ b/cmd/softpractice/project_checks_test.go @@ -0,0 +1,48 @@ +package main + +import ( + "bytes" + "context" + "errors" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" +) + +// CheckPublished runs the checks the server publishes for the lesson and +// refuses a configuration that differs from it, without running anything. +func TestCheckPublishedRunsOnlyThePublishedConfiguration(t *testing.T) { + workspaceID := uuid.NewString() + server := lessonBumpServer(t, workspaceID, http.NotFound) + defer server.Close() + root := createPinnedLinkedGitRepository(t, workspaceID, "pa-foundation-01", 2) + useCases := newLearnerUseCases(savedTestClient(t, server.URL), root) + ctx := context.Background() + + var output, errorOutput bytes.Buffer + if err := useCases.CheckPublished(ctx, &output, &errorOutput); err != nil { + t.Fatalf("published checks: %v\n%s%s", err, output.String(), errorOutput.String()) + } + if !strings.Contains(output.String(), "git version") { + t.Fatalf("check output = %q; want the command output", output.String()) + } + + marker := filepath.Join(root, "marker") + edited := `{"schema_version":1,"checks":[{"name":"test check","executable":"git","args":["init","` + + filepath.ToSlash(marker) + `"],"timeout_seconds":30}]}` + if err := os.WriteFile(filepath.Join(root, ".softpractice", "checks.json"), []byte(edited), 0o600); err != nil { + t.Fatal(err) + } + output.Reset() + err := useCases.CheckPublished(ctx, &output, &errorOutput) + if !errors.Is(err, errChecksNotPublished) { + t.Fatalf("edited checks = %v; want errChecksNotPublished", err) + } + if _, statErr := os.Stat(marker); !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("an edited check ran: %v", statErr) + } +} diff --git a/cmd/softpractice/update_plan.go b/cmd/softpractice/update_plan.go new file mode 100644 index 0000000..43b372b --- /dev/null +++ b/cmd/softpractice/update_plan.go @@ -0,0 +1,235 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/arthur-s/softpractice-cli/internal/learnercli" + "github.com/arthur-s/softpractice-cli/internal/localchecks" + "github.com/arthur-s/softpractice-cli/internal/submission" +) + +// updateKind names what `update` will do to the linked folder. +type updateKind string + +const ( + // updateLessonVersion moves the folder to the republished version of the + // lesson it holds, replacing only author-owned lesson files. + updateLessonVersion updateKind = "lesson_version" + // updateLessonTransition applies the transition from the accepted + // previous lesson to the current one. + updateLessonTransition updateKind = "lesson_transition" + // updateLocalChecks adds the missing public checks configuration. + updateLocalChecks updateKind = "local_checks" +) + +// updateOperation is one change to a project file: add, replace, or delete. +type updateOperation struct { + Kind string `json:"kind"` + Path string `json:"path"` +} + +// updatePlan is a verified project update that has not touched the folder +// yet. The learner can review Operations before ApplyUpdate; the Softpractice +// metadata under .softpractice/ is rewritten as well and is not listed. +type updatePlan struct { + Kind updateKind + Root string + HeadCommit string + FromAssignmentID string + FromAssignmentVersion int + ToAssignmentID string + ToAssignmentVersion int + Ref string + Operations []updateOperation + + link learnercli.ProjectLink + localChecks []byte + course learnercli.CourseUpdate + archivePath string +} + +// prepareProjectUpdate makes every check `softpractice update` makes and asks +// the server for the update, but changes nothing on disk. A non-empty +// expectedBaseRevisionID pins the accepted revision `project restore` staged. +func prepareProjectUpdate( + ctx context.Context, + client *learnercli.Client, + startDirectory string, + expectedBaseRevisionID string, +) (updatePlan, error) { + repository, link, workspace, err := linkedWorkspace(ctx, client, startDirectory, true) + if err != nil { + return updatePlan{}, err + } + defer os.Remove(repository.ArchivePath) + if !repository.Clean { + return updatePlan{}, errors.New(text(ctx, "рабочее дерево не чистое; сделайте commit или stash перед обновлением проекта", "working tree is not clean; commit or stash changes before updating the course project")) + } + if workspace.Workspace.State != "active" { + return updatePlan{}, fmt.Errorf(text(ctx, "workspace находится в состоянии %s и не имеет обновления курса", "workspace is %s and has no course update"), workspace.Workspace.State) + } + plan := updatePlan{ + Root: repository.Root, HeadCommit: repository.CommitSHA, + FromAssignmentID: link.AssignmentID, FromAssignmentVersion: link.AssignmentVersion, + ToAssignmentID: workspace.Assignment.ID, ToAssignmentVersion: workspace.Assignment.Version, + Operations: []updateOperation{}, + link: link, localChecks: workspace.Assignment.LocalChecks, + } + if pendingLessonVersion(link, workspace) { + return prepareLessonVersionUpdate(ctx, client, plan) + } + if link.SchemaVersion == 2 && workspace.Assignment.ID == link.AssignmentID && workspace.Assignment.Version == link.AssignmentVersion { + checksPath := filepath.Join(repository.Root, filepath.FromSlash(localchecks.RelativePath)) + if _, statErr := os.Stat(checksPath); errors.Is(statErr, os.ErrNotExist) { + plan.Kind = updateLocalChecks + plan.Operations = append(plan.Operations, updateOperation{Kind: "add", Path: localchecks.RelativePath}) + return plan, nil + } else if statErr != nil { + return updatePlan{}, fmt.Errorf("inspect local checks: %w", statErr) + } + return updatePlan{}, fmt.Errorf(text(ctx, "урок %s v%d всё ещё текущий; отправьте решение и дождитесь принятого результата перед обновлением", "lesson %s v%d is still current; submit and receive an accepted result before updating"), link.AssignmentID, link.AssignmentVersion) + } + update, err := client.PrepareCourseUpdate(ctx, link.WorkspaceID) + if err != nil { + return updatePlan{}, fmt.Errorf("prepare course update: %w", err) + } + // The transition may start from a newer version of the lesson this folder + // holds: the lesson was republished after this tree was made, and the tree + // was accepted on its own version. The server offers that only when the + // transition itself replaces every lesson file the republication changed; + // the base content check below still pins the exact accepted tree. + if (link.SchemaVersion == 2 && (update.FromAssignmentID != link.AssignmentID || update.FromAssignmentVersion < link.AssignmentVersion)) || + update.ToAssignmentID != workspace.Assignment.ID || update.ToAssignmentVersion != workspace.Assignment.Version { + return updatePlan{}, errors.New(text(ctx, "сервер вернул обновление для другого урока; проект не изменён", "server returned a course update for a different lesson; project was left unchanged")) + } + if expectedBaseRevisionID != "" && update.BaseRevisionID != expectedBaseRevisionID { + return updatePlan{}, errors.New("workspace base revision changed while restoring the project; retry") + } + normalized, err := submission.NormalizeTarGz(repository.ArchivePath, os.TempDir(), submission.DefaultArchiveLimits()) + if err != nil { + return updatePlan{}, fmt.Errorf("verify local project content: %w", err) + } + defer os.Remove(normalized.Path) + if normalized.ContentSHA256 != update.BaseContentSHA256 { + // A non-empty startDirectory means this is the staging copy driven by + // `project restore`, not a folder the learner is working in. + return updatePlan{}, courseUpdateMismatchError(ctx, repository, update, startDirectory != "") + } + plan.Kind = updateLessonTransition + plan.FromAssignmentID, plan.FromAssignmentVersion = update.FromAssignmentID, update.FromAssignmentVersion + plan.Ref = update.Ref + plan.course = update + plan.archivePath = "/v1/workspaces/" + link.WorkspaceID + "/current-assignment/course-update/archive?format=tar.gz" + for _, operation := range update.Operations { + plan.Operations = append(plan.Operations, updateOperation{Kind: operation.Kind, Path: operation.Path}) + } + return plan, nil +} + +func prepareLessonVersionUpdate(ctx context.Context, client *learnercli.Client, plan updatePlan) (updatePlan, error) { + link := plan.link + update, err := client.LessonVersionUpdate(ctx, link.WorkspaceID, link.AssignmentVersion) + if err != nil { + return updatePlan{}, fmt.Errorf("prepare lesson version update: %w", err) + } + if update.AssignmentID != link.AssignmentID || update.FromVersion != link.AssignmentVersion || + update.ToVersion != plan.ToAssignmentVersion { + return updatePlan{}, errors.New(text(ctx, "сервер вернул обновление для другого урока; проект не изменён", "server returned a course update for a different lesson; project was left unchanged")) + } + for _, operation := range update.Operations { + if operation.Kind != "replace" { + return updatePlan{}, errors.New(text(ctx, "обновление урока может только заменять файлы урока; проект не изменён", "a lesson update may only replace lesson files; project was left unchanged")) + } + plan.Operations = append(plan.Operations, updateOperation{Kind: operation.Kind, Path: operation.Path}) + } + plan.Kind = updateLessonVersion + plan.Ref = update.Ref + if len(update.Operations) > 0 { + plan.course = learnercli.CourseUpdate{ + Ref: update.Ref, + FromAssignmentID: update.AssignmentID, FromAssignmentVersion: update.FromVersion, + ToAssignmentID: update.AssignmentID, ToAssignmentVersion: update.ToVersion, + ArchiveSHA256: update.ArchiveSHA256, ArchiveSize: update.ArchiveSize, + Files: update.Files, Operations: update.Operations, + } + plan.archivePath = fmt.Sprintf("/v1/workspaces/%s/current-assignment/lesson-version-update/archive?from_version=%d&format=tar.gz", + link.WorkspaceID, link.AssignmentVersion) + } + return plan, nil +} + +// applyProjectUpdate applies a prepared update. It refuses when the folder +// moved since the plan was made: the learner confirmed that exact change. +func applyProjectUpdate(ctx context.Context, client *learnercli.Client, plan updatePlan, output io.Writer) error { + current, err := inspectGitRepository(ctx, plan.Root, false) + if err != nil { + return err + } + if current.CommitSHA != plan.HeadCommit || !current.Clean { + return errors.New(text(ctx, + "HEAD или рабочее дерево изменились после подготовки обновления; проект не изменён, повторите обновление", + "HEAD or the working tree changed after the update was prepared; project was left unchanged, run the update again")) + } + root := plan.Root + switch plan.Kind { + case updateLocalChecks: + if err := writeLocalChecks(root, plan.localChecks); err != nil { + return err + } + if err := commitLocalChecks(ctx, root, "Add Softpractice public checks"); err != nil { + return err + } + fmt.Fprintln(output, text(ctx, + "Конфигурация публичных проверок добавлена в проект. Теперь повторите `softpractice submit`.", + "Public checks configuration was added to the project. Now run `softpractice submit` again.")) + return nil + case updateLessonVersion: + link := plan.link + if len(plan.Operations) == 0 { + return refreshLessonVersion(ctx, root, link, plan.ToAssignmentVersion, plan.localChecks, output) + } + if err := downloadAndApplyCourseUpdate(ctx, client, root, link, plan.course, plan.localChecks, plan.archivePath); err != nil { + return err + } + replaced := make([]string, 0, len(plan.Operations)) + for _, operation := range plan.Operations { + replaced = append(replaced, operation.Path) + } + fmt.Fprintf(output, + text(ctx, "Урок %s обновлён: v%d → v%d. Заменены файлы урока: %s. Ваши файлы не изменены; посмотрите обновлённое задание и продолжайте.\n", + "Lesson %s updated: v%d → v%d. Lesson files replaced: %s. Your files are unchanged; review the updated assignment and continue.\n"), + link.AssignmentID, plan.FromAssignmentVersion, plan.ToAssignmentVersion, strings.Join(replaced, ", ")) + return nil + case updateLessonTransition: + if err := downloadAndApplyCourseUpdate(ctx, client, root, plan.link, plan.course, plan.localChecks, plan.archivePath); err != nil { + return err + } + fmt.Fprintf(output, text(ctx, "Проект курса обновлён в этой папке: %s\n", "Course project updated in place: %s\n"), root) + fmt.Fprintln(output, text(ctx, + "Принятая предыдущая ревизия осталась в истории Git. Просмотрите изменения урока и продолжайте с `softpractice submit`.", + "The accepted previous revision remains in Git history. Review the lesson changes, then continue with `softpractice submit`.")) + return nil + default: + return fmt.Errorf("unknown project update %q", plan.Kind) + } +} + +func updateLinkedProject( + ctx context.Context, + client *learnercli.Client, + startDirectory string, + expectedBaseRevisionID string, + output io.Writer, +) error { + plan, err := prepareProjectUpdate(ctx, client, startDirectory, expectedBaseRevisionID) + if err != nil { + return err + } + return applyProjectUpdate(ctx, client, plan, output) +} diff --git a/cmd/softpractice/update_plan_test.go b/cmd/softpractice/update_plan_test.go new file mode 100644 index 0000000..1bab3c9 --- /dev/null +++ b/cmd/softpractice/update_plan_test.go @@ -0,0 +1,115 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/google/uuid" + + "github.com/arthur-s/softpractice-cli/internal/learnercli" + "github.com/arthur-s/softpractice-cli/internal/starterbundle" +) + +// refactoredLessonServer serves a lesson version update v1 → v2 of +// pa-foundation-01 that replaces main.py with the returned bytes. +func refactoredLessonServer(t *testing.T, workspaceID string) (*httptest.Server, []byte) { + t.Helper() + payloadRoot := t.TempDir() + refactored := []byte("# refactored author module\n") + if err := os.WriteFile(filepath.Join(payloadRoot, "main.py"), refactored, 0o600); err != nil { + t.Fatal(err) + } + digest := sha256.Sum256(refactored) + var archive bytes.Buffer + metadata, err := starterbundle.Build(payloadRoot, []starterbundle.File{ + {Path: "main.py", SHA256: hex.EncodeToString(digest[:])}, + }, &archive) + if err != nil { + t.Fatal(err) + } + const ref = "pa-foundation-01-v1-to-pa-foundation-01-v2@1.0.0" + server := lessonBumpServer(t, workspaceID, func(writer http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/v1/workspaces/" + workspaceID + "/current-assignment/lesson-version-update": + writeTestJSON(writer, map[string]any{ + "ref": ref, "assignment_id": "pa-foundation-01", "from_version": 1, "to_version": 2, + "archive_sha256": metadata.SHA256, "archive_size": metadata.Size, + "files": []any{map[string]any{"path": "main.py", "sha256": hex.EncodeToString(digest[:])}}, + "operations": []any{map[string]any{"kind": "replace", "path": "main.py"}}, + }) + case "/v1/workspaces/" + workspaceID + "/current-assignment/lesson-version-update/archive": + if request.URL.Query().Get("from_version") != "1" { + http.NotFound(writer, request) + return + } + writer.Header().Set("Content-Type", "application/gzip") + writer.Header().Set("Content-Length", fmt.Sprint(metadata.Size)) + writer.Header().Set("X-Softpractice-Course-Update-SHA256", metadata.SHA256) + writer.Header().Set("X-Softpractice-Course-Update-Ref", ref) + _, _ = writer.Write(archive.Bytes()) + default: + http.NotFound(writer, request) + } + }) + return server, refactored +} + +// Preparing an update lists what will change and changes nothing; applying +// it is refused once HEAD moved, because the learner confirmed that plan. +func TestPrepareUpdateChangesNothingAndApplyRefusesAMovedHead(t *testing.T) { + workspaceID := uuid.NewString() + server, refactored := refactoredLessonServer(t, workspaceID) + defer server.Close() + client := savedTestClient(t, server.URL) + root := createPinnedLinkedGitRepository(t, workspaceID, "pa-foundation-01", 1) + if output, err := exec.Command("git", "-C", root, "update-index", "--assume-unchanged", ".softpractice/project.json").CombinedOutput(); err != nil { + t.Fatalf("ignore local project link in test repository: %v: %s", err, output) + } + useCases := newLearnerUseCases(client, root) + ctx := context.Background() + + plan, err := useCases.PrepareUpdate(ctx) + if err != nil { + t.Fatal(err) + } + if plan.Kind != updateLessonVersion || plan.FromAssignmentVersion != 1 || plan.ToAssignmentVersion != 2 || + len(plan.Operations) != 1 || plan.Operations[0] != (updateOperation{Kind: "replace", Path: "main.py"}) { + t.Fatalf("plan = %+v", plan) + } + if content, _ := os.ReadFile(filepath.Join(root, "main.py")); string(content) == string(refactored) { + t.Fatal("preparing the update replaced main.py") + } + if link, err := learnercli.LoadProjectLink(root); err != nil || link.AssignmentVersion != 1 { + t.Fatalf("preparing the update moved the project link: %+v, %v", link, err) + } + + commitLearnerWork(t, root, "solution.py", "# more work\n") + var output bytes.Buffer + if err := useCases.ApplyUpdate(ctx, plan, &output); err == nil || !strings.Contains(err.Error(), "HEAD") { + t.Fatalf("apply after HEAD moved = %v; want a refusal", err) + } + if content, _ := os.ReadFile(filepath.Join(root, "main.py")); string(content) == string(refactored) { + t.Fatal("a refused update replaced main.py") + } + + plan, err = useCases.PrepareUpdate(ctx) + if err != nil { + t.Fatal(err) + } + if err := useCases.ApplyUpdate(ctx, plan, &output); err != nil { + t.Fatal(err) + } + if content, err := os.ReadFile(filepath.Join(root, "main.py")); err != nil || string(content) != string(refactored) { + t.Fatalf("author file = %q, %v; want the refactored bytes", content, err) + } +} diff --git a/cmd/softpractice/usecase.go b/cmd/softpractice/usecase.go index d5a5064..9461083 100644 --- a/cmd/softpractice/usecase.go +++ b/cmd/softpractice/usecase.go @@ -2,7 +2,6 @@ package main import ( "context" - "encoding/json" "errors" "fmt" "io" @@ -81,14 +80,14 @@ func (u learnerUseCases) Status(ctx context.Context) (statusSnapshot, error) { snapshot := statusSnapshot{Repository: repository, Link: link, Workspace: workspace, User: user} if latest := workspace.LatestSubmission; latest != nil && (latest.JobState == "completed" || latest.JobState == "failed") { // The status stays available when only these optional reads fail. - if result, err := u.Evaluation(ctx, latest.ID, evaluationWait{}); err == nil && result.Outcome == evaluationReady { - if built, err := buildMachineResult(result.RawEvaluation, defaultReviewFeedback); err == nil { - snapshot.Latest = &latestResult{ - Status: built.Status, - QuestionsAnswerable: built.Review != nil && built.Review.QuestionsAnswerable, - } - snapshot.Latest.CommitSHA, _ = u.SubmittedCommit(ctx, latest.ID) + if result, err := u.Evaluation(ctx, latest.ID, evaluationWait{}, defaultReviewFeedback); err == nil && + result.Outcome == evaluationReady { + built := result.Result + snapshot.Latest = &latestResult{ + Status: built.Status, + QuestionsAnswerable: built.Review != nil && built.Review.QuestionsAnswerable, } + snapshot.Latest.CommitSHA, _ = u.SubmittedCommit(ctx, latest.ID) } } return snapshot, nil @@ -366,10 +365,18 @@ func (u learnerUseCases) SendSubmission(ctx context.Context, prepared preparedSu return receipt, nil } -// Update applies the pending lesson transition or lesson version update to -// the linked folder, with every check `softpractice update` makes. -func (u learnerUseCases) Update(ctx context.Context, output io.Writer) error { - return updateLinkedProject(ctx, u.client, u.startDirectory, "", output) +// PrepareUpdate makes every check `softpractice update` makes and returns +// the pending lesson transition, lesson version update, or missing public +// checks as a plan. It changes nothing: the caller shows the learner what +// will change and asks for confirmation before ApplyUpdate. +func (u learnerUseCases) PrepareUpdate(ctx context.Context) (updatePlan, error) { + return prepareProjectUpdate(ctx, u.client, u.startDirectory, "") +} + +// ApplyUpdate applies a prepared plan, provided the folder has not moved +// since it was prepared. Its report for the learner goes to output. +func (u learnerUseCases) ApplyUpdate(ctx context.Context, plan updatePlan, output io.Writer) error { + return applyProjectUpdate(ctx, u.client, plan, output) } // latestSubmission names the linked workspace's newest submission. Right @@ -431,15 +438,20 @@ const defaultEvaluationPoll = 2 * time.Second type evaluationResult struct { Outcome evaluationOutcome SubmissionID string - // Submission and RawEvaluation are set for ready and pending outcomes. - Submission machineSubmission - RawEvaluation json.RawMessage + // Submission is set for ready and pending outcomes. + Submission machineSubmission + // Result is the evaluation projected at the requested detail; it is set + // only for the ready outcome. The raw API projection, which carries the + // reviewer's directions and questions, never leaves this use case. + Result *machineResult } +// Evaluation reads or waits for one evaluation and projects it at detail. func (u learnerUseCases) Evaluation( ctx context.Context, submissionID string, wait evaluationWait, + detail reviewFeedbackDetail, ) (evaluationResult, error) { if err := validateSubmissionID(submissionID); err != nil { return evaluationResult{}, err @@ -463,9 +475,13 @@ func (u learnerUseCases) Evaluation( return evaluationResult{}, errors.New("evaluation response does not match the requested submission") } if payload.Terminal { + built, err := buildMachineResult(response.Evaluation, detail) + if err != nil { + return evaluationResult{}, err + } return evaluationResult{ Outcome: evaluationReady, SubmissionID: submissionID, - Submission: payload, RawEvaluation: response.Evaluation, + Submission: payload, Result: built, }, nil } pending = evaluationResult{Outcome: evaluationPending, SubmissionID: submissionID, Submission: payload} From a819c8211dc4d391a31ee74dce52e6a912f26139 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 09:22:27 +0000 Subject: [PATCH 09/14] cli: add softpractice mcp, a stdio MCP server for coding agents `softpractice mcp [--project DIR]` serves the lesson loop to Claude Code, Codex, Claude Desktop and other MCP clients through the same use cases as the CLI: status, task, material, hints, check, submit, result, submissions and update, plus the task and material as resources. - Tool names, descriptions and schemas are in English for the model; messages for the person follow the CLI language, Russian when none is configured, since the course content is Russian. - submit and update ask the person to confirm with a preview (commit and size; the add/replace/delete list). The confirmation is a multi round-trip tool result, so it works on protocol 2026-07-28 and, through the SDK, as elicitation/create on older clients. A client without elicitation gets confirmation_unavailable; nothing is done. The retry re-prepares the action and refuses when it changed after the person saw it; the state it carries is signed by the server process. - Evaluations are always projected without directions, and the reviewer's questions are never returned; there is no parameter for either. Hints are read-only. check runs only the configuration the server publishes. - Errors are isError results with a stable code and what the person has to do. The server never starts a sign-in, and only JSON-RPC reaches stdout. - status next_actions name the tool to call instead of a CLI command. Uses github.com/modelcontextprotocol/go-sdk v1.8.0. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XKQjpw1FmuSnc8k4EtFspp --- cmd/softpractice/machine.go | 17 +- cmd/softpractice/main.go | 8 + cmd/softpractice/mcp.go | 750 +++++++++++++++++++++++++++++ cmd/softpractice/mcp_test.go | 390 +++++++++++++++ cmd/softpractice/project_checks.go | 9 +- cmd/softpractice/ui.go | 22 + cmd/softpractice/usecase.go | 6 + go.mod | 8 + go.sum | 22 + 9 files changed, 1226 insertions(+), 6 deletions(-) create mode 100644 cmd/softpractice/mcp.go create mode 100644 cmd/softpractice/mcp_test.go diff --git a/cmd/softpractice/machine.go b/cmd/softpractice/machine.go index b2a83a1..0fe36d3 100644 --- a/cmd/softpractice/machine.go +++ b/cmd/softpractice/machine.go @@ -50,11 +50,14 @@ type machineStatus struct { } // nextAction is one step the learner can take now. Command is a CLI command; -// URL is a page for an action that happens on the site. +// Tool is the `softpractice mcp` tool for it, set only in MCP output; URL is a +// page for an action that happens on the site. type nextAction struct { - Code string `json:"code"` - Command string `json:"command,omitempty"` - URL string `json:"url,omitempty"` + Code string `json:"code"` + Command string `json:"command,omitempty"` + Tool string `json:"tool,omitempty"` + Arguments map[string]any `json:"arguments,omitempty"` + URL string `json:"url,omitempty"` } // nextActions derives the learner's next steps from the status, most @@ -186,6 +189,10 @@ type machineSubmissionSummary struct { } func writeStatusJSON(ctx context.Context, output io.Writer, snapshot statusSnapshot) error { + return writeMachineJSON(output, buildMachineStatus(ctx, snapshot)) +} + +func buildMachineStatus(ctx context.Context, snapshot statusSnapshot) machineStatus { user, link, workspace, repository := snapshot.User, snapshot.Link, snapshot.Workspace, snapshot.Repository payload := machineStatus{Kind: "softpractice.status"} payload.Account.ID = user.ID @@ -222,7 +229,7 @@ func writeStatusJSON(ctx context.Context, output io.Writer, snapshot statusSnaps } } payload.NextActions = nextActions(ctx, snapshot) - return writeMachineJSON(output, payload) + return payload } type evaluationAPIResponse struct { diff --git a/cmd/softpractice/main.go b/cmd/softpractice/main.go index bca994d..a913ac2 100644 --- a/cmd/softpractice/main.go +++ b/cmd/softpractice/main.go @@ -242,6 +242,14 @@ func run( return submit(ctx, client, "", remaining[1:], input, output, errorOutput) case "open": return openCurrent(ctx, client, "", remaining[1:], output, errorOutput) + case "mcp": + if *languageValue == "" && !languageChosen(config) { + // An MCP client often starts the server without a locale. The + // course is written in Russian, so its frame follows the content. + settings.Language = languageRussian + ctx = withSettings(ctx, settings) + } + return mcpCommand(ctx, client, remaining[1:], input, output, errorOutput) default: return usageError{message: fmt.Sprintf(text(ctx, "неизвестная команда %q", "unknown command %q"), remaining[0])} } diff --git a/cmd/softpractice/mcp.go b/cmd/softpractice/mcp.go new file mode 100644 index 0000000..7fa4044 --- /dev/null +++ b/cmd/softpractice/mcp.go @@ -0,0 +1,750 @@ +package main + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/arthur-s/softpractice-cli/internal/learnercli" +) + +// mcpReviewFeedback is the only detail at which `softpractice mcp` projects +// an evaluation. There is deliberately no tool parameter for it: an agent can +// show the learner what the review found, but it does not get the reviewer's +// directions as a list of fixes to apply for them. +const mcpReviewFeedback = feedbackWithoutDirections + +// mcpMaxWait bounds wait_seconds. A client stops waiting for a tool call at +// its own timeout, so the agent should ask for less than that. +const mcpMaxWait = 600 + +// mcpMaxCheckOutput bounds the check output a tool result carries; the end of +// the output, where a failure is reported, is kept. +const mcpMaxCheckOutput = 32 << 10 + +const mcpInstructions = `SoftPractice is an engineering practice trainer. The learner solves a lesson in this Git project and submits a commit for automated checks and review. + +Typical loop: status → task and material → hints → check → submit → result → update. Call status first: its next_actions name the tool to call next, or a page where the learner acts. + +The lesson content (task, material, review) is authored in Russian. Talk to the learner in the language they use. + +Boundaries, which are part of the course: +- Results never include the reviewer's directions (what to change) or the text of the reviewer's questions. Do not try to obtain them another way, for example with the softpractice command line; the learner reads them on the result page. +- Never answer the reviewer's questions for the learner; the learner answers them on the result page. +- Opening a new hint is the learner's decision on the assignment page. hints lists only the hints already opened. +- submit and update ask the learner to confirm. When this client cannot ask, they fail with confirmation_unavailable: ask the learner to run the command in a terminal.` + +type mcpServer struct { + useCases learnerUseCases + settings runtimeSettings + stateKey []byte +} + +func mcpCommand( + ctx context.Context, + client *learnercli.Client, + args []string, + input io.Reader, + output, errorOutput io.Writer, +) error { + flags := flag.NewFlagSet("mcp", flag.ContinueOnError) + flags.SetOutput(errorOutput) + project := flags.String("project", "", "project folder; defaults to the current directory") + if err := parseFlags(flags, args); err != nil { + return err + } + if flags.NArg() != 0 { + return usage(ctx, "использование: softpractice mcp [--project DIR]", "usage: softpractice mcp [--project DIR]") + } + directory := strings.TrimSpace(*project) + if directory != "" { + absolute, err := filepath.Abs(directory) + if err != nil { + return err + } + if info, err := os.Stat(absolute); err != nil || !info.IsDir() { + return usage(ctx, "папка проекта не найдена: "+absolute, "project folder not found: "+absolute) + } + directory = absolute + } + server, err := newMCPServer(ctx, newLearnerUseCases(client, directory), errorOutput) + if err != nil { + return err + } + // Standard output belongs to JSON-RPC: nothing else may write to it. + transport := &mcp.IOTransport{Reader: io.NopCloser(input), Writer: nopWriteCloser{output}} + return server.Run(ctx, transport) +} + +type nopWriteCloser struct{ io.Writer } + +func (nopWriteCloser) Close() error { return nil } + +func newMCPServer(ctx context.Context, useCases learnerUseCases, logOutput io.Writer) (*mcp.Server, error) { + key := make([]byte, 32) + if _, err := rand.Read(key); err != nil { + return nil, err + } + s := &mcpServer{useCases: useCases, settings: settingsFromContext(ctx), stateKey: key} + server := mcp.NewServer( + &mcp.Implementation{Name: "softpractice", Title: "SoftPractice", Version: cliVersion}, + &mcp.ServerOptions{ + Instructions: mcpInstructions, + Logger: slog.New(slog.NewTextHandler(logOutput, &slog.HandlerOptions{Level: slog.LevelWarn})), + }, + ) + s.register(server) + return server, nil +} + +func boolPointer(value bool) *bool { return &value } + +func (s *mcpServer) register(server *mcp.Server) { + readOnly := &mcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: boolPointer(true)} + mcp.AddTool(server, &mcp.Tool{ + Name: "status", + Title: "Lesson status", + Description: "Show the linked lesson project: account, lesson and its version, local Git HEAD and whether the tree is clean, " + + "the latest submission, and next_actions. Call it first and after every step. Each next action has a code and either " + + "a tool to call (with arguments), a command, or a url for a page where the learner acts.", + Annotations: readOnly, + }, s.status) + mcp.AddTool(server, &mcp.Tool{ + Name: "task", + Title: "Lesson task", + Description: "Return the assignment of the lesson this folder holds, at the lesson version its files were made for: " + + "title, estimated minutes, and instructions_markdown (authored in Russian). transition or lesson_version_update " + + "is present when the update tool has lesson files to apply.", + Annotations: readOnly, + }, s.task) + mcp.AddTool(server, &mcp.Tool{ + Name: "material", + Title: "Lesson material", + Description: "Return the theory material of the folder's lesson, or of lesson_id (for example an earlier lesson of the practicum). " + + "material is null when the lesson has none.", + Annotations: readOnly, + }, s.material) + mcp.AddTool(server, &mcp.Tool{ + Name: "hints", + Title: "Opened hints", + Description: "Return the prepared hints the learner has already opened for this lesson. status is available while a closed hint " + + "remains, exhausted when all are open, unavailable when the lesson has none. This tool cannot open a hint: the learner " + + "opens the next one on the assignment page at url.", + Annotations: readOnly, + }, s.hints) + mcp.AddTool(server, &mcp.Tool{ + Name: "check", + Title: "Run public checks", + Description: "Run the lesson's public checks (tests, linters) on the working tree, uncommitted changes included, and return " + + "passed and the output. It runs only the checks the server publishes for this lesson; if .softpractice/checks.json " + + "differs it fails with checks_not_published. A failing check is a result (passed: false), not a tool error.", + Annotations: &mcp.ToolAnnotations{DestructiveHint: boolPointer(false), IdempotentHint: true, OpenWorldHint: boolPointer(true)}, + }, s.check) + mcp.AddTool(server, &mcp.Tool{ + Name: "submit", + Title: "Submit for review", + Description: "Submit the committed HEAD of this lesson for evaluation. The working tree must be clean: commit first. " + + "The learner is asked to confirm; declined fails with declined and nothing is sent, and a client that cannot ask fails with " + + "confirmation_unavailable. Local checks are not run here: call check before. With wait_seconds it then waits for the " + + "evaluation like result.", + Annotations: &mcp.ToolAnnotations{DestructiveHint: boolPointer(false), OpenWorldHint: boolPointer(true)}, + }, s.submit) + mcp.AddTool(server, &mcp.Tool{ + Name: "result", + Title: "Evaluation result", + Description: "Return the evaluation of submission_id, or of the latest submission. outcome is ready, pending, or superseded " + + "(that evaluation will have no result; a new commit must be submitted). A ready result has the automated checks with " + + "counterexamples, the review summary, rubric, and findings (what and why, without the reviewer's directions), and the " + + "number of reviewer questions: the learner answers them on result_url when questions_answerable is true. " + + "wait_seconds waits while the evaluation is pending; keep it below this client's tool call timeout.", + Annotations: readOnly, + }, s.result) + mcp.AddTool(server, &mcp.Tool{ + Name: "submissions", + Title: "Lesson submissions", + Description: "List the recent submissions of this lesson, newest first (at most five), with their evaluation state.", + Annotations: readOnly, + }, s.submissions) + mcp.AddTool(server, &mcp.Tool{ + Name: "update", + Title: "Apply lesson update", + Description: "Apply the pending update to this folder: the transition to the next lesson after an accepted result, a new " + + "version of the current lesson, or the missing public checks configuration. The working tree must be clean. The learner " + + "is asked to confirm with the list of files to add, replace, and delete; the update is committed to Git.", + Annotations: &mcp.ToolAnnotations{DestructiveHint: boolPointer(true), OpenWorldHint: boolPointer(true)}, + }, s.update) + + server.AddResource(&mcp.Resource{ + URI: "softpractice://lesson/task", Name: "task", Title: "Lesson task", MIMEType: "text/markdown", + Description: "The assignment of the lesson this folder holds, as Markdown.", + }, s.taskResource) + server.AddResource(&mcp.Resource{ + URI: "softpractice://lesson/material", Name: "material", Title: "Lesson material", MIMEType: "text/markdown", + Description: "The theory material of the lesson this folder holds, as Markdown.", + }, s.materialResource) +} + +// context gives a handler the CLI settings: language and web URL. +func (s *mcpServer) context(ctx context.Context) context.Context { + return withSettings(ctx, s.settings) +} + +type mcpNoInput struct{} + +func (s *mcpServer) status(ctx context.Context, _ *mcp.CallToolRequest, _ mcpNoInput) (*mcp.CallToolResult, machineStatus, error) { + ctx = s.context(ctx) + snapshot, err := s.useCases.Status(ctx) + if err != nil { + return nil, machineStatus{}, mcpToolFailure(ctx, err) + } + payload := buildMachineStatus(ctx, snapshot) + payload.NextActions = mcpNextActions(payload.NextActions) + return nil, payload, nil +} + +// mcpNextActions points each next step at the tool that performs it. A step +// the learner takes on the site keeps its URL. +func mcpNextActions(actions []nextAction) []nextAction { + converted := make([]nextAction, 0, len(actions)) + for _, action := range actions { + switch action.Code { + case "submit", "resubmit": + action.Tool = "submit" + case "apply_update", "update_lesson_version": + action.Tool = "update" + case "wait_result": + action.Tool, action.Arguments = "result", map[string]any{"wait_seconds": 60} + case "read_result": + action.Tool = "result" + } + if strings.HasPrefix(action.Command, "softpractice ") { + action.Command = "" + } + converted = append(converted, action) + } + return converted +} + +func (s *mcpServer) task(ctx context.Context, _ *mcp.CallToolRequest, _ mcpNoInput) (*mcp.CallToolResult, machineTask, error) { + ctx = s.context(ctx) + lesson, assignment, err := s.useCases.Task(ctx) + if err != nil { + return nil, machineTask{}, mcpToolFailure(ctx, err) + } + payload := machineTask{ + Kind: "softpractice.task", WorkspaceID: lesson.Workspace.Workspace.ID, ProjectID: lesson.Link.ProjectID, + Assignment: assignment, URL: webLessonURL(ctx, assignment.ID, assignment.Version, ""), + } + if pendingTransition(lesson.Link, lesson.Workspace) { + payload.Transition = &machineTransition{ + FromAssignmentID: lesson.Link.AssignmentID, FromAssignmentVersion: lesson.Link.AssignmentVersion, + ToAssignmentID: lesson.Workspace.Assignment.ID, ToAssignmentVersion: lesson.Workspace.Assignment.Version, + } + } + if pendingLessonVersion(lesson.Link, lesson.Workspace) { + payload.LessonVersionUpdate = &machineLessonVersionUpdate{ + AssignmentID: lesson.Link.AssignmentID, + FromVersion: lesson.Link.AssignmentVersion, ToVersion: lesson.Workspace.Assignment.Version, + } + } + return nil, payload, nil +} + +type mcpMaterialInput struct { + LessonID string `json:"lesson_id,omitempty" jsonschema:"lesson ID, such as an earlier lesson of the practicum; defaults to the lesson of this folder"` +} + +func (s *mcpServer) material(ctx context.Context, _ *mcp.CallToolRequest, input mcpMaterialInput) (*mcp.CallToolResult, machineMaterial, error) { + ctx = s.context(ctx) + snapshot, err := s.useCases.Material(ctx, strings.TrimSpace(input.LessonID)) + if err != nil { + return nil, machineMaterial{}, mcpToolFailure(ctx, err) + } + return nil, machineMaterial{ + Kind: "softpractice.material", AssignmentID: snapshot.AssignmentID, Version: snapshot.Version, + Material: snapshot.Material, URL: webLessonURL(ctx, snapshot.AssignmentID, snapshot.Version, "/material"), + }, nil +} + +func (s *mcpServer) hints(ctx context.Context, _ *mcp.CallToolRequest, _ mcpNoInput) (*mcp.CallToolResult, machineHints, error) { + ctx = s.context(ctx) + snapshot, err := s.useCases.Hints(ctx) + if err != nil { + return nil, machineHints{}, mcpToolFailure(ctx, err) + } + return nil, machineHints{ + Kind: "softpractice.hints", AssignmentID: snapshot.Lesson.AssignmentID, Version: snapshot.Lesson.Version, + Status: snapshot.Status, Revealed: snapshot.Revealed, + URL: webLessonURL(ctx, snapshot.Lesson.AssignmentID, snapshot.Lesson.Version, ""), + }, nil +} + +type mcpCheckOutput struct { + Kind string `json:"kind"` + Passed bool `json:"passed"` + // Failure says why the checks did not pass; empty when they passed. + Failure string `json:"failure,omitempty"` + Output string `json:"output"` + // Truncated is true when the beginning of the output was cut. + Truncated bool `json:"truncated"` +} + +func (s *mcpServer) check(ctx context.Context, _ *mcp.CallToolRequest, _ mcpNoInput) (*mcp.CallToolResult, mcpCheckOutput, error) { + ctx = s.context(ctx) + var combined lockedBuffer + err := s.useCases.CheckPublished(ctx, &combined, &combined) + payload := mcpCheckOutput{Kind: "softpractice.check", Passed: err == nil} + var failed checkFailedError + if err != nil { + if !errors.As(err, &failed) { + return nil, mcpCheckOutput{}, mcpToolFailure(ctx, err) + } + payload.Failure = err.Error() + } + payload.Output, payload.Truncated = tailString(combined.String(), mcpMaxCheckOutput) + return nil, payload, nil +} + +func tailString(value string, limit int) (string, bool) { + if len(value) <= limit { + return value, false + } + cut := value[len(value)-limit:] + // Do not start in the middle of a UTF-8 sequence. + for len(cut) > 0 && cut[0]&0xC0 == 0x80 { + cut = cut[1:] + } + return cut, true +} + +// lockedBuffer collects the stdout and stderr of checks, which may be written +// concurrently. +type lockedBuffer struct { + mu sync.Mutex + buffer bytes.Buffer +} + +func (b *lockedBuffer) Write(data []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buffer.Write(data) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buffer.String() +} + +type mcpWaitInput struct { + WaitSeconds int `json:"wait_seconds,omitempty" jsonschema:"seconds to wait for a pending evaluation, 0 to 600; 0 returns at once. Keep it below this client's tool call timeout"` +} + +func (s *mcpServer) evaluationWait(ctx context.Context, request *mcp.CallToolRequest, seconds int) (evaluationWait, error) { + if seconds < 0 || seconds > mcpMaxWait { + return evaluationWait{}, usage(ctx, "wait_seconds должен быть от 0 до 600", "wait_seconds must be between 0 and 600") + } + wait := evaluationWait{Wait: seconds > 0, Timeout: time.Duration(seconds) * time.Second} + token := request.Params.GetProgressToken() + if token != nil && request.Session != nil { + polls := 0 + wait.Pending = func(pending machineSubmission) { + polls++ + _ = request.Session.NotifyProgress(ctx, &mcp.ProgressNotificationParams{ + ProgressToken: token, Progress: float64(polls), + Message: fmt.Sprintf(text(ctx, "Проверка ещё идёт: %s, попытка %d из %d", "The evaluation is still running: %s, attempt %d of %d"), + pending.JobState, pending.Attempt, pending.MaxAttempts), + }) + } + } + return wait, nil +} + +type mcpResultInput struct { + SubmissionID string `json:"submission_id,omitempty" jsonschema:"submission ID; defaults to the latest submission of this project"` + WaitSeconds int `json:"wait_seconds,omitempty" jsonschema:"seconds to wait while the evaluation is pending, 0 to 600; 0 returns at once. Keep it below this client's tool call timeout"` +} + +type mcpEvaluation struct { + machineEvaluationResult + // PreviousLessonID is set when the current lesson has no submission yet + // and the accepted submission of the previous lesson is shown instead. + PreviousLessonID string `json:"previous_lesson_id,omitempty"` +} + +func (s *mcpServer) result(ctx context.Context, request *mcp.CallToolRequest, input mcpResultInput) (*mcp.CallToolResult, mcpEvaluation, error) { + ctx = s.context(ctx) + wait, err := s.evaluationWait(ctx, request, input.WaitSeconds) + if err != nil { + return nil, mcpEvaluation{}, mcpToolFailure(ctx, err) + } + submissionID := strings.TrimSpace(input.SubmissionID) + previousLesson := "" + if submissionID != "" && validateSubmissionID(submissionID) != nil { + return nil, mcpEvaluation{}, mcpToolFailure(ctx, + usage(ctx, "ID отправки должен быть каноническим UUID", "submission ID must be a canonical UUID")) + } + if submissionID == "" { + latest, err := s.useCases.LatestSubmission(ctx) + if err != nil { + return nil, mcpEvaluation{}, mcpToolFailure(ctx, err) + } + submissionID, previousLesson = latest.SubmissionID, latest.PredecessorAssignmentID + } + evaluation, err := s.useCases.Evaluation(ctx, submissionID, wait, mcpReviewFeedback) + if err != nil { + return nil, mcpEvaluation{}, mcpToolFailure(ctx, err) + } + return nil, mcpEvaluation{ + machineEvaluationResult: buildMachineEvaluationResult(evaluation, webResultURL(ctx, submissionID)), + PreviousLessonID: previousLesson, + }, nil +} + +func (s *mcpServer) submissions(ctx context.Context, _ *mcp.CallToolRequest, _ mcpNoInput) (*mcp.CallToolResult, machineSubmissions, error) { + ctx = s.context(ctx) + snapshot, err := s.useCases.Submissions(ctx) + if err != nil { + return nil, machineSubmissions{}, mcpToolFailure(ctx, err) + } + return nil, machineSubmissions{ + Kind: "softpractice.submissions", AssignmentID: snapshot.Lesson.AssignmentID, + Version: snapshot.Lesson.Version, History: snapshot.History, + }, nil +} + +type mcpSubmission struct { + machineSubmissionReceipt + // LocalChecks is always not_run: submit does not run them, check does. + LocalChecks string `json:"local_checks"` + // Evaluation is present when wait_seconds was set. + Evaluation *machineEvaluationResult `json:"evaluation,omitempty"` + // EvaluationError says why waiting failed after the submission was sent; + // the result tool reads the evaluation later. + EvaluationError string `json:"evaluation_error,omitempty"` +} + +func (s *mcpServer) submit(ctx context.Context, request *mcp.CallToolRequest, input mcpWaitInput) (*mcp.CallToolResult, mcpSubmission, error) { + ctx = s.context(ctx) + wait, err := s.evaluationWait(ctx, request, input.WaitSeconds) + if err != nil { + return nil, mcpSubmission{}, mcpToolFailure(ctx, err) + } + prepared, err := s.useCases.PrepareSubmission(ctx) + if err != nil { + return nil, mcpSubmission{}, mcpToolFailure(ctx, err) + } + defer prepared.Close() + link, workspace, repository := prepared.Link, prepared.Workspace, prepared.Repository + version := submittedLessonVersion(link, workspace) + fingerprint := strings.Join([]string{link.WorkspaceID, workspace.Assignment.ID, fmt.Sprint(version), repository.CommitSHA}, "\n") + var message strings.Builder + fmt.Fprintf(&message, text(ctx, + "Отправить решение на проверку?\n\nУрок: %s v%d — %s\nCommit: %s\nФайлов: %d, сжатый размер: %s\n", + "Submit this solution for evaluation?\n\nLesson: %s v%d — %s\nCommit: %s\nFiles: %d, compressed size: %s\n"), + workspace.Assignment.ID, version, workspace.Assignment.Title, repository.CommitSHA, + len(repository.Files), formatBytes(repository.CompressedBytes)) + for _, path := range suspiciousPaths(repository.Files) { + fmt.Fprintf(&message, text(ctx, "Внимание: commit содержит потенциально чувствительный файл: %s\n", + "Warning: the commit contains a potentially sensitive file: %s\n"), path) + } + message.WriteString(text(ctx, "\nОтправленную попытку нельзя отозвать.", "\nA submission cannot be withdrawn.")) + confirmation, err := s.confirm(ctx, request, "submit", fingerprint, message.String()) + if err != nil || confirmation != nil { + return confirmation, mcpSubmission{}, err + } + receipt, err := s.useCases.SendSubmission(ctx, prepared) + if err != nil { + return nil, mcpSubmission{}, mcpToolFailure(ctx, err) + } + payload := mcpSubmission{ + machineSubmissionReceipt: machineSubmissionReceipt{ + Kind: "softpractice.submission", SubmissionID: receipt.SubmissionID, RevisionID: receipt.RevisionID, + EvaluationJobID: receipt.EvaluationJobID, JobState: receipt.JobState, Replayed: receipt.Replayed, + SubmittedAt: receipt.SubmittedAt, ResultURL: webResultURL(ctx, receipt.SubmissionID), + LessonUpdate: receipt.LessonUpdate, + }, + LocalChecks: "not_run", + } + if wait.Wait { + evaluation, err := s.useCases.Evaluation(ctx, receipt.SubmissionID, wait, mcpReviewFeedback) + if err != nil { + // The submission was sent: report it, and let result fetch the + // evaluation later. + payload.EvaluationError = err.Error() + return nil, payload, nil + } + built := buildMachineEvaluationResult(evaluation, payload.ResultURL) + payload.Evaluation = &built + } + return nil, payload, nil +} + +type mcpUpdate struct { + Kind string `json:"kind"` + UpdateKind updateKind `json:"update_kind"` + FromAssignmentID string `json:"from_assignment_id"` + FromAssignmentVersion int `json:"from_assignment_version"` + ToAssignmentID string `json:"to_assignment_id"` + ToAssignmentVersion int `json:"to_assignment_version"` + Operations []updateOperation `json:"operations"` + // Report is what `softpractice update` prints for the learner. + Report string `json:"report"` +} + +func (s *mcpServer) update(ctx context.Context, request *mcp.CallToolRequest, _ mcpNoInput) (*mcp.CallToolResult, mcpUpdate, error) { + ctx = s.context(ctx) + plan, err := s.useCases.PrepareUpdate(ctx) + if err != nil { + return nil, mcpUpdate{}, mcpToolFailure(ctx, err) + } + var fingerprint, message strings.Builder + fmt.Fprintf(&fingerprint, "%s\n%s\n%s\n%d\n%s\n%d\n%s", plan.Kind, plan.HeadCommit, plan.FromAssignmentID, + plan.FromAssignmentVersion, plan.ToAssignmentID, plan.ToAssignmentVersion, plan.Ref) + switch plan.Kind { + case updateLessonTransition: + fmt.Fprintf(&message, text(ctx, "Перейти к следующему уроку в этой папке?\n\n%s v%d → %s v%d\n", + "Move this folder to the next lesson?\n\n%s v%d → %s v%d\n"), + plan.FromAssignmentID, plan.FromAssignmentVersion, plan.ToAssignmentID, plan.ToAssignmentVersion) + case updateLessonVersion: + fmt.Fprintf(&message, text(ctx, "Перейти на новую версию урока?\n\n%s v%d → v%d\n", + "Move to the new version of the lesson?\n\n%s v%d → v%d\n"), + plan.FromAssignmentID, plan.FromAssignmentVersion, plan.ToAssignmentVersion) + case updateLocalChecks: + message.WriteString(text(ctx, "Добавить в проект конфигурацию публичных проверок?\n", + "Add the public checks configuration to the project?\n")) + } + if len(plan.Operations) == 0 { + message.WriteString(text(ctx, "\nФайлы проекта не меняются.\n", "\nNo project file changes.\n")) + } else { + message.WriteString(text(ctx, "\nФайлы:\n", "\nFiles:\n")) + } + for _, operation := range plan.Operations { + fmt.Fprintf(&fingerprint, "\n%s %s", operation.Kind, operation.Path) + fmt.Fprintf(&message, " %s %s\n", mcpOperationLabel(ctx, operation.Kind), operation.Path) + } + message.WriteString(text(ctx, "\nИзменения будут закоммичены в Git; прежнее состояние останется в истории.", + "\nThe changes are committed to Git; the previous state stays in history.")) + confirmation, err := s.confirm(ctx, request, "update", fingerprint.String(), message.String()) + if err != nil || confirmation != nil { + return confirmation, mcpUpdate{}, err + } + var report bytes.Buffer + if err := s.useCases.ApplyUpdate(ctx, plan, &report); err != nil { + return nil, mcpUpdate{}, mcpToolFailure(ctx, err) + } + return nil, mcpUpdate{ + Kind: "softpractice.update", UpdateKind: plan.Kind, + FromAssignmentID: plan.FromAssignmentID, FromAssignmentVersion: plan.FromAssignmentVersion, + ToAssignmentID: plan.ToAssignmentID, ToAssignmentVersion: plan.ToAssignmentVersion, + Operations: plan.Operations, Report: strings.TrimSpace(report.String()), + }, nil +} + +func mcpOperationLabel(ctx context.Context, kind string) string { + switch kind { + case "add": + return text(ctx, "добавить", "add") + case "replace": + return text(ctx, "заменить", "replace") + case "delete": + return text(ctx, "удалить", "delete") + default: + return kind + } +} + +// mcpConfirmationState is what a confirmation request carries to the retry +// that answers it: which tool asked, and what exactly the learner was shown. +type mcpConfirmationState struct { + Tool string `json:"tool"` + Fingerprint string `json:"fingerprint"` +} + +const mcpConfirmationID = "confirm" + +// confirm asks the learner to confirm an action through elicitation, as a +// multi round-trip tool result: the client asks the learner and calls the +// tool again with the answer. It returns the input request to send on the +// first call; nil and nil once the learner accepted exactly this action on +// the retry; and a tool error when the client cannot ask, the learner did not +// accept, or the action changed since the learner saw it. +func (s *mcpServer) confirm( + ctx context.Context, + request *mcp.CallToolRequest, + tool, fingerprint, message string, +) (*mcp.CallToolResult, error) { + answer, answered := request.Params.InputResponses[mcpConfirmationID] + if !answered { + if capabilities := request.ClientCapabilities(); capabilities == nil || capabilities.Elicitation == nil || + (capabilities.Elicitation.Form == nil && capabilities.Elicitation.URL != nil) { + return nil, mcpFailure("confirmation_unavailable", fmt.Sprintf(text(ctx, + "этот клиент не умеет запрашивать подтверждение у человека; ничего не сделано. Выполните в терминале: `softpractice %s`", + "this client cannot ask a person to confirm; nothing was done. Run in a terminal: `softpractice %s`"), tool)) + } + state, err := s.sealState(mcpConfirmationState{Tool: tool, Fingerprint: fingerprint}) + if err != nil { + return nil, err + } + return &mcp.CallToolResult{ + InputRequests: mcp.InputRequestMap{mcpConfirmationID: &mcp.ElicitParams{ + Mode: "form", Message: message, + RequestedSchema: map[string]any{"type": "object", "properties": map[string]any{}}, + }}, + RequestState: state, + }, nil + } + state, err := s.openState(request.Params.RequestState) + if err != nil || state.Tool != tool { + return nil, mcpFailure("confirmation_invalid", text(ctx, + "подтверждение не относится к этому действию; ничего не сделано, вызовите инструмент заново", + "the confirmation does not belong to this action; nothing was done, call the tool again")) + } + result, ok := answer.(*mcp.ElicitResult) + if !ok || result.Action != "accept" { + return nil, mcpFailure("declined", text(ctx, + "человек не подтвердил действие; ничего не сделано", + "the person did not confirm; nothing was done")) + } + if !hmac.Equal([]byte(state.Fingerprint), []byte(fingerprint)) { + return nil, mcpFailure("changed_since_confirmation", text(ctx, + "проект или урок изменились после подтверждения; ничего не сделано, вызовите инструмент заново", + "the project or lesson changed after the confirmation; nothing was done, call the tool again")) + } + return nil, nil +} + +// sealState signs the confirmation state with a key that lives only in this +// server process, so a retry cannot carry a state the server did not issue. +func (s *mcpServer) sealState(state mcpConfirmationState) (string, error) { + body, err := json.Marshal(state) + if err != nil { + return "", err + } + mac := hmac.New(sha256.New, s.stateKey) + mac.Write(body) + return base64.RawURLEncoding.EncodeToString(body) + "." + base64.RawURLEncoding.EncodeToString(mac.Sum(nil)), nil +} + +func (s *mcpServer) openState(sealed string) (mcpConfirmationState, error) { + encodedBody, encodedMAC, found := strings.Cut(sealed, ".") + body, bodyErr := base64.RawURLEncoding.DecodeString(encodedBody) + signature, macErr := base64.RawURLEncoding.DecodeString(encodedMAC) + if !found || bodyErr != nil || macErr != nil { + return mcpConfirmationState{}, errors.New("malformed confirmation state") + } + mac := hmac.New(sha256.New, s.stateKey) + mac.Write(body) + if !hmac.Equal(signature, mac.Sum(nil)) { + return mcpConfirmationState{}, errors.New("confirmation state was not issued by this server") + } + var state mcpConfirmationState + if err := json.Unmarshal(body, &state); err != nil { + return mcpConfirmationState{}, err + } + return state, nil +} + +// mcpToolError is a tool error the agent can act on: a stable code, a +// message for the person, and what the person has to do when it is theirs. +type mcpToolError struct { + Code string `json:"code"` + Message string `json:"message"` + Action string `json:"action,omitempty"` + HTTPStatus int `json:"http_status,omitempty"` + APICode string `json:"api_code,omitempty"` + SupportID string `json:"support_id,omitempty"` +} + +// Error is the JSON the tool result carries as its text. +func (e *mcpToolError) Error() string { + body, err := json.Marshal(map[string]*mcpToolError{"error": e}) + if err != nil { + return e.Message + } + return string(body) +} + +func mcpFailure(code, message string) error { + return &mcpToolError{Code: code, Message: message} +} + +// mcpToolFailure classifies err with the codes of `--json` errors and adds +// what the person has to do for those that are theirs. +func mcpToolFailure(ctx context.Context, err error) error { + failure := &mcpToolError{Code: errorCode(err), Message: err.Error()} + var statusError *learnercli.HTTPError + if errors.As(err, &statusError) { + failure.HTTPStatus, failure.APICode, failure.SupportID = statusError.Status, statusError.Code, statusError.SupportID + } + switch failure.Code { + case "login_required": + failure.Action = text(ctx, + "Попросите человека войти в терминале: `softpractice login`. Этот сервер вход не запускает.", + "Ask the person to sign in from a terminal: `softpractice login`. This server does not start a sign-in.") + case "project_not_linked", "not_git_repository": + failure.Action = text(ctx, + "Запустите MCP-сервер в папке проекта урока или передайте её: `softpractice mcp --project DIR`.", + "Start the MCP server in the lesson project folder, or pass it: `softpractice mcp --project DIR`.") + case "checks_not_published": + failure.Action = text(ctx, + "Попросите человека посмотреть команды в .softpractice/checks.json и запустить `softpractice check` в терминале.", + "Ask the person to review the commands in .softpractice/checks.json and run `softpractice check` in a terminal.") + } + return failure +} + +func (s *mcpServer) taskResource(ctx context.Context, request *mcp.ReadResourceRequest) (*mcp.ReadResourceResult, error) { + ctx = s.context(ctx) + _, assignment, err := s.useCases.Task(ctx) + if err != nil { + return nil, mcpToolFailure(ctx, err) + } + body := fmt.Sprintf("# %s\n\n%s\n", assignment.Title, strings.TrimSpace(assignment.InstructionsMarkdown)) + return &mcp.ReadResourceResult{Contents: []*mcp.ResourceContents{{ + URI: request.Params.URI, MIMEType: "text/markdown", Text: body, + }}}, nil +} + +func (s *mcpServer) materialResource(ctx context.Context, request *mcp.ReadResourceRequest) (*mcp.ReadResourceResult, error) { + ctx = s.context(ctx) + snapshot, err := s.useCases.Material(ctx, "") + if err != nil { + return nil, mcpToolFailure(ctx, err) + } + var body strings.Builder + if material := snapshot.Material; material == nil { + fmt.Fprintf(&body, text(ctx, "У урока `%s` нет теоретического материала.\n", "Lesson `%s` has no theory material.\n"), + snapshot.AssignmentID) + } else { + fmt.Fprintf(&body, "# %s\n\n%s\n", material.Title, strings.TrimSpace(material.Markdown)) + if diagram := material.Diagram; diagram.Title != "" { + fmt.Fprintf(&body, "\n## %s\n\n", diagram.Title) + for _, node := range diagram.Nodes { + fmt.Fprintf(&body, "- **%s**: %s\n", node.Title, node.Text) + } + if diagram.TextAlternative != "" { + fmt.Fprintf(&body, "\n%s\n", diagram.TextAlternative) + } + } + } + return &mcp.ReadResourceResult{Contents: []*mcp.ResourceContents{{ + URI: request.Params.URI, MIMEType: "text/markdown", Text: body.String(), + }}}, nil +} diff --git a/cmd/softpractice/mcp_test.go b/cmd/softpractice/mcp_test.go new file mode 100644 index 0000000..1c735a2 --- /dev/null +++ b/cmd/softpractice/mcp_test.go @@ -0,0 +1,390 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// The protocol versions a client may speak: the current one, where the +// confirmation travels as a multi round-trip tool result, and the previous +// one, where the server sends elicitation/create itself. +var mcpTestProtocols = []string{"2026-07-28", "2025-11-25"} + +// leakMarkers are texts of the evaluation the MCP server must never return: +// the reviewer's directions, the counterexamples' next steps, and the text of +// the reviewer's questions. +var leakMarkers = []string{reviewerDirectionMarker, "NEXT-STEP-TEXT", "DEFENSE-QUESTION-TEXT"} + +// mcpLessonAPI is a fake API for a folder on pa-foundation-01 v2 that accepts +// submissions and answers every evaluation with the revise projection. +type mcpLessonAPI struct { + server *httptest.Server + workspaceID string + submissions atomic.Int32 +} + +func newMCPLessonAPI(t *testing.T) *mcpLessonAPI { + t.Helper() + api := &mcpLessonAPI{workspaceID: uuid.NewString()} + api.server = lessonBumpServer(t, api.workspaceID, func(writer http.ResponseWriter, request *http.Request) { + switch { + case request.Method == http.MethodPost && + request.URL.Path == "/v1/workspaces/"+api.workspaceID+"/assignments/pa-foundation-01/submissions": + _, _ = io.Copy(io.Discard, request.Body) + api.submissions.Add(1) + writer.Header().Set("Content-Type", "application/json") + writer.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(writer).Encode(map[string]any{ + "submission_id": currentSubmissionID, "revision_id": uuid.NewString(), "evaluation_job_id": uuid.NewString(), + "job_state": "queued", "replayed": false, "submitted_at": time.Now().UTC(), + "submission_url": "/v1/submissions/" + currentSubmissionID, + "evaluation_url": "/v1/submissions/" + currentSubmissionID + "/evaluation", + }) + case request.URL.Path == "/v1/submissions/"+currentSubmissionID+"/evaluation": + writeTestJSON(writer, reviseEvaluationBody(currentSubmissionID)) + default: + http.NotFound(writer, request) + } + }) + t.Cleanup(api.server.Close) + return api +} + +// connectMCP starts the server on useCases and connects an in-process client +// speaking protocol. +func connectMCP(t *testing.T, useCases learnerUseCases, options *mcp.ClientOptions, protocol string) *mcp.ClientSession { + t.Helper() + ctx := withSettings(context.Background(), runtimeSettings{Language: languageEnglish, WebURL: "https://softpractice.example"}) + server, err := newMCPServer(ctx, useCases, io.Discard) + if err != nil { + t.Fatal(err) + } + serverTransport, clientTransport := mcp.NewInMemoryTransports() + serverSession, err := server.Connect(ctx, serverTransport, nil) + if err != nil { + t.Fatal(err) + } + client := mcp.NewClient(&mcp.Implementation{Name: "test-client", Version: "1"}, options) + session, err := client.Connect(ctx, clientTransport, &mcp.ClientSessionOptions{ProtocolVersion: protocol}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = session.Close() + _ = serverSession.Wait() + }) + return session +} + +func elicitationAnswer(action string, before func()) *mcp.ClientOptions { + return &mcp.ClientOptions{ElicitationHandler: func(context.Context, *mcp.ElicitRequest) (*mcp.ElicitResult, error) { + if before != nil { + before() + } + return &mcp.ElicitResult{Action: action}, nil + }} +} + +// callTool calls a tool and returns its result as JSON, failing the test on +// a protocol error. +func callTool(t *testing.T, session *mcp.ClientSession, name string, arguments map[string]any) (*mcp.CallToolResult, string) { + t.Helper() + if arguments == nil { + arguments = map[string]any{} + } + result, err := session.CallTool(context.Background(), &mcp.CallToolParams{Name: name, Arguments: arguments}) + if err != nil { + t.Fatalf("call %s: %v", name, err) + } + body, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + return result, string(body) +} + +// toolErrorCode is the code of a tool error result. +func toolErrorCode(t *testing.T, result *mcp.CallToolResult) string { + t.Helper() + if !result.IsError || len(result.Content) != 1 { + t.Fatalf("result is not a tool error: %+v", result) + } + var payload struct { + Error mcpToolError `json:"error"` + } + if err := json.Unmarshal([]byte(result.Content[0].(*mcp.TextContent).Text), &payload); err != nil { + t.Fatalf("tool error is not JSON: %v", err) + } + return payload.Error.Code +} + +func TestMCPListsTheLessonLoopToolsOnly(t *testing.T) { + session := connectMCP(t, newLearnerUseCases(nil, t.TempDir()), nil, mcpTestProtocols[0]) + tools, err := session.ListTools(context.Background(), nil) + if err != nil { + t.Fatal(err) + } + var names []string + for _, tool := range tools.Tools { + names = append(names, tool.Name) + readOnly := tool.Annotations != nil && tool.Annotations.ReadOnlyHint + wantReadOnly := !slices.Contains([]string{"check", "submit", "update"}, tool.Name) + if readOnly != wantReadOnly { + t.Errorf("tool %s read-only = %v, want %v", tool.Name, readOnly, wantReadOnly) + } + if tool.Description == "" || tool.OutputSchema == nil { + t.Errorf("tool %s has no description or output schema", tool.Name) + } + schema, _ := json.Marshal(tool.InputSchema) + if strings.Contains(string(schema), "direction") { + t.Errorf("tool %s takes a directions parameter: %s", tool.Name, schema) + } + } + slices.Sort(names) + want := []string{"check", "hints", "material", "result", "status", "submissions", "submit", "task", "update"} + if !slices.Equal(names, want) { + t.Fatalf("tools = %v, want %v", names, want) + } + resources, err := session.ListResources(context.Background(), nil) + if err != nil || len(resources.Resources) != 2 { + t.Fatalf("resources = %+v, %v", resources, err) + } +} + +// Neither result nor submit with wait_seconds returns the reviewer's +// directions, the counterexamples' next steps, or the questions' text, while +// what the review found is there. +func TestMCPNeverReturnsDirectionsOrQuestions(t *testing.T) { + api := newMCPLessonAPI(t) + root := createPinnedLinkedGitRepository(t, api.workspaceID, "pa-foundation-01", 2) + session := connectMCP(t, newLearnerUseCases(savedTestClient(t, api.server.URL), root), + elicitationAnswer("accept", nil), mcpTestProtocols[0]) + + result, body := callTool(t, session, "result", map[string]any{"submission_id": currentSubmissionID}) + _, submitted := callTool(t, session, "submit", map[string]any{"wait_seconds": 30}) + for _, output := range []string{body, submitted} { + for _, marker := range leakMarkers { + if strings.Contains(output, marker) { + t.Fatalf("tool output contains %q:\n%s", marker, output) + } + } + } + if result.IsError || !strings.Contains(body, "FINDING-TITLE-TEXT") || !strings.Contains(body, `"questions_count":1`) || + !strings.Contains(body, `"directions_hidden":true`) { + t.Fatalf("result lacks the review:\n%s", body) + } + if !strings.Contains(submitted, "REVIEW-SUMMARY-TEXT") || api.submissions.Load() != 1 { + t.Fatalf("submit with wait_seconds = %s (submissions %d)", submitted, api.submissions.Load()) + } +} + +func TestMCPSubmitAsksForConfirmation(t *testing.T) { + for _, protocol := range mcpTestProtocols { + t.Run(protocol, func(t *testing.T) { + api := newMCPLessonAPI(t) + root := createPinnedLinkedGitRepository(t, api.workspaceID, "pa-foundation-01", 2) + useCases := newLearnerUseCases(savedTestClient(t, api.server.URL), root) + + declined := connectMCP(t, useCases, elicitationAnswer("decline", nil), protocol) + result, _ := callTool(t, declined, "submit", nil) + if code := toolErrorCode(t, result); code != "declined" || api.submissions.Load() != 0 { + t.Fatalf("declined submit: code %q, submissions %d", code, api.submissions.Load()) + } + + var shown string + accepted := connectMCP(t, useCases, &mcp.ClientOptions{ + ElicitationHandler: func(_ context.Context, request *mcp.ElicitRequest) (*mcp.ElicitResult, error) { + shown = request.Params.Message + return &mcp.ElicitResult{Action: "accept"}, nil + }, + }, protocol) + result, body := callTool(t, accepted, "submit", nil) + if result.IsError || api.submissions.Load() != 1 || !strings.Contains(body, currentSubmissionID) || + !strings.Contains(body, `"local_checks":"not_run"`) { + t.Fatalf("accepted submit = %s (submissions %d)", body, api.submissions.Load()) + } + if !strings.Contains(shown, "pa-foundation-01 v2") || !strings.Contains(shown, "Commit: ") { + t.Fatalf("confirmation message = %q", shown) + } + }) + } +} + +// A client that cannot ask the person is refused: nothing is sent, and the +// person is told to use the terminal. +func TestMCPSubmitWithoutElicitationIsRefused(t *testing.T) { + for _, protocol := range mcpTestProtocols { + t.Run(protocol, func(t *testing.T) { + api := newMCPLessonAPI(t) + root := createPinnedLinkedGitRepository(t, api.workspaceID, "pa-foundation-01", 2) + session := connectMCP(t, newLearnerUseCases(savedTestClient(t, api.server.URL), root), nil, protocol) + result, body := callTool(t, session, "submit", nil) + if code := toolErrorCode(t, result); code != "confirmation_unavailable" || + !strings.Contains(body, "softpractice submit") { + t.Fatalf("submit without elicitation: %s", body) + } + if api.submissions.Load() != 0 { + t.Fatalf("submitted %d times without a confirmation", api.submissions.Load()) + } + }) + } +} + +// When HEAD moves while the person is looking at the confirmation, the +// confirmed commit is not the one that would be sent: nothing is sent. +func TestMCPSubmitRefusesAChangeAfterConfirmation(t *testing.T) { + api := newMCPLessonAPI(t) + root := createPinnedLinkedGitRepository(t, api.workspaceID, "pa-foundation-01", 2) + session := connectMCP(t, newLearnerUseCases(savedTestClient(t, api.server.URL), root), + elicitationAnswer("accept", func() { commitLearnerWork(t, root, "late.py", "# late change\n") }), mcpTestProtocols[0]) + result, _ := callTool(t, session, "submit", nil) + if code := toolErrorCode(t, result); code != "changed_since_confirmation" || api.submissions.Load() != 0 { + t.Fatalf("code %q, submissions %d", code, api.submissions.Load()) + } +} + +// update shows the files it will change and applies them only on accept. +func TestMCPUpdateConfirmsTheFileList(t *testing.T) { + workspaceID := uuid.NewString() + server, refactored := refactoredLessonServer(t, workspaceID) + defer server.Close() + root := createPinnedLinkedGitRepository(t, workspaceID, "pa-foundation-01", 1) + var shown string + session := connectMCP(t, newLearnerUseCases(savedTestClient(t, server.URL), root), &mcp.ClientOptions{ + ElicitationHandler: func(_ context.Context, request *mcp.ElicitRequest) (*mcp.ElicitResult, error) { + shown = request.Params.Message + return &mcp.ElicitResult{Action: "accept"}, nil + }, + }, mcpTestProtocols[0]) + result, body := callTool(t, session, "update", nil) + if result.IsError || !strings.Contains(body, `"update_kind":"lesson_version"`) { + t.Fatalf("update = %s", body) + } + if !strings.Contains(shown, "replace main.py") || !strings.Contains(shown, "v1 → v2") { + t.Fatalf("confirmation message = %q", shown) + } + if content, err := os.ReadFile(filepath.Join(root, "main.py")); err != nil || string(content) != string(refactored) { + t.Fatalf("main.py = %q after an accepted update", content) + } +} + +func TestMCPLoginRequiredDoesNotLeakTokens(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if strings.HasPrefix(request.URL.Path, "/v1/auth/device") { + t.Errorf("the MCP server started a sign-in: %s", request.URL.Path) + } + writeTestAPIError(writer, http.StatusUnauthorized, "invalid_credentials") + })) + defer server.Close() + root := createPinnedLinkedGitRepository(t, uuid.NewString(), "pa-foundation-01", 2) + session := connectMCP(t, newLearnerUseCases(savedTestClient(t, server.URL), root), nil, mcpTestProtocols[0]) + for _, tool := range []string{"status", "task", "result"} { + result, body := callTool(t, session, tool, nil) + if code := toolErrorCode(t, result); code != "login_required" { + t.Fatalf("%s: code %q: %s", tool, code, body) + } + if strings.Contains(body, "access-token") || strings.Contains(body, "refresh-token") { + t.Fatalf("%s leaks a token: %s", tool, body) + } + if !strings.Contains(body, "softpractice login") { + t.Fatalf("%s does not tell the person how to sign in: %s", tool, body) + } + } +} + +// status points each next step at an MCP tool, not at a CLI command. +func TestMCPNextActionsNameTools(t *testing.T) { + actions := mcpNextActions([]nextAction{ + {Code: "submit", Command: "softpractice submit --wait"}, + {Code: "wait_result", Command: "softpractice result --wait"}, + {Code: "stash_changes", Command: "git stash"}, + {Code: "answer_questions", URL: "https://softpractice.example/submissions/x/result"}, + }) + if actions[0].Tool != "submit" || actions[0].Command != "" || + actions[1].Tool != "result" || actions[1].Arguments["wait_seconds"] != 60 || + actions[2].Tool != "" || actions[2].Command != "git stash" || + actions[3].Tool != "" || actions[3].URL == "" { + t.Fatalf("next actions = %+v", actions) + } +} + +// `softpractice mcp` speaks JSON-RPC on standard output and writes nothing +// else there. +func TestMCPCommandWritesOnlyJSONRPCToStdout(t *testing.T) { + directory := t.TempDir() + t.Setenv(configDirectoryEnv, directory) + t.Setenv("SOFTPRACTICE_CREDENTIALS_DIR", directory) + t.Setenv("SOFTPRACTICE_API_URL", "http://127.0.0.1:9") + t.Setenv("SOFTPRACTICE_LANGUAGE", "") + t.Setenv("LC_ALL", "") + t.Setenv("LC_MESSAGES", "") + t.Setenv("LANG", "") + + inputReader, inputWriter := io.Pipe() + outputReader, outputWriter := io.Pipe() + var errorOutput lockedBuffer + done := make(chan error, 1) + go func() { + done <- run(context.Background(), []string{"mcp", "--project", t.TempDir()}, inputReader, outputWriter, &errorOutput) + _ = outputWriter.Close() + }() + lines := bufio.NewScanner(outputReader) + lines.Buffer(make([]byte, 1<<20), 1<<20) + send := func(message string) { + if _, err := io.WriteString(inputWriter, message+"\n"); err != nil { + t.Fatal(err) + } + } + receive := func() map[string]any { + if !lines.Scan() { + t.Fatalf("no response: %v; stderr: %s", lines.Err(), errorOutput.String()) + } + var message map[string]any + if err := json.Unmarshal(lines.Bytes(), &message); err != nil || message["jsonrpc"] != "2.0" { + t.Fatalf("stdout line is not JSON-RPC: %q", lines.Text()) + } + return message + } + send(`{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"t","version":"1"}}}`) + receive() + send(`{"jsonrpc":"2.0","method":"notifications/initialized"}`) + send(`{"jsonrpc":"2.0","id":2,"method":"tools/list"}`) + if tools := receive()["result"].(map[string]any)["tools"].([]any); len(tools) != 9 { + t.Fatalf("tools/list returned %d tools", len(tools)) + } + // Outside a Git repository the error is a tool result, in Russian by + // default when no language is configured anywhere. + send(`{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"task","arguments":{}}}`) + response := receive()["result"].(map[string]any) + text := response["content"].([]any)[0].(map[string]any)["text"].(string) + if response["isError"] != true || !strings.Contains(text, "not_git_repository") || !strings.Contains(text, "Запустите") { + t.Fatalf("task outside a project = %v", response) + } + _ = inputWriter.Close() + select { + case err := <-done: + if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, mcp.ErrConnectionClosed) { + t.Fatalf("mcp command: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("the MCP server did not stop when standard input closed") + } + for lines.Scan() { + t.Fatalf("unexpected stdout after shutdown: %q", lines.Text()) + } +} diff --git a/cmd/softpractice/project_checks.go b/cmd/softpractice/project_checks.go index b3cef5c..adc9e6b 100644 --- a/cmd/softpractice/project_checks.go +++ b/cmd/softpractice/project_checks.go @@ -131,10 +131,17 @@ func (u learnerUseCases) CheckPublished(ctx context.Context, output, errorOutput "Локальные публичные проверки рабочего дерева:", "Local public checks for the working tree:")) if err := localchecks.Run(ctx, repository.Root, checks, output, errorOutput); err != nil { - return err + return checkFailedError{err} } fmt.Fprintln(output, text(ctx, "Все локальные публичные проверки прошли.", "All local public checks passed.")) return nil } + +// checkFailedError is a check that ran and did not pass, as opposed to checks +// that could not be run at all. +type checkFailedError struct{ err error } + +func (e checkFailedError) Error() string { return e.err.Error() } +func (e checkFailedError) Unwrap() error { return e.err } diff --git a/cmd/softpractice/ui.go b/cmd/softpractice/ui.go index 1fe2551..1b8e5e4 100644 --- a/cmd/softpractice/ui.go +++ b/cmd/softpractice/ui.go @@ -111,6 +111,24 @@ func defaultSettings(store configStore) runtimeSettings { } } +// languageChosen reports whether the learner chose a language anywhere the +// CLI looks for one: the saved setting, SOFTPRACTICE_LANGUAGE, or a Russian +// or English system locale. The --lang flag is checked by the caller. +func languageChosen(store configStore) bool { + if config, err := store.Load(); err == nil && config.Language != "" { + return true + } + if strings.TrimSpace(os.Getenv("SOFTPRACTICE_LANGUAGE")) != "" { + return true + } + for _, name := range []string{"LC_ALL", "LC_MESSAGES", "LANG"} { + if languageFromLocale(os.Getenv(name)) != "" { + return true + } + } + return languageFromLocale(systemLocaleName()) != "" +} + func detectSystemLanguage() language { for _, name := range []string{"LC_ALL", "LC_MESSAGES", "LANG"} { if detected := languageFromLocale(os.Getenv(name)); detected != "" { @@ -176,6 +194,7 @@ func printHelp(ctx context.Context, output io.Writer) { submissions Показать отправки урока update Применить переход к следующему уроку open Открыть задание, материал или результат в браузере + mcp MCP-сервер для Claude Code, Codex и Claude Desktop Настройки: set api-url URL Сохранить адрес API @@ -218,6 +237,7 @@ Solution: submissions Show the lesson's submissions update Apply the next lesson transition open Open the task, material, or result in a browser + mcp MCP server for Claude Code, Codex, and Claude Desktop Settings: set api-url URL Save the API address @@ -269,6 +289,8 @@ func printCommandHelp(ctx context.Context, output io.Writer, command string) err russian, english = "Использование: softpractice check\n\nЗапускает публичные проверки для текущего рабочего дерева, включая незакоммиченные изменения. Ничего не отправляет на сервер.", "Usage: softpractice check\n\nRuns public checks against the current working tree, including uncommitted changes. Does not submit anything to the server." case "update": russian, english = "Использование: softpractice update\n\nПосле принятия решения применяет в этом же проекте переход к следующему уроку: добавляет, заменяет или удаляет только явно объявленные файлы.\n\nПереход применяется к принятому решению прошлого урока. Если текущий коммит — другой, команда ничего не меняет и показывает оба коммита и способ продолжить.", "Usage: softpractice update\n\nAfter acceptance, applies the next-lesson transition in the same project, adding, replacing, or removing only explicitly declared files.\n\nThe transition applies to the accepted solution of the previous lesson. When the current commit is a different one, the command changes nothing and shows both commits and how to continue." + case "mcp": + russian, english = "Использование: softpractice mcp [--project DIR]\n\nЗапускает локальный MCP-сервер на stdio для Claude Code, Codex, Claude Desktop и других агентов. Инструменты: status, task, material, hints, check, submit, result, submissions, update. submit и update просят подтверждения человека. Рекомендации рецензента и текст его вопросов сервер не отдаёт. Сначала войдите: softpractice login. --project задаёт папку проекта, если клиент запускает сервер не в ней. Язык сообщений: --lang, SOFTPRACTICE_LANGUAGE или set lang; по умолчанию русский.", "Usage: softpractice mcp [--project DIR]\n\nStarts a local stdio MCP server for Claude Code, Codex, Claude Desktop, and other agents. Tools: status, task, material, hints, check, submit, result, submissions, update. submit and update ask a person to confirm. The server never returns the reviewer's directions or the text of the reviewer's questions. Sign in first: softpractice login. --project sets the project folder when the client starts the server elsewhere. Message language: --lang, SOFTPRACTICE_LANGUAGE, or set lang; Russian by default." case "open": russian, english = "Использование: softpractice open [task|material|result] [--web URL] [--no-browser]\n\nОткрывает в браузере задание, материал или последний результат. Без аргумента — результат, а до первой отправки — задание.", "Usage: softpractice open [task|material|result] [--web URL] [--no-browser]\n\nOpens the task, the material, or the latest result in a browser. Without an argument: the result, or the task before the first submission." case "set": diff --git a/cmd/softpractice/usecase.go b/cmd/softpractice/usecase.go index 9461083..a245d40 100644 --- a/cmd/softpractice/usecase.go +++ b/cmd/softpractice/usecase.go @@ -430,6 +430,9 @@ const ( type evaluationWait struct { Wait bool Timeout time.Duration + // Pending, when set, is told about every pending answer before the next + // poll, so a caller can report progress while it waits. + Pending func(machineSubmission) } // defaultEvaluationPoll is used when the server names no interval. @@ -509,6 +512,9 @@ func (u learnerUseCases) Evaluation( } return pending, nil } + if wait.Pending != nil && pending.Outcome != "" { + wait.Pending(pending.Submission) + } if err := u.sleep(ctx, delay); err != nil { return evaluationResult{}, err } diff --git a/go.mod b/go.mod index 619de50..66808f2 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.26.3 require ( github.com/google/uuid v1.6.0 + github.com/modelcontextprotocol/go-sdk v1.8.0 github.com/zalando/go-keyring v0.2.8 golang.org/x/sys v0.47.0 ) @@ -11,4 +12,11 @@ require ( require ( github.com/danieljoos/wincred v1.2.3 // indirect github.com/godbus/dbus/v5 v5.2.2 // indirect + github.com/google/jsonschema-go v0.4.3 // indirect + github.com/segmentio/asm v1.1.3 // indirect + github.com/segmentio/encoding v0.5.4 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + golang.org/x/oauth2 v0.35.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/time v0.15.0 // indirect ) diff --git a/go.sum b/go.sum index a9a684a..a18b48b 100644 --- a/go.sum +++ b/go.sum @@ -4,17 +4,39 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/godbus/dbus/v5 v5.2.2 h1:TUR3TgtSVDmjiXOgAAyaZbYmIeP3DPkld3jgKGV8mXQ= github.com/godbus/dbus/v5 v5.2.2/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= +github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/modelcontextprotocol/go-sdk v1.8.0 h1:KIvahhYqwtbeniWVPs3TcXEA7b8jEtwfBpOTAI+Urx4= +github.com/modelcontextprotocol/go-sdk v1.8.0/go.mod h1:dL7u98E/zjJTGzEq+j30jQ8K2k1mb6LeAH4inEcSGts= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= +github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= +github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= +github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/zalando/go-keyring v0.2.8 h1:6sD/Ucpl7jNq10rM2pgqTs0sZ9V3qMrqfIIy5YPccHs= github.com/zalando/go-keyring v0.2.8/go.mod h1:tsMo+VpRq5NGyKfxoBVjCuMrG47yj8cmakZDO5QGii0= +golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= +golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 319fc7df4214955a320797810f258a2e066465cd Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 09:23:21 +0000 Subject: [PATCH 10/14] docs: describe softpractice mcp How to connect it to Claude Code, Codex and Claude Desktop, the tools and resources it serves, what it leaves to the learner (confirmation, directions, reviewer questions, hints, unpublished checks), the error format, and the language of its messages. The npm README is kept equal to README.md. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XKQjpw1FmuSnc8k4EtFspp --- README.md | 87 ++++++++++++++++++++++++++++++++++++++++ README_RU.md | 88 +++++++++++++++++++++++++++++++++++++++++ packaging/npm/README.md | 87 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 262 insertions(+) diff --git a/README.md b/README.md index 463037e..d450f1d 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,7 @@ softpractice update | `softpractice submissions` | Show the recent submissions of the lesson | | `softpractice update` | Apply the transition to the next lesson, or move to a newer version of the current lesson (replaces only lesson files, never yours; optional until the lesson ends) | | `softpractice open` | Open the task, material, or latest result in the browser | +| `softpractice mcp` | Run the MCP server for Claude Code, Codex, Claude Desktop, and other agents | | `softpractice project restore` | Restore a project from the latest usable revision | Run `softpractice help ` for detailed help. @@ -136,6 +137,92 @@ softpractice result --wait --timeout 10m --json Exit codes: `0` done, `1` error, `2` invalid arguments, `3` the evaluation is not ready yet, `4` the evaluation was superseded and will have no result. +## MCP server + +`softpractice mcp` is a local MCP server on stdio. It lets Claude Code, Codex, +Claude Desktop, and other MCP clients work through the lesson with you, using +the same operations as the commands above. Sign in first with +`softpractice login`: the server never starts a sign-in itself. + +The server works on the lesson project in the directory the client starts it +in. When the client starts it elsewhere, as Claude Desktop does, pass the +project folder with `--project`. + +**Claude Code**, from the project folder: + +```bash +claude mcp add softpractice -- softpractice mcp +``` + +**Codex**, in `~/.codex/config.toml`: + +```toml +[mcp_servers.softpractice] +command = "softpractice" +args = ["mcp"] +# Allow result and submit to wait for an evaluation. +tool_timeout_sec = 180 +``` + +**Claude Desktop**, in `claude_desktop_config.json`. Use the full path of the +`softpractice` executable if the app does not find it on `PATH`: + +```json +{ + "mcpServers": { + "softpractice": { + "command": "softpractice", + "args": ["mcp", "--project", "/path/to/your/lesson-project"] + } + } +} +``` + +| Tool | What it does | +| --- | --- | +| `status` | Project state and `next_actions`, each naming the tool to call next or a page for you | +| `task` | The assignment of the lesson in this folder | +| `material` | The theory material; `lesson_id` for another lesson | +| `hints` | The hints you have already opened | +| `check` | Run the public checks on the working tree | +| `submit` | Submit the committed `HEAD`, after you confirm; `wait_seconds` waits for the result | +| `result` | The evaluation result; `wait_seconds` waits for it | +| `submissions` | The recent submissions of the lesson | +| `update` | Apply the lesson update, after you confirm the list of files it changes | + +The task and the material are also available as the resources +`softpractice://lesson/task` and `softpractice://lesson/material`. + +What the server leaves to you: + +- `submit` and `update` ask you to confirm in the client, with the commit or + the list of files. If the client cannot ask (it does not support MCP + elicitation), they do nothing and return `confirmation_unavailable`; run + `softpractice submit` or `softpractice update` in a terminal instead. +- Results never include the reviewer's recommendations (`--directions`) or + the text of the reviewer's questions. You answer the questions on the result + page; the agent gets their number and the link. +- The agent sees only the hints you have opened; you open the next one on the + assignment page. +- `check` runs only the checks the server publishes for the lesson. If + `.softpractice/checks.json` differs, it returns `checks_not_published`: + review the commands and run `softpractice check` in a terminal. +- `login`, `logout`, `starter`, `project restore`, `set`, and `open` are not + available through MCP. + +These limits apply to the MCP tools. An agent that can run shell commands can +also run the `softpractice` commands themselves. + +Errors are tool results with `isError` and a JSON body +`{"error": {"code", "message", "action"}}`; `action` says what you have to do, +for example `login_required` asks you to run `softpractice login`. + +Tool names and descriptions are in English, for the model. Messages meant for +you, such as the confirmation and error messages, use the CLI language (see +below); when no language is configured, the MCP server uses Russian, the +language of the course. To change it, set `SOFTPRACTICE_LANGUAGE` in the +server's `env` in the client configuration, or pass `--lang en` before `mcp`. + ## Language SoftPractice CLI supports English and Russian and selects a language from the diff --git a/README_RU.md b/README_RU.md index 17a0e5a..813e59d 100644 --- a/README_RU.md +++ b/README_RU.md @@ -89,6 +89,7 @@ softpractice update | `softpractice submissions` | Показать последние отправки урока | | `softpractice update` | Применить переход к следующему уроку или перейти на новую версию текущего урока (заменяет только файлы урока, ваши файлы не трогает; до конца урока необязательно) | | `softpractice open` | Открыть задание, материал или последний результат в браузере | +| `softpractice mcp` | Запустить MCP-сервер для Claude Code, Codex, Claude Desktop и других агентов | | `softpractice project restore` | Восстановить проект из последней подходящей ревизии | Подробную справку можно получить командой `softpractice help <команда>`. @@ -137,6 +138,93 @@ softpractice result --wait --timeout 10m --json Коды выхода: `0` — готово, `1` — ошибка, `2` — неверные аргументы, `3` — проверка ещё не готова, `4` — проверка заменена, результата не будет. +## MCP-сервер + +`softpractice mcp` — локальный MCP-сервер на stdio. Через него Claude Code, +Codex, Claude Desktop и другие MCP-клиенты проходят урок вместе с тобой теми же +операциями, что и команды выше. Сначала войди: `softpractice login`. Сам сервер +вход никогда не запускает. + +Сервер работает с проектом урока в папке, в которой его запустил клиент. Если +клиент запускает его в другом месте, как Claude Desktop, передай папку +проекта флагом `--project`. + +**Claude Code**, из папки проекта: + +```bash +claude mcp add softpractice -- softpractice mcp +``` + +**Codex**, в `~/.codex/config.toml`: + +```toml +[mcp_servers.softpractice] +command = "softpractice" +args = ["mcp"] +# Чтобы result и submit успевали дождаться проверки. +tool_timeout_sec = 180 +``` + +**Claude Desktop**, в `claude_desktop_config.json`. Если приложение не находит +`softpractice` в `PATH`, укажи полный путь к исполняемому файлу: + +```json +{ + "mcpServers": { + "softpractice": { + "command": "softpractice", + "args": ["mcp", "--project", "/path/to/your/lesson-project"] + } + } +} +``` + +| Инструмент | Что делает | +| --- | --- | +| `status` | Состояние проекта и `next_actions`: какой инструмент вызвать дальше или какая страница нужна тебе | +| `task` | Задание урока в этой папке | +| `material` | Теоретический материал; `lesson_id` — для другого урока | +| `hints` | Подсказки, которые ты уже открыл | +| `check` | Запустить публичные проверки для рабочего дерева | +| `submit` | Отправить закоммиченный `HEAD` после твоего подтверждения; `wait_seconds` — дождаться результата | +| `result` | Результат проверки; `wait_seconds` — дождаться его | +| `submissions` | Последние отправки урока | +| `update` | Применить обновление урока после того, как ты подтвердишь список файлов | + +Задание и материал доступны также как resources +`softpractice://lesson/task` и `softpractice://lesson/material`. + +Что сервер оставляет тебе: + +- `submit` и `update` просят подтверждения в клиенте и показывают commit или + список файлов. Если клиент не умеет спрашивать (не поддерживает MCP + elicitation), они ничего не делают и возвращают `confirmation_unavailable`; + тогда выполни `softpractice submit` или `softpractice update` в терминале. +- В результатах никогда нет рекомендаций рецензента (`--directions`) и текста + его вопросов. На вопросы ты отвечаешь на странице результата; агент получает + только их число и ссылку. +- Агент видит только подсказки, которые ты уже открыл; следующую ты открываешь + на странице задания. +- `check` запускает только проверки, которые сервер публикует для урока. Если + `.softpractice/checks.json` отличается, он возвращает + `checks_not_published`: посмотри команды и запусти `softpractice check` в + терминале. +- `login`, `logout`, `starter`, `project restore`, `set` и `open` через MCP + недоступны. + +Эти ограничения относятся к инструментам MCP. Агент, который может выполнять +команды в терминале, может вызвать и сами команды `softpractice`. + +Ошибки приходят как результаты инструментов с `isError` и JSON +`{"error": {"code", "message", "action"}}`; в `action` написано, что сделать +тебе, например при `login_required` — выполнить `softpractice login`. + +Имена и описания инструментов — на английском, их читает модель. Сообщения для +тебя — подтверждение и ошибки — на языке CLI (см. ниже); если язык нигде не +задан, MCP-сервер говорит по-русски, на языке курса. Чтобы сменить язык, задай +`SOFTPRACTICE_LANGUAGE` в `env` сервера в настройках клиента или передай +`--lang en` перед `mcp`. + ## Язык SoftPractice CLI поддерживает английский и русский языки и по возможности diff --git a/packaging/npm/README.md b/packaging/npm/README.md index 463037e..d450f1d 100644 --- a/packaging/npm/README.md +++ b/packaging/npm/README.md @@ -89,6 +89,7 @@ softpractice update | `softpractice submissions` | Show the recent submissions of the lesson | | `softpractice update` | Apply the transition to the next lesson, or move to a newer version of the current lesson (replaces only lesson files, never yours; optional until the lesson ends) | | `softpractice open` | Open the task, material, or latest result in the browser | +| `softpractice mcp` | Run the MCP server for Claude Code, Codex, Claude Desktop, and other agents | | `softpractice project restore` | Restore a project from the latest usable revision | Run `softpractice help ` for detailed help. @@ -136,6 +137,92 @@ softpractice result --wait --timeout 10m --json Exit codes: `0` done, `1` error, `2` invalid arguments, `3` the evaluation is not ready yet, `4` the evaluation was superseded and will have no result. +## MCP server + +`softpractice mcp` is a local MCP server on stdio. It lets Claude Code, Codex, +Claude Desktop, and other MCP clients work through the lesson with you, using +the same operations as the commands above. Sign in first with +`softpractice login`: the server never starts a sign-in itself. + +The server works on the lesson project in the directory the client starts it +in. When the client starts it elsewhere, as Claude Desktop does, pass the +project folder with `--project`. + +**Claude Code**, from the project folder: + +```bash +claude mcp add softpractice -- softpractice mcp +``` + +**Codex**, in `~/.codex/config.toml`: + +```toml +[mcp_servers.softpractice] +command = "softpractice" +args = ["mcp"] +# Allow result and submit to wait for an evaluation. +tool_timeout_sec = 180 +``` + +**Claude Desktop**, in `claude_desktop_config.json`. Use the full path of the +`softpractice` executable if the app does not find it on `PATH`: + +```json +{ + "mcpServers": { + "softpractice": { + "command": "softpractice", + "args": ["mcp", "--project", "/path/to/your/lesson-project"] + } + } +} +``` + +| Tool | What it does | +| --- | --- | +| `status` | Project state and `next_actions`, each naming the tool to call next or a page for you | +| `task` | The assignment of the lesson in this folder | +| `material` | The theory material; `lesson_id` for another lesson | +| `hints` | The hints you have already opened | +| `check` | Run the public checks on the working tree | +| `submit` | Submit the committed `HEAD`, after you confirm; `wait_seconds` waits for the result | +| `result` | The evaluation result; `wait_seconds` waits for it | +| `submissions` | The recent submissions of the lesson | +| `update` | Apply the lesson update, after you confirm the list of files it changes | + +The task and the material are also available as the resources +`softpractice://lesson/task` and `softpractice://lesson/material`. + +What the server leaves to you: + +- `submit` and `update` ask you to confirm in the client, with the commit or + the list of files. If the client cannot ask (it does not support MCP + elicitation), they do nothing and return `confirmation_unavailable`; run + `softpractice submit` or `softpractice update` in a terminal instead. +- Results never include the reviewer's recommendations (`--directions`) or + the text of the reviewer's questions. You answer the questions on the result + page; the agent gets their number and the link. +- The agent sees only the hints you have opened; you open the next one on the + assignment page. +- `check` runs only the checks the server publishes for the lesson. If + `.softpractice/checks.json` differs, it returns `checks_not_published`: + review the commands and run `softpractice check` in a terminal. +- `login`, `logout`, `starter`, `project restore`, `set`, and `open` are not + available through MCP. + +These limits apply to the MCP tools. An agent that can run shell commands can +also run the `softpractice` commands themselves. + +Errors are tool results with `isError` and a JSON body +`{"error": {"code", "message", "action"}}`; `action` says what you have to do, +for example `login_required` asks you to run `softpractice login`. + +Tool names and descriptions are in English, for the model. Messages meant for +you, such as the confirmation and error messages, use the CLI language (see +below); when no language is configured, the MCP server uses Russian, the +language of the course. To change it, set `SOFTPRACTICE_LANGUAGE` in the +server's `env` in the client configuration, or pass `--lang en` before `mcp`. + ## Language SoftPractice CLI supports English and Russian and selects a language from the From 31ea97f6a3d818d912d7f1cdbad89d7e61d5393f Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 09:26:02 +0000 Subject: [PATCH 11/14] cli: expect the system display language in the mcp stdio test On Windows the system locale reports the display language, so the MCP server correctly answers in English there instead of the Russian default. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XKQjpw1FmuSnc8k4EtFspp --- cmd/softpractice/mcp_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/softpractice/mcp_test.go b/cmd/softpractice/mcp_test.go index 1c735a2..c94d65a 100644 --- a/cmd/softpractice/mcp_test.go +++ b/cmd/softpractice/mcp_test.go @@ -368,11 +368,16 @@ func TestMCPCommandWritesOnlyJSONRPCToStdout(t *testing.T) { t.Fatalf("tools/list returned %d tools", len(tools)) } // Outside a Git repository the error is a tool result, in Russian by - // default when no language is configured anywhere. + // default when no language is configured anywhere. The system locale + // still counts: Windows reports its display language. + wantAction := "Запустите" + if languageFromLocale(systemLocaleName()) == languageEnglish { + wantAction = "Start the MCP server" + } send(`{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"task","arguments":{}}}`) response := receive()["result"].(map[string]any) text := response["content"].([]any)[0].(map[string]any)["text"].(string) - if response["isError"] != true || !strings.Contains(text, "not_git_repository") || !strings.Contains(text, "Запустите") { + if response["isError"] != true || !strings.Contains(text, "not_git_repository") || !strings.Contains(text, wantAction) { t.Fatalf("task outside a project = %v", response) } _ = inputWriter.Close() From d49f221ef972344e3ff9428a154a6dbd0dd5a499 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 09:29:32 +0000 Subject: [PATCH 12/14] cli: turn off git housekeeping in tests After a commit git may start a detached gc or maintenance process that keeps writing to .git after the test returns, so t.TempDir cleanup fails with "directory not empty" (seen on the ubuntu runner, git 2.55). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XKQjpw1FmuSnc8k4EtFspp --- cmd/softpractice/testmain_test.go | 37 +++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 cmd/softpractice/testmain_test.go diff --git a/cmd/softpractice/testmain_test.go b/cmd/softpractice/testmain_test.go new file mode 100644 index 0000000..26bfd5d --- /dev/null +++ b/cmd/softpractice/testmain_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" + "os" + "strconv" + "testing" +) + +// TestMain turns off git's automatic housekeeping for every git command the +// tests run. After a commit git may start a detached gc or maintenance +// process that keeps writing to .git after the test has returned, and the +// temporary directory cleanup then fails with "directory not empty". +func TestMain(m *testing.M) { + settings := [][2]string{{"gc.auto", "0"}, {"maintenance.auto", "false"}} + count, _ := strconv.Atoi(os.Getenv("GIT_CONFIG_COUNT")) + for _, setting := range settings { + os.Setenv(fmt.Sprintf("GIT_CONFIG_KEY_%d", count), setting[0]) + os.Setenv(fmt.Sprintf("GIT_CONFIG_VALUE_%d", count), setting[1]) + count++ + } + os.Setenv("GIT_CONFIG_COUNT", strconv.Itoa(count)) + os.Exit(m.Run()) +} + +func TestGitHousekeepingIsOffInTests(t *testing.T) { + root := createGitRepository(t, "{}") + for _, key := range []string{"gc.auto", "maintenance.auto"} { + value, err := gitOutput(t.Context(), root, "config", "--get", key) + if err != nil { + t.Fatalf("git config %s: %v", key, err) + } + if got := string(value); got != "0\n" && got != "false\n" { + t.Fatalf("git config %s = %q", key, got) + } + } +} From 9a86fb31a8ad57eadff2f710575292b4c5e06023 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 10:27:25 +0000 Subject: [PATCH 13/14] cli: suggest a 45-second wait in mcp next actions 60 seconds equals Codex's default tool call timeout, so a suggested wait could time out the call before the result arrives. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01XKQjpw1FmuSnc8k4EtFspp --- cmd/softpractice/mcp.go | 8 +++++++- cmd/softpractice/mcp_test.go | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/softpractice/mcp.go b/cmd/softpractice/mcp.go index 7fa4044..27d96da 100644 --- a/cmd/softpractice/mcp.go +++ b/cmd/softpractice/mcp.go @@ -34,6 +34,12 @@ const mcpReviewFeedback = feedbackWithoutDirections // its own timeout, so the agent should ask for less than that. const mcpMaxWait = 600 +// mcpSuggestedWait is the wait_seconds status suggests for a pending +// evaluation. It stays below the shortest default tool call timeout among +// common clients, Codex's 60 seconds; the agent calls result again if the +// evaluation is still pending. +const mcpSuggestedWait = 45 + // mcpMaxCheckOutput bounds the check output a tool result carries; the end of // the output, where a failure is reported, is kept. const mcpMaxCheckOutput = 32 << 10 @@ -229,7 +235,7 @@ func mcpNextActions(actions []nextAction) []nextAction { case "apply_update", "update_lesson_version": action.Tool = "update" case "wait_result": - action.Tool, action.Arguments = "result", map[string]any{"wait_seconds": 60} + action.Tool, action.Arguments = "result", map[string]any{"wait_seconds": mcpSuggestedWait} case "read_result": action.Tool = "result" } diff --git a/cmd/softpractice/mcp_test.go b/cmd/softpractice/mcp_test.go index c94d65a..b2ff275 100644 --- a/cmd/softpractice/mcp_test.go +++ b/cmd/softpractice/mcp_test.go @@ -316,7 +316,7 @@ func TestMCPNextActionsNameTools(t *testing.T) { {Code: "answer_questions", URL: "https://softpractice.example/submissions/x/result"}, }) if actions[0].Tool != "submit" || actions[0].Command != "" || - actions[1].Tool != "result" || actions[1].Arguments["wait_seconds"] != 60 || + actions[1].Tool != "result" || actions[1].Arguments["wait_seconds"] != mcpSuggestedWait || actions[2].Tool != "" || actions[2].Command != "git stash" || actions[3].Tool != "" || actions[3].URL == "" { t.Fatalf("next actions = %+v", actions) From 90c516e62461d5c5f22d07f0ff7ecc6d488ac6da Mon Sep 17 00:00:00 2001 From: Arthur Sultanbekov Date: Wed, 30 Sep 2026 16:50:21 +0500 Subject: [PATCH 14/14] =?UTF-8?q?=D0=B8=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D1=8B=20=D0=BF=D1=80=D0=BE=D0=B1=D0=BB=D0=B5?= =?UTF-8?q?=D0=BC=D1=8B=20=D0=BF=D0=B0=D1=80=D0=B0=D0=BB=D0=BB=D0=B5=D0=BB?= =?UTF-8?q?=D1=8C=D0=BD=D0=BE=D0=B3=D0=BE=20=D0=B8=D1=81=D0=BF=D0=BE=D0=BB?= =?UTF-8?q?=D1=8C=D0=B7=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/softpractice/mcp.go | 56 +++-- cmd/softpractice/mcp_test.go | 38 +++ internal/learnercli/client.go | 59 +++-- .../learnercli/credential_concurrency_test.go | 219 ++++++++++++++++++ internal/learnercli/credential_lock.go | 46 ++++ internal/learnercli/credential_lock_unix.go | 18 ++ .../learnercli/credential_lock_windows.go | 19 ++ internal/learnercli/learnercli_test.go | 12 +- internal/localchecks/checks.go | 41 +++- internal/localchecks/checks_test.go | 17 ++ 10 files changed, 479 insertions(+), 46 deletions(-) create mode 100644 internal/learnercli/credential_concurrency_test.go create mode 100644 internal/learnercli/credential_lock.go create mode 100644 internal/learnercli/credential_lock_unix.go create mode 100644 internal/learnercli/credential_lock_windows.go diff --git a/cmd/softpractice/mcp.go b/cmd/softpractice/mcp.go index 27d96da..b4ff56f 100644 --- a/cmd/softpractice/mcp.go +++ b/cmd/softpractice/mcp.go @@ -323,39 +323,55 @@ func (s *mcpServer) check(ctx context.Context, _ *mcp.CallToolRequest, _ mcpNoIn } payload.Failure = err.Error() } - payload.Output, payload.Truncated = tailString(combined.String(), mcpMaxCheckOutput) + payload.Output, payload.Truncated = combined.snapshot() return nil, payload, nil } -func tailString(value string, limit int) (string, bool) { - if len(value) <= limit { - return value, false - } - cut := value[len(value)-limit:] - // Do not start in the middle of a UTF-8 sequence. - for len(cut) > 0 && cut[0]&0xC0 == 0x80 { - cut = cut[1:] - } - return cut, true -} - -// lockedBuffer collects the stdout and stderr of checks, which may be written -// concurrently. +// lockedBuffer retains only the tail of stdout and stderr, which checks may +// write concurrently. Memory stays bounded even while the process is running. type lockedBuffer struct { - mu sync.Mutex - buffer bytes.Buffer + mu sync.Mutex + buffer []byte + truncated bool } func (b *lockedBuffer) Write(data []byte) (int, error) { b.mu.Lock() defer b.mu.Unlock() - return b.buffer.Write(data) + n := len(data) + if b.buffer == nil { + b.buffer = make([]byte, 0, mcpMaxCheckOutput) + } + if len(b.buffer)+n > mcpMaxCheckOutput { + b.truncated = true + if n >= mcpMaxCheckOutput { + b.buffer = append(b.buffer[:0], data[n-mcpMaxCheckOutput:]...) + return n, nil + } + keep := mcpMaxCheckOutput - n + copy(b.buffer, b.buffer[len(b.buffer)-keep:]) + b.buffer = b.buffer[:keep] + } + b.buffer = append(b.buffer, data...) + return n, nil } -func (b *lockedBuffer) String() string { +func (b *lockedBuffer) snapshot() (string, bool) { b.mu.Lock() defer b.mu.Unlock() - return b.buffer.String() + data := b.buffer + if b.truncated { + // Do not start in the middle of a UTF-8 sequence. + for len(data) > 0 && data[0]&0xC0 == 0x80 { + data = data[1:] + } + } + return string(data), b.truncated +} + +func (b *lockedBuffer) String() string { + value, _ := b.snapshot() + return value } type mcpWaitInput struct { diff --git a/cmd/softpractice/mcp_test.go b/cmd/softpractice/mcp_test.go index b2ff275..f9260c8 100644 --- a/cmd/softpractice/mcp_test.go +++ b/cmd/softpractice/mcp_test.go @@ -12,9 +12,11 @@ import ( "path/filepath" "slices" "strings" + "sync" "sync/atomic" "testing" "time" + "unicode/utf8" "github.com/google/uuid" "github.com/modelcontextprotocol/go-sdk/mcp" @@ -25,6 +27,42 @@ import ( // one, where the server sends elicitation/create itself. var mcpTestProtocols = []string{"2026-07-28", "2025-11-25"} +func TestMCPCheckOutputRetainsABoundedTail(t *testing.T) { + var buffer lockedBuffer + var expected strings.Builder + for _, chunk := range []string{"prefix", strings.Repeat("я", mcpMaxCheckOutput), "\nFAIL: last check\n"} { + expected.WriteString(chunk) + if n, err := buffer.Write([]byte(chunk)); err != nil || n != len(chunk) { + t.Fatalf("write = %d, %v", n, err) + } + if len(buffer.buffer) > mcpMaxCheckOutput || cap(buffer.buffer) > mcpMaxCheckOutput { + t.Fatal("check output buffer exceeded its limit") + } + } + output, truncated := buffer.snapshot() + want := expected.String()[expected.Len()-mcpMaxCheckOutput:] + for len(want) > 0 && want[0]&0xC0 == 0x80 { + want = want[1:] + } + if !truncated || output != want || !utf8.ValidString(output) { + t.Fatalf("invalid retained tail: length=%d, truncated=%t", len(output), truncated) + } +} + +func TestMCPCheckOutputAcceptsConcurrentStreams(t *testing.T) { + var buffer lockedBuffer + var writers sync.WaitGroup + for _, chunk := range []string{strings.Repeat("out", 20000), strings.Repeat("err", 20000)} { + writers.Go(func() { _, _ = buffer.Write([]byte(chunk)) }) + } + writers.Wait() + _, _ = buffer.Write([]byte("\nFAIL: final marker\n")) + output, truncated := buffer.snapshot() + if !truncated || len(output) > mcpMaxCheckOutput || !strings.HasSuffix(output, "\nFAIL: final marker\n") { + t.Fatalf("unexpected tail: length=%d, truncated=%t", len(output), truncated) + } +} + // leakMarkers are texts of the evaluation the MCP server must never return: // the reviewer's directions, the counterexamples' next steps, and the text of // the reviewer's questions. diff --git a/internal/learnercli/client.go b/internal/learnercli/client.go index ddbf6b6..ac17c08 100644 --- a/internal/learnercli/client.go +++ b/internal/learnercli/client.go @@ -258,6 +258,11 @@ func (c *Client) Submit( } func (c *Client) SaveCredentials(credentials Credentials) error { + unlock, err := c.Store.lock(context.Background()) + if err != nil { + return err + } + defer unlock() if credentials.APIURL != c.BaseURL { return errors.New("credentials belong to a different API URL") } @@ -269,6 +274,11 @@ func (c *Client) SaveCredentials(credentials Credentials) error { } func (c *Client) Logout(ctx context.Context) error { + unlock, err := c.Store.lock(ctx) + if err != nil { + return err + } + defer unlock() credentials, err := c.Store.Load() if err != nil { return err @@ -295,38 +305,43 @@ func (c *Client) Logout(ctx context.Context) error { } func (c *Client) validCredentials(ctx context.Context) (Credentials, error) { - c.credentialsMu.Lock() - var credentials Credentials - if c.credentials != nil { - credentials = *c.credentials + return c.authorizedCredentials(ctx, "") +} + +func (c *Client) refresh(ctx context.Context, rejected Credentials) (Credentials, error) { + return c.authorizedCredentials(ctx, rejected.AccessToken) +} + +// Serialize rotation across MCP calls and CLI processes sharing this store. +// Reload the refresh credential under the lock: another process may have +// rotated it, signed in, or signed out since the last request. +func (c *Client) authorizedCredentials(ctx context.Context, rejectedAccessToken string) (Credentials, error) { + unlock, err := c.Store.lock(ctx) + if err != nil { + return Credentials{}, err } - c.credentialsMu.Unlock() - if credentials.RefreshToken == "" { - loaded, err := c.Store.Load() - if err != nil { - return Credentials{}, err - } - credentials = loaded + defer unlock() + credentials, err := c.Store.Load() + if err != nil { + return Credentials{}, err } if credentials.APIURL != c.BaseURL { return Credentials{}, loginRequired("stored credentials belong to a different API URL; login again") } - if time.Now().Add(30 * time.Second).Before(credentials.AccessExpiresAt) { + c.credentialsMu.Lock() + if cached := c.credentials; cached != nil && cached.APIURL == credentials.APIURL && + cached.RefreshToken == credentials.RefreshToken { + credentials = *cached + } + c.credentialsMu.Unlock() + if time.Now().Add(30*time.Second).Before(credentials.AccessExpiresAt) && + credentials.AccessToken != rejectedAccessToken { return credentials, nil } now := time.Now() - if !credentials.RefreshIdleExpiresAt.IsZero() && - !now.Before(credentials.RefreshIdleExpiresAt) { - return Credentials{}, loginRequired("CLI login expired; run `softpractice login`") - } - if !credentials.RefreshAbsoluteExpiresAt.IsZero() && - !now.Before(credentials.RefreshAbsoluteExpiresAt) { + if !now.Before(credentials.RefreshIdleExpiresAt) || !now.Before(credentials.RefreshAbsoluteExpiresAt) { return Credentials{}, loginRequired("CLI login expired; run `softpractice login`") } - return c.refresh(ctx, credentials) -} - -func (c *Client) refresh(ctx context.Context, credentials Credentials) (Credentials, error) { var tokens tokenSet if err := c.jsonRequest(ctx, http.MethodPost, "/v1/auth/tokens/refresh", "", map[string]string{ "refresh_token": credentials.RefreshToken, diff --git a/internal/learnercli/credential_concurrency_test.go b/internal/learnercli/credential_concurrency_test.go new file mode 100644 index 0000000..6740337 --- /dev/null +++ b/internal/learnercli/credential_concurrency_test.go @@ -0,0 +1,219 @@ +package learnercli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestConcurrentCredentialsDoNotReuseRefreshTokens(t *testing.T) { + for _, separateClients := range []bool{false, true} { + t.Run(fmt.Sprintf("separate_clients=%t", separateClients), func(t *testing.T) { + var mu sync.Mutex + current := "initial" + rotations := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/auth/tokens/refresh" { + _, _ = w.Write([]byte(`{}`)) + return + } + var input map[string]string + _ = json.NewDecoder(r.Body).Decode(&input) + mu.Lock() + defer mu.Unlock() + if input["refresh_token"] != current { + w.WriteHeader(401) + _, _ = w.Write([]byte(`{"error":{"code":"invalid_credentials","message":"refresh reuse"}}`)) + return + } + rotations++ + current = fmt.Sprintf("refresh-%d", rotations) + writeRefreshTokens(w, current, fmt.Sprintf("access-%d", rotations)) + })) + defer server.Close() + store := CredentialStore{Path: filepath.Join(t.TempDir(), "credentials.json"), Secrets: newMemorySecretStore()} + first, _ := NewClient(server.URL, store) + if err := first.SaveCredentials(testRefreshCredentials(server.URL, "initial")); err != nil { + t.Fatal(err) + } + clients := []*Client{first, first} + if separateClients { + clients[1], _ = NewClient(server.URL, store) + } + start := make(chan struct{}) + results := make(chan error, len(clients)) + for _, client := range clients { + go func() { + <-start + var out map[string]any + results <- client.AuthorizedJSON(context.Background(), "GET", "/v1/me", nil, &out) + }() + } + close(start) + for range clients { + if err := <-results; err != nil { + t.Fatal(err) + } + } + want := 1 + if separateClients { + want = 2 + } + mu.Lock() + defer mu.Unlock() + if rotations != want { + t.Fatalf("rotations = %d, want %d", rotations, want) + } + }) + } +} + +func TestConcurrentUnauthorizedResponsesUseTheNewAccessToken(t *testing.T) { + var rejected atomic.Int32 + var rotations atomic.Int32 + bothRejected := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/auth/tokens/refresh" { + rotations.Add(1) + writeRefreshTokens(w, "rotated", "new-access") + return + } + if r.Header.Get("Authorization") == "Bearer old-access" { + if rejected.Add(1) == 2 { + close(bothRejected) + } + select { + case <-bothRejected: + case <-r.Context().Done(): + return + } + w.WriteHeader(401) + _, _ = w.Write([]byte(`{"error":{"code":"invalid_credentials","message":"expired"}}`)) + return + } + _, _ = w.Write([]byte(`{}`)) + })) + defer server.Close() + store := CredentialStore{Path: filepath.Join(t.TempDir(), "credentials.json"), Secrets: newMemorySecretStore()} + client, _ := NewClient(server.URL, store) + credentials := testRefreshCredentials(server.URL, "initial") + credentials.AccessToken, credentials.AccessExpiresAt = "old-access", time.Now().Add(time.Hour) + if err := client.SaveCredentials(credentials); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + results := make(chan error, 2) + for range 2 { + go func() { var out map[string]any; results <- client.AuthorizedJSON(ctx, "GET", "/v1/me", nil, &out) }() + } + for range 2 { + if err := <-results; err != nil { + t.Fatal(err) + } + } + if rotations.Load() != 1 { + t.Fatalf("rotations = %d, want 1", rotations.Load()) + } +} + +func TestRunningClientObservesTerminalLoginAndLogout(t *testing.T) { + var refreshed atomic.Value + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/auth/tokens/refresh" { + var input map[string]string + _ = json.NewDecoder(r.Body).Decode(&input) + refreshed.Store(input["refresh_token"]) + if input["refresh_token"] != "terminal-login" { + w.WriteHeader(401) + return + } + writeRefreshTokens(w, "rotated", "new-access") + return + } + _, _ = w.Write([]byte(`{}`)) + })) + defer server.Close() + store := CredentialStore{Path: filepath.Join(t.TempDir(), "credentials.json"), Secrets: newMemorySecretStore()} + running, _ := NewClient(server.URL, store) + terminal, _ := NewClient(server.URL, store) + old := testRefreshCredentials(server.URL, "old-session") + old.AccessToken, old.AccessExpiresAt = "old-access", time.Now().Add(time.Hour) + if err := running.SaveCredentials(old); err != nil { + t.Fatal(err) + } + if err := terminal.SaveCredentials(testRefreshCredentials(server.URL, "terminal-login")); err != nil { + t.Fatal(err) + } + var out map[string]any + if err := running.AuthorizedJSON(context.Background(), "GET", "/v1/me", nil, &out); err != nil { + t.Fatal(err) + } + if refreshed.Load() != "terminal-login" { + t.Fatalf("refreshed %q", refreshed.Load()) + } + if err := terminal.Logout(context.Background()); err != nil { + t.Fatal(err) + } + if err := running.AuthorizedJSON(context.Background(), "GET", "/v1/me", nil, &out); !errors.Is(err, ErrLoginRequired) { + t.Fatalf("after logout: %v", err) + } +} + +func TestCredentialLockAcrossProcesses(t *testing.T) { + if path := os.Getenv("SOFTPRACTICE_TEST_LOCK_PATH"); path != "" { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + unlock, err := (CredentialStore{Path: path}).lock(ctx) + if os.Getenv("SOFTPRACTICE_TEST_LOCK_HELD") == "1" { + if err == nil { + unlock() + t.Fatal("acquired lock held by parent") + } + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatal(err) + } + } else { + if err != nil { + t.Fatal(err) + } + unlock() + } + return + } + store := CredentialStore{Path: filepath.Join(t.TempDir(), "credentials.json")} + unlock, err := store.lock(context.Background()) + if err != nil { + t.Fatal(err) + } + defer unlock() + for _, held := range []string{"1", "0"} { + if held == "0" { + unlock() + } + command := exec.Command(os.Args[0], "-test.run=^TestCredentialLockAcrossProcesses$") + command.Env = append(os.Environ(), "SOFTPRACTICE_TEST_LOCK_PATH="+store.Path, "SOFTPRACTICE_TEST_LOCK_HELD="+held) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("child: %v\n%s", err, output) + } + } +} + +func testRefreshCredentials(api, refresh string) Credentials { + return Credentials{APIURL: api, RefreshToken: refresh, RefreshIdleExpiresAt: time.Now().Add(time.Hour), RefreshAbsoluteExpiresAt: time.Now().Add(2 * time.Hour)} +} + +func writeRefreshTokens(w http.ResponseWriter, refresh, access string) { + _ = json.NewEncoder(w).Encode(map[string]any{"access_token": access, "expires_in_seconds": 900, "refresh_token": refresh, + "refresh_idle_expires_at": time.Now().Add(time.Hour), "refresh_absolute_expires_at": time.Now().Add(2 * time.Hour)}) +} diff --git a/internal/learnercli/credential_lock.go b/internal/learnercli/credential_lock.go new file mode 100644 index 0000000..e97c286 --- /dev/null +++ b/internal/learnercli/credential_lock.go @@ -0,0 +1,46 @@ +package learnercli + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" +) + +// Keep the lock file in place: deleting it could let another process lock a +// different inode while a previous holder is still using the credentials. +func (s CredentialStore) lock(ctx context.Context) (func(), error) { + if s.Path == "" { + return nil, fmt.Errorf("credential metadata path is not configured") + } + if err := os.MkdirAll(filepath.Dir(s.Path), 0o700); err != nil { + return nil, err + } + file, err := os.OpenFile(s.Path+".lock", os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("open credential lock: %w", err) + } + for { + if err := ctx.Err(); err != nil { + file.Close() + return nil, err + } + locked, err := tryCredentialLock(file) + if err != nil { + file.Close() + return nil, fmt.Errorf("lock credentials: %w", err) + } + if locked { + return func() { file.Close() }, nil + } + timer := time.NewTimer(25 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + file.Close() + return nil, ctx.Err() + case <-timer.C: + } + } +} diff --git a/internal/learnercli/credential_lock_unix.go b/internal/learnercli/credential_lock_unix.go new file mode 100644 index 0000000..73fd3d3 --- /dev/null +++ b/internal/learnercli/credential_lock_unix.go @@ -0,0 +1,18 @@ +//go:build !windows + +package learnercli + +import ( + "errors" + "os" + + "golang.org/x/sys/unix" +) + +func tryCredentialLock(file *os.File) (bool, error) { + err := unix.Flock(int(file.Fd()), unix.LOCK_EX|unix.LOCK_NB) + if errors.Is(err, unix.EWOULDBLOCK) || errors.Is(err, unix.EAGAIN) { + return false, nil + } + return err == nil, err +} diff --git a/internal/learnercli/credential_lock_windows.go b/internal/learnercli/credential_lock_windows.go new file mode 100644 index 0000000..65d345f --- /dev/null +++ b/internal/learnercli/credential_lock_windows.go @@ -0,0 +1,19 @@ +//go:build windows + +package learnercli + +import ( + "errors" + "os" + + "golang.org/x/sys/windows" +) + +func tryCredentialLock(file *os.File) (bool, error) { + err := windows.LockFileEx(windows.Handle(file.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, + 0, 1, 0, &windows.Overlapped{}) + if errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + return false, nil + } + return err == nil, err +} diff --git a/internal/learnercli/learnercli_test.go b/internal/learnercli/learnercli_test.go index f21d5ef..49a040c 100644 --- a/internal/learnercli/learnercli_test.go +++ b/internal/learnercli/learnercli_test.go @@ -13,6 +13,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "testing" "time" @@ -390,13 +391,18 @@ func TestDownloadCourseUpdateRefreshesAfterUnauthorizedResponse(t *testing.T) { } } -type memorySecretStore struct{ values map[string]string } +type memorySecretStore struct { + mu sync.Mutex + values map[string]string +} func newMemorySecretStore() *memorySecretStore { return &memorySecretStore{values: make(map[string]string)} } func (s *memorySecretStore) Get(service, account string) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() value, ok := s.values[service+"\x00"+account] if !ok { return "", keyring.ErrNotFound @@ -405,6 +411,8 @@ func (s *memorySecretStore) Get(service, account string) (string, error) { } func (s *memorySecretStore) Set(service, account, secret string) error { + s.mu.Lock() + defer s.mu.Unlock() if s.values == nil { return errors.New("secret store is not initialized") } @@ -413,6 +421,8 @@ func (s *memorySecretStore) Set(service, account, secret string) error { } func (s *memorySecretStore) Delete(service, account string) error { + s.mu.Lock() + defer s.mu.Unlock() delete(s.values, service+"\x00"+account) return nil } diff --git a/internal/localchecks/checks.go b/internal/localchecks/checks.go index df3a1ec..92208bf 100644 --- a/internal/localchecks/checks.go +++ b/internal/localchecks/checks.go @@ -12,6 +12,8 @@ import ( "path/filepath" "strings" "time" + "unicode" + "unicode/utf8" ) const RelativePath = ".softpractice/checks.json" @@ -104,8 +106,11 @@ func Run(ctx context.Context, root string, config Config, output, errorOutput io // Only waitProcess owns cancellation, so the parent is not killed before // the entire process tree can be terminated. command.Cancel = func() error { return nil } - var stdout bytes.Buffer - command.Stdout = io.MultiWriter(output, &stdout) + var stdout stdoutObserver + command.Stdout = output + if check.RequireEmptyStdout { + command.Stdout = io.MultiWriter(output, &stdout) + } command.Stderr = errorOutput if err := command.Start(); err != nil { cancel() @@ -123,9 +128,39 @@ func Run(ctx context.Context, root string, config Config, output, errorOutput io if err != nil { return fmt.Errorf("check %q failed: %w", check.Name, err) } - if check.RequireEmptyStdout && len(bytes.TrimSpace(stdout.Bytes())) != 0 { + if check.RequireEmptyStdout && stdout.hasOutput() { return fmt.Errorf("check %q failed: expected no output", check.Name) } } return nil } + +// stdoutObserver preserves the TrimSpace semantics of require_empty_stdout +// without retaining the output. A whitespace rune may span multiple writes. +type stdoutObserver struct { + pending [utf8.UTFMax]byte + n int + nonSpace bool +} + +func (s *stdoutObserver) Write(data []byte) (int, error) { + for _, value := range data { + if s.nonSpace { + break + } + s.pending[s.n] = value + s.n++ + for s.n > 0 && utf8.FullRune(s.pending[:s.n]) { + r, size := utf8.DecodeRune(s.pending[:s.n]) + if !unicode.IsSpace(r) { + s.nonSpace = true + break + } + copy(s.pending[:], s.pending[size:s.n]) + s.n -= size + } + } + return len(data), nil +} + +func (s *stdoutObserver) hasOutput() bool { return s.nonSpace || s.n != 0 } diff --git a/internal/localchecks/checks_test.go b/internal/localchecks/checks_test.go index 4bf64e4..d69ed43 100644 --- a/internal/localchecks/checks_test.go +++ b/internal/localchecks/checks_test.go @@ -31,6 +31,23 @@ func TestRunStopsInOrderAtFirstFailure(t *testing.T) { } } +func TestStdoutObserverPreservesWhitespaceSemanticsAcrossWrites(t *testing.T) { + for _, input := range []string{"", " \t\r\n", strings.Repeat("\u2003\u00a0\u3000", 10000), " \nx\t", "\xff", "\xe2\x80"} { + for _, chunkSize := range []int{1, 2, 7, 4096} { + var observer stdoutObserver + for start := 0; start < len(input); start += chunkSize { + end := min(start+chunkSize, len(input)) + if n, err := observer.Write([]byte(input[start:end])); err != nil || n != end-start { + t.Fatalf("write = %d, %v", n, err) + } + } + if want := len(bytes.TrimSpace([]byte(input))) != 0; observer.hasOutput() != want { + t.Fatalf("chunk size %d, input length %d: hasOutput = %t, want %t", chunkSize, len(input), observer.hasOutput(), want) + } + } + } +} + func TestRunReportsMissingExecutableAndEmptyOutputContract(t *testing.T) { err := Run(context.Background(), t.TempDir(), Config{Checks: []Check{{ Name: "missing", Executable: "softpractice-definitely-missing-executable", TimeoutSeconds: 1,