diff --git a/cli/cmd_export.go b/cli/cmd_export.go new file mode 100644 index 0000000..e1e9305 --- /dev/null +++ b/cli/cmd_export.go @@ -0,0 +1,40 @@ +package cli + +import ( + "os" + + "github.com/spf13/cobra" +) + +func (a *App) exportCmd() *cobra.Command { + var outFile string + cmd := &cobra.Command{ + Use: "export", + Short: "Export all TIOBE index entries as JSONL", + Long: `export fetches the current TIOBE index and writes one JSON record per line. + +Examples: + tiobe export > tiobe.jsonl + tiobe export --out tiobe.jsonl + tiobe export -o csv > tiobe.csv`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + entries, err := a.client.Index(cmd.Context()) + if err != nil { + return mapFetchErr(err) + } + if outFile != "" { + f, err := os.Create(outFile) + if err != nil { + return codeError(exitError, err) + } + defer f.Close() + r := a.newRendererTo(f) + return r.Render(entries) + } + return a.renderOrEmpty(entries, len(entries)) + }, + } + cmd.Flags().StringVar(&outFile, "out", "", "write output to FILE instead of stdout") + return cmd +} diff --git a/cli/cmd_info.go b/cli/cmd_info.go new file mode 100644 index 0000000..4f597b0 --- /dev/null +++ b/cli/cmd_info.go @@ -0,0 +1,24 @@ +package cli + +import "github.com/spf13/cobra" + +func (a *App) infoCmd() *cobra.Command { + return &cobra.Command{ + Use: "info", + Short: "Show TIOBE index statistics", + Long: `info prints aggregate statistics: total languages tracked and the current +top language. + +Examples: + tiobe info + tiobe info -o json`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + info, err := a.client.Stats(cmd.Context()) + if err != nil { + return mapFetchErr(err) + } + return a.render(info) + }, + } +} diff --git a/cli/output.go b/cli/output.go new file mode 100644 index 0000000..24fb9e3 --- /dev/null +++ b/cli/output.go @@ -0,0 +1,32 @@ +package cli + +import ( + "io" + + render "github.com/tamnd/tiobe-cli/pkg" +) + +type Format = render.Format + +const ( + FormatTable = render.FormatTable + FormatJSON = render.FormatJSON + FormatJSONL = render.FormatJSONL + FormatCSV = render.FormatCSV + FormatTSV = render.FormatTSV + FormatURL = render.FormatURL + FormatRaw = render.FormatRaw +) + +func NewRenderer(w io.Writer, format Format, fields []string, noHeader bool, tmpl string) *render.Renderer { + return render.New(w, format, fields, noHeader, tmpl) +} + +// newRendererTo builds a renderer writing to w using the App's current settings. +func (a *App) newRendererTo(w io.Writer) *render.Renderer { + format := Format(a.output) + if !format.Valid() { + format = FormatJSONL + } + return NewRenderer(w, format, a.fields, a.noHeader, a.template) +} diff --git a/cli/root.go b/cli/root.go index de84313..4d1ea84 100644 --- a/cli/root.go +++ b/cli/root.go @@ -2,30 +2,117 @@ package cli import ( + "fmt" + "os" + + "github.com/mattn/go-isatty" "github.com/spf13/cobra" + "github.com/tamnd/tiobe-cli/tiobe" ) -// Build metadata, set via -ldflags at release time. var ( Version = "dev" Commit = "none" Date = "unknown" ) -// Root builds the root command and its subtree. +const ( + exitError = 1 + exitUsage = 2 + exitNoData = 3 +) + +type ExitError struct { + Code int + Err error +} + +func (e *ExitError) Error() string { + if e.Err != nil { + return e.Err.Error() + } + return fmt.Sprintf("exit %d", e.Code) +} + +func (e *ExitError) Unwrap() error { return e.Err } + +func codeError(code int, err error) error { return &ExitError{Code: code, Err: err} } + +type App struct { + client *tiobe.Client + cfg tiobe.Config + output string + fields []string + noHeader bool + template string + limit int +} + func Root() *cobra.Command { - root := &cobra.Command{ - Use: "tiobe", - Short: "A command line for tiobe.", - Long: `A command line for tiobe. + app := &App{cfg: tiobe.DefaultConfig()} -This is a fresh scaffold. Add your commands here on top of the tiobe -library package, then wire them into Root with root.AddCommand.`, + root := &cobra.Command{ + Use: "tiobe", + Short: "Browse the TIOBE programming language index from the command line", SilenceUsage: true, SilenceErrors: true, + PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { + return app.setup() + }, } - root.AddCommand(newVersionCmd()) - // TODO: root.AddCommand(newGetCmd()), etc. + pf := root.PersistentFlags() + pf.StringVarP(&app.output, "output", "o", "auto", "output: table|json|jsonl|csv|tsv|url|raw") + pf.StringSliceVar(&app.fields, "fields", nil, "comma-separated columns to include") + pf.BoolVar(&app.noHeader, "no-header", false, "omit header row in table/csv/tsv") + pf.StringVar(&app.template, "template", "", "Go text/template per record") + pf.IntVarP(&app.limit, "limit", "n", 0, "limit number of records (0 = all)") + pf.DurationVar(&app.cfg.Rate, "delay", app.cfg.Rate, "minimum spacing between requests") + pf.DurationVar(&app.cfg.Timeout, "timeout", app.cfg.Timeout, "per-request timeout") + pf.IntVar(&app.cfg.Retries, "retries", app.cfg.Retries, "retry attempts on 429/5xx") + + root.AddCommand( + app.indexCmd(), + app.exportCmd(), + app.infoCmd(), + newVersionCmd(), + ) return root } + +func (a *App) setup() error { + if a.output == "" || a.output == "auto" { + if isatty.IsTerminal(os.Stdout.Fd()) { + a.output = string(FormatTable) + } else { + a.output = string(FormatJSONL) + } + } + if !Format(a.output).Valid() { + return codeError(exitUsage, fmt.Errorf("unknown output format %q", a.output)) + } + a.client = tiobe.NewClient(a.cfg) + return nil +} + +func (a *App) render(records any) error { + r := NewRenderer(os.Stdout, Format(a.output), a.fields, a.noHeader, a.template) + return r.Render(records) +} + +func (a *App) renderOrEmpty(records any, n int) error { + if err := a.render(records); err != nil { + return err + } + if n == 0 { + return codeError(exitNoData, nil) + } + return nil +} + +func mapFetchErr(err error) error { + if err == nil { + return nil + } + return codeError(exitError, err) +} diff --git a/tiobe/tiobe.go b/tiobe/tiobe.go index d6f3e99..03d3ee0 100644 --- a/tiobe/tiobe.go +++ b/tiobe/tiobe.go @@ -1,10 +1,4 @@ -// Package tiobe is the library behind the tiobe command line: -// the HTTP client, request shaping, and the typed data models for tiobe. -// -// The Client here is the spine every command shares. It sets a real -// User-Agent, paces requests so a busy session stays polite, and retries the -// transient failures (429 and 5xx) that any public site throws under load. -// Build your endpoint calls and JSON decoding on top of it. +// Package tiobe is the library behind the tiobe CLI. package tiobe import ( @@ -12,41 +6,125 @@ import ( "fmt" "io" "net/http" + "regexp" + "strconv" + "strings" "time" ) -// DefaultUserAgent identifies the client to tiobe. A real, honest -// User-Agent is both polite and the thing most likely to keep you unblocked. -const DefaultUserAgent = "tiobe/dev (+https://github.com/tamnd/tiobe-cli)" +const DefaultUserAgent = "tiobe-cli/dev (+https://github.com/tamnd/tiobe-cli)" -// Client talks to tiobe over HTTP. -type Client struct { - HTTP *http.Client +type Config struct { + BaseURL string + Rate time.Duration + Timeout time.Duration + Retries int UserAgent string - // Rate is the minimum gap between requests. Zero means no pacing. - Rate time.Duration - Retries int +} + +func DefaultConfig() Config { + return Config{ + BaseURL: "https://www.tiobe.com", + Rate: 500 * time.Millisecond, + Timeout: 30 * time.Second, + Retries: 3, + UserAgent: DefaultUserAgent, + } +} +type Client struct { + cfg Config + http *http.Client last time.Time } -// NewClient returns a Client with sensible defaults: a 30s timeout, a 200ms -// minimum gap between requests, and five retries on transient errors. -func NewClient() *Client { +func NewClient(cfg Config) *Client { return &Client{ - HTTP: &http.Client{Timeout: 30 * time.Second}, - UserAgent: DefaultUserAgent, - Rate: 200 * time.Millisecond, - Retries: 5, + cfg: cfg, + http: &http.Client{Timeout: cfg.Timeout}, + } +} + +var ( + tableRe = regexp.MustCompile(`(?s)]+id="([\w-]+)"[^>]*>(.*?)`) + rowRe = regexp.MustCompile(`(?s)]*>(.*?)`) + cellRe = regexp.MustCompile(`(?s)]*>(.*?)`) + tagRe = regexp.MustCompile(`<[^>]+>`) +) + +func cleanCell(s string) string { + return strings.TrimSpace(tagRe.ReplaceAllString(s, "")) +} + +// Index fetches the TIOBE programming language index page and returns +// entries from the top-20 table and the "other" (21-70) table. +func (c *Client) Index(ctx context.Context) ([]*Entry, error) { + body, err := c.get(ctx, c.cfg.BaseURL+"/tiobe-index/") + if err != nil { + return nil, err + } + html := string(body) + + tables := make(map[string]string) + for _, m := range tableRe.FindAllStringSubmatch(html, -1) { + tables[m[1]] = m[2] + } + + var entries []*Entry + rank := 0 + + // Top 20 table: cols = currentRank, prevRank, (icon), (icon), language, rating, change + if top, ok := tables["top20"]; ok { + for _, rowM := range rowRe.FindAllStringSubmatch(top, -1) { + cells := cellRe.FindAllStringSubmatch(rowM[1], -1) + if len(cells) < 7 { + continue + } + cur, err := strconv.Atoi(cleanCell(cells[0][1])) + if err != nil { + continue + } + rank++ + e := &Entry{ + Rank: cur, + PrevRank: cleanCell(cells[1][1]), + Language: cleanCell(cells[4][1]), + Rating: cleanCell(cells[5][1]), + Change: cleanCell(cells[6][1]), + } + entries = append(entries, e) + } } + + // Other (21–70) table: cols = rank, language, rating + if other, ok := tables["otherPL"]; ok { + for _, rowM := range rowRe.FindAllStringSubmatch(other, -1) { + cells := cellRe.FindAllStringSubmatch(rowM[1], -1) + if len(cells) < 3 { + continue + } + cur, err := strconv.Atoi(cleanCell(cells[0][1])) + if err != nil { + continue + } + rank++ + e := &Entry{ + Rank: cur, + PrevRank: "", + Language: cleanCell(cells[1][1]), + Rating: cleanCell(cells[2][1]), + Change: "", + } + entries = append(entries, e) + } + } + + return entries, nil } -// Get fetches url and returns the response body. It paces and retries according -// to the client's settings. The caller owns nothing extra; the body is read -// fully and closed here. -func (c *Client) Get(ctx context.Context, url string) ([]byte, error) { +func (c *Client) get(ctx context.Context, url string) ([]byte, error) { var lastErr error - for attempt := 0; attempt <= c.Retries; attempt++ { + for attempt := 0; attempt <= c.cfg.Retries; attempt++ { if attempt > 0 { select { case <-ctx.Done(): @@ -66,15 +144,15 @@ func (c *Client) Get(ctx context.Context, url string) ([]byte, error) { return nil, fmt.Errorf("get %s: %w", url, lastErr) } -func (c *Client) do(ctx context.Context, url string) (body []byte, retry bool, err error) { +func (c *Client) do(ctx context.Context, url string) ([]byte, bool, error) { c.pace() req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, false, err } - req.Header.Set("User-Agent", c.UserAgent) + req.Header.Set("User-Agent", c.cfg.UserAgent) - resp, err := c.HTTP.Do(req) + resp, err := c.http.Do(req) if err != nil { return nil, true, err } @@ -86,7 +164,6 @@ func (c *Client) do(ctx context.Context, url string) (body []byte, retry bool, e if resp.StatusCode != http.StatusOK { return nil, false, fmt.Errorf("http %d", resp.StatusCode) } - b, err := io.ReadAll(resp.Body) if err != nil { return nil, true, err @@ -94,12 +171,11 @@ func (c *Client) do(ctx context.Context, url string) (body []byte, retry bool, e return b, false, nil } -// pace blocks until at least Rate has passed since the previous request. func (c *Client) pace() { - if c.Rate <= 0 { + if c.cfg.Rate <= 0 { return } - if wait := c.Rate - time.Since(c.last); wait > 0 { + if wait := c.cfg.Rate - time.Since(c.last); wait > 0 { time.Sleep(wait) } c.last = time.Now() @@ -112,3 +188,21 @@ func backoff(attempt int) time.Duration { } return d } + +// Stats returns aggregate statistics about the current TIOBE index. +func (c *Client) Stats(ctx context.Context) (*Info, error) { + entries, err := c.Index(ctx) + if err != nil { + return nil, err + } + top := "" + if len(entries) > 0 { + top = entries[0].Language + } + return &Info{ + TotalLanguages: len(entries), + TopLanguage: top, + SiteURL: c.cfg.BaseURL, + IndexURL: c.cfg.BaseURL + "/tiobe-index/", + }, nil +} diff --git a/tiobe/types.go b/tiobe/types.go new file mode 100644 index 0000000..2643579 --- /dev/null +++ b/tiobe/types.go @@ -0,0 +1,18 @@ +package tiobe + +// Entry is one row from the TIOBE programming language index. +type Entry struct { + Rank int `json:"rank" csv:"rank" tsv:"rank"` + PrevRank string `json:"prev_rank" csv:"prev_rank" tsv:"prev_rank"` + Language string `json:"language" csv:"language" tsv:"language"` + Rating string `json:"rating" csv:"rating" tsv:"rating"` + Change string `json:"change" csv:"change" tsv:"change"` +} + +// Info holds aggregate statistics for the TIOBE index. +type Info struct { + TotalLanguages int `json:"total_languages"` + TopLanguage string `json:"top_language"` + SiteURL string `json:"site_url"` + IndexURL string `json:"index_url"` +}