diff --git a/CHANGELOG.md b/CHANGELOG.md index 85471a6..f854238 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,21 @@ # Changelog +## 0.7.0 — 2026-10-15 + +Files from Azure DevOps: read what can be read, save the rest. + +- `ado_attachments` reads PDF (text page by page, `pages=2-5`; scanned pages shown as images), PowerPoint, saved e-mails + (.eml) and ZIP files (the list, `part=` one file inside), next to Word, Excel, Outlook e-mails and text. +- The kind of a file comes from its bytes, not its name: binaries (traces, packages) are never sent as text. +- Long documents come as an outline and the start, with their size in tokens; `detail=full` for all. +- `save=true` streams a file to disk at any size; only its path, size and SHA-256 go to Claude. Files over 20 MB or not + readable get a link and the hint to save them, instead of an error. +- Build artifacts: `ado_pipeline build= artifacts=true` lists them, `artifact=` shows the files in one + (`part=` reads a file), `save=true` saves it. +- Download folder: `adolens config --download-dir ` or the settings menu (default Downloads\ADO Lens). +- A required label language with no label file in the model is reported by `ado_review` and `ado_label`. +- THIRD-PARTY-NOTICES.md lists the bundled libraries and their licenses (PdfPig: Apache 2.0); shipped in every package. + ## 0.6.0 — 2026-10-15 Screenshots and documents for Claude, D365 labels, checks before testing, ready commands, a settings menu. diff --git a/README.md b/README.md index 233b50e..cd70bd6 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,8 @@ adolens config --add-toolset pipelines # one more tool set on top of the m adolens config --confirm-above 3 # updates of more items are previewed first (0: always) adolens config --usage-log false adolens config --show-tokens true # end every answer with [~N tokens] +adolens config --download-dir D:\Traces # where save=true puts files (default: Downloads\ADO Lens) +adolens config --connection contoso --label-languages "en-US, pl" # labels must exist in these languages adolens config --connection contoso --env staging=UAT --env master=PROD # what branches mean (for ado_trace) ``` @@ -208,13 +210,13 @@ What each MCP tool answers, when Claude should reach for it, and the CLI command | `ado_changes` | the files of a changeset (`523`), pull request (`PR 482`) or commit (sha + `repo`) with X++ objects and their added / changed / removed methods; `file` gives the diff of one file — for AOT XML per method plus the metadata (fields, controls, properties) without raw XML noise | reading what exactly changed | `adolens changes 523`, `adolens changes 523 --file AxClass/SalesTable_Extension.xml` | | `ado_pr` | without id: pull requests waiting for my review and mine (`list=active`: all); with id: status, reviewers and votes, checks (build, work item linking, reviewers), linked work items, open comment threads written by people (not system notes) | code review, "what is blocking my PR" | `adolens pr`, `adolens pr 482` | | `ado_pr_update` | `create` (repo, source, target, title, markdown text, work items, reviewers, draft), `comment` (general or on a file and line), `reply`, `resolve`, `vote`, `publish`, `complete`, `abandon`; `@[name]` notifies | reviewing, opening and finishing PRs | — | -| `ado_pipeline` | recent builds (of a pipeline, a branch); with `build`: the failed stage, job and steps, their errors, and the lines of the log that report errors (MSBuild noise cut) | "why is the build red" | `adolens pipeline`, `adolens pipeline 900` | +| `ado_pipeline` | recent builds (of a pipeline, a branch); with `build`: the failed stage, job and steps, their errors, and the lines of the log that report errors (MSBuild noise cut); `artifacts=true` lists the build's artifacts, `artifact=` the files in one (`part=` reads a file), `save=true` saves it to disk | "why is the build red", getting a package or a log | `adolens pipeline 900`, `adolens pipeline 900 --artifact drop --save` | | `ado_pipeline_run` | queues a pipeline by name or id, optionally on a branch, with variables | re-running a build | — | | `ado_wiki` | the project wiki: page tree (`depth`), one page as markdown with its version and subpages, or search across the project's wikis with snippets; long pages are cut unless `detail=full` | "how does X work", finding documentation | `adolens wiki`, `adolens wiki "Team/Setup"`, `adolens wiki --search "dual write"` | | `ado_wiki_write` | `create` a page, `append` / `prepend` to it, or `replace` it with the version from the last full read — a page changed by someone else in the meantime is never overwritten; optional change comment | release notes, keeping docs current | `adolens wiki-write "Team/Release notes" --file notes.md --mode append` | | `ado_tests` | the tests of work items (ids) or of a sprint's requirements and bugs: their Test Cases with the latest outcome per plan, suite and configuration (who, when), items without tests; `detail=full` adds the steps | "is it tested", sprint review | `adolens tests 123`, `adolens tests` (current sprint) | | `ado_test_create` | Test Cases for a work item from markdown (`## title`, steps `1. action => expected`), linked as tested-by, in the item's area and iteration; `plan` + `suite` (id or name) add them to a suite | turning acceptance criteria into tests | `adolens test-create 123 --file cases.md --plan R1 --suite Login` | -| `ado_attachments` | the files of a work item and the screenshots pasted into its text or comments; `file` shows an image to Claude or gives the text of a Word, Excel, Outlook e-mail (.msg) or text file; `add` attaches a local file | specifications and e-mails attached to items, "look at the screenshot" | `adolens attachments 123`, `adolens attachments 123 --file 2` | +| `ado_attachments` | the files of a work item and the screenshots pasted into its text or comments; `file` gives the text of a PDF (`pages=2-5`; scanned pages shown as images), Word, Excel, PowerPoint, e-mail (.msg, .eml), ZIP (list; `part=` one file inside) or text file, or shows an image - the type is read from the bytes, binaries are never sent as text, long documents come as an outline and the start (`detail=full` for all); `save=true` streams the file to disk at any size and returns only its path, size and SHA-256; files over 20 MB or not readable get a link and the hint to save them; `add` attaches a local file | specifications, e-mails and screenshots of items; traces and logs for other tools | `adolens attachments 123 --file 2 --pages 1-3`, `adolens attachments 123 --file 2 --save` | | `ado_create_tree` | a tree of work items from an indented list (`- Feature: …`, children indented, `\| description`, `@person`), under an existing item if given; previewed first, one undo removes it all | breaking a feature into requirements and tasks | — | | `ado_metrics` | done items and estimate per finished sprint (velocity), cycle and lead time (median, 85%), what is in progress longest | "are we getting faster", planning | `adolens metrics --sprints 6` | | `ado_label` | a D365 message pasted as seen (values in place of %1 %2 … are fine) or `@File:Id`: the label, its text in every language (missing ones flagged) and the objects, X++ methods, fields and controls using it | "where does this error come from" | `adolens label "Item 1234: received 12, on hand 5"` | diff --git a/README.pl.md b/README.pl.md index b602d57..9543e7a 100644 --- a/README.pl.md +++ b/README.pl.md @@ -211,13 +211,13 @@ Na co odpowiada każde narzędzie MCP, kiedy Claude powinien po nie sięgnąć i | `ado_changes` | pliki changesetu (`523`), pull requesta (`PR 482`) albo commita (sha + `repo`) z obiektami X++ i ich dodanymi / zmienionymi / usuniętymi metodami; `file` daje diff jednego pliku — dla AOT XML per metoda plus metadane (pola, kontrolki, właściwości) bez szumu surowego XML-a | czytanie, co dokładnie się zmieniło | `adolens changes 523`, `adolens changes 523 --file AxClass/SalesTable_Extension.xml` | | `ado_pr` | bez id: pull requesty czekające na mój przegląd i moje (`list=active`: wszystkie); z id: stan, recenzenci i głosy, kontrole (build, podpięty work item, recenzenci), podpięte work itemy, otwarte wątki pisane przez ludzi (bez wpisów systemowych) | code review, „co blokuje mój PR” | `adolens pr`, `adolens pr 482` | | `ado_pr_update` | `create` (repo, źródło, cel, tytuł, opis w markdownie, work itemy, recenzenci, szkic), `comment` (ogólny albo na pliku i linii), `reply`, `resolve`, `vote`, `publish`, `complete`, `abandon`; `@[imię]` wysyła powiadomienie | przegląd, zakładanie i kończenie PR-ów | — | -| `ado_pipeline` | ostatnie buildy (pipeline'u, gałęzi); z `build`: nieudany etap, zadanie i kroki, ich błędy i linie logu zgłaszające błędy (bez szumu MSBuild) | „czemu build jest czerwony” | `adolens pipeline`, `adolens pipeline 900` | +| `ado_pipeline` | ostatnie buildy (pipeline'u, gałęzi); z `build`: nieudany etap, zadanie i kroki, ich błędy i linie logu zgłaszające błędy (bez szumu MSBuild); `artifacts=true` lista artefaktów buildu, `artifact=` pliki w nim (`part=` czyta plik), `save=true` zapisuje go na dysk | „czemu build jest czerwony”, pobranie paczki lub logu | `adolens pipeline 900`, `adolens pipeline 900 --artifact drop --save` | | `ado_pipeline_run` | uruchamia pipeline po nazwie albo id, opcjonalnie na gałęzi, ze zmiennymi | ponowne puszczenie buildu | — | | `ado_wiki` | wiki projektu: drzewo stron (`depth`), strona jako markdown z wersją i podstronami albo wyszukiwanie we wszystkich wiki projektu z fragmentami; długie strony są ucinane, chyba że `detail=full` | „jak działa X”, szukanie dokumentacji | `adolens wiki`, `adolens wiki "Team/Setup"`, `adolens wiki --search "dual write"` | | `ado_wiki_write` | `create` — nowa strona, `append` / `prepend` — dopisanie, `replace` — zastąpienie z wersją z ostatniego pełnego odczytu (strony zmienionej w międzyczasie przez kogoś innego nigdy nie nadpisze); opcjonalny komentarz zmiany | notatki wydania, aktualna dokumentacja | `adolens wiki-write "Team/Release notes" --file notes.md --mode append` | | `ado_tests` | testy work itemów (id) albo wymagań i błędów sprintu: ich Test Case'y z ostatnim wynikiem w każdym planie, zestawie i konfiguracji (kto, kiedy), elementy bez testów; `detail=full` dodaje kroki | „czy to przetestowane”, przegląd sprintu | `adolens tests 123`, `adolens tests` (bieżący sprint) | | `ado_test_create` | Test Case'y dla work itemu z markdownu (`## tytuł`, kroki `1. akcja => oczekiwany wynik`), podpięte jako tested-by, w obszarze i iteracji elementu; `plan` + `suite` (id albo nazwa) dodają je do zestawu | kryteria akceptacji zamienione w testy | `adolens test-create 123 --file przypadki.md --plan R1 --suite Logowanie` | -| `ado_attachments` | pliki work itemu i zrzuty ekranu wklejone w jego treść lub komentarze; `file` pokazuje obraz Claude'owi albo daje tekst pliku Word, Excel, maila Outlooka (.msg) lub tekstowego; `add` dołącza plik z dysku | specyfikacje i maile przy elementach, „zobacz zrzut” | `adolens attachments 123`, `adolens attachments 123 --file 2` | +| `ado_attachments` | pliki work itemu i zrzuty ekranu wklejone w jego treść lub komentarze; `file` daje tekst PDF-a (`pages=2-5`; zeskanowane strony pokazane jako obrazy), Worda, Excela, PowerPointa, maila (.msg, .eml), ZIP-a (lista; `part=` jeden plik ze środka) lub pliku tekstowego, albo pokazuje obraz - typ jest rozpoznawany po bajtach, pliki binarne nigdy nie idą jako tekst, długie dokumenty przychodzą jako spis treści i początek (`detail=full` całość); `save=true` zapisuje plik na dysk strumieniowo w dowolnym rozmiarze i zwraca tylko ścieżkę, rozmiar i SHA-256; pliki ponad 20 MB albo nieczytelne dostają link i podpowiedź zapisu; `add` dołącza plik z dysku | specyfikacje, maile i zrzuty przy elementach; trace'y i logi dla innych narzędzi | `adolens attachments 123 --file 2 --pages 1-3`, `adolens attachments 123 --file 2 --save` | | `ado_create_tree` | drzewo work itemów z listy z wcięciami (`- Feature: …`, dzieci wcięte, `\| opis`, `@osoba`), opcjonalnie pod istniejącym elementem; najpierw podgląd, jedno cofnięcie usuwa całość | rozbicie feature'a na wymagania i taski | — | | `ado_metrics` | zrobione elementy i szacunek w zakończonych sprintach (prędkość), czas realizacji (mediana, 85%), co najdłużej w toku | „czy przyspieszamy”, planowanie | `adolens metrics --sprints 6` | | `ado_label` | komunikat D365 wklejony tak, jak widać na ekranie (z wartościami zamiast %1 %2 …) albo `@Plik:Id`: etykieta, jej tekst w każdym języku (brakujące oznaczone) i obiekty, metody X++, pola i kontrolki, które jej używają | „skąd ten błąd” | `adolens label "Towar 1234: przyjęto 12, stan 5"` | diff --git a/SECURITY.md b/SECURITY.md index f6893da..46d447e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -40,6 +40,9 @@ organization and a `Demo` project) is enough. Credential Manager still wins. Prefer the installer when Claude Code is used too. - **Other servers:** `adolens status` reports other MCP entries in the Claude configuration that keep a PAT in plain text (for example an older Azure DevOps server). It never changes them; remove the PAT there yourself. +- **Saved files:** `save=true` streams a file from Azure DevOps into the download folder (default `Downloads\ADO Lens`, + `adolens config --download-dir`). The file name is cleaned so it cannot point outside that folder, an existing file is + never overwritten, and saved files are never opened or run. Only the path, size and SHA-256 go to Claude. - **Network:** the server talks only to `https://dev.azure.com/` (and the `vsrm` / `vssps` hosts of the same organization) over HTTPS, with the PAT in the `Authorization` header. It opens no network ports. diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md new file mode 100644 index 0000000..1c4fd87 --- /dev/null +++ b/THIRD-PARTY-NOTICES.md @@ -0,0 +1,14 @@ +# Third-party notices + +ADO Lens is distributed under the MIT license (see LICENSE). The program bundles these libraries; their licenses +apply to them. + +| Library | License | Source | +|---|---|---| +| PdfPig (UglyToad.PdfPig and its parts) | Apache License 2.0 | https://github.com/UglyToad/PdfPig | +| ModelContextProtocol (C# SDK) | MIT | https://github.com/modelcontextprotocol/csharp-sdk | +| Microsoft.Data.Sqlite, SQLitePCLRaw | MIT | https://github.com/dotnet/efcore, https://github.com/ericsink/SQLitePCL.raw | +| SQLite | Public domain | https://sqlite.org | +| Microsoft.Extensions.* and the .NET runtime | MIT | https://github.com/dotnet/runtime | + +The Apache License 2.0 text: https://www.apache.org/licenses/LICENSE-2.0. PdfPig is used unmodified. diff --git a/install.ps1 b/install.ps1 index 8bdac42..2b20b04 100644 --- a/install.ps1 +++ b/install.ps1 @@ -95,7 +95,7 @@ else { } Copy-Item $f.FullName $dst -Force } - foreach ($f in 'install.ps1', 'uninstall.ps1', 'README.md', 'README.pl.md', 'CHANGELOG.md', 'SECURITY.md', 'LICENSE') { + foreach ($f in 'install.ps1', 'uninstall.ps1', 'README.md', 'README.pl.md', 'CHANGELOG.md', 'SECURITY.md', 'LICENSE', 'THIRD-PARTY-NOTICES.md') { if (Test-Path (Join-Path $src $f)) { Copy-Item (Join-Path $src $f) $InstallDir -Force } } if (Test-Path (Join-Path $src 'skills')) { Copy-Item (Join-Path $src 'skills') $InstallDir -Recurse -Force } diff --git a/pack.ps1 b/pack.ps1 index 7335e7b..34a3069 100644 --- a/pack.ps1 +++ b/pack.ps1 @@ -43,7 +43,7 @@ Write-Host 'publishing (self-contained)...' -ForegroundColor Cyan if ($LASTEXITCODE -ne 0) { throw 'dotnet publish failed' } Remove-Item (Join-Path $stage 'bin\*.pdb') -Force -ErrorAction SilentlyContinue -foreach ($f in 'install.ps1', 'uninstall.ps1', 'README.md', 'README.pl.md', 'CHANGELOG.md', 'SECURITY.md', 'LICENSE') { +foreach ($f in 'install.ps1', 'uninstall.ps1', 'README.md', 'README.pl.md', 'CHANGELOG.md', 'SECURITY.md', 'LICENSE', 'THIRD-PARTY-NOTICES.md') { Copy-Item (Join-Path $PSScriptRoot $f) $stage -Force } # Skills for Claude (SKILL.md per folder), installed next to the server. @@ -92,6 +92,7 @@ if (-not $NoZip) { [IO.File]::WriteAllText((Join-Path $mcpbStage 'manifest.json'), $manifest, (New-Object Text.UTF8Encoding $false)) Copy-Item (Join-Path $stage 'bin') $mcpbStage -Recurse Copy-Item (Join-Path $stage 'LICENSE') $mcpbStage + Copy-Item (Join-Path $stage 'THIRD-PARTY-NOTICES.md') $mcpbStage Copy-Item (Join-Path $PSScriptRoot 'packaging\icon\icon.png') $mcpbStage $mcpb = Join-Path $OutDir "ado-lens-$version.mcpb" New-Zip $mcpbStage $mcpb $false diff --git a/skills/dev-build/SKILL.md b/skills/dev-build/SKILL.md index d5b137e..4f17f97 100644 --- a/skills/dev-build/SKILL.md +++ b/skills/dev-build/SKILL.md @@ -13,6 +13,9 @@ description: Diagnoses a failed build or release for a developer - the failing s 3. Find the code: the object or file named in the error (`ado_file`, X++: `part=`), and what changed since the last green build of the same pipeline (`ado_pipeline` lists them; `ado_changes` of the changes between). 4. Who and what: `ado_file history=true` for the last change of that object and its work item. +5. Logs, packages and traces kept as build artifacts: `ado_pipeline build= artifacts=true`, then `artifact= + part=` to read one file, or `save=true` to put it on disk for another tool (a trace parser, a diff tool) - + only the path comes back, not the content. ## Answer diff --git a/src/AdoLens/AdoClient.cs b/src/AdoLens/AdoClient.cs index 5cd00d2..a30e853 100644 --- a/src/AdoLens/AdoClient.cs +++ b/src/AdoLens/AdoClient.cs @@ -108,6 +108,31 @@ public async Task GetBytesAsync(string url, long maxBytes, CancellationT } } + /// Streams a file into without holding it in memory; returns the bytes written. + public async Task DownloadToAsync(string url, Stream target, CancellationToken ct = default) + { + url = WithApiVersion(url); + if (Uri.TryCreate(url, UriKind.Absolute, out var abs) && !IsAzureDevOpsHost(abs)) + throw new AdoApiException($"refusing to send the PAT to {abs.Host}: only Azure DevOps hosts are allowed"); + for (var attempt = 1; ; attempt++) + { + using var req = new HttpRequestMessage(HttpMethod.Get, url); + req.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/octet-stream")); + req.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/zip")); + using var res = await http.SendAsync(req, HttpCompletionOption.ResponseHeadersRead, ct); + if (attempt < MaxAttempts && RetryDelay(res) is { } delay) + { + await Delay(delay, ct); + continue; + } + if (!res.IsSuccessStatusCode || res.Content.Headers.ContentType?.MediaType == "text/html") await EnsureSuccessAsync(res, ct); + await using var source = await res.Content.ReadAsStreamAsync(ct); + var before = target.CanSeek ? target.Position : 0; + await source.CopyToAsync(target, 1 << 20, ct); + return target.CanSeek ? target.Position - before : res.Content.Headers.ContentLength ?? -1; + } + } + /// Uploads a file as a work item attachment; returns its URL. public async Task UploadAttachmentAsync(string project, string fileName, byte[] content, CancellationToken ct = default) { diff --git a/src/AdoLens/AdoLens.csproj b/src/AdoLens/AdoLens.csproj index 63d19ee..90c2fc5 100644 --- a/src/AdoLens/AdoLens.csproj +++ b/src/AdoLens/AdoLens.csproj @@ -23,7 +23,7 @@ ADO Lens for Azure DevOps Azure DevOps (work items, boards, wiki, Git and TFVC, pipelines) served to AI assistants over MCP Copyright © 2026 WillowDev - 0.6.0 + 0.7.0 @@ -34,6 +34,7 @@ + diff --git a/src/AdoLens/Attachments.cs b/src/AdoLens/Attachments.cs index c705d01..7142b24 100644 --- a/src/AdoLens/Attachments.cs +++ b/src/AdoLens/Attachments.cs @@ -1,11 +1,9 @@ -using System.IO.Compression; using System.Text; using System.Text.Json; -using System.Xml.Linq; namespace AdoLens; -/// Files attached to work items: the list, their text (Word, Excel, plain text), and adding one. +/// Files attached to work items: the list, their content for the model, saving them, adding one. public static class Attachments { const long MaxDownload = 20 * 1024 * 1024; @@ -14,13 +12,6 @@ public static class Attachments sealed record Attached(string Name, string Url, long? Size, string? Comment); - const long MaxImage = 4 * 1024 * 1024; - - static readonly Dictionary ImageTypes = new(StringComparer.OrdinalIgnoreCase) - { - [".png"] = "image/png", [".jpg"] = "image/jpeg", [".jpeg"] = "image/jpeg", [".gif"] = "image/gif", [".webp"] = "image/webp", - }; - static List List(JsonElement wi) => Json.Arr(wi, "relations").Where(r => Json.Str(r, "rel") == Rel).Select(r => { @@ -64,37 +55,46 @@ void From(string? html, string where) static string Size(long? bytes) => bytes is not { } b ? "" : b < 1024 ? $" ({b} B)" : b < 1024 * 1024 ? $" ({b / 1024} KB)" : $" ({b / 1024.0 / 1024:0.#} MB)"; /// - /// Without : the attachments of the item and the images pasted into it; with it (name or - /// number): the text of a document, or an image (returned as image bytes for the client to show to the model). + /// Without : the attachments of the item and the images pasted into it. With it (name or + /// number): its content for the model - text of a document (PDF, Word, Excel, PowerPoint, e-mail, ZIP list, text), + /// or images (pictures, scanned PDF pages); with the file is streamed to disk instead and + /// only its path comes back. Files too large or not readable get a link and the hint to save them. /// - public static async Task<(string Text, (byte[] Bytes, string Mime)? Image)> ReadAsync(Lens lens, string? connection, int id, string? file, string? detail, - CancellationToken ct = default) + public static async Task<(string Text, List<(byte[] Bytes, string Mime)> Images)> ReadAsync(Lens lens, string? connection, int id, string? file, + string? detail, string? pages = null, string? part = null, bool save = false, CancellationToken ct = default) { var session = lens.For(connection); var items = await session.Work.GetAsync([id], relations: true, ct: ct); - if (items.Count == 0) return ($"no work item #{id} visible with this PAT", null); + if (items.Count == 0) return ($"no work item #{id} visible with this PAT", []); var list = List(items[0]).Concat(await InlineImagesAsync(session, items[0], ct)).ToList(); var head = WorkItemFormat.Header(items[0]); - if (list.Count == 0) return ($"{head}: no attachments and no pasted images", null); + if (list.Count == 0) return ($"{head}: no attachments and no pasted images", []); if (string.IsNullOrWhiteSpace(file)) return ($"{head}: {list.Count} file(s)\n" + string.Join('\n', list.Select((a, i) => $"{i + 1}. {a.Name}{Size(a.Size)}{(a.Comment is { Length: > 0 } c ? $" - {c}" : "")}")) + - "\nfile= reads one: documents as text (Word, Excel, e-mail, text), images are shown.", null); + "\nfile= reads one (documents as text, images and scans shown); save=true saves it to disk (any size).", []); var pick = int.TryParse(file, out var n) && n >= 1 && n <= list.Count ? list[n - 1] : list.FirstOrDefault(a => a.Name.Equals(file.Trim(), StringComparison.OrdinalIgnoreCase)) ?? list.FirstOrDefault(a => a.Name.Contains(file.Trim(), StringComparison.OrdinalIgnoreCase)); - if (pick is null) return ($"no file '{file}' on #{id}; files: {string.Join(", ", list.Select(a => a.Name))}", null); - var bytes = await session.Client.GetBytesAsync(pick.Url + (pick.Url.Contains('?') ? "&" : "?") + "download=true", MaxDownload, ct); - if (ImageTypes.TryGetValue(Path.GetExtension(pick.Name), out var mime) || ImageMime(bytes) is { } sniffed && (mime = sniffed) is not null) - { - if (bytes.Length > MaxImage) return ($"{pick.Name}{Size(bytes.Length)}: the image is too large to show (over {MaxImage / 1024 / 1024} MB)", null); - return ($"{pick.Name}{Size(bytes.Length)} from {head}{(pick.Comment is { Length: > 0 } where ? $" ({where})" : "")} - image attached", (bytes, mime)); - } - var text = Extract(pick.Name, bytes); - if (text is null) return ($"{pick.Name}{Size(bytes.Length)}: this kind of file cannot be read as text (open it from the work item)", null); - var full = detail == "full"; - return ($"{pick.Name}{Size(bytes.Length)} from {head}\n\n" + - (full || text.Length <= MaxText ? text : text[..MaxText] + $"\n\n… {text.Length - MaxText} more characters (detail=full)"), null); + if (pick is null) return ($"no file '{file}' on #{id}; files: {string.Join(", ", list.Select(a => a.Name))}", []); + var download = pick.Url + (pick.Url.Contains('?') ? "&" : "?") + "download=true"; + if (save) + return (await Downloads.SaveAsync(session.Client, download, Downloads.BaseDir(lens.Settings), id.ToString(System.Globalization.CultureInfo.InvariantCulture), pick.Name, ct), []); + + string Link(string why) => $"{pick.Name}{Size(pick.Size)}: {why}.\nsave=true saves it to disk (streamed, any size); in a browser signed in to Azure DevOps: {pick.Url}"; + if (pick.Size > MaxDownload) return (Link($"larger than {MaxDownload / 1024 / 1024} MB, too large to read here"), []); + byte[] bytes; + try { bytes = await session.Client.GetBytesAsync(download, MaxDownload, ct); } + catch (AdoApiException ex) when (ex.Message.Contains("larger than", StringComparison.Ordinal)) { return (Link(ex.Message), []); } + + var content = Documents.Read(pick.Name, bytes, pages, part); + if (content.Text is null && content.Images.Count == 0) return (Link(content.Note ?? "nothing readable as text"), []); + var sb = new StringBuilder($"{pick.Name} ({Documents.Size(bytes.Length)}, {content.Kind}) from {head}"); + if (pick.Comment is { Length: > 0 } where) sb.Append($" ({where})"); + if (content.Note is not null) sb.Append($"\n{content.Note}"); + if (content.Images.Count > 0) sb.Append($"\n{content.Images.Count} image(s) attached"); + if (content.Text is not null) sb.Append("\n\n").Append(Documents.Fit(content.Text, MaxText, detail == "full")); + return (sb.ToString(), content.Images); } /// An image recognized by its first bytes (pasted images may have no usable file name). @@ -105,90 +105,8 @@ void From(string? html, string where) : b.Length > 12 && b[0] == 'R' && b[1] == 'I' && b[2] == 'F' && b[3] == 'F' && b[8] == 'W' && b[9] == 'E' && b[10] == 'B' && b[11] == 'P' ? "image/webp" : null; - /// The text of a file: Word and Excel (Office Open XML), or anything that is plain text. - internal static string? Extract(string name, byte[] bytes) - { - var ext = Path.GetExtension(name).ToLowerInvariant(); - try - { - switch (ext) - { - case ".docx" or ".docm": - return Word(bytes); - case ".xlsx" or ".xlsm": - return Excel(bytes); - case ".msg": - return OutlookMsg.Read(bytes); - case ".txt" or ".md" or ".csv" or ".json" or ".xml" or ".log" or ".sql" or ".cs" or ".xpp" or ".yml" or ".yaml" or ".html" or ".htm": - return Encoding.UTF8.GetString(bytes).TrimStart('\uFEFF'); - } - } - catch (Exception ex) when (ex is InvalidDataException or System.Xml.XmlException) { return null; } - // Unknown extension: text when it has no zero bytes in the start. - return bytes.Take(4096).Contains((byte)0) ? null : Encoding.UTF8.GetString(bytes); - } - - static XDocument? Part(ZipArchive zip, string path) => - zip.GetEntry(path) is { } e ? XDocument.Load(e.Open()) : null; - - /// Paragraphs as lines, headings as markdown, tables as "a | b" rows. - static string Word(byte[] bytes) - { - using var zip = new ZipArchive(new MemoryStream(bytes)); - var doc = Part(zip, "word/document.xml") ?? throw new InvalidDataException(); - XNamespace w = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"; - var sb = new StringBuilder(); - var body = doc.Root?.Element(w + "body"); - if (body is null) return ""; - foreach (var block in body.Elements()) - { - if (block.Name == w + "p") sb.AppendLine(Paragraph(block, w)); - else if (block.Name == w + "tbl") - { - foreach (var row in block.Elements(w + "tr")) - sb.AppendLine("| " + string.Join(" | ", row.Elements(w + "tc").Select(c => string.Join(" ", c.Elements(w + "p").Select(p => Paragraph(p, w))).Trim())) + " |"); - sb.AppendLine(); - } - } - // Collapse runs of empty lines. - return System.Text.RegularExpressions.Regex.Replace(sb.ToString(), @"\n{3,}", "\n\n").Trim(); - } - - static string Paragraph(XElement p, XNamespace w) - { - var text = string.Concat(p.Descendants().Select(e => e.Name == w + "t" ? e.Value : e.Name == w + "tab" ? "\t" : e.Name == w + "br" ? "\n" : "")); - var style = p.Element(w + "pPr")?.Element(w + "pStyle")?.Attribute(w + "val")?.Value ?? ""; - var level = style.StartsWith("Heading", StringComparison.OrdinalIgnoreCase) && int.TryParse(style[7..], out var l) ? l - : style.StartsWith("Nagwek", StringComparison.OrdinalIgnoreCase) && int.TryParse(style[6..], out var l2) ? l2 : 0; - if (level > 0 && text.Length > 0) return new string('#', Math.Min(level, 6)) + " " + text; - if (p.Element(w + "pPr")?.Element(w + "numPr") is not null && text.Length > 0) return "- " + text; - return text; - } - - /// Each sheet as rows of "a | b | c" (shared strings resolved). - static string Excel(byte[] bytes) - { - using var zip = new ZipArchive(new MemoryStream(bytes)); - XNamespace x = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; - var shared = Part(zip, "xl/sharedStrings.xml")?.Root?.Elements(x + "si").Select(si => string.Concat(si.Descendants(x + "t").Select(t => t.Value))).ToList() ?? []; - var sb = new StringBuilder(); - foreach (var entry in zip.Entries.Where(e => e.FullName.StartsWith("xl/worksheets/sheet", StringComparison.OrdinalIgnoreCase)).OrderBy(e => e.FullName.Length).ThenBy(e => e.FullName)) - { - sb.AppendLine($"## {Path.GetFileNameWithoutExtension(entry.FullName)}"); - var sheet = XDocument.Load(entry.Open()); - foreach (var row in sheet.Descendants(x + "row").Take(500)) - { - var cells = row.Elements(x + "c").Select(c => - { - var v = c.Element(x + "v")?.Value ?? string.Concat(c.Descendants(x + "t").Select(t => t.Value)); - return (string?)c.Attribute("t") == "s" && int.TryParse(v, out var i) && i < shared.Count ? shared[i] : v; - }).ToList(); - if (cells.Any(c => c.Length > 0)) sb.AppendLine("| " + string.Join(" | ", cells) + " |"); - } - sb.AppendLine(); - } - return sb.ToString().Trim(); - } + /// The text of a file, or null when it has none (images, binaries). + internal static string? Extract(string name, byte[] bytes) => Documents.Read(name, bytes).Text; /// Attaches a local file to a work item. public static async Task AddAsync(Lens lens, string? connection, int id, string path, string? comment, CancellationToken ct = default) diff --git a/src/AdoLens/Cli.cs b/src/AdoLens/Cli.cs index bd8b27e..c8246f8 100644 --- a/src/AdoLens/Cli.cs +++ b/src/AdoLens/Cli.cs @@ -30,6 +30,7 @@ config show or change settings --default-connection --mode po|dev|full --usage-log true|false --show-tokens true|false --add-toolset --remove-toolset --confirm-above --language pl|en|auto (settings console and installer texts) + --download-dir |default (where save=true puts files; default Downloads\ADO Lens) --connection --label-languages "en-US, pl"|all (languages review and label require) status configuration, connection check, metadata and Claude registrations --connection --offline --quiet @@ -53,6 +54,7 @@ board [] --team --move --column --detail summary|norma changes <523 | C523 | "PR 482" | sha --repo > files, X++ methods; --file for a diff pr [] PRs waiting for my review and mine (--list active: all); with id: reviewers, checks, threads pipeline [] recent builds (--definition --branch --top 10); with id: why it failed + --artifacts (list) | --artifact [--part ] [--save [--to ]] wiki [] --search --depth 2 --detail full --wiki wiki-write --file (or standard input) --mode create|append|prepend|replace --version (for replace, from 'wiki ') --comment @@ -62,7 +64,8 @@ wiki [] --search --depth 2 --detail full --wiki skills puts the skills of the mode into Claude Code (po-*, dev-*, all for full), removes the others review checks before testing: work item linked, labels in every language, X++ messages from labels, no TODO; --item : every change of a work item and its tasks - attachments files of a work item; --file reads one (Word, Excel, text); --add attaches + attachments files of a work item; --file reads one (PDF, Word, Excel, PowerPoint, e-mail, + ZIP, text; --pages 2-5, --part ); --save [--to ] saves it; --add label "" a D365 message as seen (values for %1 are fine) or @File:Id: label, languages, where used health backlog hygiene of the team --team release-notes [current|previous|] or --since 14d [--until yyyy-MM-dd] markdown for the wiki @@ -297,6 +300,11 @@ public int Config() settings.ExtraToolsets.Remove(Toolsets.Normalize(rem)); changed = true; } + if (cl.Opt("download-dir") is { } dd) + { + settings.DownloadDir = string.IsNullOrWhiteSpace(dd) || dd.Trim().Equals("default", StringComparison.OrdinalIgnoreCase) ? null : dd.Trim(); + changed = true; + } if (cl.Opt("language") is { } lang) { settings.Language = lang.Trim().ToLowerInvariant() switch @@ -732,6 +740,7 @@ async Task SetupMenuAsync() Console.WriteLine(Ui.T(" 5. Status (is everything working)", " 5. Status (czy wszystko działa)")); Console.WriteLine(Ui.T(" 6. Language: Polish / English", " 6. Język: polski / angielski")); Console.WriteLine(Ui.T(" 7. Required label languages (checks of D365 changes)", " 7. Wymagane języki etykiet (kontrola zmian D365)")); + Console.WriteLine(Ui.T($" 8. Download folder: {Downloads.BaseDir(settings)}", $" 8. Folder pobierania: {Downloads.BaseDir(settings)}")); var choice = Prompt(Ui.T("Choice (Enter = done): ", "Wybór (Enter = koniec): ")); switch (choice?.Trim()) { @@ -771,6 +780,16 @@ async Task SetupMenuAsync() settings.Save(SettingsPath); Ui.Polish = settings.Language == "pl" || settings.Language is null && Ui.Polish; break; + case "8": + var folder = Prompt(Ui.T(" folder for saved files ('default' = Downloads\\ADO Lens, Enter = keep): ", + " folder na pobrane pliki ('default' = Pobrane\\ADO Lens, Enter = bez zmian): ")); + if (!string.IsNullOrWhiteSpace(folder)) + { + settings.DownloadDir = folder.Trim().Equals("default", StringComparison.OrdinalIgnoreCase) ? null : folder.Trim().Trim('"'); + settings.Save(SettingsPath); + Console.WriteLine($" {Downloads.BaseDir(settings)}"); + } + break; case "7": var cn = PickConnection(settings); if (cn is null) break; @@ -831,8 +850,12 @@ public async Task PrAsync() public async Task PipelineAsync() { var lens = NewLens(); + if (cl.Opt("to") is { } to) lens.Settings.DownloadDir = to; + var save = cl.Flag("save") || cl.Peek("to") is not null; Print(Int(cl.Positional.FirstOrDefault()) is { } build - ? await PipelineQueries.BuildAsync(lens, cl.Opt("connection"), cl.Opt("project"), build, cl.Opt("detail")) + ? cl.Flag("artifacts") || cl.Peek("artifact") is not null + ? await PipelineQueries.ArtifactsAsync(lens, cl.Opt("connection"), cl.Opt("project"), build, cl.Opt("artifact"), cl.Opt("part"), save) + : await PipelineQueries.BuildAsync(lens, cl.Opt("connection"), cl.Opt("project"), build, cl.Opt("detail")) : await PipelineQueries.ListAsync(lens, cl.Opt("connection"), cl.Opt("project"), cl.Opt("definition"), cl.Opt("branch"), Int(cl.Opt("top")) ?? 10)); return 0; } @@ -866,14 +889,16 @@ public async Task AttachmentsAsync() if (Int(cl.Positional.FirstOrDefault()) is not { } id) { Console.Error.WriteLine("usage: adolens attachments [--file ] [--add --comment ]"); return 2; } var lens = NewLens(); if (cl.Opt("add") is { } add) { Print(await Attachments.AddAsync(lens, cl.Opt("connection"), id, add, cl.Opt("comment"))); return 0; } - var (text, image) = await Attachments.ReadAsync(lens, cl.Opt("connection"), id, cl.Opt("file"), cl.Opt("detail")); + if (cl.Opt("to") is { } to) lens.Settings.DownloadDir = to; + var (text, images) = await Attachments.ReadAsync(lens, cl.Opt("connection"), id, cl.Opt("file"), cl.Opt("detail"), cl.Opt("pages"), cl.Opt("part"), + cl.Flag("save") || cl.Peek("to") is not null); Print(text); - if (image is { } i) + foreach (var i in images) { - // In a terminal an image is saved and opened instead. + // A terminal cannot show images: they are written to the temporary folder. var path = Path.Combine(Path.GetTempPath(), $"adolens-{id}-{Guid.NewGuid():N}"[..24] + (i.Mime == "image/jpeg" ? ".jpg" : "." + i.Mime[6..])); await File.WriteAllBytesAsync(path, i.Bytes); - Console.WriteLine($"saved: {path}"); + Console.WriteLine($"image: {path}"); } return 0; } diff --git a/src/AdoLens/CommandLine.cs b/src/AdoLens/CommandLine.cs index 98e981b..7a2a76d 100644 --- a/src/AdoLens/CommandLine.cs +++ b/src/AdoLens/CommandLine.cs @@ -21,9 +21,9 @@ public sealed partial class CommandLine ["connections"] = [], ["trace"] = ["connection=", "detail="], ["pr"] = ["connection=", "project=", "repo=", "list=", "detail="], - ["pipeline"] = ["connection=", "project=", "definition=", "branch=", "top=", "detail="], + ["pipeline"] = ["connection=", "project=", "definition=", "branch=", "top=", "detail=", "artifacts", "artifact=", "part=", "save", "to="], ["changes"] = ["connection=", "repo=", "file=", "detail="], - ["config"] = ["env=", "remove-env=", "connection=", "default-connection=", "project=", "team=", "mode=", "usage-log=", "add-toolset=", "remove-toolset=", "confirm-above=", "show-tokens=", "language=", "label-languages="], + ["config"] = ["env=", "remove-env=", "connection=", "default-connection=", "project=", "team=", "mode=", "usage-log=", "add-toolset=", "remove-toolset=", "confirm-above=", "show-tokens=", "language=", "label-languages=", "download-dir="], ["item"] = ["connection=", "include=", "detail="], ["query"] = ["connection=", "project=", "wiql=", "query=", "top="], ["create"] = ["connection=", "project=", "type=", "title=", "description=", "criteria=", "assign=", "parent=", "iteration=", "area=", "tags=", "state=", "field=", "template="], @@ -45,7 +45,7 @@ public sealed partial class CommandLine ["api"] = ["connection="], ["skills"] = [], ["review"] =["connection=", "repo=", "item="], - ["attachments"] =["connection=", "file=", "add=", "comment=", "detail="], + ["attachments"] = ["connection=", "file=", "add=", "comment=", "detail=", "pages=", "part=", "save", "to="], ["label"] =["connection=", "project=", "branch="], ["search"] =["connection=", "project=", "scope=", "top="], ["file"] = ["connection=", "project=", "repo=", "version=", "part=", "history", "top=", "detail="], diff --git a/src/AdoLens/Documents.cs b/src/AdoLens/Documents.cs new file mode 100644 index 0000000..d1da283 --- /dev/null +++ b/src/AdoLens/Documents.cs @@ -0,0 +1,308 @@ +using System.IO.Compression; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml.Linq; + +namespace AdoLens; + +/// What a file is, found from its first bytes (and the inside of a ZIP), not from its name. +public enum FileKind { Text, Image, Word, Excel, PowerPoint, Pdf, OutlookMsg, Email, Zip, OldOffice, Binary } + +/// A file read for the model: its text, page images (scans), and what else can be asked for. +public sealed record DocumentContent(FileKind Kind, string? Text, List<(byte[] Bytes, string Mime)> Images, string? Note = null); + +/// +/// Reads files from Azure DevOps (attachments, artifacts) for the model: documents as text, images and scanned PDF +/// pages as images. Binary files are never turned into text; they are saved instead. +/// +public static partial class Documents +{ + public const int MaxImageBytes = 4 * 1024 * 1024; + public const int MaxPdfImagePages = 3; + + public static FileKind Detect(string name, byte[] b) + { + bool Starts(params byte[] sig) => b.Length >= sig.Length && sig.Select((x, i) => b[i] == x).All(x => x); + if (Attachments.ImageMime(b) is not null) return FileKind.Image; + if (Starts(0x25, 0x50, 0x44, 0x46)) return FileKind.Pdf; // %PDF + if (Starts(0x50, 0x4B, 0x03, 0x04) || Starts(0x50, 0x4B, 0x05, 0x06)) + { + try + { + using var zip = new ZipArchive(new MemoryStream(b)); + if (zip.GetEntry("word/document.xml") is not null) return FileKind.Word; + if (zip.GetEntry("xl/workbook.xml") is not null || zip.Entries.Any(e => e.FullName.StartsWith("xl/worksheets/", StringComparison.OrdinalIgnoreCase))) + return FileKind.Excel; + if (zip.GetEntry("ppt/presentation.xml") is not null || zip.Entries.Any(e => e.FullName.StartsWith("ppt/slides/", StringComparison.OrdinalIgnoreCase))) + return FileKind.PowerPoint; + return FileKind.Zip; + } + catch (InvalidDataException) { return FileKind.Binary; } + } + if (Starts(0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1)) + return OutlookMsg.IsMessage(b) ? FileKind.OutlookMsg : FileKind.OldOffice; + if (LooksLikeText(b)) + return name.EndsWith(".eml", StringComparison.OrdinalIgnoreCase) || IsEml(Encoding.UTF8.GetString(b, 0, Math.Min(b.Length, 4000))) + ? FileKind.Email : FileKind.Text; + return FileKind.Binary; + } + + /// Text: UTF-16 with a BOM, or no zero bytes and mostly printable characters in the first 8 KB. + internal static bool LooksLikeText(byte[] b) + { + if (b.Length >= 2 && (b[0] == 0xFF && b[1] == 0xFE || b[0] == 0xFE && b[1] == 0xFF)) return true; + var n = Math.Min(b.Length, 8192); + if (n == 0) return true; + var control = 0; + for (var i = 0; i < n; i++) + { + if (b[i] == 0) return false; + if (b[i] < 0x09 || b[i] is > 0x0D and < 0x20) control++; + } + return control * 100 / n < 2; + } + + static string DecodeText(byte[] b) => + b.Length >= 2 && b[0] == 0xFF && b[1] == 0xFE ? Encoding.Unicode.GetString(b, 2, b.Length - 2) + : b.Length >= 2 && b[0] == 0xFE && b[1] == 0xFF ? Encoding.BigEndianUnicode.GetString(b, 2, b.Length - 2) + : Encoding.UTF8.GetString(b).TrimStart(''); + + /// + /// Reads a file. ("3", "2-5") limits a PDF; names a file inside a + /// ZIP. Null text with no images: nothing readable (the caller offers saving or a link). + /// + public static DocumentContent Read(string name, byte[] bytes, string? pages = null, string? part = null) + { + var kind = Detect(name, bytes); + try + { + return kind switch + { + FileKind.Text => new(kind, DecodeText(bytes), []), + FileKind.Image => bytes.Length > MaxImageBytes + ? new(kind, null, [], $"the image is too large to show (over {MaxImageBytes / 1024 / 1024} MB)") + : new(kind, null, [(bytes, Attachments.ImageMime(bytes)!)]), + FileKind.Word => new(kind, Word(bytes), []), + FileKind.Excel => new(kind, Excel(bytes), []), + FileKind.PowerPoint => new(kind, PowerPoint(bytes), []), + FileKind.OutlookMsg => new(kind, OutlookMsg.Read(bytes), []), + FileKind.Email => new(kind, Eml.Read(DecodeText(bytes)), []), + FileKind.Pdf => Pdf(bytes, pages), + FileKind.Zip => Zip(name, bytes, part, pages), + FileKind.OldOffice => new(kind, null, [], "an old Office format (.doc / .xls / .ppt) - save it and open it, or ask for it as .docx / .xlsx"), + _ => new(kind, null, [], "a binary file - nothing to read as text"), + }; + } + catch (Exception ex) when (ex is InvalidDataException or System.Xml.XmlException or IOException or InvalidOperationException or ArgumentException + or IndexOutOfRangeException or UglyToad.PdfPig.Core.PdfDocumentFormatException) + { + return new(kind, null, [], $"the {kind} file could not be read ({ex.Message})"); + } + } + + /// "3" → (3,3); "2-5" → (2,5); null → (1, max). + internal static (int From, int To) PageRange(string? pages, int count) + { + if (string.IsNullOrWhiteSpace(pages)) return (1, count); + var m = Regex.Match(pages, @"^\s*(\d+)\s*(?:-\s*(\d+))?\s*$"); + if (!m.Success) throw new ArgumentException("pages is a number or a range, e.g. 3 or 2-5"); + var from = Math.Max(1, int.Parse(m.Groups[1].Value)); + var to = m.Groups[2].Success ? int.Parse(m.Groups[2].Value) : from; + return (from, Math.Min(Math.Max(from, to), count)); + } + + static XDocument? Part(ZipArchive zip, string path) => + zip.GetEntry(path) is { } e ? XDocument.Load(e.Open()) : null; + + /// Paragraphs as lines, headings as markdown, tables as "a | b" rows. + internal static string Word(byte[] bytes) + { + using var zip = new ZipArchive(new MemoryStream(bytes)); + var doc = Part(zip, "word/document.xml") ?? throw new InvalidDataException("no word/document.xml"); + XNamespace w = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"; + var sb = new StringBuilder(); + var body = doc.Root?.Element(w + "body"); + if (body is null) return ""; + foreach (var block in body.Elements()) + { + if (block.Name == w + "p") sb.AppendLine(Paragraph(block, w)); + else if (block.Name == w + "tbl") + { + foreach (var row in block.Elements(w + "tr")) + sb.AppendLine("| " + string.Join(" | ", row.Elements(w + "tc").Select(c => string.Join(" ", c.Elements(w + "p").Select(p => Paragraph(p, w))).Trim())) + " |"); + sb.AppendLine(); + } + } + return Regex.Replace(sb.ToString(), @"\n{3,}", "\n\n").Trim(); + } + + static string Paragraph(XElement p, XNamespace w) + { + var text = string.Concat(p.Descendants().Select(e => e.Name == w + "t" ? e.Value : e.Name == w + "tab" ? "\t" : e.Name == w + "br" ? "\n" : "")); + var style = p.Element(w + "pPr")?.Element(w + "pStyle")?.Attribute(w + "val")?.Value ?? ""; + // "Heading1" in English Word, "Nagwek1" in a Polish one. + var digits = new string(style.Where(char.IsDigit).ToArray()); + var level = (style.StartsWith("Heading", StringComparison.OrdinalIgnoreCase) || style.StartsWith("Nagwek", StringComparison.OrdinalIgnoreCase)) + && int.TryParse(digits, out var l) ? l : 0; + if (level > 0 && text.Length > 0) return new string('#', Math.Min(level, 6)) + " " + text; + if (p.Element(w + "pPr")?.Element(w + "numPr") is not null && text.Length > 0) return "- " + text; + return text; + } + + /// Each sheet (by its name) as rows of "a | b | c". + internal static string Excel(byte[] bytes) + { + using var zip = new ZipArchive(new MemoryStream(bytes)); + XNamespace x = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; + XNamespace r = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; + var shared = Part(zip, "xl/sharedStrings.xml")?.Root?.Elements(x + "si").Select(si => string.Concat(si.Descendants(x + "t").Select(t => t.Value))).ToList() ?? []; + // Sheet names: workbook.xml names each sheet's relationship, workbook.xml.rels gives its file. + var rels = Part(zip, "xl/_rels/workbook.xml.rels")?.Root?.Elements().ToDictionary(e => (string?)e.Attribute("Id") ?? "", e => (string?)e.Attribute("Target") ?? "") ?? []; + var sheets = Part(zip, "xl/workbook.xml")?.Descendants(x + "sheet") + .Select(s => (Name: (string?)s.Attribute("name") ?? "", Path: rels.GetValueOrDefault((string?)s.Attribute(r + "id") ?? ""))) + .Where(s => s.Path is not null).Select(s => (s.Name, Path: "xl/" + s.Path!.TrimStart('/').Replace("xl/", ""))).ToList() ?? []; + if (sheets.Count == 0) + sheets = zip.Entries.Where(e => e.FullName.StartsWith("xl/worksheets/sheet", StringComparison.OrdinalIgnoreCase)) + .Select(e => (Path.GetFileNameWithoutExtension(e.FullName), e.FullName)).ToList(); + var sb = new StringBuilder(); + foreach (var (sheetName, path) in sheets) + { + if (zip.GetEntry(path) is not { } entry) continue; + sb.AppendLine($"## {sheetName}"); + var sheet = XDocument.Load(entry.Open()); + foreach (var row in sheet.Descendants(x + "row").Take(500)) + { + var cells = row.Elements(x + "c").Select(c => + { + var v = c.Element(x + "v")?.Value ?? string.Concat(c.Descendants(x + "t").Select(t => t.Value)); + return (string?)c.Attribute("t") == "s" && int.TryParse(v, out var i) && i < shared.Count ? shared[i] : v; + }).ToList(); + if (cells.Any(c => c.Length > 0)) sb.AppendLine("| " + string.Join(" | ", cells) + " |"); + } + sb.AppendLine(); + } + return sb.ToString().Trim(); + } + + /// A saved e-mail starts with header lines, among them MIME-Version and From or Subject. + internal static bool IsEml(string head) + { + var lines = head.Replace("\r", "").Split('\n').TakeWhile(l => l.Length > 0).ToList(); + if (lines.Count < 2 || !Regex.IsMatch(lines[0], @"^[A-Za-z][A-Za-z0-9-]*:")) return false; + bool Has(string h) => lines.Any(l => l.StartsWith(h + ":", StringComparison.OrdinalIgnoreCase)); + return Has("MIME-Version") && (Has("From") || Has("Subject")); + } + + [GeneratedRegex(@"slide(\d+)\.xml$")] + private static partial Regex SlideNumber(); + + /// Each slide's text, in slide order, with its notes. + internal static string PowerPoint(byte[] bytes) + { + using var zip = new ZipArchive(new MemoryStream(bytes)); + XNamespace a = "http://schemas.openxmlformats.org/drawingml/2006/main"; + var slides = zip.Entries.Where(e => e.FullName.StartsWith("ppt/slides/slide", StringComparison.OrdinalIgnoreCase) && e.FullName.EndsWith(".xml", StringComparison.OrdinalIgnoreCase)) + .Select(e => (N: int.TryParse(SlideNumber().Match(e.FullName).Groups[1].Value, out var n) ? n : 0, Entry: e)).OrderBy(s => s.N).ToList(); + var sb = new StringBuilder(); + foreach (var (n, entry) in slides) + { + var doc = XDocument.Load(entry.Open()); + var paragraphs = doc.Descendants(a + "p").Select(p => string.Concat(p.Descendants(a + "t").Select(t => t.Value)).Trim()).Where(t => t.Length > 0).ToList(); + sb.AppendLine($"## Slide {n}"); + foreach (var p in paragraphs) sb.AppendLine(p); + if (zip.GetEntry($"ppt/notesSlides/notesSlide{n}.xml") is { } notes) + { + var text = string.Join(" ", XDocument.Load(notes.Open()).Descendants(a + "t").Select(t => t.Value)).Trim(); + if (text.Length > 0 && !Regex.IsMatch(text, @"^\d+$")) sb.AppendLine($"(notes: {text})"); + } + sb.AppendLine(); + } + return sb.ToString().Trim(); + } + + /// + /// PDF text page by page. Pages with (almost) no text are scans: their embedded page images are returned instead, + /// a few at a time. + /// + static DocumentContent Pdf(byte[] bytes, string? pages) + { + using var doc = UglyToad.PdfPig.PdfDocument.Open(bytes); + var count = doc.NumberOfPages; + var (from, to) = PageRange(pages, count); + var sb = new StringBuilder(); + var images = new List<(byte[], string)>(); + var scanned = new List(); + for (var i = from; i <= to; i++) + { + var page = doc.GetPage(i); + var text = UglyToad.PdfPig.DocumentLayoutAnalysis.TextExtractor.ContentOrderTextExtractor.GetText(page).Trim(); + if (text.Length >= 20) + { + sb.AppendLine($"## Page {i}").AppendLine(text).AppendLine(); + continue; + } + scanned.Add(i); + if (images.Count >= MaxPdfImagePages) continue; + foreach (var img in page.GetImages()) + { + byte[]? data = null; + string? mime = null; + if (img.TryGetPng(out var png)) { data = png; mime = "image/png"; } + else if (Attachments.ImageMime(img.RawBytes.ToArray()) is { } m) { data = img.RawBytes.ToArray(); mime = m; } + if (data is null || data.Length > MaxImageBytes) continue; + images.Add((data, mime!)); + sb.AppendLine($"## Page {i}: scanned - shown as an image"); + break; + } + } + var note = new StringBuilder($"PDF, {count} page(s)"); + if (from != 1 || to != count) note.Append($", pages {from}-{to} shown"); + else if (count > 3) note.Append("; pages=3 or pages=2-5 reads a part"); + if (scanned.Count > 0) + { + note.Append($"; no text on page(s) {string.Join(", ", scanned.Take(20))} (scanned)"); + var notShown = scanned.Count - images.Count; + if (notShown > 0) note.Append($" - {notShown} not shown: ask for them with pages="); + } + return new DocumentContent(FileKind.Pdf, sb.Length > 0 ? sb.ToString().Trim() : null, images, note.ToString()); + } + + /// The list of files in a ZIP; with , one file inside it read like any other. + static DocumentContent Zip(string name, byte[] bytes, string? part, string? pages) + { + using var zip = new ZipArchive(new MemoryStream(bytes)); + var files = zip.Entries.Where(e => !e.FullName.EndsWith('/')).ToList(); + if (string.IsNullOrWhiteSpace(part)) + { + var sb = new StringBuilder($"{files.Count} file(s) in {name}:\n"); + foreach (var e in files.Take(200)) sb.AppendLine($"- {e.FullName} ({Size(e.Length)})"); + if (files.Count > 200) sb.AppendLine($"- … {files.Count - 200} more"); + return new(FileKind.Zip, sb.Append("part= reads one of them.").ToString(), []); + } + var entry = files.FirstOrDefault(e => e.FullName.Equals(part.Trim(), StringComparison.OrdinalIgnoreCase)) + ?? files.FirstOrDefault(e => e.FullName.EndsWith(part.Trim(), StringComparison.OrdinalIgnoreCase)); + if (entry is null) return new(FileKind.Zip, null, [], $"no '{part}' in {name}"); + if (entry.Length > 50L * 1024 * 1024) return new(FileKind.Zip, null, [], $"{entry.FullName} is {Size(entry.Length)} - save the ZIP and open it"); + using var ms = new MemoryStream(); + using (var s = entry.Open()) s.CopyTo(ms); + var inner = Read(entry.Name, ms.ToArray(), pages); + return inner with { Note = $"{entry.FullName} from {name}" + (inner.Note is null ? "" : $"; {inner.Note}") }; + } + + public static string Size(long bytes) => + bytes < 1024 ? $"{bytes} B" : bytes < 1024 * 1024 ? $"{bytes / 1024} KB" : $"{bytes / 1024.0 / 1024:0.#} MB"; + + /// + /// A long text cut for the model: the headings (an outline) and the start; the rest on request. Short texts and + /// detail=full pass whole. + /// + public static string Fit(string text, int max, bool full) + { + if (full || text.Length <= max) return text; + // Real headings only: "## Page 3" / "## Slide 2" are markers this reader adds, not the document's outline. + var headings = text.Split('\n').Where(l => l.StartsWith('#') && !Regex.IsMatch(l, @"^#+\s*(Page|Slide) \d+")).Take(40).ToList(); + var sb = new StringBuilder($"{text.Length:N0} characters (~{text.Length / 4:N0} tokens) - the start below; detail=full for all"); + if (headings.Count > 0) sb.Append($"\noutline: {string.Join(" | ", headings.Select(h => h.TrimStart('#', ' ')))}"); + return sb.Append("\n\n").Append(text[..max]).Append("\n…").ToString(); + } +} diff --git a/src/AdoLens/Downloads.cs b/src/AdoLens/Downloads.cs new file mode 100644 index 0000000..90400cf --- /dev/null +++ b/src/AdoLens/Downloads.cs @@ -0,0 +1,111 @@ +using System.Security.Cryptography; + +namespace AdoLens; + +/// +/// Files from Azure DevOps saved on disk, streamed (any size, nothing held in memory, nothing sent to the model but the +/// path). Saved files are never opened or run. +/// +public static class Downloads +{ + /// The folder from the settings, else Downloads\ADO Lens. + public static string BaseDir(Settings settings) => + !string.IsNullOrWhiteSpace(settings.DownloadDir) ? Environment.ExpandEnvironmentVariables(settings.DownloadDir) + : Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), "Downloads", "ADO Lens"); + + /// A file name that stays a file name: no folders, no characters Windows refuses, not empty, not too long. + public static string SafeName(string? name) + { + // ":" first: "a:b.txt" is a name with a colon, not drive "a:". + var n = Path.GetFileName((name ?? "").Replace(':', '_').Replace('/', '\\')); + foreach (var c in Path.GetInvalidFileNameChars()) n = n.Replace(c, '_'); + n = n.Trim().Trim('.'); + if (n.Length == 0) n = "file"; + if (n.Length > 150) n = n[..(150 - Path.GetExtension(n).Length)] + Path.GetExtension(n); + // Reserved device names (CON, NUL, COM1…) cannot be files. + if (System.Text.RegularExpressions.Regex.IsMatch(Path.GetFileNameWithoutExtension(n), @"^(CON|PRN|AUX|NUL|COM\d|LPT\d)$", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) + n = "_" + n; + return n; + } + + /// A path in the folder that does not exist yet: "name.ext", then "name (2).ext"… + public static string FreePath(string folder, string name) + { + var path = Path.Combine(folder, name); + for (var i = 2; File.Exists(path); i++) + path = Path.Combine(folder, $"{Path.GetFileNameWithoutExtension(name)} ({i}){Path.GetExtension(name)}"); + return path; + } + + /// + /// Streams into \<sub>\<name>; the SHA-256 is computed on the way. + /// A partly written file is removed when the download fails. + /// + public static async Task SaveAsync(AdoClient client, string url, string folder, string sub, string name, CancellationToken ct) + { + var root = Path.GetFullPath(folder); + var dir = Path.GetFullPath(Path.Combine(root, SafeName(sub))); + if (!dir.StartsWith(root, StringComparison.OrdinalIgnoreCase)) throw new ArgumentException("the target folder is outside the download folder"); + Directory.CreateDirectory(dir); + var path = FreePath(dir, SafeName(name)); + long size; + string sha; + try + { + await using (var file = new FileStream(path, FileMode.CreateNew, FileAccess.Write, FileShare.None, 1 << 20, useAsync: true)) + await using (var hashing = new HashingStream(file)) + { + size = await client.DownloadToAsync(url, hashing, ct); + sha = hashing.Hash(); + } + } + catch + { + try { File.Delete(path); } catch (IOException) { } + throw; + } + return $"saved: {path} ({Documents.Size(new FileInfo(path).Length)}, sha256 {sha})"; + } + + /// Writes through to a file and hashes what passes. + sealed class HashingStream(Stream inner) : Stream + { + readonly IncrementalHash hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + long written; + + public string Hash() => Convert.ToHexString(hash.GetHashAndReset()).ToLowerInvariant(); + public override bool CanRead => false; + public override bool CanSeek => false; + public override bool CanWrite => true; + public override long Length => written; + public override long Position { get => written; set => throw new NotSupportedException(); } + public override void Flush() => inner.Flush(); + public override Task FlushAsync(CancellationToken ct) => inner.FlushAsync(ct); + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) + { + hash.AppendData(buffer, offset, count); + inner.Write(buffer, offset, count); + written += count; + } + + public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken ct = default) + { + hash.AppendData(buffer.Span); + await inner.WriteAsync(buffer, ct); + written += buffer.Length; + } + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken ct) => + WriteAsync(buffer.AsMemory(offset, count), ct).AsTask(); + + protected override void Dispose(bool disposing) + { + if (disposing) hash.Dispose(); + base.Dispose(disposing); + } + } +} diff --git a/src/AdoLens/Eml.cs b/src/AdoLens/Eml.cs new file mode 100644 index 0000000..b3573a3 --- /dev/null +++ b/src/AdoLens/Eml.cs @@ -0,0 +1,139 @@ +using System.Text; +using System.Text.RegularExpressions; + +namespace AdoLens; + +/// A saved e-mail (.eml, MIME): headers, the text body (plain, else HTML as text), the names of its attachments. +public static partial class Eml +{ + static Eml() => Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + + sealed record Part(Dictionary Headers, string Body); + + public static string Read(string raw) + { + var root = Parse(raw.Replace("\r\n", "\n")); + var sb = new StringBuilder(); + void Line(string label, string header) { if (root.Headers.TryGetValue(header, out var v) && v.Length > 0) sb.AppendLine($"{label}: {Words(v)}"); } + Line("Subject", "subject"); + Line("From", "from"); + Line("To", "to"); + Line("Cc", "cc"); + Line("Sent", "date"); + var leaves = Leaves(root).ToList(); + var files = leaves.Select(FileName).Where(n => n is not null).ToList(); + if (files.Count > 0) sb.AppendLine($"Attachments: {string.Join(", ", files)}"); + sb.AppendLine(); + var plain = leaves.FirstOrDefault(p => FileName(p) is null && ContentType(p).StartsWith("text/plain", StringComparison.OrdinalIgnoreCase)); + var html = leaves.FirstOrDefault(p => FileName(p) is null && ContentType(p).StartsWith("text/html", StringComparison.OrdinalIgnoreCase)); + var body = plain is not null ? Decode(plain) : html is not null ? Html.ToMarkdown(Decode(html)) : null; + sb.Append(string.IsNullOrWhiteSpace(body) ? "(no text body)" : body.Replace("\r\n", "\n").Trim()); + return sb.ToString(); + } + + static Part Parse(string text) + { + var split = text.IndexOf("\n\n", StringComparison.Ordinal); + var head = split < 0 ? text : text[..split]; + var body = split < 0 ? "" : text[(split + 2)..]; + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + string? last = null; + foreach (var line in head.Split('\n')) + { + if (line.Length > 0 && (line[0] == ' ' || line[0] == '\t') && last is not null) headers[last] += " " + line.Trim(); + else if (line.IndexOf(':') is var c and > 0) + { + last = line[..c].Trim(); + headers[last] = headers.TryGetValue(last, out var old) ? old : line[(c + 1)..].Trim(); + } + } + return new Part(headers, body); + } + + static string ContentType(Part p) => p.Headers.GetValueOrDefault("content-type") ?? "text/plain"; + + static string? Param(string header, string name) => + Regex.Match(header, $@"\b{name}\*?=\s*(?:""(?[^""]*)""|(?[^;\s]+))", RegexOptions.IgnoreCase) is { Success: true } m ? m.Groups["v"].Value : null; + + static IEnumerable Leaves(Part p) + { + var type = ContentType(p); + if (!type.StartsWith("multipart/", StringComparison.OrdinalIgnoreCase) || Param(type, "boundary") is not { } boundary) + { + yield return p; + yield break; + } + var sections = p.Body.Split("--" + boundary); + foreach (var s in sections.Skip(1)) + { + if (s.StartsWith("--", StringComparison.Ordinal)) break; + foreach (var leaf in Leaves(Parse(s.TrimStart('\n')))) + yield return leaf; + } + } + + static string? FileName(Part p) + { + var disposition = p.Headers.GetValueOrDefault("content-disposition") ?? ""; + var name = Param(disposition, "filename") ?? Param(ContentType(p), "name"); + if (name is null && !disposition.StartsWith("attachment", StringComparison.OrdinalIgnoreCase)) return null; + return name is null ? "(unnamed)" : Words(name); + } + + static Encoding Charset(string? name) + { + try { return name is null ? Encoding.UTF8 : Encoding.GetEncoding(name.Trim('"')); } + catch (ArgumentException) { return Encoding.UTF8; } + } + + static string Decode(Part p) + { + var encoding = (p.Headers.GetValueOrDefault("content-transfer-encoding") ?? "").Trim().ToLowerInvariant(); + var charset = Charset(Param(ContentType(p), "charset")); + // 7bit / 8bit: the body is text already (the file was read as UTF-8). + return encoding switch + { + "base64" => charset.GetString(SafeBase64(p.Body)), + "quoted-printable" => charset.GetString(QuotedPrintable(p.Body, header: false)), + _ => p.Body, + }; + } + + static byte[] SafeBase64(string s) + { + try { return Convert.FromBase64String(Regex.Replace(s, @"[^A-Za-z0-9+/=]", "")); } + catch (FormatException) { return []; } + } + + static byte[] QuotedPrintable(string s, bool header) + { + var bytes = new List(); + s = Regex.Replace(s, @"=\n", ""); // soft line breaks + for (var i = 0; i < s.Length; i++) + { + if (s[i] == '=' && i + 2 < s.Length && Uri.IsHexDigit(s[i + 1]) && Uri.IsHexDigit(s[i + 2])) + { + bytes.Add(Convert.ToByte(s.Substring(i + 1, 2), 16)); + i += 2; + } + else if (header && s[i] == '_') bytes.Add((byte)' '); + else bytes.AddRange(Encoding.Latin1.GetBytes(s[i].ToString())); + } + return [.. bytes]; + } + + [GeneratedRegex(@"=\?(?[^?]+)\?(?[BbQq])\?(?[^?]*)\?=")] + private static partial Regex EncodedWord(); + + /// "=?UTF-8?B?…?=" and "=?ISO-8859-2?Q?…?=" in headers. + internal static string Words(string header) + { + var joined = Regex.Replace(header, @"\?=\s+=\?", "?==?"); + return EncodedWord().Replace(joined, m => + { + var cs = Charset(m.Groups["cs"].Value); + var bytes = m.Groups["enc"].Value is "B" or "b" ? SafeBase64(m.Groups["text"].Value) : QuotedPrintable(m.Groups["text"].Value, header: true); + return cs.GetString(bytes); + }).Trim(); + } +} diff --git a/src/AdoLens/Labels.cs b/src/AdoLens/Labels.cs index f8c2d6a..dedb4a9 100644 --- a/src/AdoLens/Labels.cs +++ b/src/AdoLens/Labels.cs @@ -133,7 +133,10 @@ public static async Task FindAsync(Lens lens, string? connection, string sb.AppendLine($"{label} ({root})"); // Every language of the label file: the text, or that it is missing. var folder = file.Path[..file.Path.IndexOf("/LabelResources/", StringComparison.OrdinalIgnoreCase)] + "/LabelResources"; - foreach (var lf in await LanguagesAsync(client, folder, file, ct)) + var languageFiles = await LanguagesAsync(client, folder, file, ct); + foreach (var lang in session.Config.LabelLanguages.Where(l => !languageFiles.Any(x => x.Language.Equals(l, StringComparison.OrdinalIgnoreCase)))) + sb.AppendLine($" {lang}: NO LABEL FILE (required, but {file.FileId} has no {lang} file)"); + foreach (var lf in languageFiles) { var content = await ReadAsync(client, lf, ct); var value = content is null ? null : Parse(content).GetValueOrDefault(id); diff --git a/src/AdoLens/McpTools.cs b/src/AdoLens/McpTools.cs index 4a8c3db..ae5784f 100644 --- a/src/AdoLens/McpTools.cs +++ b/src/AdoLens/McpTools.cs @@ -333,9 +333,13 @@ public static Task PrUpdate(Lens lens, public static class PipelineTools { [McpServerTool(Name = "ado_pipeline", ReadOnly = true, Idempotent = true)] - [Description("Builds: recent runs (optionally of one pipeline or branch); with build: why it failed - failed steps, errors and the error lines of the log.")] + [Description("Builds: recent runs (optionally of one pipeline or branch); with build: why it failed - failed steps, errors and the error lines of the log; artifacts=true lists its artifacts, artifact= its files (part= reads one, save=true saves it to disk).")] public static Task Pipeline(Lens lens, int? build = null, + bool artifacts = false, + string? artifact = null, + string? part = null, + bool save = false, [Description("Pipeline name or id.")] string? definition = null, string? branch = null, int top = 10, @@ -343,8 +347,11 @@ public static Task Pipeline(Lens lens, string? project = null, string? connection = null, CancellationToken ct = default) - => McpTools.Run("ado_pipeline", new { build, definition, branch, top, detail, project, connection }, - () => build is { } b ? PipelineQueries.BuildAsync(lens, connection, project, b, detail, ct) : PipelineQueries.ListAsync(lens, connection, project, definition, branch, top, ct)); + => McpTools.Run("ado_pipeline", new { build, artifacts, artifact, part, save, definition, branch, top, detail, project, connection }, + () => build is { } b + ? artifacts || artifact is not null ? PipelineQueries.ArtifactsAsync(lens, connection, project, b, artifact, part, save, ct) + : PipelineQueries.BuildAsync(lens, connection, project, b, detail, ct) + : PipelineQueries.ListAsync(lens, connection, project, definition, branch, top, ct)); [McpServerTool(Name = "ado_pipeline_run")] [Description("Queues a run of a pipeline, optionally on a branch.")] @@ -499,25 +506,28 @@ public static Task CreateTree(Lens lens, string items, () => WorkTree.CreateAsync(lens, connection, project, parent, items, iteration, area, confirm, ct)); [McpServerTool(Name = "ado_attachments")] - [Description("Files of a work item and screenshots pasted into its text or comments: the list; file= shows an image or the text of a document (Word, Excel, Outlook e-mail, text); add= attaches a file.")] + [Description("Files of a work item and screenshots pasted into it: the list; file= gives a document's text (PDF, Word, Excel, PowerPoint, e-mail, ZIP list, text) or shows images and scanned pages; save=true streams it to disk (any size) and returns the path; add= attaches a file.")] public static async Task> AttachmentsTool(Lens lens, int id, string? file = null, + [Description("PDF pages: 3 or 2-5.")] string? pages = null, + [Description("A file inside a ZIP.")] string? part = null, + bool save = false, [Description("Local file to attach.")] string? add = null, string? comment = null, [Description("full: the whole text.")] string? detail = null, string? connection = null, CancellationToken ct = default) { - (byte[] Bytes, string Mime)? image = null; - var text = await McpTools.Run("ado_attachments", new { id, file, add, connection }, async () => + List<(byte[] Bytes, string Mime)> images = []; + var text = await McpTools.Run("ado_attachments", new { id, file, pages, part, save, add, connection }, async () => { if (add is not null) return await Attachments.AddAsync(lens, connection, id, add, comment, ct); - var (t, img) = await Attachments.ReadAsync(lens, connection, id, file, detail, ct); - image = img; + var (t, imgs) = await Attachments.ReadAsync(lens, connection, id, file, detail, pages, part, save, ct); + images = imgs; return t; }); var blocks = new List { new ModelContextProtocol.Protocol.TextContentBlock { Text = text } }; - if (image is { } i) blocks.Add(ModelContextProtocol.Protocol.ImageContentBlock.FromBytes(i.Bytes, i.Mime)); + blocks.AddRange(images.Select(i => ModelContextProtocol.Protocol.ImageContentBlock.FromBytes(i.Bytes, i.Mime))); return blocks; } diff --git a/src/AdoLens/OutlookMsg.cs b/src/AdoLens/OutlookMsg.cs index aa804e1..cfcc653 100644 --- a/src/AdoLens/OutlookMsg.cs +++ b/src/AdoLens/OutlookMsg.cs @@ -112,6 +112,18 @@ public List Children(Entry storage) } } + /// Whether a compound file is an Outlook item (it has the message property streams at the top). + public static bool IsMessage(byte[] bytes) + { + try + { + var cfb = new Cfb(bytes); + return cfb.Entries.Count > 0 && cfb.Children(cfb.Entries[0]).Any(e => e.Name.StartsWith("__substg1.0_", StringComparison.OrdinalIgnoreCase) + || e.Name == "__properties_version1.0"); + } + catch (Exception ex) when (ex is InvalidDataException or ArgumentException or IndexOutOfRangeException) { return false; } + } + /// The e-mail as text, or null when the file is not an Outlook message. public static string? Read(byte[] bytes) { diff --git a/src/AdoLens/Pipelines.cs b/src/AdoLens/Pipelines.cs index 049eea8..ff09789 100644 --- a/src/AdoLens/Pipelines.cs +++ b/src/AdoLens/Pipelines.cs @@ -74,6 +74,41 @@ internal static string Excerpt(string log, int max) return string.Join('\n', hits.Select(x => $"{x.i + 1}: {Formatting.Collapse(x.l, 300)}")); } + const long MaxArtifactRead = 20 * 1024 * 1024; + + /// + /// The artifacts of a build (packages, logs, reports): the list; with the files in it, + /// one file read as text, the whole artifact streamed to disk. + /// + public static async Task ArtifactsAsync(Lens lens, string? connection, string? project, int build, string? artifact, string? part, bool save, + CancellationToken ct = default) + { + var session = lens.For(connection); + var scope = AdoClient.Scope(session.Project(project)); + var list = await session.Client.GetListAsync($"{scope}_apis/build/builds/{build}/artifacts", ct); + static long? SizeOf(JsonElement a) => + long.TryParse(Json.Str(Json.Obj(Json.Obj(a, "resource"), "properties"), "artifactsize"), out var n) ? n : null; + if (list.Count == 0) return $"build {build} has no artifacts"; + if (string.IsNullOrWhiteSpace(artifact)) + return $"artifacts of build {build}:\n" + string.Join('\n', list.Select(a => + $"- {Json.Str(a, "name")} ({Json.Str(Json.Obj(a, "resource"), "type")}{(SizeOf(a) is { } s ? ", " + Documents.Size(s) : "")})")) + + "\nartifact= lists its files; part= reads one; save=true saves the artifact (ZIP) to disk."; + var pick = list.FirstOrDefault(a => string.Equals(Json.Str(a, "name"), artifact.Trim(), StringComparison.OrdinalIgnoreCase)); + if (pick.ValueKind != JsonValueKind.Object) return $"no artifact '{artifact}' in build {build}; artifacts: {string.Join(", ", list.Select(a => Json.Str(a, "name")))}"; + var name = Json.Str(pick, "name")!; + var url = Json.Str(Json.Obj(pick, "resource"), "downloadUrl"); + if (url is null) return $"artifact {name} has no download link (type {Json.Str(Json.Obj(pick, "resource"), "type")}) - open the build in the browser"; + if (save) + return await Downloads.SaveAsync(session.Client, url, Downloads.BaseDir(lens.Settings), $"build {build}", name + ".zip", ct); + if (SizeOf(pick) > MaxArtifactRead) + return $"{name} is {Documents.Size(SizeOf(pick)!.Value)} - too large to read here; save=true saves it to disk"; + byte[] bytes; + try { bytes = await session.Client.GetBytesAsync(url, MaxArtifactRead, ct); } + catch (AdoApiException ex) when (ex.Message.Contains("larger than", StringComparison.Ordinal)) { return $"{name}: {ex.Message}; save=true saves it to disk"; } + var content = Documents.Read(name + ".zip", bytes, null, part); + return (content.Note is null ? "" : content.Note + "\n") + (content.Text is null ? "nothing readable as text; save=true saves it to disk" : Documents.Fit(content.Text, 12000, false)); + } + public static async Task BuildAsync(Lens lens, string? connection, string? project, int id, string? detail, CancellationToken ct = default) { var session = lens.For(connection); diff --git a/src/AdoLens/Review.cs b/src/AdoLens/Review.cs index d5faccc..c28794a 100644 --- a/src/AdoLens/Review.cs +++ b/src/AdoLens/Review.cs @@ -117,6 +117,9 @@ static async Task ReviewOneAsync(Session session, string change, string? var folder = folders.GetValueOrDefault(fileId) ?? await FindFolderAsync(session, c, fileId, ct); if (folder is null) continue; // a label file of another model (Microsoft's): not checked var languages = await LanguageFilesAsync(session, c, folder, fileId, ct); + // A required language with no label file at all: every label is missing in it. + foreach (var lang in session.Config.LabelLanguages.Where(l => !languages.Any(x => x.Language.Equals(l, StringComparison.OrdinalIgnoreCase)))) + missing.Add($"{lang}: no label file {fileId}.{lang}.label.txt in the model - none of its labels are translated"); foreach (var (lang, path) in languages) { if (!session.Config.LabelLanguageRequired(lang)) continue; diff --git a/src/AdoLens/Settings.cs b/src/AdoLens/Settings.cs index bbeee87..25125e5 100644 --- a/src/AdoLens/Settings.cs +++ b/src/AdoLens/Settings.cs @@ -49,6 +49,9 @@ public sealed partial class Settings /// Language of the settings console and installer texts: "pl", "en", or null for the language of Windows. public string? Language { get; set; } + /// Where files are saved (save=true); null: Downloads\ADO Lens. + public string? DownloadDir { get; set; } + static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web) { WriteIndented = true, diff --git a/tests/AdoLens.Tests/DocumentTests.cs b/tests/AdoLens.Tests/DocumentTests.cs new file mode 100644 index 0000000..5c35ea8 --- /dev/null +++ b/tests/AdoLens.Tests/DocumentTests.cs @@ -0,0 +1,178 @@ +using System.IO.Compression; +using System.Net; +using System.Text; +using UglyToad.PdfPig.Content; +using UglyToad.PdfPig.Core; +using UglyToad.PdfPig.Fonts.Standard14Fonts; +using UglyToad.PdfPig.Writer; + +namespace AdoLens.Tests; + +public class DocumentTests : IDisposable +{ + readonly string dir = Path.Combine(Path.GetTempPath(), $"adolens-docs-{Guid.NewGuid():N}"); + + public void Dispose() + { + if (Directory.Exists(dir)) Directory.Delete(dir, recursive: true); + } + + static readonly byte[] Png = Convert.FromBase64String("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="); + + static byte[] Zip(params (string Path, string Content)[] parts) + { + var ms = new MemoryStream(); + using (var zip = new ZipArchive(ms, ZipArchiveMode.Create, leaveOpen: true)) + foreach (var (path, content) in parts) + { + using var w = new StreamWriter(zip.CreateEntry(path).Open(), new UTF8Encoding(false)); + w.Write(content); + } + return ms.ToArray(); + } + + /// Page 1 with text, page 2 a "scan" (only an image), page 3 text again. + static byte[] Pdf() + { + var builder = new PdfDocumentBuilder(); + var font = builder.AddStandard14Font(Standard14Font.Helvetica); + var p1 = builder.AddPage(PageSize.A4); + p1.AddText("Credit limit check: orders above the limit are blocked.", 12, new PdfPoint(40, 700), font); + var p2 = builder.AddPage(PageSize.A4); + p2.AddPng(Png, new PdfRectangle(40, 400, 400, 760)); + var p3 = builder.AddPage(PageSize.A4); + p3.AddText("The third page explains the settings of the check.", 12, new PdfPoint(40, 700), font); + return builder.Build(); + } + + [Fact] + public void Kinds_come_from_the_bytes_not_the_name() + { + Assert.Equal(FileKind.Pdf, Documents.Detect("spec.bin", Pdf())); + Assert.Equal(FileKind.Word, Documents.Detect("x.zip", Zip(("word/document.xml", "")))); + Assert.Equal(FileKind.Excel, Documents.Detect("x", Zip(("xl/workbook.xml", "")))); + Assert.Equal(FileKind.PowerPoint, Documents.Detect("x", Zip(("ppt/presentation.xml", "

")))); + Assert.Equal(FileKind.Zip, Documents.Detect("logs.docx", Zip(("a.log", "x")))); + Assert.Equal(FileKind.Image, Documents.Detect("shot.txt", Png)); + Assert.Equal(FileKind.Email, Documents.Detect("mail.txt", Encoding.UTF8.GetBytes("From: a@b.c\nSubject: x\nMIME-Version: 1.0\n\nhi"))); + Assert.Equal(FileKind.Text, Documents.Detect("notes", Encoding.UTF8.GetBytes("plain notes\nsecond line"))); + // A trace or any binary: never read as text, whatever its name. + Assert.Equal(FileKind.Binary, Documents.Detect("trace.txt", [0x45, 0x6C, 0x66, 0x46, 0x69, 0x6C, 0x65, 0, 3, 0, 1, 2, 0, 0])); + Assert.Null(Documents.Read("trace.etl", [1, 2, 3, 0, 5]).Text); + } + + [Fact] + public void Pdf_gives_text_per_page_and_scanned_pages_as_images() + { + var all = Documents.Read("spec.pdf", Pdf()); + Assert.Contains("## Page 1", all.Text); + Assert.Contains("orders above the limit are blocked", all.Text); + Assert.Contains("## Page 2: scanned - shown as an image", all.Text); + Assert.Contains("third page", all.Text); + Assert.Single(all.Images); + Assert.Contains("PDF, 3 page(s)", all.Note); + Assert.Contains("no text on page(s) 2 (scanned)", all.Note); + + var third = Documents.Read("spec.pdf", Pdf(), pages: "3"); + Assert.DoesNotContain("Page 1", third.Text); + Assert.Contains("third page", third.Text); + Assert.Empty(third.Images); + Assert.Throws(() => Documents.PageRange("a-b", 3)); + Assert.Equal((2, 3), Documents.PageRange("2-9", 3)); + } + + [Fact] + public void PowerPoint_slides_come_in_order_with_their_notes() + { + const string a = "http://schemas.openxmlformats.org/drawingml/2006/main"; + var pptx = Zip( + ("ppt/presentation.xml", "

"), + ("ppt/slides/slide10.xml", $"""Last slide"""), + ("ppt/slides/slide2.xml", $"""Second slide"""), + ("ppt/notesSlides/notesSlide2.xml", $"""Say it slowly""")); + var text = Documents.Read("deck.pptx", pptx).Text!.Replace("\r", ""); + Assert.True(text.IndexOf("## Slide 2", StringComparison.Ordinal) < text.IndexOf("## Slide 10", StringComparison.Ordinal)); + Assert.Contains("## Slide 2\nSecond slide\n(notes: Say it slowly)", text); + } + + [Fact] + public void Saved_emails_are_decoded() + { + const string eml = """ + From: =?UTF-8?B?QW5uYSBOb3dhaw==?= + To: ben@contoso.com + Subject: =?ISO-8859-2?Q?Zam=F3wienie_przyj=EAte?= + Date: Wed, 15 Oct 2026 10:00:00 +0200 + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="XYZ" + + --XYZ + Content-Type: text/plain; charset=utf-8 + Content-Transfer-Encoding: quoted-printable + + Dzie=C5=84 dobry,= + zam=C3=B3wienie jest przyj=C4=99te. + --XYZ + Content-Type: application/pdf; name="invoice.pdf" + Content-Disposition: attachment; filename="invoice.pdf" + Content-Transfer-Encoding: base64 + + JVBERi0= + --XYZ-- + """; + var text = Documents.Read("mail.eml", Encoding.UTF8.GetBytes(eml)).Text!.Replace("\r", ""); + Assert.Contains("Subject: Zamówienie przyjęte", text); + Assert.Contains("From: Anna Nowak ", text); + Assert.Contains("Attachments: invoice.pdf", text); + Assert.Contains("Dzień dobry, zamówienie jest przyjęte.", text); + } + + [Fact] + public void Zip_lists_its_files_and_reads_one() + { + var zip = Zip(("logs/app.log", "ERROR something broke"), ("readme.txt", "hello")); + var list = Documents.Read("logs.zip", zip); + Assert.Contains("- logs/app.log", list.Text); + var one = Documents.Read("logs.zip", zip, part: "app.log"); + Assert.Equal("ERROR something broke", one.Text); + Assert.StartsWith("logs/app.log from logs.zip", one.Note); + } + + [Fact] + public void Long_texts_are_cut_with_an_outline_without_page_markers() + { + var text = "## Page 1\n# Goal\nx\n## Page 2\n# Settings\n" + new string('y', 500); + var cut = Documents.Fit(text, 100, full: false); + Assert.Contains("outline: Goal | Settings", cut); + Assert.Contains("detail=full for all", cut); + Assert.Equal(text, Documents.Fit(text, 100, full: true)); + } + + [Fact] + public void Saved_files_keep_their_name_inside_the_folder() + { + Assert.Equal("evil.exe", Downloads.SafeName(@"..\..\Windows\evil.exe")); + Assert.Equal("_CON.txt", Downloads.SafeName("CON.txt")); + Assert.Equal("a_b.txt", Downloads.SafeName("a:b.txt")); + Assert.Equal("file", Downloads.SafeName(" .. ")); + Directory.CreateDirectory(dir); + File.WriteAllText(Path.Combine(dir, "x.log"), ""); + Assert.Equal(Path.Combine(dir, "x (2).log"), Downloads.FreePath(dir, "x.log")); + } + + [Fact] + public async Task Saving_streams_to_disk_with_a_checksum() + { + var body = new byte[3 * 1024 * 1024]; + new Random(1).NextBytes(body); + var ado = new FakeAdo().On("/contoso/_apis/wit/attachments/abc", _ => new HttpResponseMessage(HttpStatusCode.OK) { Content = new ByteArrayContent(body) }); + var text = await Downloads.SaveAsync(ado.Client(), "_apis/wit/attachments/abc?download=true", dir, "123", @"..\trace.etl", default); + var path = Path.Combine(dir, "123", "trace.etl"); + Assert.StartsWith($"saved: {path} (3 MB, sha256 ", text); + Assert.Equal(body, File.ReadAllBytes(path)); + Assert.Contains(Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(body)).ToLowerInvariant(), text); + // A second save of the same name does not overwrite. + await Downloads.SaveAsync(ado.Client(), "_apis/wit/attachments/abc", dir, "123", "trace.etl", default); + Assert.True(File.Exists(Path.Combine(dir, "123", "trace (2).etl"))); + } +} diff --git a/tests/AdoLens.Tests/KnowledgeTests.cs b/tests/AdoLens.Tests/KnowledgeTests.cs index 768226a..22d43e2 100644 --- a/tests/AdoLens.Tests/KnowledgeTests.cs +++ b/tests/AdoLens.Tests/KnowledgeTests.cs @@ -230,4 +230,19 @@ public async Task A_recipe_is_saved_previewed_and_run() Assert.Contains("carried over", (string)items.Items[4]["fields"]!["System.Tags"]!); Assert.Equal("deleted recipe close-sprint", Recipes.Delete(lens, "close-sprint")); } + + [Fact] + public async Task A_large_attachment_gets_a_link_and_is_not_downloaded() + { + ((JsonArray)items.Items[1]["relations"]!).Add(JsonNode.Parse(""" + {"rel":"AttachedFile","url":"https://dev.azure.com/contoso/_apis/wit/attachments/big","attributes":{"name":"trace.etl","resourceSize":52428800}} + """)); + var (list, _) = await Attachments.ReadAsync(lens, null, 1, null, null); + Assert.Contains("1. trace.etl (50 MB)", list); + var (text, images) = await Attachments.ReadAsync(lens, null, 1, "trace.etl", null); + Assert.Contains("larger than 20 MB", text); + Assert.Contains("save=true saves it to disk", text); + Assert.Contains("https://dev.azure.com/contoso/_apis/wit/attachments/big", text); + Assert.Empty(images); + } }