Skip to content

Commit 5a36845

Browse files
KSemenenkoKSemenenko
committed
Stream PDF sources and bound native image decoding
1 parent 2c3a792 commit 5a36845

12 files changed

Lines changed: 267 additions & 35 deletions

‎CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22

33
All notable changes to ManagedCode.FileContext are documented here.
44

5+
## 1.0.13
6+
7+
- Parse PDF text, pages and embedded images from bounded seekable streams instead of whole-document arrays.
8+
- Stage non-seekable storage sources to automatically deleted temporary files with cancellation and size limits.
9+
- Validate total source-image pixels before native decoding, independent of rendered output scale.
10+
511
## 1.0.12
612

713
- Bound native PDF reads and rendering before source buffering through a shared, configurable PDF processor.

‎Directory.Build.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
<AnalysisMode>Recommended</AnalysisMode>
1313
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
1414
<NoWarn>$(NoWarn);CS1591;MAAI001</NoWarn>
15-
<Version>1.0.12</Version>
15+
<Version>1.0.13</Version>
1616
<PackageVersion>$(Version)</PackageVersion>
1717
</PropertyGroup>
1818

‎docs/Architecture.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,8 @@ source and retaining it through parsing/rendering. `MaximumConcurrentPdfOperatio
8282
`FileContextPdfRenderDocument` exposes page count and sequential page rendering from one parsed PDF;
8383
dispose it after the batch. The low-level synchronous image helpers remain caller-scheduled APIs.
8484

85+
PDF text, page rendering and embedded-image APIs accept bounded seekable streams. Storage-backed PDF tools keep seekable provider streams directly and stage non-seekable sources to an automatically deleted temporary file, never a whole-document managed array. Native raster decoding has a separate per-page source-image pixel budget; lowering output scale does not reduce source bitmap allocation.
86+
8587
All potentially large operations are controlled by `IOptions<FileContextOptions>`: PDF source/page/image budgets, full-read bytes, range bytes, files scanned, bytes per searched file, matches per file, total search results, graph documents, graph source bytes, and exported graph characters. Non-seekable cloud streams are supported by sequential streaming.
8688

8789
## Start here

‎src/ManagedCode.FileContext/FileContextDefaults.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ public static class FileContextDefaults
99
public const int MaximumPdfReadBytes = 100 * 1024 * 1024;
1010
public const int MaximumConcurrentPdfOperations = 1;
1111
public const int MaximumImageBytes = 8 * 1024 * 1024;
12+
public const int MaximumDecodedPdfImagePixels = 32_000_000;
1213
public const int MaximumRenderedPagePixels = 4_000_000;
1314
public const int MaximumImagesPerPdfPage = 20;
1415
public const double DefaultPdfPageScale = 1.5;

‎src/ManagedCode.FileContext/FileContextOptions.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@ public sealed class FileContextOptions
2828

2929
public int MaximumImageBytes { get; set; } = FileContextDefaults.MaximumImageBytes;
3030

31+
/// <summary>Maximum total source-image pixels decoded on one PDF page, independent of output scale.</summary>
32+
public int MaximumDecodedPdfImagePixels { get; set; } = FileContextDefaults.MaximumDecodedPdfImagePixels;
33+
3134
public int MaximumRenderedPagePixels { get; set; } = FileContextDefaults.MaximumRenderedPagePixels;
3235

3336
public int MaximumImagesPerPdfPage { get; set; } = FileContextDefaults.MaximumImagesPerPdfPage;
@@ -80,6 +83,7 @@ internal void Validate()
8083
ValidatePositive(MaximumPdfReadBytes, nameof(MaximumPdfReadBytes));
8184
ValidatePositive(MaximumConcurrentPdfOperations, nameof(MaximumConcurrentPdfOperations));
8285
ValidatePositive(MaximumImageBytes, nameof(MaximumImageBytes));
86+
ValidatePositive(MaximumDecodedPdfImagePixels, nameof(MaximumDecodedPdfImagePixels));
8387
ValidatePositive(MaximumRenderedPagePixels, nameof(MaximumRenderedPagePixels));
8488
ValidatePositive(MaximumImagesPerPdfPage, nameof(MaximumImagesPerPdfPage));
8589
if (!double.IsFinite(MinimumPdfPageScale) || MinimumPdfPageScale <= 0

‎src/ManagedCode.FileContext/FileContextService.Pdf.cs‎

Lines changed: 16 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ namespace ManagedCode.FileContext;
66
public sealed partial class FileContextService
77
{
88
private const string PdfExtension = ".pdf";
9-
private const int CopyBufferSize = 81920;
109

1110
public Task<FileContextPdfText> ReadPdfTextAsync(string path,
1211
int maxCharacters = FileContextDefaults.MaximumPdfTextCharacters,
@@ -19,7 +18,9 @@ public Task<FileContextPdfText> ReadPdfTextAsync(string path,
1918
return FileContextOperation.RunAsync(_options.OperationTimeout, async token =>
2019
{
2120
using var permit = await _pdfProcessor.AcquireAsync(token).ConfigureAwait(false);
22-
return FileContextPdfTextExtractor.Extract(await ReadPdfBytesAsync(path, token).ConfigureAwait(false), maxCharacters);
21+
var source = await OpenPdfAsync(path, token).ConfigureAwait(false);
22+
await using var sourceLifetime = source.ConfigureAwait(false);
23+
return FileContextPdfTextExtractor.Extract(source.Stream, maxCharacters, _options);
2324
}, cancellationToken);
2425
}
2526

@@ -28,17 +29,20 @@ public Task<DataContent> RenderPdfPageAsync(string path, int pageNumber,
2829
FileContextOperation.RunAsync(_options.OperationTimeout, async token =>
2930
{
3031
using var permit = await _pdfProcessor.AcquireAsync(token).ConfigureAwait(false);
32+
var source = await OpenPdfAsync(path, token).ConfigureAwait(false);
33+
await using var sourceLifetime = source.ConfigureAwait(false);
3134
return FileContextImageContent.FromPngBytes(FileContextPdfImages.RenderPagePng(
32-
await ReadPdfBytesAsync(path, token).ConfigureAwait(false), pageNumber, _options), _options);
35+
source.Stream, pageNumber, _options), _options);
3336
}, cancellationToken);
3437

3538
public Task<int> CountPdfPageImagesAsync(string path, int pageNumber,
3639
CancellationToken cancellationToken = default) =>
3740
FileContextOperation.RunAsync(_options.OperationTimeout, async token =>
3841
{
3942
using var permit = await _pdfProcessor.AcquireAsync(token).ConfigureAwait(false);
40-
return FileContextPdfImages.ExtractPageImagesPng(
41-
await ReadPdfBytesAsync(path, token).ConfigureAwait(false), pageNumber, _options).Count;
43+
var source = await OpenPdfAsync(path, token).ConfigureAwait(false);
44+
await using var sourceLifetime = source.ConfigureAwait(false);
45+
return FileContextPdfImages.ExtractPageImagesPng(source.Stream, pageNumber, _options).Count;
4246
}, cancellationToken);
4347

4448
public Task<DataContent> ExtractPdfImageAsync(string path, int pageNumber, int imageNumber,
@@ -48,8 +52,9 @@ public Task<DataContent> ExtractPdfImageAsync(string path, int pageNumber, int i
4852
return FileContextOperation.RunAsync(_options.OperationTimeout, async token =>
4953
{
5054
using var permit = await _pdfProcessor.AcquireAsync(token).ConfigureAwait(false);
51-
var images = FileContextPdfImages.ExtractPageImagesPng(
52-
await ReadPdfBytesAsync(path, token).ConfigureAwait(false), pageNumber, _options);
55+
var source = await OpenPdfAsync(path, token).ConfigureAwait(false);
56+
await using var sourceLifetime = source.ConfigureAwait(false);
57+
var images = FileContextPdfImages.ExtractPageImagesPng(source.Stream, pageNumber, _options);
5358
if (imageNumber > images.Count)
5459
{
5560
throw new InvalidOperationException("The embedded image number is outside this PDF page.");
@@ -58,7 +63,7 @@ public Task<DataContent> ExtractPdfImageAsync(string path, int pageNumber, int i
5863
}, cancellationToken);
5964
}
6065

61-
private async Task<byte[]> ReadPdfBytesAsync(string path, CancellationToken cancellationToken)
66+
private async Task<FileContextPdfSource> OpenPdfAsync(string path, CancellationToken cancellationToken)
6267
{
6368
if (!string.Equals(Path.GetExtension(StoragePathScope.Normalize(path)), PdfExtension, StringComparison.OrdinalIgnoreCase))
6469
{
@@ -70,21 +75,8 @@ private async Task<byte[]> ReadPdfBytesAsync(string path, CancellationToken canc
7075
{
7176
throw new IOException("The PDF exceeds the read limit.");
7277
}
73-
var source = await _fileStore.OpenReadAsync(path, cancellationToken).ConfigureAwait(false);
74-
await using (source.ConfigureAwait(false))
75-
{
76-
using var output = new MemoryStream();
77-
var buffer = new byte[CopyBufferSize];
78-
int read;
79-
while ((read = await source.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0)
80-
{
81-
if (output.Length + read > _options.MaximumPdfReadBytes)
82-
{
83-
throw new IOException("The PDF exceeds the read limit.");
84-
}
85-
await output.WriteAsync(buffer.AsMemory(0, read), cancellationToken).ConfigureAwait(false);
86-
}
87-
return output.ToArray();
88-
}
78+
return await FileContextPdfSource.OpenAsync(
79+
await _fileStore.OpenReadAsync(path, cancellationToken).ConfigureAwait(false), _options, cancellationToken)
80+
.ConfigureAwait(false);
8981
}
9082
}

‎src/ManagedCode.FileContext/Pdf/FileContextPdfImageBudget.cs‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
using UglyToad.PdfPig.Content;
2+
3+
namespace ManagedCode.FileContext.Pdf;
4+
5+
internal static class FileContextPdfImageBudget
6+
{
7+
public static void Validate(Page page, FileContextOptions options)
8+
{
9+
long pixels = 0;
10+
var count = 0;
11+
foreach (var image in page.GetImages())
12+
{
13+
count++;
14+
var imagePixels = (long)image.WidthInSamples * image.HeightInSamples;
15+
if (image.WidthInSamples <= 0 || image.HeightInSamples <= 0
16+
|| imagePixels > options.MaximumDecodedPdfImagePixels - pixels)
17+
{
18+
throw new IOException("The PDF page exceeds the decoded image pixel limit.");
19+
}
20+
pixels += imagePixels;
21+
if (count > options.MaximumImagesPerPdfPage)
22+
{
23+
throw new IOException("The PDF page exceeds the embedded image count limit.");
24+
}
25+
}
26+
}
27+
}

‎src/ManagedCode.FileContext/Pdf/FileContextPdfImages.cs‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,14 @@ public static byte[] RenderPagePng(byte[] pdf, int pageNumber, FileContextOption
2828
return document.RenderPagePng(pageNumber, scale);
2929
}
3030

31+
/// <summary>Renders a seekable PDF source without buffering the complete document.</summary>
32+
public static byte[] RenderPagePng(Stream pdf, int pageNumber, FileContextOptions? options = null,
33+
double? scale = null)
34+
{
35+
using var document = new FileContextPdfRenderDocument(pdf, options);
36+
return document.RenderPagePng(pageNumber, scale);
37+
}
38+
3139
/// <summary>Extracts the embedded images on one page; these do not include page text or vector drawings.</summary>
3240
public static IReadOnlyList<FileContextPdfImage> ExtractPageImagesPng(byte[] pdf, int pageNumber) =>
3341
ExtractPageImagesPng(pdf, pageNumber, new FileContextOptions());
@@ -37,11 +45,23 @@ public static IReadOnlyList<FileContextPdfImage> ExtractPageImagesPng(byte[] pdf
3745
FileContextOptions options)
3846
{
3947
ArgumentNullException.ThrowIfNull(options);
40-
var settings = ValidateInput(pdf, options);
48+
ValidateInput(pdf, options);
49+
using var source = new MemoryStream(pdf, writable: false);
50+
return ExtractPageImagesPng(source, pageNumber, options);
51+
}
52+
53+
/// <summary>Extracts embedded images from a bounded seekable source. The caller owns the stream.</summary>
54+
public static IReadOnlyList<FileContextPdfImage> ExtractPageImagesPng(Stream pdf, int pageNumber,
55+
FileContextOptions? options = null)
56+
{
57+
var settings = options ?? new FileContextOptions();
58+
FileContextPdfSource.Validate(pdf, settings);
4159
using var document = PdfDocument.Open(pdf, SkiaRenderingParsingOptions.Instance);
4260
ValidatePage(pageNumber, document.NumberOfPages);
61+
var page = document.GetPage(pageNumber);
62+
FileContextPdfImageBudget.Validate(page, settings);
4363
var result = new List<FileContextPdfImage>();
44-
foreach (var image in document.GetPage(pageNumber).GetImages())
64+
foreach (var image in page.GetImages())
4565
{
4666
if (result.Count >= settings.MaximumImagesPerPdfPage)
4767
{
@@ -60,15 +80,14 @@ public static IReadOnlyList<FileContextPdfImage> ExtractPageImagesPng(byte[] pdf
6080
return result;
6181
}
6282

63-
private static FileContextOptions ValidateInput(byte[] pdf, FileContextOptions options)
83+
private static void ValidateInput(byte[] pdf, FileContextOptions options)
6484
{
6585
ArgumentNullException.ThrowIfNull(pdf);
6686
options.Validate();
6787
if (pdf.Length > options.MaximumPdfReadBytes)
6888
{
6989
throw new IOException("The PDF exceeds the read limit.");
7090
}
71-
return options;
7291
}
7392

7493
private static void ValidatePage(int pageNumber, int pageCount)

‎src/ManagedCode.FileContext/Pdf/FileContextPdfRenderDocument.cs‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,22 @@ public sealed class FileContextPdfRenderDocument : IDisposable
88
{
99
private readonly PdfDocument _document;
1010
private readonly FileContextOptions _options;
11+
private readonly Stream _source;
12+
private readonly bool _ownedSource;
1113

1214
public FileContextPdfRenderDocument(byte[] pdf, FileContextOptions? options = null)
15+
: this(new MemoryStream(pdf ?? throw new ArgumentNullException(nameof(pdf)), writable: false), options)
16+
{
17+
_ownedSource = true;
18+
}
19+
20+
/// <summary>Parses a seekable source without copying it. The caller owns the stream.</summary>
21+
public FileContextPdfRenderDocument(Stream pdf, FileContextOptions? options = null)
1322
{
1423
ArgumentNullException.ThrowIfNull(pdf);
1524
_options = (options ?? new FileContextOptions()).Clone();
16-
_options.Validate();
17-
if (pdf.Length > _options.MaximumPdfReadBytes)
18-
{
19-
throw new IOException("The PDF exceeds the read limit.");
20-
}
25+
FileContextPdfSource.Validate(pdf, _options);
26+
_source = pdf;
2127
_document = PdfDocument.Open(pdf, SkiaRenderingParsingOptions.Instance);
2228
_document.AddSkiaPageFactory();
2329
}
@@ -41,6 +47,7 @@ public byte[] RenderPagePng(int pageNumber, double? scale = null)
4147
{
4248
throw new IOException("The rendered PDF page exceeds the pixel limit.");
4349
}
50+
FileContextPdfImageBudget.Validate(page, _options);
4451
using var image = _document.GetPageAsPng(pageNumber, (float)resolvedScale, _options.PdfPngQuality);
4552
if (image.Length > _options.MaximumImageBytes)
4653
{
@@ -49,5 +56,9 @@ public byte[] RenderPagePng(int pageNumber, double? scale = null)
4956
return image.ToArray();
5057
}
5158

52-
public void Dispose() => _document.Dispose();
59+
public void Dispose()
60+
{
61+
_document.Dispose();
62+
if (_ownedSource) { _source.Dispose(); }
63+
}
5364
}

‎src/ManagedCode.FileContext/Pdf/FileContextPdfSource.cs‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
namespace ManagedCode.FileContext.Pdf;
2+
3+
/// <summary>Owns a bounded seekable PDF source; non-seekable inputs are staged to disk.</summary>
4+
public sealed class FileContextPdfSource : IAsyncDisposable
5+
{
6+
private const int CopyBufferBytes = 81920;
7+
private const string TemporaryFilePrefix = "filecontext-pdf-";
8+
9+
private FileContextPdfSource(Stream stream) => Stream = stream;
10+
11+
public Stream Stream { get; }
12+
13+
/// <summary>Takes ownership of the input, including on failure. Dispose the result after parsing.</summary>
14+
public static async Task<FileContextPdfSource> OpenAsync(Stream source, FileContextOptions options,
15+
CancellationToken cancellationToken = default)
16+
{
17+
ArgumentNullException.ThrowIfNull(source);
18+
ArgumentNullException.ThrowIfNull(options);
19+
Stream? staged = null;
20+
try
21+
{
22+
options.Validate();
23+
cancellationToken.ThrowIfCancellationRequested();
24+
if (source.CanSeek)
25+
{
26+
Validate(source, options);
27+
source.Position = 0;
28+
return new FileContextPdfSource(source);
29+
}
30+
staged = CreateTemporaryFile();
31+
var buffer = new byte[CopyBufferBytes];
32+
int read;
33+
while ((read = await source.ReadAsync(buffer, cancellationToken).ConfigureAwait(false)) > 0)
34+
{
35+
if (staged.Length + read > options.MaximumPdfReadBytes)
36+
{
37+
throw new IOException("The PDF exceeds the read limit.");
38+
}
39+
await staged.WriteAsync(buffer.AsMemory(0, read), cancellationToken).ConfigureAwait(false);
40+
}
41+
staged.Position = 0;
42+
await source.DisposeAsync().ConfigureAwait(false);
43+
return new FileContextPdfSource(staged);
44+
}
45+
catch
46+
{
47+
if (staged is not null) { await staged.DisposeAsync().ConfigureAwait(false); }
48+
await source.DisposeAsync().ConfigureAwait(false);
49+
throw;
50+
}
51+
}
52+
53+
internal static void Validate(Stream source, FileContextOptions options)
54+
{
55+
if (!source.CanRead || !source.CanSeek)
56+
{
57+
throw new ArgumentException("PDF parsing requires a readable seekable stream. Stage it with FileContextPdfSource.OpenAsync.", nameof(source));
58+
}
59+
options.Validate();
60+
if (source.Length > options.MaximumPdfReadBytes)
61+
{
62+
throw new IOException("The PDF exceeds the read limit.");
63+
}
64+
source.Position = 0;
65+
}
66+
67+
private static FileStream CreateTemporaryFile() => new(
68+
Path.Combine(Path.GetTempPath(), TemporaryFilePrefix + Guid.NewGuid().ToString("N")),
69+
FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, CopyBufferBytes,
70+
FileOptions.Asynchronous | FileOptions.DeleteOnClose);
71+
72+
public ValueTask DisposeAsync() => Stream.DisposeAsync();
73+
}

0 commit comments

Comments
 (0)