From 09018402ebb909dcbc0a306c2ac67805776cd5d0 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Fri, 26 Jun 2026 14:59:46 +1000 Subject: [PATCH 1/3] Fix gap in coverage for HttpDownloadFile Avoid model remove/clash by picking model not is use by other integration test fixtures. --- sdk_v2/cpp/run_coverage.ps1 | 8 +- sdk_v2/cpp/test/CMakeLists.txt | 1 + .../test/internal_api/http_download_test.cc | 125 ++++++++++++++++++ sdk_v2/cpp/test/sdk_api/download_test.cc | 18 ++- sdk_v2/cpp/test/sdk_api/shared_test_env.h | 15 +++ 5 files changed, 162 insertions(+), 5 deletions(-) create mode 100644 sdk_v2/cpp/test/internal_api/http_download_test.cc diff --git a/sdk_v2/cpp/run_coverage.ps1 b/sdk_v2/cpp/run_coverage.ps1 index 7c53012f..290b3fff 100644 --- a/sdk_v2/cpp/run_coverage.ps1 +++ b/sdk_v2/cpp/run_coverage.ps1 @@ -5,7 +5,9 @@ # # Prerequisites: # - OpenCppCoverage installed (https://github.com/OpenCppCoverage/OpenCppCoverage) -# - A RelWithDebInfo build: .\build.bat --build --parallel +# - A Debug (or RelWithDebInfo) build: python build.py --build --config Debug --parallel +# Debug is preferred for coverage — its line tables map 1:1 to source lines, so per-line +# hit/miss reporting is more accurate than an optimized RelWithDebInfo build. # # Usage: # .\run_coverage.ps1 # default Debug for better matching of lines @@ -63,6 +65,10 @@ $unitCov = Join-Path $covDir "unit.cov" $unitArgs = $commonArgs + @("--export_type", "binary:$unitCov", "--") $unitArgs += $unitExe +# Include DISABLED_ tests (e.g. the real-network http_download manifest test) so the unit step +# covers code paths that are network-gated off by default. Mirrors the integration step below. +$unitArgs += "--gtest_also_run_disabled_tests" + if ($TestFilter) { $unitArgs += "--gtest_filter=$TestFilter" } diff --git a/sdk_v2/cpp/test/CMakeLists.txt b/sdk_v2/cpp/test/CMakeLists.txt index efa1bf1c..2e350f92 100644 --- a/sdk_v2/cpp/test/CMakeLists.txt +++ b/sdk_v2/cpp/test/CMakeLists.txt @@ -30,6 +30,7 @@ add_executable(foundry_local_tests internal_api/file_lock_test.cc internal_api/file_uri_test.cc internal_api/genai_config_test.cc + internal_api/http_download_test.cc internal_api/http_retry_test.cc internal_api/item_test.cc internal_api/local_model_scanner_test.cc diff --git a/sdk_v2/cpp/test/internal_api/http_download_test.cc b/sdk_v2/cpp/test/internal_api/http_download_test.cc new file mode 100644 index 00000000..69945804 --- /dev/null +++ b/sdk_v2/cpp/test/internal_api/http_download_test.cc @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// +// Real-network tests for HttpDownloadFile — the generic HTTPS file downloader used by the +// execution-provider bootstrappers (WebGPU / CUDA). These are DISABLED by default because they +// require network access; the coverage run enables them via --gtest_also_run_disabled_tests. +// +// The positive case downloads the small WebGPU EP manifest JSON (a few hundred bytes) from the +// same CDN that serves the EP packages, rather than a multi-hundred-MB EP zip. That keeps the +// test fast while still exercising the full curl transport + Content-Length + progress + success +// path. The negative case targets an unresolvable host so the failure path is deterministic and +// independent of any server's error-page behavior. +#include "http/http_download.h" + +#include "logger.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; +using namespace fl; + +namespace { + +// Mirrors kManifestUrl in src/ep_detection/webgpu_ep_bootstrapper.cc. The manifest is a small +// JSON document served from the same CDN as the EP packages, so it validates the real download +// path without pulling a large binary. +constexpr const char* kWebGpuManifestUrl = + "https://foundrypackages-ffhrdhbxb7gpdreh.b02.azurefd.net/webgpu_ep_prod.json"; + +constexpr const char* kUserAgent = "FoundryLocal"; + +/// Captures log output so a failed download surfaces the downloader's own diagnostics. +class RecordingLogger : public ILogger { + public: + void Log(LogLevel level, std::string_view message) override { + entries.emplace_back(level, std::string(message)); + } + + std::string Dump() const { + std::ostringstream oss; + for (const auto& [level, msg] : entries) { + oss << " [" << static_cast(level) << "] " << msg << "\n"; + } + return oss.str(); + } + + std::vector> entries; +}; + +/// RAII temp file that removes itself on construction and destruction so each run starts clean. +class TempFile { + public: + explicit TempFile(const std::string& name) { + path_ = fs::temp_directory_path() / name; + std::error_code ec; + fs::remove(path_, ec); + } + + ~TempFile() { + std::error_code ec; + fs::remove(path_, ec); + } + + const fs::path& path() const { return path_; } + + private: + fs::path path_; +}; + +std::string ReadFile(const fs::path& path) { + std::ifstream in(path, std::ios::binary); + return {std::istreambuf_iterator(in), std::istreambuf_iterator()}; +} + +} // namespace + +// Downloads the real WebGPU EP manifest and validates the success path end-to-end: returns +// true, writes a non-empty file, reports a terminal 100% progress callback, and the payload is +// the expected JSON shape. +TEST(DISABLED_HttpDownload, DownloadsWebGpuManifest) { + RecordingLogger logger; + TempFile dest("fl_webgpu_manifest_test.json"); + + std::vector progress; + std::atomic cancel{false}; + + bool ok = HttpDownloadFile(kWebGpuManifestUrl, dest.path(), kUserAgent, &cancel, [&progress](float pct) { progress.push_back(pct); }, logger); + + ASSERT_TRUE(ok) << "WebGPU manifest download failed. Logger output:\n" + << logger.Dump(); + + ASSERT_TRUE(fs::exists(dest.path())); + EXPECT_GT(fs::file_size(dest.path()), 0u); + + // The final progress callback is always 100% on success. + ASSERT_FALSE(progress.empty()); + EXPECT_FLOAT_EQ(progress.back(), 100.0f); + + // The payload is the EP manifest — parses as JSON and carries a 'packages' object. + auto manifest = nlohmann::json::parse(ReadFile(dest.path())); + EXPECT_TRUE(manifest.contains("packages")); +} + +// A transport failure (unresolvable host) returns false and leaves no output file behind. +// The reserved `.invalid` TLD (RFC 2606) never resolves, so the DNS failure is deterministic and +// fast regardless of network conditions to real hosts. +TEST(DISABLED_HttpDownload, ReturnsFalseAndWritesNoFileOnUnresolvableHost) { + RecordingLogger logger; + TempFile dest("fl_webgpu_manifest_unresolvable.json"); + + bool ok = HttpDownloadFile("https://foundry-local-test.invalid/webgpu_ep_prod.json", dest.path(), + kUserAgent, /*cancel_flag=*/nullptr, /*progress_cb=*/{}, logger); + + EXPECT_FALSE(ok); + EXPECT_FALSE(fs::exists(dest.path())); +} diff --git a/sdk_v2/cpp/test/sdk_api/download_test.cc b/sdk_v2/cpp/test/sdk_api/download_test.cc index 9ebb33d3..87a0c7cf 100644 --- a/sdk_v2/cpp/test/sdk_api/download_test.cc +++ b/sdk_v2/cpp/test/sdk_api/download_test.cc @@ -31,9 +31,15 @@ class DISABLED_DownloadFixture : public ::testing::Test { }; TEST_F(DISABLED_DownloadFixture, RemoveAndRedownloadSmallestModel) { - // Find the smallest CPU model that is NOT already loaded by SharedTestEnv. - // Iterate all variants of each alias group — GetModels() only returns the - // selected variant, which may hide smaller CPU variants behind a GPU pick. + // Find the smallest CPU model to remove+redownload, EXCLUDING the models the shared + // environment reserves for its modality fixtures. Those (e.g. whisper for the audio + // fixture, qwen for chat) are loaded by other suites; removing one here would race the + // fixture that depends on it. Skipping the whole reserved set — not just "not currently + // loaded" — is what makes this robust: a reserved model may be unloaded right now but + // loaded by a later suite. Iterate all variants of each alias group — GetModels() only + // returns the selected variant, which may hide smaller CPU variants behind a GPU pick. + auto reserved = SharedTestEnv::Get().ReservedModels(); + foundry_local::IModel* target = nullptr; int64_t target_size = std::numeric_limits::max(); @@ -42,6 +48,10 @@ TEST_F(DISABLED_DownloadFixture, RemoveAndRedownloadSmallestModel) { continue; } + if (reserved.count(m.get()) != 0) { + continue; + } + auto variants = m->GetVariants(); for (const auto& v : variants) { auto vi = v->GetInfo(); @@ -59,7 +69,7 @@ TEST_F(DISABLED_DownloadFixture, RemoveAndRedownloadSmallestModel) { } } - ASSERT_NE(target, nullptr) << "No unloaded CPU model found in catalog"; + ASSERT_NE(target, nullptr) << "No unreserved, unloaded CPU model found in catalog"; auto info = target->GetInfo(); std::cout << "Download test model: " << info.Name() diff --git a/sdk_v2/cpp/test/sdk_api/shared_test_env.h b/sdk_v2/cpp/test/sdk_api/shared_test_env.h index 482d3c76..cc8e9ed7 100644 --- a/sdk_v2/cpp/test/sdk_api/shared_test_env.h +++ b/sdk_v2/cpp/test/sdk_api/shared_test_env.h @@ -598,6 +598,21 @@ class SharedTestEnv : public ::testing::Environment { bool has_cuda() const { return has_cuda_; } bool has_webgpu() const { return has_webgpu_; } + // Models this environment reserves for its modality fixtures (chat, tool, audio, streaming + // audio, embeddings, vision, reasoning). Returned regardless of current load state: a reserved + // model may be unloaded right now but loaded by a later suite via AcquireModels(). Tests that + // mutate the cache (remove/redownload) must skip this set so they don't race the fixtures that + // depend on these models. + std::set ReservedModels() const { + std::set reserved; + for (auto* p : AllSelectedModels()) { + if (p) { + reserved.insert(p); + } + } + return reserved; + } + private: // Return `m` only if the suite acquired it. Reflects "did // SetUpTestSuite ask for this and did the load succeed?", NOT the From e613cc6ee1f75a525047e8e8994b74e8b05004c2 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Fri, 26 Jun 2026 17:28:52 +1000 Subject: [PATCH 2/3] Fix long line --- sdk_v2/cpp/test/internal_api/http_download_test.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdk_v2/cpp/test/internal_api/http_download_test.cc b/sdk_v2/cpp/test/internal_api/http_download_test.cc index 69945804..2190aafa 100644 --- a/sdk_v2/cpp/test/internal_api/http_download_test.cc +++ b/sdk_v2/cpp/test/internal_api/http_download_test.cc @@ -93,7 +93,10 @@ TEST(DISABLED_HttpDownload, DownloadsWebGpuManifest) { std::vector progress; std::atomic cancel{false}; - bool ok = HttpDownloadFile(kWebGpuManifestUrl, dest.path(), kUserAgent, &cancel, [&progress](float pct) { progress.push_back(pct); }, logger); + bool ok = HttpDownloadFile( + kWebGpuManifestUrl, dest.path(), kUserAgent, &cancel, + [&progress](float pct) { progress.push_back(pct); }, + logger); ASSERT_TRUE(ok) << "WebGPU manifest download failed. Logger output:\n" << logger.Dump(); From 76a2e541c667cdbaab7e6c96d294f12d3f25763d Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Wed, 1 Jul 2026 09:23:26 +1000 Subject: [PATCH 3/3] Address PR comment --- .../test/internal_api/http_download_test.cc | 44 +++++++------------ 1 file changed, 15 insertions(+), 29 deletions(-) diff --git a/sdk_v2/cpp/test/internal_api/http_download_test.cc b/sdk_v2/cpp/test/internal_api/http_download_test.cc index 2190aafa..29977205 100644 --- a/sdk_v2/cpp/test/internal_api/http_download_test.cc +++ b/sdk_v2/cpp/test/internal_api/http_download_test.cc @@ -5,16 +5,14 @@ // execution-provider bootstrappers (WebGPU / CUDA). These are DISABLED by default because they // require network access; the coverage run enables them via --gtest_also_run_disabled_tests. // -// The positive case downloads the small WebGPU EP manifest JSON (a few hundred bytes) from the -// same CDN that serves the EP packages, rather than a multi-hundred-MB EP zip. That keeps the -// test fast while still exercising the full curl transport + Content-Length + progress + success -// path. The negative case targets an unresolvable host so the failure path is deterministic and -// independent of any server's error-page behavior. +// The positive case downloads the real WebGPU EP zip from the production CDN and validates the +// full binary download path (curl transport + Content-Length + progress + success). The negative +// case targets an unresolvable host so the failure path is deterministic and independent of any +// server's error-page behavior. #include "http/http_download.h" #include "logger.h" -#include #include #include @@ -30,11 +28,9 @@ using namespace fl; namespace { -// Mirrors kManifestUrl in src/ep_detection/webgpu_ep_bootstrapper.cc. The manifest is a small -// JSON document served from the same CDN as the EP packages, so it validates the real download -// path without pulling a large binary. -constexpr const char* kWebGpuManifestUrl = - "https://foundrypackages-ffhrdhbxb7gpdreh.b02.azurefd.net/webgpu_ep_prod.json"; +// Production WebGPU EP package URL used for exercising large binary downloads. +constexpr const char* kWebGpuZipUrl = + "https://foundrypackages-ffhrdhbxb7gpdreh.b02.azurefd.net/webgpu_ep_0.1.0_win-x64.zip"; constexpr const char* kUserAgent = "FoundryLocal"; @@ -76,29 +72,23 @@ class TempFile { fs::path path_; }; -std::string ReadFile(const fs::path& path) { - std::ifstream in(path, std::ios::binary); - return {std::istreambuf_iterator(in), std::istreambuf_iterator()}; -} - } // namespace -// Downloads the real WebGPU EP manifest and validates the success path end-to-end: returns -// true, writes a non-empty file, reports a terminal 100% progress callback, and the payload is -// the expected JSON shape. -TEST(DISABLED_HttpDownload, DownloadsWebGpuManifest) { +// Downloads the real WebGPU EP zip and validates the success path end-to-end: returns +// true, writes a non-empty file, and reports a terminal 100% progress callback. +TEST(DISABLED_HttpDownload, DownloadsWebGpuZip) { RecordingLogger logger; - TempFile dest("fl_webgpu_manifest_test.json"); + TempFile dest("fl_webgpu_ep_test.zip"); std::vector progress; std::atomic cancel{false}; bool ok = HttpDownloadFile( - kWebGpuManifestUrl, dest.path(), kUserAgent, &cancel, + kWebGpuZipUrl, dest.path(), kUserAgent, &cancel, [&progress](float pct) { progress.push_back(pct); }, logger); - ASSERT_TRUE(ok) << "WebGPU manifest download failed. Logger output:\n" + ASSERT_TRUE(ok) << "WebGPU zip download failed. Logger output:\n" << logger.Dump(); ASSERT_TRUE(fs::exists(dest.path())); @@ -107,10 +97,6 @@ TEST(DISABLED_HttpDownload, DownloadsWebGpuManifest) { // The final progress callback is always 100% on success. ASSERT_FALSE(progress.empty()); EXPECT_FLOAT_EQ(progress.back(), 100.0f); - - // The payload is the EP manifest — parses as JSON and carries a 'packages' object. - auto manifest = nlohmann::json::parse(ReadFile(dest.path())); - EXPECT_TRUE(manifest.contains("packages")); } // A transport failure (unresolvable host) returns false and leaves no output file behind. @@ -118,9 +104,9 @@ TEST(DISABLED_HttpDownload, DownloadsWebGpuManifest) { // fast regardless of network conditions to real hosts. TEST(DISABLED_HttpDownload, ReturnsFalseAndWritesNoFileOnUnresolvableHost) { RecordingLogger logger; - TempFile dest("fl_webgpu_manifest_unresolvable.json"); + TempFile dest("fl_webgpu_zip_unresolvable.zip"); - bool ok = HttpDownloadFile("https://foundry-local-test.invalid/webgpu_ep_prod.json", dest.path(), + bool ok = HttpDownloadFile("https://foundry-local-test.invalid/webgpu_ep_0.1.0_win-x64.zip", dest.path(), kUserAgent, /*cancel_flag=*/nullptr, /*progress_cb=*/{}, logger); EXPECT_FALSE(ok);