Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions .npmrc

This file was deleted.

2 changes: 2 additions & 0 deletions eng/tsp-core/pipelines/jobs/build-packages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ jobs:
variables:
TYPESPEC_SKIP_WEBSITE_BUILD: true # Disable docusaurus build
TYPESPEC_SKIP_VS_BUILD: true # VS extension is built in the dedicated build-vs (Windows) job
TYPESPEC_NPM_REGISTRY: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js/npm/registry/
NODE_USE_ENV_PROXY: 1

steps:
- template: /eng/tsp-core/pipelines/templates/install.yml
Expand Down
2 changes: 2 additions & 0 deletions eng/tsp-core/pipelines/jobs/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ jobs:
variables:
TYPESPEC_VS_CI_BUILD: false # Enable official Visual Studio extension build
TYPESPEC_SKIP_WEBSITE_BUILD: true # Disable docusaurus build
TYPESPEC_NPM_REGISTRY: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js/npm/registry/
NODE_USE_ENV_PROXY: 1
DISPLAY: ":99" # Set DISPLAY for Linux GUI applications

pool:
Expand Down
2 changes: 1 addition & 1 deletion eng/tsp-core/pipelines/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ extends:
- template: /eng/tsp-core/pipelines/jobs/build-packages.yml@self
- template: /eng/tsp-core/pipelines/jobs/build-vs.yml@self
- template: /eng/tsp-core/pipelines/jobs/cli/build-tsp-cli-all.yml@self
# - template: /eng/tsp-core/pipelines/jobs/e2e.yml@self
- template: /eng/tsp-core/pipelines/jobs/e2e.yml@self
parameters:
azLogin: true

Expand Down
2 changes: 2 additions & 0 deletions packages/compiler/src/init/init.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import type { Diagnostic } from "../core/types.js";
import { NoTarget } from "../core/types.js";
import { installTypeSpecDependencies } from "../install/install.js";
import { MANIFEST } from "../manifest.js";
import { loadNpmRegistryConfig } from "../package-manger/npm-registry-config.js";
import type { ValidationResult } from "./init-template-validate.js";
import { validateTemplateDefinitions } from "./init-template-validate.js";
import type { EmitterTemplate, InitTemplate, InitTemplateInput } from "./init-template.js";
Expand Down Expand Up @@ -119,6 +120,7 @@ export async function initTypeSpecProjectWorker(
directory,
parameters,
emitters,
npmRegistryConfig: await loadNpmRegistryConfig(),

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the compiler changes need to wait for some input from Timothee Guerin (@timotheeguerin)

});

await scaffoldNewProject(host, scaffoldingConfig);
Expand Down
32 changes: 25 additions & 7 deletions packages/compiler/src/init/scaffold.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,10 @@ import { stringify } from "yaml";
import type { TypeSpecRawConfig } from "../config/types.js";
import { getDirectoryPath, joinPaths } from "../core/path-utils.js";
import type { SystemHost } from "../core/types.js";
import { fetchLatestPackageManifest } from "../package-manger/npm-registry.js";
import {
fetchLatestPackageManifest,
type NpmRegistryConfig,
} from "../package-manger/npm-registry.js";
import type { PackageJson } from "../types/package-json.js";
import {
createFileTemplatingContext,
Expand Down Expand Up @@ -58,6 +61,9 @@ export interface ScaffoldingConfig {
* Selected emitters the tempalates.
*/
emitters: Record<string, any>;

/** Configuration used to fetch package metadata from the npm registry. */
npmRegistryConfig?: NpmRegistryConfig;
}

export function normalizeLibrary(library: InitTemplateLibrary): InitTemplateLibrarySpec {
Expand Down Expand Up @@ -115,18 +121,26 @@ async function writePackageJson(host: SystemHost, config: ScaffoldingConfig) {

if (!config.template.skipCompilerPackage) {
versionResolutions.push(
resolvePackageVersion("@typespec/compiler").then((v) => ["@typespec/compiler", v]),
resolvePackageVersion("@typespec/compiler", config.npmRegistryConfig).then((v) => [
"@typespec/compiler",
v,
]),
);
}

for (const library of config.libraries) {
versionResolutions.push(
getPackageVersion(library.name, library).then((v) => [library.name, v]),
getPackageVersion(library.name, library, config.npmRegistryConfig).then((v) => [
library.name,
v,
]),
);
}

for (const key of Object.keys(config.emitters)) {
versionResolutions.push(getPackageVersion(key, config.emitters[key]).then((v) => [key, v]));
versionResolutions.push(
getPackageVersion(key, config.emitters[key], config.npmRegistryConfig).then((v) => [key, v]),
);
}

const dependencies: Record<string, string> = Object.fromEntries(
Expand Down Expand Up @@ -264,16 +278,20 @@ async function writeFile(
async function getPackageVersion(
packageName: string,
templatePackageConfig: { version?: string },
npmRegistryConfig?: NpmRegistryConfig,
): Promise<string> {
if (templatePackageConfig.version !== undefined) {
return templatePackageConfig.version;
}
return resolvePackageVersion(packageName);
return resolvePackageVersion(packageName, npmRegistryConfig);
}

async function resolvePackageVersion(packageName: string): Promise<string> {
async function resolvePackageVersion(
packageName: string,
npmRegistryConfig?: NpmRegistryConfig,
): Promise<string> {
try {
const manifest = await fetchLatestPackageManifest(packageName);
const manifest = await fetchLatestPackageManifest(packageName, npmRegistryConfig);
return `^${manifest.version}`;
} catch {
return "latest";
Expand Down
19 changes: 16 additions & 3 deletions packages/compiler/src/install/install.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,12 @@ import { createDiagnosticCollector } from "../core/diagnostics.js";
import { getDirectoryPath, joinPaths } from "../core/path-utils.js";
import { NoTarget, type Diagnostic, type Tracer } from "../core/types.js";
import { downloadAndExtractPackage } from "../package-manger/npm-package-download.js";
import { fetchPackageManifest, type NpmManifest } from "../package-manger/npm-registry.js";
import { loadNpmRegistryConfig } from "../package-manger/npm-registry-config.js";
import {
fetchPackageManifest,
type NpmManifest,
type NpmRegistryConfig,
} from "../package-manger/npm-registry.js";
import { mkTempDir } from "../utils/fs-utils.js";
import type { SupportedPackageManager } from "./config.js";
import { getPackageManagerConfig, type PackageManagerConfig } from "./config.js";
Expand Down Expand Up @@ -91,6 +96,7 @@ async function installPackageManager(
spec: Descriptor,
installDir: string,
manifest: NpmManifest,
npmRegistryConfig: NpmRegistryConfig,
) {
await rm(installDir, { recursive: true, force: true });
const tempDir = await mkTempDir(host, pmDir, `tsp-pm-${packageManager}-${manifest.version}`);
Expand All @@ -99,7 +105,12 @@ async function installPackageManager(
"downloading-extracting",
`Downloading and extracting ${packageManager} at version ${manifest.version} in ${tempDir}`,
);
const extractResult = await downloadAndExtractPackage(manifest, tempDir, spec.hash?.algorithm);
const extractResult = await downloadAndExtractPackage(
manifest,
tempDir,
spec.hash?.algorithm,
npmRegistryConfig,
);
if (spec.hash) {
if (spec.hash.value !== extractResult.hash.value) {
throw new InstallDependenciesError(
Expand Down Expand Up @@ -165,7 +176,8 @@ export async function installTypeSpecDependencies(
);
const packageManager = spec.name;
const packageManagerConfig = getPackageManagerConfig(packageManager);
const manifest = await fetchPackageManifest(packageManager, spec.range);
const npmRegistryConfig = await loadNpmRegistryConfig();
const manifest = await fetchPackageManifest(packageManager, spec.range, npmRegistryConfig);
tracer.trace(
"fetched-manifest",
`Resolved manifest for ${packageManager} at version ${manifest.version}`,
Expand All @@ -182,6 +194,7 @@ export async function installTypeSpecDependencies(
spec,
installDir,
manifest,
npmRegistryConfig,
);
if (savePackageManager) {
await updatePackageManagerInPackageJson(host, packageJsonPath, {
Expand Down
24 changes: 19 additions & 5 deletions packages/compiler/src/package-manger/npm-package-download.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,36 @@ import { createHash } from "crypto";
import { Readable } from "stream";
import { extract as tarX } from "tar/extract";
import type { Hash } from "../install/spec.js";
import { fetchPackageManifest, type NpmManifest } from "./npm-registry.js";
import { loadNpmRegistryConfig } from "./npm-registry-config.js";
import {
fetchPackageManifest,
getNpmRequestHeaders,
type NpmManifest,
type NpmRegistryConfig,
} from "./npm-registry.js";

export async function downloadPackageVersion(
packageName: string,
version: string,
dest: string,
): Promise<ExtractedTarballResult> {
const manifest = await fetchPackageManifest(packageName, version);
return downloadAndExtractTarball(manifest.dist.tarball, dest);
const config = await loadNpmRegistryConfig();
const manifest = await fetchPackageManifest(packageName, version, config);
return downloadAndExtractTarball(manifest.dist.tarball, dest, "sha512", config);
}

export async function downloadAndExtractPackage(
manifest: NpmManifest,
dest: string,
hashAlgorithm: string = "sha512",
config?: NpmRegistryConfig,
): Promise<ExtractedTarballResult> {
return downloadAndExtractTarball(manifest.dist.tarball, dest, hashAlgorithm);
return downloadAndExtractTarball(
manifest.dist.tarball,
dest,
hashAlgorithm,
config ?? (await loadNpmRegistryConfig()),
);
}

export interface ExtractedTarballResult {
Expand All @@ -31,8 +44,9 @@ async function downloadAndExtractTarball(
url: string,
dest: string,
hashAlgorithm: string = "sha512",
config: NpmRegistryConfig = {},
): Promise<ExtractedTarballResult> {
const res = await fetch(url);
const res = await fetch(url, { headers: getNpmRequestHeaders(url, config) });
const tarballStream = Readable.fromWeb(res.body as any);
const hash = tarballStream.pipe(createHash(hashAlgorithm));
const extractor = tarX({
Expand Down
101 changes: 101 additions & 0 deletions packages/compiler/src/package-manger/npm-registry-config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
import { readFile } from "fs/promises";
import { homedir } from "os";
import { join } from "path";
import type { NpmRegistryConfig } from "./npm-registry.js";

interface NpmrcAuthFields {
auth?: string;
authToken?: string;
password?: string;
username?: string;
}

export async function loadNpmRegistryConfig(): Promise<NpmRegistryConfig> {
const npmrcPath = process.env["NPM_CONFIG_USERCONFIG"] ?? join(homedir(), ".npmrc");
let content: string;
try {
content = await readFile(npmrcPath, "utf8");
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
return {};
}
throw error;
}

const values = parseNpmrc(content);
const registry = values.get("registry");
const authFields = new Map<string, NpmrcAuthFields>();

for (const [key, value] of values) {
const separatorIndex = key.lastIndexOf(":");
if (!key.startsWith("//") || separatorIndex === -1) {
continue;
}

const scope = key.slice(0, separatorIndex);
const field = key.slice(separatorIndex + 1);
const fields = authFields.get(scope) ?? {};
switch (field) {
case "_auth":
fields.auth = value;
break;
case "_authToken":
fields.authToken = value;
break;
case "_password":
fields.password = value;
break;
case "username":
fields.username = value;
break;
default:
continue;
}
authFields.set(scope, fields);
}

return {
registry,
authentication: [...authFields].flatMap(([scope, fields]) => {
const authorization = createAuthorizationHeader(fields);
return authorization === undefined ? [] : [{ scope, authorization }];
}),
};
}

function parseNpmrc(content: string): Map<string, string> {
const values = new Map<string, string>();
for (const line of content.split(/\r?\n/)) {
const trimmed = line.trim();
if (trimmed.length === 0 || trimmed.startsWith("#") || trimmed.startsWith(";")) {
continue;
}

const separatorIndex = trimmed.indexOf("=");
if (separatorIndex === -1) {
continue;
}

const key = trimmed.slice(0, separatorIndex).trim();
const value = trimmed
.slice(separatorIndex + 1)
.trim()
.replace(/\$\{([^}]+)\}/g, (_, name: string) => process.env[name] ?? "");
values.set(key, value);
}
return values;
}

function createAuthorizationHeader(fields: NpmrcAuthFields): string | undefined {
if (fields.authToken !== undefined) {
return `Bearer ${fields.authToken}`;
}
if (fields.auth !== undefined) {
return `Basic ${fields.auth}`;
}
if (fields.username !== undefined && fields.password !== undefined) {
const password = Buffer.from(fields.password, "base64").toString("utf8");
return `Basic ${Buffer.from(`${fields.username}:${password}`).toString("base64")}`;
}
return undefined;
}
Loading
Loading