diff --git a/packages/bake/src/objects/acquisition/operations-acquisition.test.ts b/packages/bake/src/objects/acquisition/operations-acquisition.test.ts index 9a3f885d33e..e7cb097fa30 100644 --- a/packages/bake/src/objects/acquisition/operations-acquisition.test.ts +++ b/packages/bake/src/objects/acquisition/operations-acquisition.test.ts @@ -5,7 +5,7 @@ import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { execFileSync } from 'node:child_process'; import { randomUUID } from 'node:crypto'; -import { executeAcquisition, parseAcquisitionPlan, convertMappedComposition, parseMappedCompositionRecipe } from '@cssearth/bake/objects/acquisition'; +import { commandOutput, executeAcquisition, parseAcquisitionPlan, convertMappedComposition, parseMappedCompositionRecipe } from '@cssearth/bake/objects/acquisition'; import { acquirePinnedDownloads, verifySources, type SourceManifest } from '@cssearth/bake/objects/sources'; import { gzipSync } from 'node:zlib'; const test = sourceTest(); @@ -285,6 +285,19 @@ test('ZIP restoration verifies both the streamed archive and its exact extracted } }); +// The ZIP test above restored an empty file on CI: unzip had finished a 59-byte member before the pinned write began to +// read it, and Node discards what a finished child wrote that nothing reads yet. The reader here arrives late on purpose. +test('a command\'s output is kept for a reader that arrives after the command has finished', async () => { + const output = commandOutput(process.execPath, ['-e', 'process.stdout.write("a small member")'], 'node'); + await new Promise(done => { setTimeout(done, 400); }); + const parts: Buffer[] = []; + for await (const chunk of output) parts.push(chunk as Buffer); + assert.equal(Buffer.concat(parts).toString(), 'a small member'); + // A command that fails ends the stream with the first line of its complaint. + const failing = commandOutput(process.execPath, ['-e', 'process.stdout.write("part");console.error("caution: filename not matched\\nmore");process.exit(11)'], 'unzip could not read member.bin'); + await assert.rejects(async () => { for await (const _chunk of failing) { /* read to the end */ } }, /^Error: unzip could not read member\.bin: caution: filename not matched\.$/u); +}); + // A pinned slice of an archive member too large to keep whole: the request must be honoured as 206 Partial Content, // and a server that answers with the whole body is refused instead of downloaded. const slice=Buffer.from('exactly the pinned slice of a very large archive member'); diff --git a/packages/bake/src/objects/acquisition/operations-acquisition.ts b/packages/bake/src/objects/acquisition/operations-acquisition.ts index c4f2e0bd2e6..4a5369eccc2 100644 --- a/packages/bake/src/objects/acquisition/operations-acquisition.ts +++ b/packages/bake/src/objects/acquisition/operations-acquisition.ts @@ -3,7 +3,7 @@ import sharp from 'sharp'; import { lstat, readFile, mkdir, rename, rm } from 'node:fs/promises'; import { createWriteStream } from 'node:fs'; import { basename, dirname, resolve } from 'node:path'; -import { Readable } from 'node:stream'; +import { PassThrough, Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import { execFile, spawn } from 'node:child_process'; import { promisify } from 'node:util'; @@ -31,6 +31,19 @@ const rangedEntry=(manifest:SourceManifest,path:string)=>[...manifest.inputs,... // default transport names this project; a step's own headers win. const SOURCE_USER_AGENT='cssEarth/0.6 (https://github.com/layoutit/css.earth; source restore)'; const NAMED_TRANSPORT:AcquisitionTransport={fetch:(url,init)=>fetch(url,{...init,headers:{'user-agent':SOURCE_USER_AGENT,...Object.fromEntries(new Headers(init?.headers))}})}; +/** A command's output as a stream. It is piped on at once: Node discards what a finished child wrote that nothing reads yet, so a reader + * that arrives late (publishPinnedSourceStream makes the directory first) got none of a small member, and an empty file was pinned. + * A command that fails ends the stream with the first line of its complaint, under `what`. */ +export function commandOutput(command:string,args:readonly string[],what:string):Readable { + const child=spawn(command,args,{stdio:['ignore','pipe','pipe']}),output=child.stdout.pipe(new PassThrough()); + let complaint='';child.stderr.on('data',(chunk:Buffer)=>{complaint+=chunk.toString();}); + const exit=new Promise(done=>{child.on('close',done);}); + child.on('error',error=>{output.destroy(error);}); + return Readable.from((async function*(){ + for await(const chunk of output)yield chunk as Buffer; + if(await exit!==0)throw new Error(`${what}: ${complaint.trim().split('\n')[0]||'no message'}.`); + })()); +} export async function executeAcquisition({sourceRoot,manifest,plan,group='refresh',transport=NAMED_TRANSPORT,mirrorOrigin=null,objectId=basename(dirname(sourceRoot))}:{sourceRoot:string;manifest:SourceManifest;plan:AcquisitionPlan;group?:string;transport?:AcquisitionTransport;mirrorOrigin?:string|null;objectId?:string}) { const selected=plan.operations.filter(step=>step.groups.includes(group));if(!selected.length)throw new Error(`Acquisition group ${group} is undeclared.`); const request=async(url:string,init?:RequestInit)=>{const response=await transport.fetch(url,init);if(!response.ok)throw new Error(`Source request failed ${response.status}: ${url}.`);return response;}; @@ -91,13 +104,7 @@ export async function executeAcquisition({sourceRoot,manifest,plan,group='refres const entry=[...manifest.inputs,...manifest.generatedIntermediates,...manifest.documents].find(entry=>entry.path===step.path);if(!entry)throw new Error(`Undeclared acquisition target: ${step.path}.`); // unzip writes the member straight into the pinned file, so one of hundreds of megabytes is never held in memory. A // member unzip could not read whole fails the stream before the file takes its place. - const child=spawn('unzip',['-p',archivePath,step.member],{stdio:['ignore','pipe','pipe']}); - let complaint='';child.stderr.on('data',(chunk:Buffer)=>{complaint+=chunk.toString();}); - const exit=new Promise((done,fail)=>{child.on('close',done);child.on('error',fail);}); - await publishPinnedSourceStream({sourceRoot,entry,stream:Readable.from((async function*(){ - for await(const chunk of child.stdout)yield chunk as Buffer; - if(await exit!==0)throw new Error(`unzip could not read ${step.member} from ${step.url}: ${complaint.trim().split('\n')[0]??'no message'}.`); - })())}); + await publishPinnedSourceStream({sourceRoot,entry,stream:commandOutput('unzip',['-p',archivePath,step.member],`unzip could not read ${step.member} from ${step.url}`)}); } else if(step.kind==='tar-gz-member'){ const cache=resolve('.local/source-archives');await mkdir(cache,{recursive:true}); diff --git a/packages/telescope-cli/src/new-object/new-object-cli.test.mts b/packages/telescope-cli/src/new-object/new-object-cli.test.mts index 24c23d5f1e8..4edf53422c6 100644 --- a/packages/telescope-cli/src/new-object/new-object-cli.test.mts +++ b/packages/telescope-cli/src/new-object/new-object-cli.test.mts @@ -45,7 +45,7 @@ test('prose the scaffold cannot know is marked, and the package it writes matche } assert.deepEqual(JSON.parse(files.get('src/objects/antares/source/presentation/solar-system.json')!), JSON.parse(await readFile(resolve(root, 'src/objects/antares/source/presentation/solar-system.json'), 'utf8'))); assert.throws(() => scaffoldStarFiles({ ...spec, temperatureK: Number.NaN }, record, EPOCH), /effectiveTemperatureK/u); - assert.throws(() => scaffoldStarFiles({ ...spec, temperatureK: 100 }, record, EPOCH), /1,000 and 40,000 K/u); + assert.throws(() => scaffoldStarFiles({ ...spec, temperatureK: 100 }, record, EPOCH), /1,000 and 250,000 K/u); assert.throws(() => scaffoldStarFiles({ ...spec, temperatureSource: 'Ohnaka et al. 2013' }, record, EPOCH), /URL/u); assert.throws(() => scaffoldStarFiles(spec, { ...record, star: { ...record.star, presentationUp: undefined } }, EPOCH), /presentationUp/u); }); diff --git a/packages/telescope-cli/src/new-object/simulation/simulation-dataset.mts b/packages/telescope-cli/src/new-object/simulation/simulation-dataset.mts index 39c5f2caed2..920896affb5 100644 --- a/packages/telescope-cli/src/new-object/simulation/simulation-dataset.mts +++ b/packages/telescope-cli/src/new-object/simulation/simulation-dataset.mts @@ -30,7 +30,7 @@ import { containedPath, publishPinnedSourceStream } from '@cssearth/bake/objects import { assertRangeResponse, rangeRequestHeader } from '@cssearth/objects/node'; import type { Archive } from '../archives/archives.mts'; import { bindInputs, json, openOnMap, type PackageFiles } from '../dataset.mts'; -import { MAX_RECORDS, ZENODO_RECORDS, namesObject, parseZenodoRecord, parseZenodoSearch, reuseLicense, sizeText, speaksOfSimulation, zenodoQuery, type ReuseLicense } from '../../simulations/simulations.mts'; +import { MAX_RECORDS, ZENODO_RECORDS, nameForms, namesObject, parseZenodoRecord, parseZenodoSearch, reuseLicense, sizeText, speaksOfSimulation, zenodoQuery, type ReuseLicense } from '../../simulations/simulations.mts'; export interface SimulationEntry { /** Dataset id, and the dataset key of its reader text. */ @@ -318,8 +318,22 @@ export function installSimulationDataset(files: PackageFiles, id: string, name: return { minimum, maximum, promoted: openOnMap(files, id, entry.dataset) }; } -/** Zenodo's guest search answers 30 requests a minute: one is made every PACE_MS, for NAMES_AT_ONCE names, up to PAGES pages. */ -const PACE_MS = 2100, NAMES_AT_ONCE = 40, PAGES = 4; +/** Zenodo's guest search answers 30 requests a minute: one is made every PACE_MS, up to PAGES pages for a question. + * It refuses a long question with HTTP 500, so the survey of 40 names at once had stopped working. A question is sized by + * the spellings it asks for (`nameForms`: two for "TOI-1266 c", four for "HD 189733 b", eight for "HD 135344 Ab"), since + * that is what its length follows. Measured 2026-10-05 on 110 questions: every one of up to 33 spellings was answered, + * those of 40 were answered or refused, and every one of 44 or more was refused. */ +const PACE_MS = 2100, SPELLINGS_AT_ONCE = 32, PAGES = 4; + +/** `names` in questions Zenodo answers: in order, each as full as SPELLINGS_AT_ONCE allows; a name with more goes alone. */ +export function surveyQuestions(names: readonly string[]): string[][] { + const questions: string[][] = []; + for (const name of names) { + const open = questions.at(-1); + if (open && nameForms([...open, name]).length <= SPELLINGS_AT_ONCE) open.push(name); else questions.push([name]); + } + return questions; +} /** Which of `names` have simulation records on Zenodo, asked once for a whole draft run so no sweep passes one by unseen. * `note` gives a host's report the records of its planets. It is a note for a person: a record becomes a dataset only @@ -328,8 +342,7 @@ export async function simulationSurvey(archive: Archive, names: readonly string[ const found = new Map(), unique = [...new Set(names.filter(Boolean))]; let failure: string | undefined, asked = 0; try { - for (let start = 0; start < unique.length; start += NAMES_AT_ONCE) { - const group = unique.slice(start, start + NAMES_AT_ONCE); + for (const group of surveyQuestions(unique)) { for (let page = 1; page <= PAGES; page++) { if (asked++) await wait(PACE_MS); const records = parseZenodoSearch(JSON.parse(await archive.text(`${zenodoQuery(group)}&page=${page}`))); diff --git a/packages/telescope-cli/src/new-object/simulation/simulation-dataset.test.mts b/packages/telescope-cli/src/new-object/simulation/simulation-dataset.test.mts index 77104b7f6e1..edc93c207ad 100644 --- a/packages/telescope-cli/src/new-object/simulation/simulation-dataset.test.mts +++ b/packages/telescope-cli/src/new-object/simulation/simulation-dataset.test.mts @@ -12,7 +12,7 @@ import { loadNetcdfLonLatField } from '@cssearth/bake/objects/raster'; import { WORKSPACE } from '@cssearth/telescope/node'; import type { Archive } from '../archives/archives.mts'; import { rebuildExistingDatasets } from '../planet-datasets.mts'; -import { installSimulationDataset, parseSimulationEntries, restoreSimulationField, restoreSimulationMember, roundedRange, simulationPaths, simulationRecipe, simulationRelease, simulationSurvey } from './simulation-dataset.mts'; +import { installSimulationDataset, parseSimulationEntries, restoreSimulationField, restoreSimulationMember, roundedRange, simulationPaths, simulationRecipe, simulationRelease, simulationSurvey, surveyQuestions } from './simulation-dataset.mts'; const id = 'trappist-1f', o = `src/objects/${id}`; const fixture = resolve(WORKSPACE, 'packages/bake/src/objects/raster/netcdf/fixtures/field-cdf2.nc'); @@ -240,12 +240,18 @@ test('a draft run asks Zenodo once for all its planets, paces its requests and s assert.equal(survey.note(['TRAPPIST-1 e', 'TRAPPIST-1 f']), '; published simulations on Zenodo, to read before any is shown: TRAPPIST-1 e 10.5281/zenodo.5532765, 10.5281/zenodo.7752337, 10.5281/zenodo.10209661'); assert.equal(survey.note(['TRAPPIST-1 f']), ''); assert.equal(survey.failure, undefined); - // Forty names go in one request; a full page asks for the next, 2.1 seconds later. - const many = Array.from({ length: 41 }, (_, index) => `Planet-${index + 1} b`), full = { hits: { hits: Array.from({ length: 25 }, () => body.hits.hits[3]!) } }; + // A question holds 32 spellings, the size Zenodo answers: sixteen of these planets; a full page asks for the next, 2.1 seconds later. + const many = Array.from({ length: 17 }, (_, index) => `Planet-${index + 1} b`), full = { hits: { hits: Array.from({ length: 25 }, () => body.hits.hits[3]!) } }; let calls = 0; - const paged: Archive = { ...archive(null), text: async () => JSON.stringify(calls++ === 0 ? full : { hits: { hits: [] } }) }; + const questions: string[] = []; + const paged: Archive = { ...archive(null), text: async url => { questions.push(url); return JSON.stringify(calls++ === 0 ? full : { hits: { hits: [] } }); } }; assert.equal((await simulationSurvey(paged, many, async ms => { waits.push(ms); })).requests, 3); assert.deepEqual(waits, [2100, 2100]); + // The first sixteen planets twice, a page apart, then the seventeenth alone. + assert.deepEqual(questions.map(url => [(decodeURIComponent(url).match(/Planet-\d+b/gu) ?? []).length, new URL(url).searchParams.get('page')]), [[16, '1'], [16, '2'], [1, '1']]); + // A name written more ways takes more of a question: four spellings for a catalogue number, eight for a planet of a lettered star. + assert.deepEqual(surveyQuestions(['HD 1 b', 'HD 2 b', 'HD 3 b', 'HD 4 b', 'HD 5 b', 'HD 6 b', 'HD 7 b', 'HD 8 b', 'HD 9 b']).map(question => question.length), [8, 1]); + assert.deepEqual(surveyQuestions(['HD 135344 Ab', 'HD 93963 A b', 'HD 93963 A c', 'HD 202772 A b', 'HD 1 b']).map(question => question.length), [4, 1]); const failing: Archive = { ...archive(null), text: async () => { throw new Error('HTTP 429 from https://zenodo.org/api/records'); } }; const failed = await simulationSurvey(failing, ['TRAPPIST-1 e']); assert.equal(failed.failure, 'Zenodo could not be asked for published simulations of every planet (HTTP 429 from https://zenodo.org/api/records); ask with telescope simulations OBJECT.'); diff --git a/packages/telescope-cli/src/simulations/simulations.mts b/packages/telescope-cli/src/simulations/simulations.mts index b9960b1bba3..c1745dbfa86 100644 --- a/packages/telescope-cli/src/simulations/simulations.mts +++ b/packages/telescope-cli/src/simulations/simulations.mts @@ -74,11 +74,17 @@ export function parseZenodoSearch(value: unknown): readonly ZenodoRecord[] { const words = (value: string): string => ` ${value.normalize('NFKD').replace(/\p{Diacritic}/gu, '').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, ' ').trim()} `; /** A planet is written with and without a space before its letter ("TRAPPIST-1e", "TRAPPIST-1 e"), and a catalogue number - * with and without one after its letters ("GJ 1214b", "GJ1214b"): every spelling is searched and matched. */ + * with and without one after its letters ("GJ 1214b", "GJ1214b"): every spelling is searched and matched. So is a planet of + * a lettered star: Zenodo's record of WASP-94 A b's model writes "WASP-94A b", and went unfound until this covered it. */ export function nameForms(names: readonly string[]): string[] { return [...new Set(names.map(name => name.trim()).filter(Boolean).flatMap(name => { const planet = /^(.*[^\s])\s?([b-z])$/u.exec(name); - const forms = planet && /\d$/u.test(planet[1]!) ? [`${planet[1]}${planet[2]}`, `${planet[1]} ${planet[2]}`] : [name]; + // A planet of a lettered star (A to D) is written with the star's letter joined or set off, on either side: "WASP-94 A b", + // "WASP-94A b", "WASP-94 Ab", "WASP-94Ab". A host that ends in a letter keeps the space before its star's letter ("DS Tuc Ab"). + const lettered = /^(.*\d)\s?([A-D])\s?([b-z])$/u.exec(name) ?? /^(.*[^\s\d])\s([A-D])\s?([b-z])$/u.exec(name); + const stars = lettered ? [...(/\d$/u.test(lettered[1]!) ? [`${lettered[1]}${lettered[2]}`] : []), `${lettered[1]} ${lettered[2]}`] : []; + const forms = lettered ? stars.flatMap(star => [`${star}${lettered[3]}`, `${star} ${lettered[3]}`]) + : planet && /\d$/u.test(planet[1]!) ? [`${planet[1]}${planet[2]}`, `${planet[1]} ${planet[2]}`] : [name]; return forms.flatMap(form => /^[A-Za-z]+ \d/u.test(form) ? [form, form.replace(' ', '')] : [form]); }))]; } diff --git a/packages/telescope-cli/src/simulations/simulations.test.mts b/packages/telescope-cli/src/simulations/simulations.test.mts index 7b5f43119a4..5645c59bd04 100644 --- a/packages/telescope-cli/src/simulations/simulations.test.mts +++ b/packages/telescope-cli/src/simulations/simulations.test.mts @@ -47,6 +47,23 @@ test('a planet is matched with and without the space before its letter, as a phr assert.match(zenodoQuery(['TRAPPIST-1e']), /^https:\/\/zenodo\.org\/api\/records\?q=%22TRAPPIST-1e%22\+OR\+%22TRAPPIST-1\+e%22&type=dataset&size=25&sort=bestmatch$/u); }); +test('a planet of a lettered star is matched with the star\'s letter joined or set off', () => { + const forms = ['WASP-94Ab', 'WASP-94A b', 'WASP-94 Ab', 'WASP-94 A b']; + for (const written of forms) assert.deepEqual(nameForms([written]), forms, written); + // A catalogue number is still also written without the space after its letters. + assert.deepEqual(nameForms(['GJ 667 C c']), ['GJ 667Cc', 'GJ667Cc', 'GJ 667C c', 'GJ667C c', 'GJ 667 Cc', 'GJ667 Cc', 'GJ 667 C c', 'GJ667 C c']); + // A host that ends in a letter keeps the space before its star's letter. + assert.deepEqual(nameForms(['DS Tuc A b']), ['DS Tuc Ab', 'DS Tuc A b']); + assert.deepEqual(nameForms(['DS Tuc Ab']), ['DS Tuc Ab', 'DS Tuc A b']); + // Names without a star's letter before the planet's are left as they were. + assert.deepEqual(nameForms(['AU Mic b', 'Proxima Cen b', 'HD 19467 B', 'TOI-1266 c']), ['AU Mic b', 'Proxima Cen b', 'HD 19467 B', 'HD19467 B', 'TOI-1266c', 'TOI-1266 c']); + // Zenodo's record of WASP-94 A b's model (10.5281/zenodo.15085825) writes the planet "WASP-94A b". + const record = { title: 'Cloudy Mornings and Clear Evenings on a Giant Extrasolar World', description: 'the reduced transmission spectrum of the morning and evening terminators of planet WASP-94A b.' }; + assert.equal(namesObject(record, ['WASP-94 A b']), true); + assert.equal(namesObject({ title: 'Radial velocities of WASP-94 B b', description: '' }, ['WASP-94 A b']), false, 'the other star\'s planet is another planet'); + assert.equal(new URL(zenodoQuery(['WASP-94 A b'])).searchParams.get('q'), '"WASP-94Ab" OR "WASP-94A b" OR "WASP-94 Ab" OR "WASP-94 A b"'); +}); + test('only a license known to allow a derived picture counts as reuse', () => { assert.deepEqual(reuseLicense('cc-by-4.0'), { name: 'CC BY 4.0', url: 'https://creativecommons.org/licenses/by/4.0/' }); assert.equal(reuseLicense('CC-BY-NC-4.0')?.name, 'CC BY-NC 4.0');