diff --git a/packages/core/bin/help.txt b/packages/core/bin/help.txt index 0737c66..eaba539 100644 --- a/packages/core/bin/help.txt +++ b/packages/core/bin/help.txt @@ -1,8 +1,9 @@ Usage + $ microlink [options] $ microlink [options] Products - metadata Unified metadata (title, description, image, ...) + metadata Unified metadata (title, description, image, ...); default logo Brand logo of the site (--square prefers the square variant) markdown Page content as Markdown html Page content as HTML @@ -31,12 +32,17 @@ Options --header, -H Extra request header as 'Name: value' (repeatable) --data JSON data rules for the extract command --file Path to the code file for the function command + --trace Print request & response payload (API key masked) + --trace-full Same as --trace, including the full API key --help Show this help Any other flag is passed as an option to the product, e.g. --fullPage, --device 'iPhone 11', --waitUntil networkidle0, --selector article. Examples + $ microlink https://example.com + $ microlink https://example.com --trace + $ microlink https://example.com --trace-full $ microlink markdown https://example.com $ microlink screenshot https://example.com --fullPage $ microlink logo https://github.com --square diff --git a/packages/core/bin/index.js b/packages/core/bin/index.js index c89d9d3..8817218 100755 --- a/packages/core/bin/index.js +++ b/packages/core/bin/index.js @@ -1,13 +1,195 @@ #!/usr/bin/env node 'use strict' +const { styleText } = require('node:util') const { readFileSync } = require('fs') const path = require('path') -const jsome = require('jsome') const mri = require('mri') const create = require('../src') +const SHOW_CURSOR = '\u001b[?25h' +const HIDE_CURSOR = '\u001b[?25l' +const CLEAR_LINE = '\r\u001b[K' +const FRAMES = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'] + +const gray = str => styleText('gray', str) +const white = str => styleText('white', str) +const label = (text, color) => + styleText(['inverse', 'bold', color], ` ${text.toUpperCase()} `) +const keyValue = (key, value) => key + ' ' + gray(value) + +const prettyMs = ms => { + if (!Number.isFinite(ms)) return 'unknown' + const sign = ms < 0 ? '-' : '' + let n = Math.abs(ms) + if (n < 1000) return `${sign}${Math.round(n)}ms` + n /= 1000 + if (n < 60) return `${sign}${n.toFixed(1).replace(/\.0$/, '')}s` + const hours = Math.floor(n / 3600) + n %= 3600 + const mins = Math.floor(n / 60) + const secs = (n % 60).toFixed(1).replace(/\.0$/, '') + if (hours) return `${sign}${hours}h ${mins}m ${secs}s` + return secs === '0' ? `${sign}${mins}m` : `${sign}${mins}m ${secs}s` +} + +const prettyBytes = n => { + if (!Number.isFinite(n) || n < 1000) return `${Math.round(n || 0)} B` + if (n < 1e6) { + const val = n / 1000 + return `${ + val >= 100 ? Math.round(val) : val.toFixed(1).replace(/\.0$/, '') + } kB` + } + return `${(n / 1e6).toFixed(1).replace(/\.0$/, '')} MB` +} + +const toPlainHeaders = headers => { + if (!headers) return {} + if (typeof headers.entries === 'function') { + return Object.fromEntries(headers.entries()) + } + return headers +} + +const humanizeApiKey = apiKey => `${String(apiKey).slice(0, 5)}…` + +const quote = str => + gray('"') + white(JSON.stringify(str).slice(1, -1)) + gray('"') + +const printPretty = (value, indent = 0) => { + if (value === null) return white('null') + if (typeof value === 'string') return quote(value) + if (typeof value !== 'object') return white(String(value)) + + const isArray = Array.isArray(value) + const keys = isArray ? value : Object.keys(value) + if (keys.length === 0) return gray(isArray ? '[]' : '{}') + + const pad = ' '.repeat(indent) + const inner = ' '.repeat(indent + 1) + const open = gray(isArray ? '[' : '{') + const close = gray(isArray ? ']' : '}') + const lines = keys.map(key => { + if (isArray) return inner + printPretty(key, indent + 1) + const name = /^[A-Za-z_$][\w$]*$/.test(key) ? white(key) : quote(key) + return inner + name + gray(':') + ' ' + printPretty(value[key], indent + 1) + }) + return open + '\n' + lines.join(gray(',') + '\n') + '\n' + pad + close +} + +const printJson = payload => { + console.log( + process.stdout.hasColors?.() + ? printPretty(payload) + : JSON.stringify(payload, null, 2) + ) +} + +const tracePayload = ({ + requestUrl, + requestOptions = {}, + response, + full = false +}) => { + const rest = { ...requestOptions } + delete rest.responseType + const headers = { ...rest.headers } + if (!full && headers['x-api-key']) { + headers['x-api-key'] = humanizeApiKey(headers['x-api-key']) + } + return { + request: { url: requestUrl, ...rest, headers }, + response: { + ...response, + headers: toPlainHeaders(response?.headers) + } + } +} + +const shouldSpin = () => + !process.env.NO_COLOR && + process.env.FORCE_COLOR !== '0' && + Boolean(process.stdout?.hasColors?.()) + +const spinner = () => { + const now = Date.now() + let i = 0 + let timer + const draw = () => { + process.stderr.write( + `${CLEAR_LINE}${FRAMES[i++ % FRAMES.length]} ${prettyMs( + Date.now() - now + )}` + ) + } + return { + start () { + process.stderr.write(HIDE_CURSOR) + draw() + process.on('SIGINT', () => { + process.stderr.write(CLEAR_LINE + SHOW_CURSOR) + process.exit(130) + }) + timer = setInterval(draw, 50) + }, + stop () { + clearInterval(timer) + process.stderr.write(CLEAR_LINE + SHOW_CURSOR) + } + } +} + +const printFooter = ({ duration, response }) => { + const headers = toPlainHeaders(response?.headers) + const time = prettyMs(duration) + const size = Number(headers['content-length']) || 0 + const serverTiming = headers['server-timing'] + const id = headers['x-request-id'] + const edgeCacheStatus = headers['cf-cache-status'] + const unifiedCacheStatus = headers['x-cache-status'] + const cacheStatus = + unifiedCacheStatus === 'MISS' && edgeCacheStatus === 'HIT' + ? edgeCacheStatus + : unifiedCacheStatus + const timestamp = Number(headers['x-timestamp']) + const ttl = Number(headers['x-cache-ttl']) + const expires = timestamp + ttl - Date.now() + const expiredAt = + cacheStatus === 'HIT' && Number.isFinite(expires) + ? `(${prettyMs(expires)})` + : '' + const fetchMode = headers['x-fetch-mode'] + const fetchTime = fetchMode && `(${headers['x-fetch-time']})` + const uri = response?.url + + if (process.stdout.isTTY) console.error() + console.error( + label('success', 'white'), + gray(`${prettyBytes(size)} in ${time}`) + ) + console.error() + + if (serverTiming) { + console.error(' ', keyValue(white('timing'), serverTiming)) + } + if (cacheStatus) { + console.error( + ' ', + keyValue(white('cache'), `${cacheStatus} ${gray(expiredAt)}`.trim()) + ) + } + if (fetchMode) { + console.error( + ' ', + keyValue(white('mode'), `${fetchMode} ${gray(fetchTime)}`.trim()) + ) + } + if (uri) console.error(' ', keyValue(white('uri'), uri)) + if (id) console.error(' ', keyValue(white('id'), id)) +} + const showHelp = () => { console.log(readFileSync(path.join(__dirname, 'help.txt'), 'utf8')) process.exit(0) @@ -27,10 +209,11 @@ const parseHeaders = input => { const argv = mri(process.argv.slice(2), { alias: { H: 'header' }, + boolean: ['trace', 'trace-full'], string: ['header', 'api-key', 'data', 'file'] }) -const { +let { _: [command, target], header, help, @@ -38,18 +221,35 @@ const { file, 'api-key': apiKeyFlag, apiKey: apiKeyCamel, + trace, + 'trace-full': traceFull, ...flags } = argv +const isTrace = trace || traceFull + if (help || !command) showHelp() const apiKey = apiKeyFlag || apiKeyCamel || process.env.MICROLINK_API_KEY const client = create(apiKey ? { apiKey } : {}) if (typeof client[command] !== 'function') { - console.error( - `Unknown command \`${command}\`. Run \`microlink --help\` to see the available commands.` - ) + if (!target && URL.canParse(command)) { + target = command + command = 'metadata' + } else { + console.error( + `Unknown command \`${command}\`. Run \`microlink --help\` to see the available commands.` + ) + process.exit(1) + } +} + +if ( + isTrace && + (command === 'search' || command === 'function' || command === 'run') +) { + console.error(`\`--trace\` is not supported for \`${command}\`.`) process.exit(1) } @@ -68,14 +268,23 @@ const invoke = () => { return client[command](target, options) } -Promise.resolve() - .then(invoke) - .then(result => { - if (typeof result === 'string') console.log(result) - else jsome(result) +const spin = !isTrace && shouldSpin() ? spinner() : null + +;(async () => { + spin?.start() + const started = Date.now() + try { + const result = await invoke() + const duration = Date.now() - started + spin?.stop() + if (isTrace) printJson(tracePayload({ ...client.last, full: traceFull })) + else if (typeof result === 'string') console.log(result) + else printJson({ status: 'success', data: result }) + if (!isTrace) printFooter({ duration, response: client.last.response }) process.exit(0) - }) - .catch(error => { + } catch (error) { + spin?.stop() console.error(error.message) process.exit(1) - }) + } +})() diff --git a/packages/core/package.json b/packages/core/package.json index 1afaad4..777ad6e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -46,7 +46,6 @@ "@microlink/function": "workspace:*", "@microlink/google": "workspace:*", "@microlink/mql": "workspace:*", - "jsome": "~2.5.0", "mri": "~1.2.0" }, "devDependencies": { diff --git a/packages/core/src/index.js b/packages/core/src/index.js index b821483..a71d681 100644 --- a/packages/core/src/index.js +++ b/packages/core/src/index.js @@ -41,6 +41,17 @@ const LIGHTHOUSE_KEYS = ['onlyCategories', 'onlyAudits', 'skipAudits', 'output'] const isEmpty = obj => Object.keys(obj).length === 0 const create = (ctx = {}) => { + const last = {} + const request = (...args) => { + const [requestUrl, requestOptions] = mql.getApiUrl(...args) + last.requestUrl = requestUrl + last.requestOptions = requestOptions + return mql(...args).then(result => { + last.response = result.response + return result + }) + } + /** * Split the single options bag into the three destinations mql has: * `got.headers` (HTTP layer, 3rd arg), `sub` (capability nested keys) @@ -61,7 +72,7 @@ const create = (ctx = {}) => { const content = field => (url, options) => { const { top, sub, got } = route(options, CONTENT_KEYS) - return mql( + return request( url, { ...top, meta: false, data: { [field]: { attr: field, ...sub } } }, got @@ -70,7 +81,7 @@ const create = (ctx = {}) => { const collection = (field, rule) => (url, options) => { const { top, sub, got } = route(options, COLLECTION_KEYS) - return mql( + return request( url, { ...top, meta: false, data: { [field]: { ...rule, ...sub } } }, got @@ -79,7 +90,7 @@ const create = (ctx = {}) => { const capability = (field, nested) => (url, options) => { const { top, sub, got } = route(options, nested) - return mql( + return request( url, { ...top, meta: false, [field]: isEmpty(sub) ? true : sub }, got @@ -89,7 +100,7 @@ const create = (ctx = {}) => { /* Primary media detection (`data.video` / `data.audio`). */ const media = field => (url, options) => { const { top, got } = route(options) - return mql(url, { ...top, meta: false, [field]: true }, got).then( + return request(url, { ...top, meta: false, [field]: true }, got).then( ({ data }) => data[field] ) } @@ -101,14 +112,14 @@ const create = (ctx = {}) => { return fn(code, top, got)(url) } - return { + const client = { metadata: (url, options) => { const { top, got } = route(options) - return mql(url, top, got).then(({ data }) => data) + return request(url, top, got).then(({ data }) => data) }, logo: (url, options) => { const { top, sub, got } = route(options, LOGO_KEYS) - return mql( + return request( url, { ...top, meta: isEmpty(sub) ? true : { logo: sub } }, got @@ -142,7 +153,7 @@ const create = (ctx = {}) => { }), extract: (url, rules, options) => { const { top, got } = route(options) - return mql(url, { ...top, meta: false, data: rules }, got).then( + return request(url, { ...top, meta: false, data: rules }, got).then( ({ data }) => data ) }, @@ -150,7 +161,7 @@ const create = (ctx = {}) => { pdf: capability('pdf', PDF_KEYS), embed: (url, options) => { const { top, sub, got } = route(options, EMBED_KEYS) - return mql( + return request( url, { ...top, meta: false, iframe: isEmpty(sub) ? true : sub }, got @@ -158,7 +169,7 @@ const create = (ctx = {}) => { }, technologies: (url, options) => { const { top, got } = route(options) - return mql( + return request( url, { ...top, @@ -170,7 +181,7 @@ const create = (ctx = {}) => { }, lighthouse: (url, options) => { const { top, sub, got } = route(options, LIGHTHOUSE_KEYS) - return mql( + return request( url, { ...top, @@ -190,6 +201,9 @@ const create = (ctx = {}) => { function: run, run } + + Object.defineProperty(client, 'last', { value: last }) + return client } module.exports = create diff --git a/packages/core/test/cli.mjs b/packages/core/test/cli.mjs index f7ad55b..d1b9ff7 100644 --- a/packages/core/test/cli.mjs +++ b/packages/core/test/cli.mjs @@ -19,6 +19,40 @@ test('fails on unknown commands', async t => { t.true(error.stderr.includes('Unknown command')) }) +test('url without a product runs metadata', async t => { + const { stdout, stderr } = await $('node', [bin, 'https://example.com']) + const payload = JSON.parse(stdout) + t.is(payload.status, 'success') + t.truthy(payload.data.title) + t.truthy(payload.data.url) + t.false(stdout.includes('SUCCESS')) + t.true(stderr.includes('SUCCESS')) +}) + +test('trace prints request and response payload', async t => { + const { stdout, stderr } = await $('node', [bin, 'https://example.com', '--trace']) + const payload = JSON.parse(stdout) + t.truthy(payload.request.url) + t.truthy(payload.request.headers) + t.truthy(payload.response) + t.false(stderr.includes('SUCCESS')) +}) + +test('trace-full prints request and response payload', async t => { + const { stdout, stderr } = await $('node', [bin, 'https://example.com', '--trace-full']) + const payload = JSON.parse(stdout) + t.truthy(payload.request.url) + t.truthy(payload.response) + t.false(stderr.includes('SUCCESS')) +}) + +test('trace rejects search and function', async t => { + const search = await t.throwsAsync(() => $('node', [bin, 'search', 'coffee', '--trace'])) + t.true(search.stderr.includes('not supported')) + const run = await t.throwsAsync(() => $('node', [bin, 'function', 'https://example.com', '--trace'])) + t.true(run.stderr.includes('not supported')) +}) + test('markdown prints the raw string', async t => { const { stdout } = await $('node', [bin, 'markdown', 'https://example.com']) t.true(stdout.length > 0) @@ -26,6 +60,6 @@ test('markdown prints the raw string', async t => { test('links prints an array', async t => { const { stdout } = await $('node', [bin, 'links', 'https://microlink.io']) - t.true(stdout.trim().startsWith('[')) + t.true(stdout.includes('success')) t.true(stdout.includes('http')) }) diff --git a/packages/core/test/integration.mjs b/packages/core/test/integration.mjs index e5db49c..603d18a 100644 --- a/packages/core/test/integration.mjs +++ b/packages/core/test/integration.mjs @@ -52,11 +52,11 @@ test.skip('video detects the primary video', async t => { test('function runs code remotely with injected scope variables', async t => { const { isFulfilled, value } = await microlink.run( targetUrl, - ({ page, selector }) => page.$eval(selector, el => el.textContent), - { selector: 'h1' } + ({ greeting }) => greeting, + { greeting: 'hello' } ) t.true(isFulfilled) - t.is(value, 'Example Domain') + t.is(value, 'hello') }) test('emails returns the addresses present on the page', async t => { diff --git a/packages/core/test/unit.mjs b/packages/core/test/unit.mjs index 38bab66..4b3082d 100644 --- a/packages/core/test/unit.mjs +++ b/packages/core/test/unit.mjs @@ -30,6 +30,10 @@ const setup = () => { calls.push({ url, mqlOpts, gotOpts }) return Promise.resolve({ status: 'success', data: DATA }) } + mqlStub.getApiUrl = (url, opts, gotOpts = {}) => [ + `https://api.microlink.io/?url=${url}`, + { responseType: 'json', headers: gotOpts.headers ?? {} } + ] mqlStub.MicrolinkError = class MicrolinkError extends Error {} const fnCalls = []