'use strict' const { URL } = require('url') const http = require('http') const https = require('https') const zlib = require('minizlib') const Minipass = require('minipass') const Body = require('./body.js') const { writeToStream, getTotalBytes } = Body const Response = require('./response.js') const Headers = require('./headers.js') const { createHeadersLenient } = Headers const Request = require('./request.js') const { getNodeRequestOptions } = Request const FetchError = require('./fetch-error.js') const AbortError = require('./abort-error.js') // XXX this should really be split up and unit-ized for easier testing // and better DRY implementation of data/http request aborting const fetch = async (url, opts) => { if (/^data:/.test(url)) { const request = new Request(url, opts) // delay 1 promise tick so that the consumer can abort right away return Promise.resolve().then(() => new Promise((resolve, reject) => { let type, data try { const { pathname, search } = new URL(url) const split = pathname.split(',') if (split.length < 2) { throw new Error('invalid data: URI') } const mime = split.shift() const base64 = /;base64$/.test(mime) type = base64 ? mime.slice(0, -1 * ';base64'.length) : mime const rawData = decodeURIComponent(split.join(',') + search) data = base64 ? Buffer.from(rawData, 'base64') : Buffer.from(rawData) } catch (er) { return reject(new FetchError(`[${request.method}] ${ request.url} invalid URL, ${er.message}`, 'system', er)) } const { signal } = request if (signal && signal.aborted) { return reject(new AbortError('The user aborted a request.')) } const headers = { 'Content-Length': data.length } if (type) { headers['Content-Type'] = type } return resolve(new Response(data, { headers })) })) } return new Promise((resolve, reject) => { // build request object const request = new Request(url, opts) let options try { options = getNodeRequestOptions(request) } catch (er) { return reject(er) } const send = (options.protocol === 'https:' ? https : http).request const { signal } = request let response = null const abort = () => { const error = new AbortError('The user aborted a request.') reject(error) if (Minipass.isStream(request.body) && typeof request.body.destroy === 'function') { request.body.destroy(error) } if (response && response.body) { response.body.emit('error', error) } } if (signal && signal.aborted) { return abort() } const abortAndFinalize = () => { abort() finalize() } const finalize = () => { req.abort() if (signal) { signal.removeEventListener('abort', abortAndFinalize) } clearTimeout(reqTimeout) } // send request const req = send(options) if (signal) { signal.addEventListener('abort', abortAndFinalize) } let reqTimeout = null if (request.timeout) { req.once('socket', socket => { reqTimeout = setTimeout(() => { reject(new FetchError(`network timeout at: ${ request.url}`, 'request-timeout')) finalize() }, request.timeout) }) } req.on('error', er => { // if a 'response' event is emitted before the 'error' event, then by the // time this handler is run it's too late to reject the Promise for the // response. instead, we forward the error event to the response stream // so that the error will surface to the user when they try to consume // the body. this is done as a side effect of aborting the request except // for in windows, where we must forward the event manually, otherwise // there is no longer a ref'd socket attached to the request and the // stream never ends so the event loop runs out of work and the process // exits without warning. // coverage skipped here due to the difficulty in testing // istanbul ignore next if (req.res) { req.res.emit('error', er) } reject(new FetchError(`request to ${request.url} failed, reason: ${ er.message}`, 'system', er)) finalize() }) req.on('response', res => { clearTimeout(reqTimeout) const headers = createHeadersLenient(res.headers) // HTTP fetch step 5 if (fetch.isRedirect(res.statusCode)) { // HTTP fetch step 5.2 const location = headers.get('Location') // HTTP fetch step 5.3 const locationURL = location === null ? null : (new URL(location, request.url)).toString() // HTTP fetch step 5.5 if (request.redirect === 'error') { reject(new FetchError('uri requested responds with a redirect, ' + `redirect mode is set to error: ${request.url}`, 'no-redirect')) finalize() return } else if (request.redirect === 'manual') { // node-fetch-specific step: make manual redirect a bit easier to // use by setting the Location header value to the resolved URL. if (locationURL !== null) { // handle corrupted header try { headers.set('Location', locationURL) } catch (err) { /* istanbul ignore next: nodejs server prevent invalid response headers, we can't test this through normal request */ reject(err) } } } else if (request.redirect === 'follow' && locationURL !== null) { // HTTP-redirect fetch step 5 if (request.counter >= request.follow) { reject(new FetchError(`maximum redirect reached at: ${ request.url}`, 'max-redirect')) finalize() return } // HTTP-redirect fetch step 9 if (res.statusCode !== 303 && request.body && getTotalBytes(request) === null) { reject(new FetchError( 'Cannot follow redirect with body being a readable stream', 'unsupported-redirect' )) finalize() return } // Update host due to redirection request.headers.set('host', (new URL(locationURL)).host) // HTTP-redirect fetch step 6 (counter increment) // Create a new Request object. const requestOpts = { headers: new Headers(request.headers), follow: request.follow, counter: request.counter + 1, agent: request.agent, compress: request.compress, method: request.method, body: request.body, signal: request.signal, timeout: request.timeout, } // if the redirect is to a new hostname, strip the authorization and cookie headers const parsedOriginal = new URL(request.url) const parsedRedirect = new URL(locationURL) if (parsedOriginal.hostname !== parsedRedirect.hostname) { requestOpts.headers.delete('authorization') requestOpts.headers.delete('cookie') } // HTTP-redirect fetch step 11 if (res.statusCode === 303 || ( (res.statusCode === 301 || res.statusCode === 302) && request.method === 'POST' )) { requestOpts.method = 'GET' requestOpts.body = undefined requestOpts.headers.delete('content-length') } // HTTP-redirect fetch step 15 resolve(fetch(new Request(locationURL, requestOpts))) finalize() return } } // end if(isRedirect) // prepare response res.once('end', () => signal && signal.removeEventListener('abort', abortAndFinalize)) const body = new Minipass() // if an error occurs, either on the response stream itself, on one of the // decoder streams, or a response length timeout from the Body class, we // forward the error through to our internal body stream. If we see an // error event on that, we call finalize to abort the request and ensure // we don't leave a socket believing a request is in flight. // this is difficult to test, so lacks specific coverage. body.on('error', finalize) // exceedingly rare that the stream would have an error, // but just in case we proxy it to the stream in use. res.on('error', /* istanbul ignore next */ er => body.emit('error', er)) res.on('data', (chunk) => body.write(chunk)) res.on('end', () => body.end()) const responseOptions = { url: request.url, status: res.statusCode, statusText: res.statusMessage, headers: headers, size: request.size, timeout: request.timeout, counter: request.counter, trailer: new Promise(resolveTrailer => res.on('end', () => resolveTrailer(createHeadersLenient(res.trailers)))), } // HTTP-network fetch step 12.1.1.3 const codings = headers.get('Content-Encoding') // HTTP-network fetch step 12.1.1.4: handle content codings // in following scenarios we ignore compression support // 1. compression support is disabled // 2. HEAD request // 3. no Content-Encoding header // 4. no content response (204) // 5. content not modified response (304) if (!request.compress || request.method === 'HEAD' || codings === null || res.statusCode === 204 || res.statusCode === 304) { response = new Response(body, responseOptions) resolve(response) return } // Be less strict when decoding compressed responses, since sometimes // servers send slightly invalid responses that are still accepted // by common browsers. // Always using Z_SYNC_FLUSH is what cURL does. const zlibOptions = { flush: zlib.constants.Z_SYNC_FLUSH, finishFlush: zlib.constants.Z_SYNC_FLUSH, } // for gzip if (codings === 'gzip' || codings === 'x-gzip') { const unzip = new zlib.Gunzip(zlibOptions) response = new Response( // exceedingly rare that the stream would have an error, // but just in case we proxy it to the stream in use. body.on('error', /* istanbul ignore next */ er => unzip.emit('error', er)).pipe(unzip), responseOptions ) resolve(response) return } // for deflate if (codings === 'deflate' || codings === 'x-deflate') { // handle the infamous raw deflate response from old servers // a hack for old IIS and Apache servers const raw = res.pipe(new Minipass()) raw.once('data', chunk => { // see http://stackoverflow.com/questions/37519828 const decoder = (chunk[0] & 0x0F) === 0x08 ? new zlib.Inflate() : new zlib.InflateRaw() // exceedingly rare that the stream would have an error, // but just in case we proxy it to the stream in use. body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder) response = new Response(decoder, responseOptions) resolve(response) }) return } // for br if (codings === 'br') { // ignoring coverage so tests don't have to fake support (or lack of) for brotli // istanbul ignore next try { var decoder = new zlib.BrotliDecompress() } catch (err) { reject(err) finalize() return } // exceedingly rare that the stream would have an error, // but just in case we proxy it to the stream in use. body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder) response = new Response(decoder, responseOptions) resolve(response) return } // otherwise, use response as-is response = new Response(body, responseOptions) resolve(response) }) writeToStream(req, request) }) } module.exports = fetch fetch.isRedirect = code => code === 301 || code === 302 || code === 303 || code === 307 || code === 308 fetch.Headers = Headers fetch.Request = Request fetch.Response = Response fetch.FetchError = FetchError fetch.AbortError = AbortError