diff --git a/snowpack/package.json b/snowpack/package.json index f242b66600..2b9db6c8ab 100644 --- a/snowpack/package.json +++ b/snowpack/package.json @@ -77,6 +77,7 @@ "got": "^11.1.4", "http-proxy": "^1.18.1", "is-builtin-module": "^3.0.0", + "isbinaryfile": "^4.0.6", "jsonschema": "^1.2.5", "kleur": "^4.1.0", "mime-types": "^2.1.26", diff --git a/snowpack/src/build/build-pipeline.ts b/snowpack/src/build/build-pipeline.ts index af6b13bff1..2f09b54617 100644 --- a/snowpack/src/build/build-pipeline.ts +++ b/snowpack/src/build/build-pipeline.ts @@ -1,9 +1,8 @@ -import {promises as fs} from 'fs'; import path from 'path'; import {validatePluginLoadResult} from '../config'; import {logger} from '../logger'; import {SnowpackBuildMap, SnowpackConfig, SnowpackPlugin} from '../types/snowpack'; -import {getEncodingType, getExt, replaceExt} from '../util'; +import {getExt, readFile, replaceExt} from '../util'; export interface BuildFileOptions { isDev: boolean; @@ -97,7 +96,7 @@ async function runPipelineLoadStep( return { [srcExt]: { - code: await fs.readFile(srcPath, getEncodingType(srcExt)), + code: await readFile(srcPath), }, }; } diff --git a/snowpack/src/commands/build.ts b/snowpack/src/commands/build.ts index bff0861e64..5a7c265d3e 100644 --- a/snowpack/src/commands/build.ts +++ b/snowpack/src/commands/build.ts @@ -21,7 +21,7 @@ import {removeLeadingSlash} from '../config'; import {logger} from '../logger'; import {transformFileImports} from '../rewrite-imports'; import {CommandOptions, ImportMap, SnowpackConfig, SnowpackSourceFile} from '../types/snowpack'; -import {cssSourceMappingURL, getEncodingType, jsSourceMappingURL, replaceExt} from '../util'; +import {cssSourceMappingURL, jsSourceMappingURL, readFile, replaceExt} from '../util'; import {getInstallTargets, run as installRunner} from './install'; const CONCURRENT_WORKERS = require('os').cpus().length; @@ -160,7 +160,12 @@ class FileBuilder { async resolveImports(importMap: ImportMap) { let isSuccess = true; this.filesToProxy = []; - for (const [outLoc, file] of Object.entries(this.filesToResolve)) { + for (const [outLoc, rawFile] of Object.entries(this.filesToResolve)) { + // don’t transform binary file contents + if (Buffer.isBuffer(rawFile.contents)) { + continue; + } + const file = rawFile as SnowpackSourceFile; const resolveImportSpecifier = createImportResolver({ fileLoc: file.locOnDisk!, // we’re confident these are reading from disk because we just read them dependencyImportMap: importMap, @@ -174,7 +179,7 @@ class FileBuilder { // Until supported, just exit here. if (!resolvedImportUrl) { isSuccess = false; - logger.error(`${file.locOnDisk} - Could not resolve unkonwn import "${spec}".`); + logger.error(`${file.locOnDisk} - Could not resolve unknown import "${spec}".`); return spec; } // Ignore "http://*" imports @@ -227,7 +232,8 @@ class FileBuilder { async writeToDisk() { mkdirp.sync(this.outDir); for (const [outLoc, code] of Object.entries(this.output)) { - await fs.writeFile(outLoc, code, getEncodingType(path.extname(outLoc))); + const encoding = typeof code === 'string' ? 'utf-8' : undefined; + await fs.writeFile(outLoc, code, encoding); } } @@ -243,7 +249,7 @@ class FileBuilder { hmr: false, config: this.config, }); - await fs.writeFile(importProxyFileLoc, proxyCode, getEncodingType('.js')); + await fs.writeFile(importProxyFileLoc, proxyCode, 'utf-8'); } } @@ -317,7 +323,7 @@ export async function command(commandOptions: CommandOptions) { !installedFileLoc.endsWith('import-map.json') && path.extname(installedFileLoc) !== '.js' ) { - const proxiedCode = await fs.readFile(installedFileLoc, {encoding: 'utf-8'}); + const proxiedCode = await readFile(installedFileLoc); const importProxyFileLoc = installedFileLoc + '.proxy.js'; const proxiedUrl = installedFileLoc.substr(buildDirectoryLoc.length).replace(/\\/g, '/'); const proxyCode = await wrapImportProxy({ @@ -326,7 +332,7 @@ export async function command(commandOptions: CommandOptions) { hmr: false, config: config, }); - await fs.writeFile(importProxyFileLoc, proxyCode, getEncodingType('.js')); + await fs.writeFile(importProxyFileLoc, proxyCode, 'utf-8'); } } return installResult; diff --git a/snowpack/src/commands/dev.ts b/snowpack/src/commands/dev.ts index b26e611fd5..ef9ef45780 100644 --- a/snowpack/src/commands/dev.ts +++ b/snowpack/src/commands/dev.ts @@ -68,11 +68,11 @@ import { checkLockfileHash, cssSourceMappingURL, DEV_DEPENDENCIES_DIR, - getEncodingType, getExt, jsSourceMappingURL, openInBrowser, parsePackageImportSpecifier, + readFile, replaceExt, resolveDependencyManifest, updateLockfileHash, @@ -166,7 +166,7 @@ const sendFile = ( } res.writeHead(200, headers); - res.write(body, getEncodingType(ext) as BufferEncoding); + res.write(body); res.end(); }; @@ -242,6 +242,7 @@ export async function command(commandOptions: CommandOptions) { // Set the proper install options, in case an install is needed. const dependencyImportMapLoc = path.join(DEV_DEPENDENCIES_DIR, 'import-map.json'); + logger.debug(`Using cache folder: ${path.relative(cwd, DEV_DEPENDENCIES_DIR)}`); const installCommandOptions = merge(commandOptions, { config: { installOptions: { @@ -254,9 +255,12 @@ export async function command(commandOptions: CommandOptions) { // Start with a fresh install of your dependencies, if needed. if (!(await checkLockfileHash(DEV_DEPENDENCIES_DIR)) || !existsSync(dependencyImportMapLoc)) { + logger.debug('Cache out of date or missing. Updating…'); logger.info(colors.yellow('! updating dependencies...')); await installCommand(installCommandOptions); await updateLockfileHash(DEV_DEPENDENCIES_DIR); + } else { + logger.debug(`Cache up-to-date. Using existing cache`); } let dependencyImportMap: ImportMap = {imports: {}}; @@ -678,7 +682,7 @@ If Snowpack is having trouble detecting the import, add ${colors.bold( } // 2. Load the file from disk. We'll need it to check the cold cache or build from scratch. - const fileContents = await fs.readFile(fileLoc, getEncodingType(requestedFileExt)); + const fileContents = await readFile(fileLoc); // 3. Send dependencies directly, since they were already build & resolved at install time. if (reqPath.startsWith(config.buildOptions.webModulesUrl) && !isProxyModule) { diff --git a/snowpack/src/commands/install.ts b/snowpack/src/commands/install.ts index cd1358a999..fab5422fca 100644 --- a/snowpack/src/commands/install.ts +++ b/snowpack/src/commands/install.ts @@ -446,6 +446,7 @@ export async function getInstallTargets( if (webDependencies) { installTargets.push(...scanDepList(Object.keys(webDependencies), cwd)); } + // TODO: remove this if block; move logic inside scanImports if (scannedFiles) { installTargets.push(...(await scanImportsFromFiles(scannedFiles, config))); } else { @@ -457,14 +458,19 @@ export async function getInstallTargets( export async function command(commandOptions: CommandOptions) { const {cwd, config} = commandOptions; + logger.debug('Starting install'); const installTargets = await getInstallTargets(config); + logger.debug('Received install targets'); if (installTargets.length === 0) { logger.error('Nothing to install.'); return; } + logger.debug('Running install command'); const finalResult = await run({...commandOptions, installTargets}); + logger.debug('Install command successfully ran'); if (finalResult.newLockfile) { await writeLockfile(path.join(cwd, 'snowpack.lock.json'), finalResult.newLockfile); + logger.debug('Successfully wrote lockfile'); } if (finalResult.stats) { logger.info(printStats(finalResult.stats)); diff --git a/snowpack/src/config.ts b/snowpack/src/config.ts index 1b593b69c1..2a64956b21 100644 --- a/snowpack/src/config.ts +++ b/snowpack/src/config.ts @@ -2,7 +2,6 @@ import buildScriptPlugin from '@snowpack/plugin-build-script'; import runScriptPlugin from '@snowpack/plugin-run-script'; import {cosmiconfigSync} from 'cosmiconfig'; import {all as merge} from 'deepmerge'; -import fs from 'fs'; import http from 'http'; import {validate, ValidatorResult} from 'jsonschema'; import path from 'path'; @@ -22,6 +21,7 @@ import { LegacySnowpackPlugin, PluginLoadResult, } from './types/snowpack'; +import {readFile} from './util'; const CONFIG_NAME = 'snowpack'; const ALWAYS_EXCLUDE = ['**/node_modules/**/*', '**/.types/**/*']; @@ -267,7 +267,7 @@ function loadPlugins( plugin.load = async (options: PluginLoadOptions) => { const result = await build({ ...options, - contents: fs.readFileSync(options.filePath, 'utf-8'), + contents: await readFile(options.filePath), }).catch((err) => { logger.error( `[${plugin.name}] There was a problem running this older plugin. Please update the plugin to the latest version.`, diff --git a/snowpack/src/rewrite-imports.ts b/snowpack/src/rewrite-imports.ts index 6c99e2edec..e95d5a9fa8 100644 --- a/snowpack/src/rewrite-imports.ts +++ b/snowpack/src/rewrite-imports.ts @@ -81,7 +81,7 @@ async function transformCssImports(code: string, replaceImport: (specifier: stri } export async function transformFileImports( - {baseExt, contents}: SnowpackSourceFile, + {baseExt, contents}: SnowpackSourceFile, replaceImport: (specifier: string) => string, ) { if (baseExt === '.js') { diff --git a/snowpack/src/scan-imports.ts b/snowpack/src/scan-imports.ts index 6eee4f4a69..a691d0bdab 100644 --- a/snowpack/src/scan-imports.ts +++ b/snowpack/src/scan-imports.ts @@ -1,11 +1,7 @@ import {ImportSpecifier, init as initESModuleLexer, parse} from 'es-module-lexer'; -import fs from 'fs'; import glob from 'glob'; -import * as colors from 'kleur/colors'; -import mime from 'mime-types'; import path from 'path'; import stripComments from 'strip-comments'; -import srcFileExtensionMapping from './build/src-file-extension-mapping'; import {logger} from './logger'; import {InstallTarget, SnowpackConfig, SnowpackSourceFile} from './types/snowpack'; import { @@ -14,6 +10,7 @@ import { getExt, HTML_JS_REGEX, isTruthy, + readFile, SVELTE_VUE_REGEX, } from './util'; @@ -155,23 +152,38 @@ function parseFileForInstallTargets({ locOnDisk, baseExt, contents, -}: SnowpackSourceFile): InstallTarget[] { +}: SnowpackSourceFile): InstallTarget[] { + const relativeLoc = path.relative(process.cwd(), locOnDisk); + try { switch (baseExt) { case '.css': case '.less': case '.sass': case '.scss': { + logger.debug(`Scanning ${relativeLoc} for imports as CSS`); return parseCssForInstallTargets(contents); } case '.html': case '.svelte': case '.vue': { + logger.debug(`Scanning ${relativeLoc} for imports as HTML`); return parseJsForInstallTargets(extractJSFromHTML({contents, baseExt})); } - default: { + case '.js': + case '.jsx': + case '.mjs': + case '.ts': + case '.tsx': { + logger.debug(`Scanning ${relativeLoc} for imports as JS`); return parseJsForInstallTargets(contents); } + default: { + logger.debug( + `Skip scanning ${relativeLoc} for imports (unknown file extension ${baseExt})`, + ); + return []; + } } } catch (err) { // Another error! No hope left, just abort. @@ -235,32 +247,11 @@ export async function scanImports(cwd: string, config: SnowpackConfig): Promise< const loadedFiles: (SnowpackSourceFile | null)[] = await Promise.all( includeFiles.map(async (filePath) => { const {baseExt, expandedExt} = getExt(filePath); - // Always ignore dotfiles - if (filePath.startsWith('.')) { - return null; - } - - // Probably a license, a README, etc - if (baseExt === '') { - return null; - } - - // If we don't recognize the file type, it could be source. Warn just in case. - if (!Object.keys(srcFileExtensionMapping).includes(baseExt) && !mime.lookup(baseExt)) { - logger.warn( - colors.dim( - `ignoring unsupported file "${path - .relative(process.cwd(), filePath) - .replace(/\\/g, '/')}"`, - ), - ); - } - return { baseExt, expandedExt, locOnDisk: filePath, - contents: await fs.promises.readFile(filePath, 'utf-8'), + contents: await readFile(filePath), }; }), ); @@ -273,7 +264,8 @@ export async function scanImportsFromFiles( config: SnowpackConfig, ): Promise { return loadedFiles - .map(parseFileForInstallTargets) + .filter((sourceFile) => !Buffer.isBuffer(sourceFile.contents)) // filter out binary files from import scanning + .map((sourceFile) => parseFileForInstallTargets(sourceFile as SnowpackSourceFile)) .reduce((flat, item) => flat.concat(item), []) .filter((target) => { const aliasEntry = findMatchingAliasEntry(config, target.specifier); diff --git a/snowpack/src/types/snowpack.ts b/snowpack/src/types/snowpack.ts index 1d63ece9ee..a2a3330e09 100644 --- a/snowpack/src/types/snowpack.ts +++ b/snowpack/src/types/snowpack.ts @@ -15,11 +15,11 @@ export type SnowpackBuiltFile = {code: string | Buffer; map?: string}; export type SnowpackBuildMap = Record; /** Standard file interface */ -export interface SnowpackSourceFile { +export interface SnowpackSourceFile { /** base extension (e.g. `.js`) */ baseExt: string; /** file contents */ - contents: string; + contents: Type; /** expanded extension (e.g. `.proxy.js` or `.module.css`) */ expandedExt: string; /** if no location on disk, assume this exists in memory */ @@ -81,7 +81,7 @@ export interface SnowpackPlugin { export interface LegacySnowpackPlugin { defaultBuildScript: string; - build?(options: PluginLoadOptions & {contents: string}): Promise; + build?(options: PluginLoadOptions & {contents: string | Buffer}): Promise; bundle?(options: { srcDirectory: string; destDirectory: string; diff --git a/snowpack/src/util.ts b/snowpack/src/util.ts index 03b44b5901..4e90da992f 100644 --- a/snowpack/src/util.ts +++ b/snowpack/src/util.ts @@ -6,6 +6,7 @@ import projectCacheDir from 'find-cache-dir'; import findUp from 'find-up'; import fs from 'fs'; import got, {CancelableRequest, Response} from 'got'; +import {isBinaryFile} from 'isbinaryfile'; import mkdirp from 'mkdirp'; import open from 'open'; import path from 'path'; @@ -37,9 +38,11 @@ export const SVELTE_VUE_REGEX = /(]*>)(.*?)<\/script>/gims; export const URL_HAS_PROTOCOL_REGEX = /^(\w+:)?\/\//; -const UTF8_FORMATS = ['.css', '.html', '.js', '.map', '.mjs', '.json', '.svg', '.txt', '.xml']; -export function getEncodingType(ext: string): 'utf-8' | undefined { - return UTF8_FORMATS.includes(ext) ? 'utf-8' : undefined; +/** Read file from disk; return a string if it’s a code file */ +export async function readFile(filepath: string): Promise { + const data = await fs.promises.readFile(filepath); + const isBinary = await isBinaryFile(data); + return isBinary ? data : data.toString('utf-8'); } export async function readLockfile(cwd: string): Promise { diff --git a/test/integration/error-include-ignore-unsupported-files/expected-output.txt b/test/integration/error-include-ignore-unsupported-files/expected-output.txt index 8a59d2e2e1..1c544efb93 100644 --- a/test/integration/error-include-ignore-unsupported-files/expected-output.txt +++ b/test/integration/error-include-ignore-unsupported-files/expected-output.txt @@ -1,2 +1 @@ -[snowpack] ignoring unsupported file "src/c.abcdefg" [snowpack] Nothing to install. \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 19f315983c..0254a4089a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8335,6 +8335,11 @@ isarray@^2.0.1: resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.5.tgz#8af1e4c1221244cc62459faf38940d4e644a5723" integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw== +isbinaryfile@^4.0.6: + version "4.0.6" + resolved "https://registry.yarnpkg.com/isbinaryfile/-/isbinaryfile-4.0.6.tgz#edcb62b224e2b4710830b67498c8e4e5a4d2610b" + integrity sha512-ORrEy+SNVqUhrCaal4hA4fBzhggQQ+BaLntyPOdoEiwlKZW9BZiJXjg3RMiruE4tPEI3pyVPpySHQF/dKWperg== + isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"