diff --git a/packages/ember-auto-import/ts/bundler.ts b/packages/ember-auto-import/ts/bundler.ts
index 9dbb9f718..29ba3e947 100644
--- a/packages/ember-auto-import/ts/bundler.ts
+++ b/packages/ember-auto-import/ts/bundler.ts
@@ -35,7 +35,6 @@ export interface BundlerHook {
 export default class Bundler extends Plugin {
   private lastDeps: Map<string, BundleDependencies> | undefined;
   private cachedBundlerHook: BundlerHook | undefined;
-  private didEnsureDirs: boolean;
   private options: BundlerPluginOptions;
   private isWatchingSomeDeps: boolean;
 
@@ -46,7 +45,6 @@ export default class Bundler extends Plugin {
       needsCache: true,
     });
     this.options = options;
-    this.didEnsureDirs = false;
     this.isWatchingSomeDeps = deps.length > 1;
   }
 
@@ -107,27 +105,23 @@ export default class Bundler extends Plugin {
   }
 
   async build() {
-    this.ensureDirs();
     reloadDevPackages();
     let { splitter } = this.options;
     let bundleDeps = await splitter.deps();
     if (bundleDeps !== this.lastDeps || this.isWatchingSomeDeps) {
       let buildResult = await this.bundlerHook.build(bundleDeps);
+      this.emptyDirs();
       this.addEntrypoints(buildResult);
       this.addLazyAssets(buildResult);
       this.lastDeps = bundleDeps;
     }
   }
 
-  private ensureDirs() {
-    if (this.didEnsureDirs) {
-      return;
-    }
+  private emptyDirs() {
     emptyDirSync(join(this.outputPath, 'lazy'));
     for (let bundle of this.options.bundles.names) {
       emptyDirSync(join(this.outputPath, 'entrypoints', bundle));
     }
-    this.didEnsureDirs = true;
   }
 
   private addEntrypoints({ entrypoints, dir }: BuildResult) {
diff --git a/packages/ember-auto-import/ts/webpack.ts b/packages/ember-auto-import/ts/webpack.ts
index 96f6fb934..fee815d9b 100644
--- a/packages/ember-auto-import/ts/webpack.ts
+++ b/packages/ember-auto-import/ts/webpack.ts
@@ -114,9 +114,7 @@ export default class WebpackBundler implements BundlerHook {
       },
       output: {
         path: this.outputDir,
-        // entry chunks need to have stable names, so we can more easily gather
-        // them all up to append to Ember's vendor.js
-        filename: `chunk.[id].js`,
+        filename: `chunk.[id].[chunkhash].js`,
         chunkFilename: `chunk.[id].[chunkhash].js`,
         libraryTarget: 'var',
         library: '__ember_auto_import__',
@@ -124,12 +122,6 @@ export default class WebpackBundler implements BundlerHook {
       optimization: {
         splitChunks: {
           chunks: 'all',
-          cacheGroups: {
-            // similar to above, entry vendor chunks need to have stable names
-            vendors: {
-              filename: 'chunk.[name].js',
-            } as any, // typings are missing a valid documented option
-          },
         },
       },
       resolveLoader: {
diff --git a/test-scenarios/common-chunk-test.ts b/test-scenarios/common-chunk-test.ts
index 79ca6b940..15e2f80b3 100644
--- a/test-scenarios/common-chunk-test.ts
+++ b/test-scenarios/common-chunk-test.ts
@@ -77,8 +77,8 @@ appScenarios
 
             module('Unit | common chunk', function () {
               test('can use two dynamic imports that share a common chunk', async function(assert) {
-                assert.equal(await useLeft(), 'The message is left');
-                assert.equal(await useRight(), 'The message is right');
+                assert.equal(await useLeft(), 'the message is left');
+                assert.equal(await useRight(), 'the message is right');
               });
             });
           `,