diff --git a/flow-server/src/main/resources/plugins/application-theme-plugin/application-theme-plugin.js b/flow-server/src/main/resources/plugins/application-theme-plugin/application-theme-plugin.js
index a75abbb443e..99cab2437de 100644
--- a/flow-server/src/main/resources/plugins/application-theme-plugin/application-theme-plugin.js
+++ b/flow-server/src/main/resources/plugins/application-theme-plugin/application-theme-plugin.js
@@ -17,7 +17,7 @@
 const fs = require('fs');
 const path = require('path');
 const generateThemeFile = require('./theme-generator');
-const copyStaticAssets = require('./theme-copy');
+const { copyStaticAssets } = require('./theme-copy');
 
 let logger;
 
diff --git a/flow-server/src/main/resources/plugins/application-theme-plugin/theme-copy.js b/flow-server/src/main/resources/plugins/application-theme-plugin/theme-copy.js
index 377cb184a4f..ca14ba691e9 100644
--- a/flow-server/src/main/resources/plugins/application-theme-plugin/theme-copy.js
+++ b/flow-server/src/main/resources/plugins/application-theme-plugin/theme-copy.js
@@ -61,6 +61,12 @@ function copyStaticAssets(themeName, themeProperties, projectStaticAssetsOutputF
   fs.mkdirSync(projectStaticAssetsOutputFolder, {
     recursive: true
   });
+  const missingModules = checkModules(Object.keys(assets));
+  if (missingModules.length > 0) {
+    throw Error("Missing npm modules '" + missingModules.join("', '")
+      + "' for assets marked in 'theme.json'.\n" +
+      "Install package(s) by adding a @NpmPackage annotation or install it using 'npm/pnpm i'");
+  }
   Object.keys(assets).forEach((module) => {
 
     const copyRules = assets[module];
@@ -84,4 +90,16 @@ function copyStaticAssets(themeName, themeProperties, projectStaticAssetsOutputF
   });
 };
 
-module.exports = copyStaticAssets;
+function checkModules(modules) {
+  const missing = [];
+
+  modules.forEach((module) => {
+    if (!fs.existsSync(path.resolve('node_modules/', module))) {
+      missing.push(module);
+    }
+  });
+
+  return missing;
+}
+
+module.exports = {checkModules, copyStaticAssets};
diff --git a/flow-server/src/main/resources/plugins/application-theme-plugin/theme-generator.js b/flow-server/src/main/resources/plugins/application-theme-plugin/theme-generator.js
index 3637aed386c..d584b2c4e65 100644
--- a/flow-server/src/main/resources/plugins/application-theme-plugin/theme-generator.js
+++ b/flow-server/src/main/resources/plugins/application-theme-plugin/theme-generator.js
@@ -20,6 +20,8 @@
  */
 const glob = require('glob');
 const path = require('path');
+const fs = require('fs');
+const { checkModules } = require('./theme-copy');
 
 // Special folder inside a theme for component themes that go inside the component shadow root
 const themeComponentsFolder = 'components';
@@ -82,6 +84,13 @@ function generateThemeFile(themeFolder, themeName, themeProperties) {
 
   let i = 0;
   if (themeProperties.importCss) {
+    const missingModules = checkModules(themeProperties.importCss);
+    if(missingModules.length > 0) {
+      throw Error("Missing npm modules or files '" + missingModules.join("', '")
+        + "' for importCss marked in 'theme.json'.\n" +
+        "Install or update package(s) by adding a @NpmPackage annotation or install it using 'npm/pnpm i'");
+
+    }
     themeProperties.importCss.forEach((cssPath) => {
       const variable = 'module' + i++;
       imports.push(`import ${variable} from '${cssPath}';\n`);
diff --git a/flow-tests/test-embedding/test-embedding-application-theme/src/main/java/com/vaadin/flow/webcomponent/ThemedComponentExporter.java b/flow-tests/test-embedding/test-embedding-application-theme/src/main/java/com/vaadin/flow/webcomponent/ThemedComponentExporter.java
index d29b05d1c48..61ef9788faf 100644
--- a/flow-tests/test-embedding/test-embedding-application-theme/src/main/java/com/vaadin/flow/webcomponent/ThemedComponentExporter.java
+++ b/flow-tests/test-embedding/test-embedding-application-theme/src/main/java/com/vaadin/flow/webcomponent/ThemedComponentExporter.java
@@ -16,10 +16,12 @@
 package com.vaadin.flow.webcomponent;
 
 import com.vaadin.flow.component.WebComponentExporter;
+import com.vaadin.flow.component.dependency.NpmPackage;
 import com.vaadin.flow.component.webcomponent.WebComponent;
 import com.vaadin.flow.theme.Theme;
 
 @Theme(themeFolder = "embedded-theme")
+@NpmPackage(value = "@fortawesome/fontawesome-free", version = "5.15.1")
 public class ThemedComponentExporter
         extends WebComponentExporter<ThemedComponent> {
     public ThemedComponentExporter() {
diff --git a/flow-tests/test-themes/README.md b/flow-tests/test-themes/README.md
new file mode 100644
index 00000000000..bbaeb2f8707
--- /dev/null
+++ b/flow-tests/test-themes/README.md
@@ -0,0 +1,2 @@
+To test #9615 change the importCss and/or asset to a non installed npm module,
+execute `mvn verify` and verify the exception message thrown for failing webpack build.