Implement the project install step

waspc/data/Cli/templates/basic/package.json

@@ -2,9 +2,10 @@
   "name": "prototype",
   "dependencies": {
     "wasp": "file:.wasp/out/sdk/wasp",
-    "react": "18.2.0"
+    "react": "^18.2.0"
   },
   "devDependencies": {
-    "@types/react": "^18.0.37"
+    "@types/react": "^18.0.37",
+    "prisma": "4.16.2"
   }
 }

waspc/data/Generator/templates/sdk/dependencies.txt

@@ -125,3 +125,8 @@ depsRequiredByWebSockets spec,
             ("@types/cors", "^2.8.5")
           ]
     }
+
+
+LOG:
+- react moved from web-app to project package.json
+- react-dom moved from web-app to project package.json

waspc/examples/todo-typescript/package.json

@@ -1,12 +1,12 @@
 {
   "name": "prototype",
   "dependencies": {
-    "@prisma/client": "^4.16.2",
-    "react": "18.2.0",
-    "wasp": "file:.wasp/out/sdk/wasp"
+    "wasp": "file:.wasp/out/sdk/wasp",
+    "@prisma/client": "4.16.2",
+    "react": "^18.2.0"
   },
   "devDependencies": {
     "@types/react": "^18.0.37",
-    "prisma": "^4.16.2"
+    "prisma": "4.16.2"
   }
 }

waspc/src/Wasp/AppSpec.hs

@@ -42,6 +42,7 @@ import qualified Wasp.AppSpec.ExternalCode as ExternalCode
 import Wasp.AppSpec.Job (Job)
 import Wasp.AppSpec.Operation (Operation)
 import qualified Wasp.AppSpec.Operation as AS.Operation
+import Wasp.AppSpec.PackageJson (PackageJson)
 import Wasp.AppSpec.Page (Page)
 import Wasp.AppSpec.Query (Query)
 import Wasp.AppSpec.Route (Route)
@@ -56,6 +57,8 @@ import Wasp.Project.Db.Migrations (DbMigrationsDir)
 data AppSpec = AppSpec
   { -- | List of declarations like App, Page, Route, ... that describe the web app.
     decls :: [Decl],
+    -- | The contents of the package.json file found in the root directory of the wasp project.
+    packageJson :: PackageJson,
     -- | Absolute path to the directory containing the wasp project.
     waspProjectDir :: Path' Abs (Dir WaspProjectDir),
     -- | List of external server code files (they are referenced/used in the declarations).

waspc/src/Wasp/AppSpec/App.hs

@@ -6,7 +6,6 @@ import Data.Data (Data)
 import Wasp.AppSpec.App.Auth (Auth)
 import Wasp.AppSpec.App.Client (Client)
 import Wasp.AppSpec.App.Db (Db)
-import Wasp.AppSpec.App.Dependency (Dependency)
 import Wasp.AppSpec.App.EmailSender (EmailSender)
 import Wasp.AppSpec.App.Server (Server)
 import Wasp.AppSpec.App.Wasp (Wasp)
@@ -22,7 +21,6 @@ data App = App
     client :: Maybe Client,
     db :: Maybe Db,
     emailSender :: Maybe EmailSender,
-    dependencies :: Maybe [Dependency],
     webSocket :: Maybe WebSocket
   }
   deriving (Show, Eq, Data)

waspc/src/Wasp/AppSpec/PackageJson.hs

@@ -0,0 +1,31 @@
+{-# LANGUAGE DeriveGeneric #-}
+{-# LANGUAGE TemplateHaskell #-}
+
+module Wasp.AppSpec.PackageJson where
+
+import Control.Applicative (liftA2)
+import Data.Aeson.TH
+import Data.Map (Map)
+import qualified Data.Map as M
+import GHC.Generics (Generic)
+import Wasp.AppSpec.App.Dependency (Dependency)
+import qualified Wasp.AppSpec.App.Dependency as D
+
+data PackageJson = PackageJson
+  { _name :: !String,
+    -- todo(filip): do this properly once you merge martin's PR
+    _dependencies :: !(Map String String),
+    _devDependencies :: !(Map String String)
+  }
+  deriving (Show, Generic)
+
+$(deriveJSON defaultOptions {fieldLabelModifier = drop 1} ''PackageJson)
+
+dependencies :: PackageJson -> [Dependency]
+dependencies packageJson = D.fromList $ M.toList $ _dependencies packageJson
+
+devDependencies :: PackageJson -> [Dependency]
+devDependencies packageJson = D.fromList $ M.toList $ _devDependencies packageJson
+
+allDependencies :: PackageJson -> [Dependency]
+allDependencies = liftA2 (++) dependencies devDependencies

waspc/src/Wasp/AppSpec/Valid.hs

@@ -174,13 +174,13 @@ validateAuthUserEntityHasCorrectFieldsIfUsernameAndPasswordAuthIsUsed spec = cas
       usernameAttributeValidationErrors
         | isFieldUnique "username" userEntity == Just True = []
         | otherwise =
-            [ GenericValidationError $
-                "The field 'username' on entity '"
-                  ++ userEntityName
-                  ++ "' (referenced by "
-                  ++ authUserEntityPath
-                  ++ ") must be marked with the '@unique' attribute."
-            ]
+          [ GenericValidationError $
+              "The field 'username' on entity '"
+                ++ userEntityName
+                ++ "' (referenced by "
+                ++ authUserEntityPath
+                ++ ") must be marked with the '@unique' attribute."
+          ]
       userEntityFields = Entity.getFields userEntity
       authUserEntityPath = "app.auth.userEntity"
       (userEntityName, userEntity) = AS.resolveRef spec (Auth.userEntity auth)
@@ -370,7 +370,7 @@ validateWebAppBaseDir :: AppSpec -> [ValidationError]
 validateWebAppBaseDir spec = case maybeBaseDir of
   Just baseDir
     | not (startsWithSlash baseDir) ->
-        [GenericValidationError "The app.client.baseDir should start with a slash e.g. \"/test\""]
+      [GenericValidationError "The app.client.baseDir should start with a slash e.g. \"/test\""]
   _anyOtherCase -> []
   where
     maybeBaseDir = Client.baseDir =<< AS.App.client (snd $ getApp spec)

waspc/src/Wasp/Generator/Job.hs

@@ -28,4 +28,4 @@ data JobMessageData
 
 data JobOutputType = Stdout | Stderr deriving (Show, Eq)
 
-data JobType = WebApp | Server | Db deriving (Show, Eq, Ord, Bounded, Enum)
+data JobType = WebApp | Server | Db | Wasp deriving (Show, Eq, Ord, Bounded, Enum)

waspc/src/Wasp/Generator/Job/IO/PrefixedWriter.hs

@@ -172,8 +172,8 @@ makeJobMessagePrefix jobMsg =
   (T.pack . buildPrefix . concat)
     [ [("[", jobStyles)],
       [(jobName, jobStyles)],
-      styledFlags,
-      [("]", jobStyles)]
+      [("]", jobStyles)],
+      styledFlags
     ]
   where
     buildPrefix :: [StyledText] -> String
@@ -187,9 +187,10 @@ makeJobMessagePrefix jobMsg =
         minPrefixLength = 10
 
     (jobName, jobStyles) = case J._jobType jobMsg of
+      J.Wasp -> (" Wasp ", [Term.Yellow])
       J.Server -> ("Server", [Term.Magenta])
       J.WebApp -> ("Client", [Term.Cyan])
-      J.Db -> ("Db", [Term.Blue])
+      J.Db -> ("  Db  ", [Term.Blue])
 
     styledFlags :: [StyledText]
     styledFlags =

waspc/src/Wasp/Generator/NpmDependencies.hs

@@ -19,13 +19,12 @@ where
 import Data.Aeson
 import Data.List (intercalate, sort)
 import qualified Data.Map as Map
-import Data.Maybe (fromMaybe)
 import qualified Data.Maybe as Maybe
 import GHC.Generics
 import Wasp.AppSpec (AppSpec)
-import qualified Wasp.AppSpec.App as AS.App
+import qualified Wasp.AppSpec as AS
 import qualified Wasp.AppSpec.App.Dependency as D
-import qualified Wasp.AppSpec.Valid as ASV
+import qualified Wasp.AppSpec.PackageJson as AS.PackageJson
 import Wasp.Generator.Monad (Generator, GeneratorError (..), logAndThrowGeneratorError)
 
 data NpmDepsForFullStack = NpmDepsForFullStack
@@ -108,9 +107,10 @@ buildNpmDepsForFullStack spec forServer forWebApp =
 getUserNpmDepsForPackage :: AppSpec -> NpmDepsForUser
 getUserNpmDepsForPackage spec =
   NpmDepsForUser
-    { userDependencies = fromMaybe [] $ AS.App.dependencies $ snd $ ASV.getApp spec,
+    { -- todo(filip): what if package.json has no dependencies field?
+      userDependencies = AS.PackageJson.dependencies $ AS.packageJson spec,
       -- Should we allow user devDependencies? https://github.com/wasp-lang/wasp/issues/456
-      userDevDependencies = []
+      userDevDependencies = AS.PackageJson.devDependencies $ AS.packageJson spec
     }
 
 conflictErrorToMessage :: DependencyConflictError -> String
@@ -141,9 +141,11 @@ combineNpmDepsForPackage npmDepsForWasp npmDepsForUser =
   if null conflictErrors && null devConflictErrors
     then
       Right $
+        -- todo(filip): check whether dependency updates and npm install work properly
+        -- todo(filip): reconsider whether we want to change the {sever,web-app}/package.json dynamically
         NpmDepsForPackage
-          { dependencies = waspDependencies npmDepsForWasp ++ remainingUserDeps,
-            devDependencies = waspDevDependencies npmDepsForWasp ++ remainingUserDevDeps
+          { dependencies = Map.elems remainingWapsDeps,
+            devDependencies = Map.elems remainingWaspDevDeps
           }
     else
       Left $
@@ -159,8 +161,8 @@ combineNpmDepsForPackage npmDepsForWasp npmDepsForUser =
     allWaspDepsByName = waspDepsByName `Map.union` waspDevDepsByName
     conflictErrors = determineConflictErrors allWaspDepsByName userDepsByName
     devConflictErrors = determineConflictErrors allWaspDepsByName userDevDepsByName
-    remainingUserDeps = getRemainingUserDeps allWaspDepsByName userDepsByName
-    remainingUserDevDeps = getRemainingUserDeps allWaspDepsByName userDevDepsByName
+    remainingWapsDeps = allWaspDepsByName `Map.difference` userDepsByName
+    remainingWaspDevDeps = allWaspDepsByName `Map.difference` userDevDepsByName
 
 type DepsByName = Map.Map String D.Dependency
 
@@ -179,12 +181,6 @@ determineConflictErrors waspDepsByName userDepsByName =
         then Just $ DependencyConflictError waspDep userDep
         else Nothing
 
--- Given a map of wasp dependencies and a map of user dependencies, construct a
--- a list of user dependencies that remain once any overlapping wasp dependencies
--- have been removed. This assumes conflict detection was already passed.
-getRemainingUserDeps :: DepsByName -> DepsByName -> [D.Dependency]
-getRemainingUserDeps waspDepsByName userDepsByName = Map.elems $ userDepsByName `Map.difference` waspDepsByName
-
 -- Construct a map of dependency keyed by dependency name.
 makeDepsByName :: [D.Dependency] -> DepsByName
 makeDepsByName = Map.fromList . fmap (\d -> (D.name d, d))

waspc/src/Wasp/Generator/NpmInstall.hs

@@ -4,26 +4,31 @@ module Wasp.Generator.NpmInstall
   )
 where
 
-import Control.Concurrent (Chan, newChan, readChan)
+import Control.Concurrent (Chan, newChan, readChan, threadDelay, writeChan)
 import Control.Concurrent.Async (concurrently)
 import Control.Monad (when)
 import Control.Monad.IO.Class (liftIO)
 import qualified Data.Aeson as Aeson
 import qualified Data.ByteString.Lazy as B
+import qualified Data.Text as T
 import StrongPath (Abs, Dir, File', Path', Rel, relfile, (</>))
 import qualified StrongPath as SP
 import System.Directory (doesFileExist, removeFile)
 import System.Exit (ExitCode (..))
+import UnliftIO (race)
 import Wasp.AppSpec (AppSpec)
 import Wasp.Generator.Common (ProjectRootDir)
+import Wasp.Generator.Job (Job, JobMessage, JobType)
 import qualified Wasp.Generator.Job as J
 import Wasp.Generator.Job.IO.PrefixedWriter (PrefixedWriter, printJobMessagePrefixed, runPrefixedWriter)
 import Wasp.Generator.Monad (GeneratorError (..), GeneratorWarning (..))
 import qualified Wasp.Generator.NpmDependencies as N
+import qualified Wasp.Generator.SdkGenerator as SdkGenerator
 import Wasp.Generator.ServerGenerator as SG
 import qualified Wasp.Generator.ServerGenerator.Setup as ServerSetup
 import Wasp.Generator.WebAppGenerator as WG
 import qualified Wasp.Generator.WebAppGenerator.Setup as WebAppSetup
+import Wasp.Project.Common (WaspProjectDir)
 
 -- | Figure out if npm install is needed.
 --
@@ -59,14 +64,18 @@ isNpmInstallNeeded spec dstDir = do
 
 -- Run npm install for desired AppSpec dependencies, recording what we installed
 -- Installation may fail, in which the installation record is removed.
-installNpmDependenciesWithInstallRecord :: N.NpmDepsForFullStack -> Path' Abs (Dir ProjectRootDir) -> IO ([GeneratorWarning], [GeneratorError])
-installNpmDependenciesWithInstallRecord npmDepsForFullStack dstDir = do
+installNpmDependenciesWithInstallRecord ::
+  N.NpmDepsForFullStack ->
+  Path' Abs (Dir WaspProjectDir) ->
+  Path' Abs (Dir ProjectRootDir) ->
+  IO ([GeneratorWarning], [GeneratorError])
+installNpmDependenciesWithInstallRecord npmDepsForFullStack waspProjectDir dstDir = do
   -- in case anything fails during installation that would leave node modules in
   -- a broken state, we remove the file before we start npm install
   fileExists <- doesFileExist dependenciesInstalledFp
   when fileExists $ removeFile dependenciesInstalledFp
   -- now actually do the installation
-  npmInstallResult <- installNpmDependencies dstDir
+  npmInstallResult <- installNpmDependencies waspProjectDir dstDir
   case npmInstallResult of
     Left npmInstallError -> do
       return ([], [GenericGeneratorError $ "npm install failed: " ++ npmInstallError])
@@ -100,34 +109,81 @@ loadInstalledFullStackNpmDependencies dstDir = do
       return (Aeson.decode fileContents :: Maybe N.NpmDepsForFullStack)
     else return Nothing
 
+reportInstallationProgress :: Chan JobMessage -> JobType -> IO ()
+reportInstallationProgress chan jobType = reportPeriodically allPossibleMessages
+  where
+    reportPeriodically messages = do
+      threadDelay $ secToMicroSec 5
+      writeChan chan $ J.JobMessage {J._data = J.JobOutput (T.append (head messages) "\n") J.Stdout, J._jobType = jobType}
+      threadDelay $ secToMicroSec 5
+      reportPeriodically (if hasLessThan2Elems messages then messages else drop 1 messages)
+    secToMicroSec = (* 1000000)
+    hasLessThan2Elems = null . drop 1
+    allPossibleMessages =
+      [ "Still installing npm dependencies!",
+        "Installation going great - we'll get there soon!",
+        "The installation is taking a while, but we'll get there!",
+        "Yup, still not done installing.",
+        "We're getting closer and closer, everything will be installed soon!",
+        "Still waiting for the installation to finish? You should! We got too far to give up now!",
+        "You've been waiting so patiently, just wait a little longer (for the installation to finish)..."
+      ]
+
+installNpmDependenciesAndReport :: Job -> Chan JobMessage -> JobType -> IO ExitCode
+installNpmDependenciesAndReport installF chan jobType = do
+  writeChan chan $ J.JobMessage {J._data = J.JobOutput "Starting npm install\n" J.Stdout, J._jobType = jobType}
+  result <- installF chan `race` reportInstallationProgress chan jobType
+  case result of
+    Left exitCode -> return exitCode
+    Right _ -> error "This should be impossible"
+
 -- Run the individual `npm install` commands for both server and webapp projects
 -- It runs these concurrently, collects the output produced by these commands
 -- to pass them along to IO with a prefix
-installNpmDependencies :: Path' Abs (Dir ProjectRootDir) -> IO (Either String ())
-installNpmDependencies projectDir = do
-  chan <- newChan
-  let runSetupJobs =
-        ServerSetup.installNpmDependencies projectDir chan
-          `concurrently` WebAppSetup.installNpmDependencies projectDir chan
-  (_, result) <- concurrently (handleJobMessages chan) runSetupJobs
-  case result of
-    (ExitSuccess, ExitSuccess) -> return $ Right ()
-    exitCodes -> return $ Left $ setupFailedMessage exitCodes
+installNpmDependencies :: Path' Abs (Dir WaspProjectDir) -> Path' Abs (Dir ProjectRootDir) -> IO (Either String ())
+installNpmDependencies projectDir dstDir = do
+  messagesChan <- newChan
+  (_, exitCode) <-
+    concurrently
+      (handleProjectInstallMessage messagesChan)
+      (installNpmDependenciesAndReport (SdkGenerator.installNpmDependencies projectDir) messagesChan J.Wasp)
+  case exitCode of
+    ExitFailure code -> return $ Left $ "Project setup failed with exit code " ++ show code ++ "."
+    _ -> do
+      let handleMessagesJob = handleJobMessages messagesChan
+      let runSetupJobs =
+            concurrently
+              (installNpmDependenciesAndReport (ServerSetup.installNpmDependencies dstDir) messagesChan J.Server)
+              (installNpmDependenciesAndReport (WebAppSetup.installNpmDependencies dstDir) messagesChan J.WebApp)
+      (_, results) <- concurrently handleMessagesJob runSetupJobs
+      case results of
+        (ExitSuccess, ExitSuccess) -> return $ Right ()
+        exitCodes -> return $ Left $ setupFailedMessage exitCodes
   where
-    handleJobMessages = runPrefixedWriter . go (False, False)
+    handleProjectInstallMessage :: Chan J.JobMessage -> IO ()
+    handleProjectInstallMessage = runPrefixedWriter . processMessages
+      where
+        processMessages :: Chan J.JobMessage -> PrefixedWriter ()
+        processMessages chan = do
+          jobMsg <- liftIO $ readChan chan
+          case J._data jobMsg of
+            J.JobOutput {} -> printJobMessagePrefixed jobMsg >> processMessages chan
+            J.JobExit {} -> return ()
+    handleJobMessages = runPrefixedWriter . processMessages (False, False)
       where
-        go :: (Bool, Bool) -> Chan J.JobMessage -> PrefixedWriter ()
-        go (True, True) _ = return ()
-        go (isWebAppDone, isServerDone) chan = do
+        processMessages :: (Bool, Bool) -> Chan J.JobMessage -> PrefixedWriter ()
+        processMessages (True, True) _ = return ()
+        processMessages (isWebAppDone, isServerDone) chan = do
           jobMsg <- liftIO $ readChan chan
           case J._data jobMsg of
             J.JobOutput {} ->
               printJobMessagePrefixed jobMsg
-                >> go (isWebAppDone, isServerDone) chan
+                >> processMessages (isWebAppDone, isServerDone) chan
             J.JobExit {} -> case J._jobType jobMsg of
-              J.WebApp -> go (True, isServerDone) chan
-              J.Server -> go (isWebAppDone, True) chan
+              J.WebApp -> processMessages (True, isServerDone) chan
+              J.Server -> processMessages (isWebAppDone, True) chan
               J.Db -> error "This should never happen. No db job should be active."
+              J.Wasp -> error "This should never happen. No db job should be active."
 
     setupFailedMessage (serverExitCode, webAppExitCode) =
       let serverErrorMessage = case serverExitCode of