diff --git a/CHANGELOG.md b/CHANGELOG.md
index 95fa043060..29b98b4b28 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -99,6 +99,9 @@ Canonical reference for changes, improvements, and bugfixes for Boundary.
   well as the error message)
   ([issue](https://github.com/hashicorp/boundary/issues/1305),
   [PR](https://github.com/hashicorp/boundary/pull/1384))
+* server: Fix panic on worker startup failure when the server was not also
+  configured as a controller
+  ([PR](https://github.com/hashicorp/boundary/pull/1432))
 
 ### New and Improved
 
diff --git a/internal/cmd/commands/server/server.go b/internal/cmd/commands/server/server.go
index 29cf554caa..8ea04e033b 100644
--- a/internal/cmd/commands/server/server.go
+++ b/internal/cmd/commands/server/server.go
@@ -463,6 +463,11 @@ func (c *Command) Run(args []string) int {
 	if c.Config.Worker != nil {
 		if err := c.StartWorker(); err != nil {
 			c.UI.Error(err.Error())
+			if c.controller != nil {
+				if err := c.controller.Shutdown(false); err != nil {
+					c.UI.Error(fmt.Errorf("Error with controller shutdown: %w", err).Error())
+				}
+			}
 			return base.CommandCliError
 		}
 	}