diff --git a/Makefile b/Makefile
index 960922fc8d..87344d1338 100644
--- a/Makefile
+++ b/Makefile
@@ -26,8 +26,8 @@ clean-build:
 	rm -fr dist/
 	rm -rf .kalite_dist_tmp
 	rm -fr .eggs/
-	rm -fr dist-packages/
-	rm -fr dist-packages-temp/
+	rm -fr kalite/packages/dist/*
+	rm -fr .pip-temp/
 	rm -fr kalite/database/templates
 	rm -fr kalite/static-libraries/docs
 	find . -name '*.egg-info' -exec rm -fr {} +
@@ -106,9 +106,7 @@ sdist: clean docs assets
 	python setup.py sdist --formats=$(format) --static
 
 dist: clean docs assets
-	# python setup.py sdist --formats=$(format)
 	python setup.py bdist_wheel
-	# python setup.py sdist --formats=$(format) --static
 	python setup.py bdist_wheel --static  # --no-clean
 	ls -l dist
 
diff --git a/kalite/packages/dist/__init__.py b/kalite/packages/dist/__init__.py
deleted file mode 100644
index 8b13789179..0000000000
--- a/kalite/packages/dist/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/requirements.txt b/requirements.txt
index 02924647cb..e3255451ba 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,7 +5,8 @@
 # https://github.com/django/django/blob/master/tests/staticfiles_tests/apps/test/static/test/%E2%8A%97.txt
 # ...to explain: Some of the below packages depend on django, if we don't
 # specify a version here, we'll get the latest.
-django<1.6
+# See also: https://github.com/learningequality/ka-lite/issues/5419
+django==1.5.12
 
 docopt>=0.6,<0.7
 South==1.0.2
diff --git a/setup.py b/setup.py
index 50129d51eb..e3f358d0ec 100644
--- a/setup.py
+++ b/setup.py
@@ -239,6 +239,7 @@ def install_distributions(distributions):
         opts.build_dir = STATIC_DIST_PACKAGES_TEMP
         opts.download_cache = STATIC_DIST_PACKAGES_DOWNLOAD_CACHE
         opts.isolated = True
+        opts.ignore_installed = True
         opts.compile = False
         opts.ignore_dependencies = False
         # This is deprecated and will disappear in Pip 10
@@ -247,14 +248,18 @@ def install_distributions(distributions):
         # opts.no_binary = ':all:'  # Do not use any binary files (whl)
         opts.no_clean = NO_CLEAN
         command.run(opts, distributions)
-        # requirement_set.source_dir = STATIC_DIST_PACKAGES_TEMP
-        # requirement_set.install(opts)
 
     # Install requirements into kalite/packages/dist
     if DIST_BUILDING_COMMAND:
         install_distributions(RAW_REQUIREMENTS)
-        # Now remove Django because it's bundled
-        shutil.rmtree(os.path.join(STATIC_DIST_PACKAGES, "django"))
+
+        # Now remove Django because it's bundled. It gets installed as an egg
+        # so unfortunately, the path is a bit dependent on the specific
+        # version installed (we pinned it for reliability in requirements.txt)
+        shutil.rmtree(
+            os.path.join(STATIC_DIST_PACKAGES, "Django-1.5.12-py2.7.egg-info"),
+            ignore_errors=False
+        )
 
 # It's not a build command with --static or it's not a build command at all
 else: