Modularise makefiles (#1724)

The main issues addressed by this PR are:

1. Improve the user experience when building so that dependencies and rebuilding are handled automatically. No swapping between project directory and Sming directory.
2. Make parallel building work without issue
3. Improve maintainability and portability by moving the build logic out of the main makefiles and into the related Components.
4. Build out of source tree
5. Effectively document the build system
6. Only pull in external sources required for the project being built. There are quite a number of submodules, etc. and that is only likely to grow.
7. As far as possible, keep Components independent and avoid 'upward dependencies'. For example, a driver Component shouldn't use any framework definitions like `String`, etc.

All of this should have no bearing on which build system is actually used (make / CMake).

Further details can be found in the `building.md` file which has been updated as part of this PR.

.appveyor/build.cmd

@@ -1,37 +1,40 @@
 REM Windows build script
 
-SET SMING_HOME=%APPVEYOR_BUILD_FOLDER%\Sming
+set SMING_HOME=%APPVEYOR_BUILD_FOLDER%\Sming
 
-IF "%SMING_ARCH%" == "Esp8266" SET ESP_HOME=c:\Espressif
+if "%SMING_ARCH%" == "Esp8266" set ESP_HOME=c:\Espressif
 
 cd %SMING_HOME%
 gcc -v
 
+set MAKE_PARALLEL=make -j2
+
+REM Move samples and tests into directory outside of the Sming repo.
+set SMING_PROJECTS_DIR=%APPVEYOR_BUILD_FOLDER%\..
+move ../samples %SMING_PROJECTS_DIR%
+move ../tests %SMING_PROJECTS_DIR%
+
+REM This will build the Basic_Blink application and most of the framework Components
+cd %SMING_PROJECTS_DIR%/samples/Basic_Blink
 make help
 make list-config
-
-REM Compile the tools first
-make tools V=1 || goto :error
+%MAKE_PARALLEL% || goto :error
 
 cd %SMING_HOME%
-make STRICT=1 || goto :error
-
 
 if "%SMING_ARCH%" == "Host" (
 
 	REM Build a couple of basic applications
-	make Basic_Serial || goto :error
-	make Basic_ProgMem || goto :error
-
+	%MAKE_PARALLEL% Basic_Serial Basic_ProgMem STRICT=1 V=1 || goto :error
+
 	REM Run basic tests
-	cd %SMING_HOME%\..\tests\HostTests
-	make flash || goto :error
+	set SMING_TARGET_OPTIONS="--flashfile=$(FLASH_BIN) --flashsize=$(SPI_SIZE)"
+	%MAKE_PARALLEL% tests || goto :error
 
-) ELSE (
+) else (
 
-	make Basic_Blink V=1 || goto :error
-	make Basic_Ssl || goto :error
-	make Basic_SmartConfig || goto :error
+	%MAKE_PARALLEL% Basic_Ssl || goto :error
+	%MAKE_PARALLEL% Basic_SmartConfig || goto :error
 
 )
 

.gitignore

@@ -1,6 +1,5 @@
 # generated binaries
 out
-Sming/Arch/**/Compiler/lib/**/lib*.a
 
 # generated documentation
 docs/api
@@ -17,8 +16,6 @@ GPATH
 
 # Microsoft Visual SourceSafe files
 vssver2.scc
-
-.metadata
-.submodule
-/Sming/Arch/Host/Components/lwip/Windows/src
-/Sming/Arch/Host/Components/lwip/Windows/tmp
+
+.metadata
+.submodule

.gitmodules

@@ -3,7 +3,7 @@
 	url = https://github.com/SmingHub/Sming.wiki.git
 	ignore = dirty
 [submodule "spiffs"]
-	path = Sming/Components/spiffs
+	path = Sming/Components/spiffs/spiffs
 	url = https://github.com/pellepl/spiffs.git
 	ignore = dirty
 [submodule "http-parser"]
@@ -28,7 +28,7 @@
 	url = https://github.com/raburton/rboot.git
 	ignore = dirty
 [submodule "ESP8266.new-pwm"]
-	path = Sming/Arch/Esp8266/Components/pwm/new-pwm
+	path = Sming/Arch/Esp8266/Components/pwm_open/new-pwm
 	url = https://github.com/StefanBruens/ESP8266_new_pwm.git
 	ignore = dirty
 [submodule "ESP8266.axtls-8266"]
@@ -49,11 +49,11 @@
 	ignore = dirty
 
 [submodule "ESP8266.Sdk"]
-	path = Sming/Arch/Esp8266/Components/Sdk/ESP8266_NONOS_SDK
+	path = Sming/Arch/Esp8266/Components/esp8266/ESP8266_NONOS_SDK
 	url = https://github.com/espressif/ESP8266_NONOS_SDK.git
 	ignore = dirty
 [submodule "ESP8266.esptool2"]
-	path = Sming/Arch/Esp8266/Tools/esptool2
+	path = Sming/Arch/Esp8266/Components/rboot/esptool2
 	url = https://github.com/raburton/esptool2.git
 	ignore = dirty
 

.travis/build.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -ex # exit with nonzero exit code if anything fails
 
+# Build times benefit from parallel building
+export MAKE_PARALLEL="make -j3"
+
 env
 unset SPIFFY
 unset ESPTOOL2
@@ -25,24 +28,32 @@ fi
 # Full compile checks please
 export STRICT=1
 
-# Diagnostic info
 cd $SMING_HOME
+
+# Move samples and tests into directory outside of the Sming repo.
+export SMING_PROJECTS_DIR=$HOME/projects
+mkdir $SMING_PROJECTS_DIR
+mv ../samples $SMING_PROJECTS_DIR
+mv ../tests $SMING_PROJECTS_DIR
+
+# Diagnostic info
+cd $SMING_PROJECTS_DIR/samples/Basic_Blink
 make help
 make list-config
 
-# Build the framework
-make
+# This will build the Basic_Blink application and most of the framework Components
+$MAKE_PARALLEL
+
+cd $SMING_HOME
 
 if [ "$TRAVIS_BUILD_STAGE_NAME" == "Test" ]; then
-	make Basic_Blink Basic_DateTime Basic_Delegates Basic_Interrupts Basic_ProgMem Basic_Serial Basic_Servo LiveDebug DEBUG_VERBOSE_LEVEL=3
-	cd ../tests/HostTests
-	make flash
+	$MAKE_PARALLEL Basic_DateTime Basic_Delegates Basic_Interrupts Basic_ProgMem Basic_Serial Basic_Servo LiveDebug DEBUG_VERBOSE_LEVEL=3
+	# Build and run tests
+	export SMING_TARGET_OPTIONS='--flashfile=$(FLASH_BIN) --flashsize=$(SPI_SIZE)'
+	$MAKE_PARALLEL tests
 else
-	make samples
+	$MAKE_PARALLEL samples
 	make clean samples-clean
-	make ENABLE_CUSTOM_HEAP=1 STRICT=1
-	make Basic_Blink ENABLE_CUSTOM_HEAP=1 DEBUG_VERBOSE_LEVEL=3
-
-	make dist-clean
-	make HttpServer_ConfigNetwork ENABLE_CUSTOM_LWIP=2 STRICT=1
+	$MAKE_PARALLEL Basic_Blink ENABLE_CUSTOM_HEAP=1 DEBUG_VERBOSE_LEVEL=3
+	$MAKE_PARALLEL HttpServer_ConfigNetwork ENABLE_CUSTOM_LWIP=2 STRICT=1
 fi

.travis/tools/clang/format-pr.sh

@@ -10,7 +10,8 @@ $CLANG_FORMAT --version
 cd $SMING_HOME/..
 
 # Get list of modified files
-files_to_lint="$(git diff --name-only --diff-filter=ACMRTUXB $TRAVIS_COMMIT_RANGE | grep '^[^.]*[.]\(c\|cpp\|h\)$' | grep -e Sming/Core -e Sming/Platform -e Sming/Arch/.+/Core -e Sming/Arch/.+/Platform -e samples || true)"
+files_to_lint="$(git diff --name-only --diff-filter=ACMRTUXB $TRAVIS_COMMIT_RANGE | grep '^[^.]*[.]\(c\|cpp\|h\)$' | \
+	grep -e Sming/Core -e Sming/Platform -e Sming/Arch/.+/Core -e Sming/Arch/.+/Platform -e samples -e tests || true)"
 
 for f in $files_to_lint; do
 

Sming/Arch/Esp8266/Compiler/ld/common.ld

@@ -88,10 +88,6 @@ SECTIONS
 
     *libsmartconfig.a:(.literal .text .literal.* .text.*)
     *libstdc++.a:(.literal .text .literal.* .text.*)
-    *liblwip_open.a:(.literal .text .literal.* .text.*)
-    *liblwip_full.a:(.literal .text .literal.* .text.*)
-    *liblwip2.a:(.literal .text .literal.* .text.*)
-    *libaxtls.a:(.literal .text .literal.* .text.*)
     *libat.a:(.literal.* .text.*)
     *libcrypto.a:(.literal.* .text.*)
     *libespnow.a:(.literal.* .text.*)
@@ -110,12 +106,12 @@ SECTIONS
 
     *libm.a:(.literal .text .literal.* .text.*)
 
-     *(.rodata._ZTV*) /* C++ vtables */
+    *(.rodata._ZTV*) /* C++ vtables */
     *(.irom0.literal .irom.literal .irom.text.literal .irom0.text .irom.text .irom.debug.*)
-    */app.a:*(.literal .text .literal.* .text.* .stub .gnu.warning .gnu.linkonce.literal.* .gnu.linkonce.t.*.literal .gnu.linkonce.t.* .irom.debug.*)
-  *libsming.a:*(.literal .text .literal.* .text.* .stub .gnu.warning .gnu.linkonce.literal.* .gnu.linkonce.t.*.literal .gnu.linkonce.t.* .irom.debug.*)
-  *libsmingssl.a:*(.literal .text .literal.* .text.* .stub .gnu.warning .gnu.linkonce.literal.* .gnu.linkonce.t.*.literal .gnu.linkonce.t.* .irom.debug.*)
-  *libmqttc.a:*(.literal .text .literal.* .text.* .stub .gnu.warning .gnu.linkonce.literal.* .gnu.linkonce.t.*.literal .gnu.linkonce.t.* .irom.debug.*)
+
+    /* Generated libraries */
+    *liblwip2.a:(.literal .text .literal.* .text.*)
+    */clib-*.a:*(.literal .text .literal.* .text.* .stub .gnu.warning .gnu.linkonce.literal.* .gnu.linkonce.t.*.literal .gnu.linkonce.t.* .irom.debug.*)
 
 	/* __FUNCTION__ locals */
     *(.rodata._ZZ*__FUNCTION__)