Generated index file containing reference info. and include README

* Use m2r instead of recommonmark as it provides `mdinclude` directive
* Rename `api-file` role to `source`, make relative to root directory

Sming/Arch/Esp8266/Components/axtls-8266/README.rst

@@ -2,7 +2,3 @@ AXTLS 8266
 ==========
 
 SSL support using the AXTLS library
-
-.. toctree::
-
-   axtls-8266/README

Sming/Arch/Esp8266/Components/custom_heap/README.rst

@@ -5,7 +5,3 @@ Provides an alternative heap memory manager using UMM Malloc.
 
 .. warning::
    Do not enable custom heap allocation and -mforce-l32 compiler flag at the same time.
-
-.. toctree::
-
-   umm_malloc/README

Sming/Arch/Esp8266/Components/esptool/README.rst

@@ -34,10 +34,3 @@ Options
 .. envvar:: COM_SPEED_ESPTOOL
 
    Baud rate to use for flashing device. Default is :envvar:`COM_SPEED`.
-
-Documentation
--------------
-
-.. toctree::
-
-   esptool/README

Sming/Arch/Esp8266/Components/lwip2/README.rst

@@ -2,7 +2,3 @@ Esp8266 LWIP Version 2
 ======================
 
 This Component implements the current Version 2 LWIP stack. Note that at present espconn\_* functions are not supported.
-
-.. toctree::
-
-   lwip2/README

Sming/Arch/Esp8266/Components/pwm_open/README.rst

@@ -2,7 +2,3 @@ Esp8266 Open PWM
 ================
 
 Implements a drop-in replacement for the PWM support provided in the Espressif SDK.
-
-.. toctree::
-
-   new-pwm/README

Sming/Arch/Esp8266/Components/rboot/README.rst

@@ -7,8 +7,3 @@ ease of use.
 This Component also includes esptool2 which rBoot uses to manipulate ROM
 images. Use ``$(ESPTOOL2)`` if you want to use it within your own
 projects.
-
-.. toctree::
-
-   rboot/README
-   esptool2/README

Sming/Arch/Esp8266/README.rst

@@ -11,5 +11,5 @@ Components
    :maxdepth: 1
    :titlesonly:
 
-   Components/sming-arch/README
-   Components/*/README
+   Components/sming-arch/index
+   Components/*/index

Sming/Arch/Host/Components/lwip/README.rst

@@ -13,8 +13,3 @@ Options:
       Standard build
    1
       Enable debugging output
-
-
-.. toctree::
-
-   lwip/README

Sming/Arch/Host/README.rst

@@ -224,8 +224,8 @@ Components
    :maxdepth: 1
    :titlesonly:
 
-   Components/sming-arch/README
-   Components/*/README
+   Components/sming-arch/index
+   Components/*/index
 
 
 todo

Sming/Components/spiffs/README.rst

@@ -2,7 +2,3 @@ SPIFFS for Sming
 ================
 
 This Component provides additional functionality to support SPIFFS running on both Esp8266 and Host architectures.
-
-.. toctree::
-
-   spiffs/README

Sming/Libraries/ArduinoJson5/README.rst

@@ -2,10 +2,3 @@ ArduinoJson Version 5
 =====================
 
 Provided to support existing applications. New projects should use :library:`ArduinoJson6`
-
-Arduino JSON
-------------
-
-.. toctree::
-
-   ArduinoJson/README

Sming/README.rst

@@ -93,5 +93,6 @@ Components
 
  .. toctree::
    :glob:
+   :maxdepth: 1
 
-   Components/*/README
+   Components/*/index

docs/Makefile

@@ -4,6 +4,8 @@
 # Don't bother with implicit checks
 .SUFFIXES:
 
+SHELL = /bin/bash
+
 SMINGDIR := ..
 override SMING_HOME := $(SMINGDIR)/Sming
 
@@ -13,6 +15,9 @@ SPHINXBUILD		?= sphinx-build
 SOURCEDIR		= source
 BUILDDIR		= build
 
+# Documentation is copied into this directory so it gets pulled into build
+SOURCE_INCDIR	:= $(SOURCEDIR)/_inc
+
 V ?= $(VERBOSE)
 ifeq ("$(V)","1")
 Q			:=
@@ -26,6 +31,24 @@ define Sphinx
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 endef
 
+# Display variable and list values, e.g. $(call PrintVariable,LIBS)
+# $1 -> Name of variable containing values
+define PrintVariable
+	$(info $1)
+	$(foreach item,$($1),$(info - $(item)))
+endef
+
+# $1 -> Directories to look for README and image files
+define FindSourceFiles
+$(foreach d,$1,$(wildcard $d/*.rst $d/README.md $d/*.svg $d/*.png $d/*.jpg))
+endef
+
+# Translate a source path into the corresponding INCDIR location
+# $1 -> Path(s) to source file in code tree
+define GetIncPath
+$(patsubst $(SMINGDIR)/%,$(SOURCE_INCDIR)/%,$1)
+endef
+
 # Put it first so that "make" without argument is like "make help".
 .PHONY: help
 help:
@@ -42,98 +65,185 @@ latexpdf: latex
 	@echo "Running LaTeX files through pdflatex..."
 	$(Q) $(MAKE) -C $(BUILDDIR)/latex all-pdf
 
-# List of documentation files to be included from source tree
-SOURCE_FILES		:= CONTRIBUTING.md
-
 # List immediate sub-directories for a list of root directories
 # Results are sorted and without trailing path separator
 # $1 -> Root paths
 define ListSubDirs
 $(sort $(foreach d,$(dir $(wildcard $1/*/.)),$(d:/=)))
 endef
 
-SRC_INCDIR			:= $(SOURCEDIR)/_inc
+# Non-component locations to copy docs from
+SOURCE_DIRS := \
+	$(SMINGDIR)/docs \
+	$(call ListSubDirs,$(SMING_HOME)/Arch)
+
+# List of documentation files to be included from source tree
+SOURCE_FILES := \
+	$(SMINGDIR)/CONTRIBUTING.md \
+	$(call FindSourceFiles,$(SOURCE_DIRS))
 
 # Each sample, library and Component MUST have a README.rst or README.md file
 # This is a list of directories where such files may exist
 # If a component.mk file exists, then it is parsed and any dependencies handled
-SOURCE_DIRS := \
-	$(SMING_HOME)/Arch \
-	$(SMING_HOME)/Arch/*/Components \
+COMPONENT_ROOT_DIRS := \
+	$(wildcard $(SMING_HOME)/Arch/*/Components) \
 	$(SMING_HOME)/Components \
 	$(SMING_HOME)/Libraries \
 	$(SMINGDIR)/samples
 
-SOURCE_DIRS := $(sort $(SMINGDIR)/docs $(SMING_HOME) \
-	$(foreach d,$(wildcard $(SOURCE_DIRS)),$(call ListSubDirs,$d)))
+# All directories where a README file should/must exist, and where a component.mk file could exist
+COMPONENT_DIRS := $(SMING_HOME) $(foreach d,$(COMPONENT_ROOT_DIRS),$(call ListSubDirs,$d))
+
+# Fetch a list of URLs once rather than invoking shell each time we need a lookup
+SUBMODULE_URLS := $(shell awk '/path = .*/ { path = $$3 } /url = .*/ { printf "%s=%s ", path, $$3 } ' $(SMINGDIR)/.gitmodules)
+
+# Lookup the URL for a submodule given its path
+# awk '/^path = .*/ { path = $3 } /^url = .*/ { printf "%s = %s", path, $3 }' $(SMINGDIR)/.gitmodules
+# $1 -> path
+define GetSubmoduleURL
+$(subst $1=,,$(filter $1=%,$(SUBMODULE_URLS)))
+endef
+
+# Generate index.rst file for sample, library or Component
+# This provides a template to automate as much as possible
+# $1 -> name
+# $2 -> Path
+# $3 -> Source path relative to working root directory (SMINGDIR)
+define GenIndex
+$(if $(filter $2/README.rst,$(SOURCE_FILES)),
+.. include:: README.rst,
+.. mdinclude:: README.md
+)
+
+References
+------------
+* :source:`Source Code <$3>`
+$(if $(findstring $3=,$(SUBMODULE_URLS)),
+* This is a submodule: `GIT repository <$(call GetSubmoduleURL,$3)>`__ for details.
+)
+$(foreach d,$(sort $(COMPONENT_DEPEND_DIRS)),
+* :doc:`$d/index` Component
+)
+$(foreach l,$(sort $(ARDUINO_LIBRARIES)),
+* :library:`$l` Library
+)
+
+$(if $(COMPONENT_ENVVARS),
+Environment Variables
+---------------------
+$(foreach v,$(COMPONENT_ENVVARS),
+* :envvar:`$v`
+))
+
+$(foreach m,$(COMPONENT_SUBMODULES),
+Submodule: `$m <$(call GetSubmoduleURL,$3/$m)>`__
+-----------------------------------------------------------------------------------------------------
+
+.. toctree::
+
+   $m/README
+
+)
+endef
+
+
+# Used to escape above generated text
+define NewLine
+
+
+endef
+
+# Additional library dependencies
+-include $(SMING_HOME)/Libraries/components.mk
 
 # This macro sets the default component variables before including the (optional) component.mk file.
-# $1 -> Component path
+# $1 -> name
+# $2 -> path
 define ParseComponent
-$(if $V,$(info -- Parsing $1))
+$(if $V,$(info -- Parsing $2))
+COMPONENT_SUBMODULES	:=
+COMPONENT_DOCFILES		:=
 # Process any component.mk file (optional)
-ifneq (,$(wildcard $1/component.mk))
 COMPONENT_RULE			:= __no_build__
-COMPONENT_NAME			:= $(notdir $1)
-COMPONENT_PATH			:= $1
+COMPONENT_NAME			:= $1
+COMPONENT_PATH			:= $2
 COMPONENT_VARS			:=
 CONFIG_VARS				:=
 RELINK_VARS				:=
 DEBUG_VARS				:=
-COMPONENT_SUBMODULES	:=
-COMPONENT_DOCFILES		:=
 COMPONENT_DEPENDS		:=
+ARDUINO_LIBRARIES		:=
 ENABLE_ESPCONN			:=
 ENABLE_CUSTOM_LWIP		:=
-include $1/component.mk
-CMP_$1_DEPENDS			:= $$(COMPONENT_DEPENDS)
-CMP_$1_VARS				:= $$(sort $$(COMPONENT_VARS))
-SOURCE_DIRS				+= $$(addprefix $$(COMPONENT_PATH)/,$$(COMPONENT_SUBMODULES))
-SOURCE_FILES			+= $$(addprefix $$(COMPONENT_PATH)/,$$(COMPONENT_DOCFILES))
-endif
-endef # ParseComponent
+-include $2/component.mk
+CMP_$2_VARS				:= $$(sort $$(COMPONENT_VARS))
+CMP_$2_DIRS := \
+	$$(COMPONENT_PATH) \
+	$$(addprefix $$(COMPONENT_PATH)/,$$(COMPONENT_SUBMODULES))
+CMP_$2_FILES := \
+	$$(call FindSourceFiles,$$(CMP_$2_DIRS)) \
+	$$(wildcard $$(addprefix $$(COMPONENT_PATH)/,$$(COMPONENT_DOCFILES)))
+SOURCE_FILES			+= $$(CMP_$2_FILES)
 
-$(foreach d,$(SOURCE_DIRS),$(eval $(call ParseComponent,$d)))
-
-SOURCE_FILES			+= $(foreach d,$(SOURCE_DIRS),$(wildcard $d/*.rst $d/README.md $d/*.svg $d/*.png $d/*.jpg))
-SOURCE_FILES			:= $(patsubst $(SMINGDIR)/%,%,$(SOURCE_FILES))
-SOURCE_DIRS				:= $(patsubst $(SMINGDIR)/%,%,$(SOURCE_DIRS))
+#
+COMPONENT_ENVVARS		:= $$(sort $$(COMPONENT_VARS) $$(CONFIG_VARS) $$(RELINK_VARS) $$(DEBUG_VARS))
 
-SOURCE_FILE_DIRS		:= $(dir $(SOURCE_FILES))
-MISSING_README_DIRS		:= $(filter-out $(SOURCE_FILE_DIRS:/=),$(SOURCE_DIRS))
+CMP_$2_DEPENDS := $$(COMPONENT_DEPENDS)
 
-SOURCE_DIRS				:= $(patsubst %,$(SRC_INCDIR)/%,$(SOURCE_DIRS))
+# Resolve dependencies
+#
+# If this is an Arch component, then match this Arch only, or **main components** (e.g. Host/heap -> malloc_count)
+#
+ifneq (,$(findstring /Arch/,$2))
+COMPONENT_DEPEND_DIRS := $$(addprefix $(patsubst $(SMINGDIR)/%,/_inc/%,$(dir $2)),$$(COMPONENT_DEPENDS))
+else
+COMPONENT_DEPEND_DIRS := \
+	$(if $(findstring /samples/,$2),/_inc/Sming) \
+	$$(foreach d,$$(patsubst $(SMINGDIR)/%,/_inc/%,$$(foreach c,$$(COMPONENT_DEPENDS),$$(filter %/$$c,$(COMPONENT_DIRS)))),$$d)
+endif
 
-# Display variable and list values, e.g. $(call PrintVariable,LIBS)
-# $1 -> Name of variable containing values
-define PrintVariable
-	$(info $1)
-	$(foreach item,$($1),$(info - $(item)))
-endef
+CMP_$2_INDEX			:= $$(subst $$(NewLine),\n,$$(call GenIndex,$1,$2,$(patsubst $(SMINGDIR)/%,%,$2)))
 
+$$(call GetIncPath,$$(COMPONENT_PATH))/index.rst: $$(call GetIncPath,$$(CMP_$2_FILES))
 ifeq ($V,1)
-$(eval $(call PrintVariable,SOURCE_DIRS))
-$(eval $(call PrintVariable,SOURCE_FILES))
+	$$(info Creating $$@...)
 endif
+	@echo -e '$$(CMP_$2_INDEX)' > $$@
+
+endef # ParseComponent
+
+$(foreach d,$(COMPONENT_DIRS),$(eval $(call ParseComponent,$(notdir $d),$d)))
+
+SOURCE_FILE_DIRS		:= $(sort $(dir $(SOURCE_FILES)))
+MISSING_README_DIRS		:= $(filter-out $(SOURCE_FILE_DIRS:/=),$(SOURCE_DIRS))
 
 .PHONY: clean
 clean:
 	$(call Sphinx)
-	$(Q) rm -rf $(SRC_INCDIR) api
+	$(Q) rm -rf $(SOURCE_INCDIR) api
+
+SOURCE_INCDIRS			:= $(call GetIncPath,$(SOURCE_FILE_DIRS))
+SOURCE_INCFILES			:= $(call GetIncPath,$(SOURCE_FILES)) $(addsuffix /index.rst,$(call GetIncPath,$(COMPONENT_DIRS)))
 
 .PHONY: setup
-setup: $(addprefix $(SRC_INCDIR)/,$(SOURCE_FILES))
-	$(call PrintVariable,MISSING_README_DIRS)
+setup: checkdirs $(SOURCE_INCFILES)
+ifeq ($V,1)
+	$(call PrintVariable,SOURCE_DIRS)
+	$(call PrintVariable,SOURCE_FILES)
+endif
+	$(if $(MISSING_README_DIRS),$(call PrintVariable,MISSING_README_DIRS))
 
-# $1 -> source file (in incdir)
+# $1 -> document file in Sming code tree
 define CopyFileTarget
-$(SRC_INCDIR)/$1: $(SMINGDIR)/$1 | $(patsubst %/,%,$(dir $(SRC_INCDIR)/$1))
+$(call GetIncPath,$1): $1
 	$(Q) cp $$< $$@
 endef
 
 $(foreach f,$(SOURCE_FILES),$(eval $(call CopyFileTarget,$f)))
 
-$(SRC_INCDIR) $(SOURCE_DIRS):
+.PHONY: checkdirs
+checkdirs: | $(SOURCE_INCDIRS)
+$(SOURCE_INCDIRS):
 	$(Q) mkdir -p $@
 
 .PHONY: api

docs/requirements.txt

@@ -2,6 +2,5 @@
 # list of Python packages used in documentation build
 sphinx==1.8.5
 sphinx-rtd-theme
-recommonmark
-sphinx-markdown-tables
+m2r
 breathe==4.11.1