various adjustments

.gitignore

@@ -22,6 +22,9 @@
 # Generated files
 _Version.h
 bdf2u8g
+marlin_config.json
+mczip.h
+*.gen
 
 #
 # OS
@@ -142,7 +145,6 @@ vc-fileutils.settings
 .vscode/c_cpp_properties.json
 .vscode/launch.json
 .vscode/*.db
-marlin_config.json
 
 #Simulation
 imgui.ini

Marlin/Configuration_adv.h

@@ -1594,10 +1594,12 @@
   #endif
 
   /**
-   * If you are short of Flash space, this will prevent Marlin from storing your customized
-   * settings and export them to the SD Card as mc.zip with 'M503 C'.
+   * Enable this option if you have more than ~3K of unused flash space.
+   * Marlin will embed all settings in the firmware binary as compressed data.
+   * Use 'M503 C' to write the settings out to the SD Card as 'mc.zip'.
+   * See docs/ConfigEmbedding.md for details on how to use 'mc-apply.py'.
    */
-  //#define CONFIG_SKIP_EMBED_AND_SAVE_TO_SD
+  //#define CONFIGURATION_EMBEDDING
 
   // Add an optimized binary file transfer mode, initiated with 'M28 B1'
   //#define BINARY_FILE_TRANSFER

Marlin/src/HAL/AVR/inc/Conditionals_post.h

@@ -20,6 +20,3 @@
  *
  */
 #pragma once
-
-// AVR are too limited in resources to store the configuration into the binary
-#define CONFIG_SKIP_EMBED_AND_SAVE_TO_SD 1

Marlin/src/gcode/eeprom/M500-M504.cpp

@@ -25,9 +25,9 @@
 #include "../../core/serial.h"
 #include "../../inc/MarlinConfig.h"
 
-#if DISABLED(CONFIG_SKIP_EMBED_AND_SAVE_TO_SD) && ENABLED(SDSUPPORT)
+#if ENABLED(CONFIGURATION_EMBEDDING)
   #include "../../sd/SdBaseFile.h"
-  #include "../../../../.pio/build/mc.cpp"
+  #include "../../mczip.h"
 #endif
 
 /**
@@ -56,18 +56,19 @@ void GcodeSuite::M502() {
   /**
    * M503: print settings currently in memory
    *
-   *   C<flag> : Save "mc.zip" containing Marlin configuration options to the SD Card
+   * With CONFIGURATION_EMBEDDING:
+   *   C<flag> : Save the full Marlin configuration to SD Card as "mc.zip"
    */
   void GcodeSuite::M503() {
     (void)settings.report(!parser.boolval('S', true));
 
-    #if DISABLED(CONFIG_SKIP_EMBED_AND_SAVE_TO_SD) && ENABLED(SDSUPPORT)
-      if (parser.boolval('C')) {
+    #if ENABLED(CONFIGURATION_EMBEDDING)
+      if (parser.seen_test('C')) {
         SdBaseFile file;
         const uint16_t size = sizeof(mc_zip);
         // Need to create the config size on the SD card
         if (file.open("mc.zip", O_WRITE|O_CREAT) && file.write(pgm_read_ptr(mc_zip), size) != -1 && file.close())
-          SERIAL_ECHO_MSG("Configuration saved to 'mc.zip' on SD card");
+          SERIAL_ECHO_MSG("Configuration saved as 'mc.zip'");
       }
     #endif
   }

Marlin/src/inc/Conditionals_adv.h

@@ -1004,3 +1004,9 @@
 #if EITHER(MEATPACK_ON_SERIAL_PORT_1, MEATPACK_ON_SERIAL_PORT_2)
   #define HAS_MEATPACK 1
 #endif
+
+// AVR are (usually) too limited in resources to store the configuration into the binary
+#if !defined(FORCE_CONFIG_EMBED) && (defined(__AVR__) || DISABLED(SDSUPPORT) || EITHER(SDCARD_READONLY, DISABLE_M503))
+  #undef CONFIGURATION_EMBEDDING
+  #define CANNOT_EMBED_CONFIGURATION defined(__AVR__)
+#endif

Marlin/src/inc/Warnings.cpp

@@ -549,3 +549,7 @@
 #elif !USE_SENSORLESS && ENABLED(USES_DIAG_PINS)
   #warning "Driver DIAG pins must be physically removed unless SENSORLESS_HOMING is enabled. (See https://bit.ly/2ZPRlt0)"
 #endif
+
+#if CANNOT_EMBED_CONFIGURATION
+  #warning "Disabled CONFIGURATION_EMBEDDING because AVR usually have too little flash to store the configuration. Define FORCE_CONFIG_EMBED to override."
+#endif

buildroot/share/PlatformIO/scripts/SAMD51_grandcentral_m4.py

@@ -6,7 +6,6 @@
 if pioutil.is_pio_build():
 	from os.path import join, isfile
 	import shutil
-	from pprint import pprint
 
 	Import("env")
 

buildroot/share/PlatformIO/scripts/create_config_from_dict.py → buildroot/share/PlatformIO/scripts/mc-apply.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 #
 # Create a Configuration from marlin_config.json
 #
@@ -6,10 +7,8 @@
 import shutil
 import re
 
-if '--bare-output' in sys.argv:
-	output_suffix = ''
-else:
-	output_suffix = '.gen'
+opt_output = '--opt' in sys.argv
+output_suffix = '.sh' if opt_output else '' if '--bare-output' in sys.argv else '.gen'
 
 try:
 	with open('marlin_config.json', 'r') as infile:
@@ -26,7 +25,17 @@
 			outfile = open('Marlin/' + key + output_suffix, 'w')
 			for k, v in sorted(conf[key].items()):
 				# Make define line now
-				define = '#define ' + k + ' ' + v + '\n'
+				if opt_output:
+					if v != '':
+						if '"' in v:
+							v = "'%s'" % v
+						elif ' ' in v:
+							v = '"%s"' % v
+						define = 'opt_set ' + k + ' ' + v + '\n'
+					else:
+						define = 'opt_enable ' + k + '\n'
+				else:
+					define = '#define ' + k + ' ' + v + '\n'
 				outfile.write(define)
 			outfile.close()
 

buildroot/share/PlatformIO/scripts/signature.py

@@ -1,7 +1,7 @@
 #
 # signature.py
 #
-import subprocess,re,json,hashlib
+import os,subprocess,re,json,hashlib
 
 #
 # The dumbest preprocessor in the world
@@ -48,30 +48,31 @@ def compute_build_signature(env):
 	if 'BUILD_SIGNATURE' in env:
 		return
 
-	# Definition from these files will be kept
-	files_to_keep = [ 'Marlin/Configuration.h',
-					  'Marlin/Configuration_adv.h',
-	]
+	# Definitions from these files will be kept
+	files_to_keep = [ 'Marlin/Configuration.h', 'Marlin/Configuration_adv.h' ]
+
+	build_dir=os.path.join(env['PROJECT_BUILD_DIR'], env['PIOENV'])
 
 	# Check if we can skip processing
 	hashes = ''
 	for header in files_to_keep:
-		hashes = hashes + get_file_sha256sum(header)
+		hashes += get_file_sha256sum(header)[0:10]
 
-	marlin_json = 'marlin_config.json'
+	marlin_json = os.path.join(build_dir, 'marlin_config.json')
+	marlin_zip = os.path.join(build_dir, 'mc')
 
 	# Read existing config file
 	try:
 		with open(marlin_json, 'r') as infile:
 			conf = json.load(infile)
 			if conf['__INITIAL_HASH'] == hashes:
 				# Same configuration, skip recomputing the building signature
-				compress_file(marlin_json, '.pio/build/mc')
+				compress_file(marlin_json, marlin_zip)
 				return
 	except:
 		pass
 
-	# Need to parse all valid defines in the configuration files
+	# Get enabled config options based on preprocessor
 	from preprocessor import run_preprocessor
 	complete_cfg = run_preprocessor(env)
 
@@ -85,7 +86,6 @@ def compute_build_signature(env):
 		# To remember from which file it cames from
 		real_defines[header.split('/')[-1]] = defines
 
-
 	r = re.compile(r"\(+(\s*-*\s*_.*)\)+")
 
 	# First step is to collect all valid macros
@@ -109,6 +109,8 @@ def compute_build_signature(env):
 
 		defines[key] = value if len(value) else ""
 
+	if not 'CONFIGURATION_EMBEDDING' in defines:
+		return
 
 	# Second step is to filter useless macro
 	resolved_defines = {}
@@ -123,7 +125,7 @@ def compute_build_signature(env):
 		if key[-11:] == "_T_DECLARED":
 			continue
 		# Remove keys that are not in the #define list in the Configuration list
-		if not(key in all_defines) and key != "DETAILED_BUILD_VERSION" and key != "STRING_DISTRIBUTION_DATE":
+		if not (key in all_defines) and key != "DETAILED_BUILD_VERSION" and key != "STRING_DISTRIBUTION_DATE":
 			continue
 
 		# Don't be that smart guy here
@@ -156,12 +158,19 @@ def compute_build_signature(env):
 	with open(marlin_json, 'w') as outfile:
 		json.dump(data, outfile, separators=(',', ':'))
 
-	# Now compress the JSON file to as much as we can
-	compress_file(marlin_json, '.pio/build/mc')
-
-	# And generate a C source file for storing this array
-	with open('.pio/build/mc.cpp','wb') as result_file:
-		result_file.write(b'const unsigned char mc_zip[] PROGMEM = {')
-		for b in open('.pio/build/mc.zip', 'rb').read():
-			result_file.write(b'0x%02X,' % b)
-		result_file.write(b'};')
+	# Compress the JSON file as much as we can
+	compress_file(marlin_json, marlin_zip)
+
+	# Generate a C source file for storing this array
+	with open('Marlin/src/mczip.h','wb') as result_file:
+		result_file.write(b'#warning "Generated file \'mc.zip\' is embedded"\n')
+		result_file.write(b'const unsigned char mc_zip[] PROGMEM = {\n ')
+		count = 0
+		for b in open(os.path.join(build_dir, 'mc.zip'), 'rb').read():
+			result_file.write(b' 0x%02X,' % b)
+			count += 1
+			if (count % 16 == 0):
+			 	result_file.write(b'\n ')
+		if (count % 16):
+			result_file.write(b'\n')
+		result_file.write(b'};\n')

docs/ConfigEmbedding.md

@@ -0,0 +1,19 @@
+# Configuration Embedding
+
+Starting with version 2.0.9.3, Marlin automatically extracts the configuration used to generate the firmware and stores it in the firmware binary. This is enabled by defining `CONFIGURATION_EMBEDDING` in `Configuration_adv.h`.
+
+## How it's done
+To create the embedded configuration, we do a compiler pass to process the Configuration files and extract all active options. The active options are parsed into key/value pairs, serialized to JSON format, and stored in a file called `marlin_config.json`, which also includes specific build information (like the git revision, the build date, and some version information. The JSON file is then compressed in a ZIP archive called `.pio/build/mc.zip` which is converted into a C array and stored in a C++ file called `mc.h` which is included in the build.
+
+## Extracting configurations from a Marlin binary
+To get the configuration out of a binary firmware, you'll need a non-write-protected SD card inserted into the printer while running the firmware.
+Send the command `M503 C` to write the file `mc.zip` to the SD card. Copy the file to your computer, ideally in the same folder as the Marlin repository.
+
+Run the following commands to extract and apply the configuration:
+```
+$ git checkout -f
+$ unzip mc.zip 
+$ python buildroot/share/PlatformIO/scripts/mc-apply.py
+```
+
+This will attempt to update the configuration files to match the settings used for the original build. It will also dump the git reference used to build the code (which may be accessible if the firmware was built from the main repository. As a fallback it also includes the `STRING_DISTRIBUTION_DATE` which is unlikely to be modified in a fork).