Skip to content

Commit

Permalink
various adjustments
Browse files Browse the repository at this point in the history
  • Loading branch information
thinkyhead committed Dec 14, 2021
1 parent c1291b4 commit a544c1a
Show file tree
Hide file tree
Showing 11 changed files with 91 additions and 66 deletions.
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
# Generated files
_Version.h
bdf2u8g
marlin_config.json
mczip.h
*.gen

#
# OS
Expand Down Expand Up @@ -142,7 +145,6 @@ vc-fileutils.settings
.vscode/c_cpp_properties.json
.vscode/launch.json
.vscode/*.db
marlin_config.json

#Simulation
imgui.ini
Expand Down
8 changes: 5 additions & 3 deletions Marlin/Configuration_adv.h
Original file line number Diff line number Diff line change
Expand Up @@ -1594,10 +1594,12 @@
#endif

/**
* If you are short of Flash space, this will prevent Marlin from storing your customized
* settings and export them to the SD Card as mc.zip with 'M503 C'.
* Enable this option if you have more than ~3K of unused flash space.
* Marlin will embed all settings in the firmware binary as compressed data.
* Use 'M503 C' to write the settings out to the SD Card as 'mc.zip'.
* See docs/ConfigEmbedding.md for details on how to use 'mc-apply.py'.
*/
//#define CONFIG_SKIP_EMBED_AND_SAVE_TO_SD
//#define CONFIGURATION_EMBEDDING

// Add an optimized binary file transfer mode, initiated with 'M28 B1'
//#define BINARY_FILE_TRANSFER
Expand Down
3 changes: 0 additions & 3 deletions Marlin/src/HAL/AVR/inc/Conditionals_post.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,3 @@
*
*/
#pragma once

// AVR are too limited in resources to store the configuration into the binary
#define CONFIG_SKIP_EMBED_AND_SAVE_TO_SD 1
17 changes: 11 additions & 6 deletions Marlin/src/gcode/eeprom/M500-M504.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@
#include "../../core/serial.h"
#include "../../inc/MarlinConfig.h"

#if DISABLED(CONFIG_SKIP_EMBED_AND_SAVE_TO_SD) && ENABLED(SDSUPPORT)
#if ENABLED(CONFIGURATION_EMBEDDING)
#include "../../sd/SdBaseFile.h"
#include "../../../../.pio/build/mc.cpp"
#include "../../mczip.h"
#endif

/**
Expand Down Expand Up @@ -56,18 +56,23 @@ void GcodeSuite::M502() {
/**
* M503: print settings currently in memory
*
* C<flag> : Save "mc.zip" containing Marlin configuration options to the SD Card
* With CONFIGURATION_EMBEDDING:
* C<flag> : Save the full Marlin configuration to SD Card as "mc.zip"
*/
void GcodeSuite::M503() {
(void)settings.report(!parser.boolval('S', true));

#if DISABLED(CONFIG_SKIP_EMBED_AND_SAVE_TO_SD) && ENABLED(SDSUPPORT)
if (parser.boolval('C')) {
#if ENABLED(CONFIGURATION_EMBEDDING)
if (parser.seen_test('C')) {
SdBaseFile file;
const uint16_t size = sizeof(mc_zip);
// Need to create the config size on the SD card
SERIAL_ECHO_START();
SERIAL_ECHOPGM("Configuration save");
if (file.open("mc.zip", O_WRITE|O_CREAT) && file.write(pgm_read_ptr(mc_zip), size) != -1 && file.close())
SERIAL_ECHO_MSG("Configuration saved to 'mc.zip' on SD card");
SERIAL_ECHOLNPGM("d as 'mc.zip'");
else
SERIAL_ECHOLNPGM(" failed");
}
#endif
}
Expand Down
6 changes: 6 additions & 0 deletions Marlin/src/inc/Conditionals_adv.h
Original file line number Diff line number Diff line change
Expand Up @@ -1004,3 +1004,9 @@
#if EITHER(MEATPACK_ON_SERIAL_PORT_1, MEATPACK_ON_SERIAL_PORT_2)
#define HAS_MEATPACK 1
#endif

// AVR are (usually) too limited in resources to store the configuration into the binary
#if !defined(FORCE_CONFIG_EMBED) && (defined(__AVR__) || DISABLED(SDSUPPORT) || EITHER(SDCARD_READONLY, DISABLE_M503))
#undef CONFIGURATION_EMBEDDING
#define CANNOT_EMBED_CONFIGURATION defined(__AVR__)
#endif
4 changes: 4 additions & 0 deletions Marlin/src/inc/Warnings.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -549,3 +549,7 @@
#elif !USE_SENSORLESS && ENABLED(USES_DIAG_PINS)
#warning "Driver DIAG pins must be physically removed unless SENSORLESS_HOMING is enabled. (See https://bit.ly/2ZPRlt0)"
#endif

#if CANNOT_EMBED_CONFIGURATION
#warning "Disabled CONFIGURATION_EMBEDDING because AVR usually have too little flash to store the configuration. Define FORCE_CONFIG_EMBED to override."
#endif
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
if pioutil.is_pio_build():
from os.path import join, isfile
import shutil
from pprint import pprint

Import("env")

Expand Down
19 changes: 14 additions & 5 deletions ...formIO/scripts/create_config_from_dict.py → ...root/share/PlatformIO/scripts/mc-apply.py
100644 → 100755
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#!/usr/bin/env python
#
# Create a Configuration from marlin_config.json
#
Expand All @@ -6,10 +7,8 @@
import shutil
import re

if '--bare-output' in sys.argv:
output_suffix = ''
else:
output_suffix = '.gen'
opt_output = '--opt' in sys.argv
output_suffix = '.sh' if opt_output else '' if '--bare-output' in sys.argv else '.gen'

try:
with open('marlin_config.json', 'r') as infile:
Expand All @@ -26,7 +25,17 @@
outfile = open('Marlin/' + key + output_suffix, 'w')
for k, v in sorted(conf[key].items()):
# Make define line now
define = '#define ' + k + ' ' + v + '\n'
if opt_output:
if v != '':
if '"' in v:
v = "'%s'" % v
elif ' ' in v:
v = '"%s"' % v
define = 'opt_set ' + k + ' ' + v + '\n'
else:
define = 'opt_enable ' + k + '\n'
else:
define = '#define ' + k + ' ' + v + '\n'
outfile.write(define)
outfile.close()

Expand Down
49 changes: 29 additions & 20 deletions buildroot/share/PlatformIO/scripts/signature.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#
# signature.py
#
import subprocess,re,json,hashlib
import os,subprocess,re,json,hashlib

#
# The dumbest preprocessor in the world
Expand Down Expand Up @@ -48,30 +48,31 @@ def compute_build_signature(env):
if 'BUILD_SIGNATURE' in env:
return

# Definition from these files will be kept
files_to_keep = [ 'Marlin/Configuration.h',
'Marlin/Configuration_adv.h',
]
# Definitions from these files will be kept
files_to_keep = [ 'Marlin/Configuration.h', 'Marlin/Configuration_adv.h' ]

build_dir=os.path.join(env['PROJECT_BUILD_DIR'], env['PIOENV'])

# Check if we can skip processing
hashes = ''
for header in files_to_keep:
hashes = hashes + get_file_sha256sum(header)
hashes += get_file_sha256sum(header)[0:10]

marlin_json = 'marlin_config.json'
marlin_json = os.path.join(build_dir, 'marlin_config.json')
marlin_zip = os.path.join(build_dir, 'mc')

# Read existing config file
try:
with open(marlin_json, 'r') as infile:
conf = json.load(infile)
if conf['__INITIAL_HASH'] == hashes:
# Same configuration, skip recomputing the building signature
compress_file(marlin_json, '.pio/build/mc')
compress_file(marlin_json, marlin_zip)
return
except:
pass

# Need to parse all valid defines in the configuration files
# Get enabled config options based on preprocessor
from preprocessor import run_preprocessor
complete_cfg = run_preprocessor(env)

Expand All @@ -85,7 +86,6 @@ def compute_build_signature(env):
# To remember from which file it cames from
real_defines[header.split('/')[-1]] = defines


r = re.compile(r"\(+(\s*-*\s*_.*)\)+")

# First step is to collect all valid macros
Expand All @@ -109,6 +109,8 @@ def compute_build_signature(env):

defines[key] = value if len(value) else ""

if not 'CONFIGURATION_EMBEDDING' in defines:
return

# Second step is to filter useless macro
resolved_defines = {}
Expand All @@ -123,7 +125,7 @@ def compute_build_signature(env):
if key[-11:] == "_T_DECLARED":
continue
# Remove keys that are not in the #define list in the Configuration list
if not(key in all_defines) and key != "DETAILED_BUILD_VERSION" and key != "STRING_DISTRIBUTION_DATE":
if not (key in all_defines) and key != "DETAILED_BUILD_VERSION" and key != "STRING_DISTRIBUTION_DATE":
continue

# Don't be that smart guy here
Expand Down Expand Up @@ -156,12 +158,19 @@ def compute_build_signature(env):
with open(marlin_json, 'w') as outfile:
json.dump(data, outfile, separators=(',', ':'))

# Now compress the JSON file to as much as we can
compress_file(marlin_json, '.pio/build/mc')

# And generate a C source file for storing this array
with open('.pio/build/mc.cpp','wb') as result_file:
result_file.write(b'const unsigned char mc_zip[] PROGMEM = {')
for b in open('.pio/build/mc.zip', 'rb').read():
result_file.write(b'0x%02X,' % b)
result_file.write(b'};')
# Compress the JSON file as much as we can
compress_file(marlin_json, marlin_zip)

# Generate a C source file for storing this array
with open('Marlin/src/mczip.h','wb') as result_file:
result_file.write(b'#warning "Generated file \'mc.zip\' is embedded"\n')
result_file.write(b'const unsigned char mc_zip[] PROGMEM = {\n ')
count = 0
for b in open(os.path.join(build_dir, 'mc.zip'), 'rb').read():
result_file.write(b' 0x%02X,' % b)
count += 1
if (count % 16 == 0):
result_file.write(b'\n ')
if (count % 16):
result_file.write(b'\n')
result_file.write(b'};\n')
27 changes: 0 additions & 27 deletions docs/AutoConfBackup.md

This file was deleted.

19 changes: 19 additions & 0 deletions docs/ConfigEmbedding.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Configuration Embedding

Starting with version 2.0.9.3, Marlin automatically extracts the configuration used to generate the firmware and stores it in the firmware binary. This is enabled by defining `CONFIGURATION_EMBEDDING` in `Configuration_adv.h`.

## How it's done
To create the embedded configuration, we do a compiler pass to process the Configuration files and extract all active options. The active options are parsed into key/value pairs, serialized to JSON format, and stored in a file called `marlin_config.json`, which also includes specific build information (like the git revision, the build date, and some version information. The JSON file is then compressed in a ZIP archive called `.pio/build/mc.zip` which is converted into a C array and stored in a C++ file called `mc.h` which is included in the build.

## Extracting configurations from a Marlin binary
To get the configuration out of a binary firmware, you'll need a non-write-protected SD card inserted into the printer while running the firmware.
Send the command `M503 C` to write the file `mc.zip` to the SD card. Copy the file to your computer, ideally in the same folder as the Marlin repository.

Run the following commands to extract and apply the configuration:
```
$ git checkout -f
$ unzip mc.zip
$ python buildroot/share/PlatformIO/scripts/mc-apply.py
```

This will attempt to update the configuration files to match the settings used for the original build. It will also dump the git reference used to build the code (which may be accessible if the firmware was built from the main repository. As a fallback it also includes the `STRING_DISTRIBUTION_DATE` which is unlikely to be modified in a fork).

0 comments on commit a544c1a

Please sign in to comment.