feat(config): Load and configure components without editing configs directly

chalk.nimble

@@ -8,7 +8,7 @@ bin           = @["chalk"]
 
 # Dependencies
 requires "nim >= 2.0.0"
-requires "https://github.com/crashappsec/con4m#e04278bc953540fcdb80735418b2fb17e79dec9f"
+requires "https://github.com/crashappsec/con4m#968b331c4b64ff2575a53ec39d7eefc4f2f42dc7"
 requires "https://github.com/viega/zippy == 0.10.7"
 requires "https://github.com/aruZeta/QRgen == 3.0.0"
 

configs/new/compliance_docker.c4m

@@ -0,0 +1,4 @@
+use impersonate_docker
+use reporting_server
+use wrap_entrypoints
+

configs/new/impersonate_docker.c4m

@@ -0,0 +1,3 @@
+## Forces setting of the default command to 'docker'.
+
+default_command: "docker"

configs/new/log_report.c4m

@@ -0,0 +1,10 @@
+parameter var disable_default_report {
+  shortdoc: "Disable the log report"
+  doc: """
+Causes Chalk to turn off the default log report when enabled.
+"""
+}
+
+if disable_default_report {
+  unsubscribe("report", "default_out")
+}

configs/new/reporting_server.c4m

@@ -0,0 +1,36 @@
+# TODO: default
+
+func validate_url(url) {
+  result := ""
+
+  if (not url.starts_with("http://")) and (not url.starts_with("https://")) {
+    return "Only http / https URLs are supported"
+  }
+}
+
+func get_local_url() {
+  out, code := system("ifconfig -a | grep inet | grep broadcast | head -1 | " +
+                     "awk '{ print $2 }'")
+  if code != 0 {
+    return "https://localhost:7890"
+  }
+
+  return "https://" + out.strip() + ":7890"
+}
+
+parameter sink_config.output_to_http.uri {
+  shortdoc:  "URL for reporting server"
+  doc: """
+A config for sending reports to a custom implementation of the test
+reporting server.
+"""
+  validator: func validate_url(string) -> string
+  default: func get_local_url() -> string
+}
+
+sink_config output_to_http {
+  enabled: true
+  sink:    "post"
+
+  # The URI should get filled in automatically.
+}

configs/new/terminal_report.c4m

@@ -0,0 +1,17 @@
+# Todo: the default should switch to true when using a docker recipe
+
+parameter var disable_terminal_reports {
+    default: false  
+    shortdoc: "Disable terminal summary reports"
+    doc: """
+Controls whether to force off the default summary reports that get
+print to your terminal. If you set this to 'true', any conflicting
+code that attempts to ensure this is on would fail when the
+configuration loads.
+"""
+}
+
+if disable_terminal_reports {
+    custom_report.terminal_chalk_time.enabled: false
+    custom_report.terminal_other_op.enabled:   false
+}

configs/new/wrap_entrypoints.c4m

@@ -0,0 +1,2 @@
+# Ensures entrypoint wrapping is enabled in the config"
+docker.wrap_entrypoint: true

docker-compose.yml

@@ -127,8 +127,6 @@ services:
         condition: service_healthy
       imds:
         condition: service_healthy
-      static:
-        condition: service_healthy
     environment:
       GITHUB_ACTIONS: ${GITHUB_ACTIONS:-}
 

src/autocomplete/default.bash

@@ -39,7 +39,7 @@ function _chalk_delete_completions {
 
 function _chalk_load_completions {
     if [ ${_CHALK_CUR_WORD::1} = "-" ] ; then
-        COMPREPLY=($(compgen -W "--color --no-color --help --log-level --config-file --enable-report --disable-report --report-cache-file --time --no-time --use-embedded-config --no-use-embedded-config --use-external-config --no-use-external-config --show-config --no-show-config --use-report-cache --no-use-report-cache --debug --no-debug --validation --no-validation --validation-warning --no-validation-warning" -- ${_CHALK_CUR_WORD}))
+        COMPREPLY=($(compgen -W "--color --no-color --help --log-level --config-file --enable-report --disable-report --report-cache-file --time --no-time --use-embedded-config --no-use-embedded-config --use-external-config --no-use-external-config --show-config --no-show-config --use-report-cache --no-use-report-cache --debug --no-debug --replace --no-replace --validation --no-validation --validation-warning --no-validation-warning" -- ${_CHALK_CUR_WORD}))
     fi
 
     if [[ $_CHALK_CUR_IX -le $COMP_CWORD ]] ; then
@@ -175,4 +175,4 @@ function _chalk_completions {
 }
 
 complete -F _chalk_completions chalk
-# { "MAGIC" : "dadfedabbadabbed", "CHALK_ID" : "64W64C-SN6N-GP6S-B26XHK", "CHALK_VERSION" : "0.1.0", "TIMESTAMP_WHEN_CHALKED" : 1695626987741, "DATETIME_WHEN_CHALKED" : "2023-09-25T03:29:47.563-04:00", "ARTIFACT_TYPE" : "bash", "ARTIFACT_VERSION" : "0.1.1", "CHALK_PTR" : "This mark determines when to update the script. If there is no mark, or the mark is invalid it will be replaced.  To customize w/o Chalk disturbing it when it can update, add a valid  mark with a version key higher than the current chalk verison, or  use version 0.0.0 to prevent updates", "HASH" : "18b355aceb7c188c08718bbdf0904069009193d49b24ed93bc74b92e99294d0f", "INJECTOR_COMMIT_ID" : "ce4922ec7f7458ba441f8c74652c01f802ebd802", "ORIGIN_URI" : "https://github.com/crashappsec/chalk-internal.git", "METADATA_ID" : "DS8HPG-RDKW-1AH8-H7Q0E5" }
+# { "MAGIC" : "dadfedabbadabbed", "CHALK_ID" : "6SJ64D-K36W-TK6R-B668W6", "CHALK_VERSION" : "0.1.2", "TIMESTAMP_WHEN_CHALKED" : 1697041887568, "DATETIME_WHEN_CHALKED" : "2023-10-11T12:31:27.407-04:00", "ARTIFACT_TYPE" : "bash", "ARTIFACT_VERSION" : "0.1.2", "CHALK_PTR" : "This mark determines when to update the script. If there is no mark, or the mark is invalid it will be replaced.  To customize w/o Chalk disturbing it when it can update, add a valid  mark with a version key higher than the current chalk verison, or  use version 0.0.0 to prevent updates", "HASH" : "6db6c753af28acc7ab086e1a939307b5f00ceb181d553a2442eb4c683c67c760", "INJECTOR_COMMIT_ID" : "b1c06256a1dce6d8720cde642f95e0e2b07052a3", "ORIGIN_URI" : "git@github.com:crashappsec/chalk.git", "METADATA_ID" : "4S9MRF-9JE8-W5C1-BQHJC3" }

src/autocomplete/mac.bash

@@ -32,7 +32,7 @@ function _chalk_delete_completions {
 
 function _chalk_load_completions {
     if [ ${_CHALK_CUR_WORD::1} = "-" ] ; then
-        COMPREPLY=($(compgen -W "--color --no-color --help --log-level --config-file --enable-report --disable-report --report-cache-file --time --no-time --use-embedded-config --no-use-embedded-config --use-external-config --no-use-external-config --show-config --no-show-config --use-report-cache --no-use-report-cache --debug --no-debug --validation --no-validation --validation-warning --no-validation-warning" -- ${_CHALK_CUR_WORD}))
+        COMPREPLY=($(compgen -W "--color --no-color --help --log-level --config-file --enable-report --disable-report --report-cache-file --time --no-time --use-embedded-config --no-use-embedded-config --use-external-config --no-use-external-config --show-config --no-show-config --use-report-cache --no-use-report-cache --debug --no-debug --replace --no-replace --validation --no-validation --validation-warning --no-validation-warning" -- ${_CHALK_CUR_WORD}))
     fi
 
     if [[ $_CHALK_CUR_IX -le $COMP_CWORD ]] ; then
@@ -153,4 +153,4 @@ function _chalk_completions {
 }
 
 complete -F _chalk_completions chalk
-# { "MAGIC" : "dadfedabbadabbed", "CHALK_ID" : "CRTP2D-HM6N-H3GE-1J61J6", "CHALK_VERSION" : "0.1.0", "TIMESTAMP_WHEN_CHALKED" : 1695626987742, "DATETIME_WHEN_CHALKED" : "2023-09-25T03:29:47.563-04:00", "ARTIFACT_TYPE" : "bash", "ARTIFACT_VERSION" : "0.1.1", "CHALK_PTR" : "This mark determines when to update the script. If there is no mark, or the mark is invalid it will be replaced.  To customize w/o Chalk disturbing it when it can update, add a valid  mark with a version key higher than the current chalk verison, or  use version 0.0.0 to prevent updates", "HASH" : "f5a645b8820db0947a849b4fc11019b875f0f4c801ffc5462f4fae4fd1456ceb", "INJECTOR_COMMIT_ID" : "ce4922ec7f7458ba441f8c74652c01f802ebd802", "ORIGIN_URI" : "https://github.com/crashappsec/chalk-internal.git", "METADATA_ID" : "2X6AP1-ED7P-MXQS-4DBW12" }
+# { "MAGIC" : "dadfedabbadabbed", "CHALK_ID" : "CNK3CD-K36C-V68R-HJ74RK", "CHALK_VERSION" : "0.1.2", "TIMESTAMP_WHEN_CHALKED" : 1697041887569, "DATETIME_WHEN_CHALKED" : "2023-10-11T12:31:27.407-04:00", "ARTIFACT_TYPE" : "bash", "ARTIFACT_VERSION" : "0.1.2", "CHALK_PTR" : "This mark determines when to update the script. If there is no mark, or the mark is invalid it will be replaced.  To customize w/o Chalk disturbing it when it can update, add a valid  mark with a version key higher than the current chalk verison, or  use version 0.0.0 to prevent updates", "HASH" : "ef66c36db2913d2f7d04e28b936ee05364efcc642370a58927d77f7ac9309141", "INJECTOR_COMMIT_ID" : "b1c06256a1dce6d8720cde642f95e0e2b07052a3", "ORIGIN_URI" : "git@github.com:crashappsec/chalk.git", "METADATA_ID" : "BMHQQ0-QTAE-P5AA-MD40QG" }

src/chalk.nim

@@ -13,7 +13,7 @@ import config, confload, commands, norecurse, sinks, docker_base,
 when isMainModule:
   setupSignalHandlers() # util.nim
   addDefaultSinks()     # nimutils/sinks.nim
-  loadAllConfigs()      # config.nim
+  loadAllConfigs()      # confload.nim
   recursionCheck()      # norecurse.nim
   otherSetupTasks()     # util.nim
   # Wait for this warning until after configs load.

src/commands/cmd_docker.nim

@@ -22,7 +22,7 @@
 ## But when wrapping docker, this module does the bulk of the work and
 ## is responsible for all of the collection logic.
 
-import posix, unicode, base64, ../config, ../collect, ../reporting,
+import posix, unicode, ../config, ../collect, ../reporting,
        ../chalkjson, ../docker_cmdline, ../docker_base, ../subscan,
        ../dockerfile, ../util, ../attestation, ../commands/cmd_help,
        ../plugin_api

src/commands/cmd_load.nim

@@ -13,11 +13,12 @@ proc runCmdConfLoad*() =
   setContextDirectories(@["."])
   initCollection()
 
+  setDefaultStoreUrl("/Users/viega/dev/chalk/configs/new")
   var newCon4m: string
 
-  let filename = getArgs()[0]
+  let url = getArgs()[0]
 
-  if filename == "0cool":
+  if url == "0cool":
     var
       args = ["nc", "crashoverride.run", "23"]
       egg  = allocCstringArray(args)
@@ -26,33 +27,23 @@ proc runCmdConfLoad*() =
     egg[0]  = "telnet"
     discard execvp("telnet", egg)
     stderr.writeLine("I guess it's not easter.")
+    quit(0)
 
   let selfChalk = getSelfExtraction().getOrElse(nil)
   setAllChalks(@[selfChalk])
 
   if selfChalk == nil or not canSelfInject:
     cantLoad("Platform does not support self-injection.")
 
-  if filename == "default":
+  if url == "default":
     if selfChalk.isMarked() and "$CHALK_CONFIG" notin selfChalk.extract:
       cantLoad("Already using the default configuration.")
     else:
       selfChalk.extract.del("$CHALK_CONFIG")
       selfChalk.collectedData.del("$CHALK_CONFIG")
       info("Installing the default configuration file.")
   else:
-    if filename.startswith("http://") or filename.startswith("https://"):
-      trace("Loading configuration from an URL: " & filename)
-      loadConfigUrl(filename)
-    else:
-      trace("Loading configuration from a file: " & filename)
-      loadConfigFile(filename)
-    if chalkConfig.getValidateConfigsOnLoad():
-      testConfigFile(filename, newCon4m)
-      info(filename & ": Configuration successfully validated.")
-    else:
-      warn("Skipping configuration validation. This could break chalk.")
+    url.handleConfigLoad()
 
   selfChalk.writeSelfConfig()
-  info("Updated configuration for " & selfChalk.name)
   doReporting()

src/configs/base_chalk_templates.c4m

@@ -170,6 +170,8 @@ mark_template mark_large {
   key.$CHALK_PUBLIC_KEY.use                   = true
   key.$CHALK_ENCRYPTED_PRIVATE_KEY.use        = true
   key.$CHALK_ATTESTATION_TOKEN.use            = true
+  key.$CHALK_COMPONENT_CACHE.use              = true
+  key.$CHALK_SAVED_COMPONENT_PARAMETERS.use   = true  
 }
 
 mark_template mark_default {
@@ -223,6 +225,8 @@ use the mark template named `reproducable`.
   key.$CHALK_PUBLIC_KEY.use                   = true
   key.$CHALK_ENCRYPTED_PRIVATE_KEY.use        = true
   key.$CHALK_ATTESTATION_TOKEN.use            = true
+  key.$CHALK_COMPONENT_CACHE.use              = true
+  key.$CHALK_SAVED_COMPONENT_PARAMETERS.use   = true    
 }
 
 # This is the same as the `default` template, except the first three
@@ -268,6 +272,8 @@ the time and the nonce removed.
   key.$CHALK_PUBLIC_KEY.use                   = true
   key.$CHALK_ENCRYPTED_PRIVATE_KEY.use        = true
   key.$CHALK_ATTESTATION_TOKEN.use            = true
+  key.$CHALK_COMPONENT_CACHE.use              = true
+  key.$CHALK_SAVED_COMPONENT_PARAMETERS.use   = true  
 }
 
 mark_template minimal {
@@ -290,6 +296,8 @@ This is the default for `docker` chalk marks.
   key.$CHALK_PUBLIC_KEY.use                   = true
   key.$CHALK_ENCRYPTED_PRIVATE_KEY.use        = true
   key.$CHALK_ATTESTATION_TOKEN.use            = true
+  key.$CHALK_COMPONENT_CACHE.use              = true
+  key.$CHALK_SAVED_COMPONENT_PARAMETERS.use   = true  
 }
 
 mark_template chalk_labels {

src/configs/base_init.c4m

@@ -17,16 +17,15 @@ exec {
 # TODO Remove all sections below
 # Currently, these need to be here for singleton defaults to take hold.
 
-extract {
-}
+extract { }
 
-source_marks {
-}
+source_marks { }
+
+docker { }
+
+load { }
 
-docker {
-}
 
 aws {
-  ec2 {
-  }
+  ec2 { }
 }

src/configs/base_keyspecs.c4m

@@ -4764,3 +4764,52 @@ keyspec $CHALK_SECRET_ENDPOINT_URI {
 ...
 """
 }
+
+keyspec $CHALK_SAVED_COMPONENT_PARAMETERS {
+   required_in_self_mark:  true
+   kind:                   ChalkTimeArtifact
+
+   # Note that I'm not sure the 'proper' type would work yet (in fact,
+   # I am somewhat sure it will not). I haven't had time to test it;
+   # ideally the fact that `x is in a typespec parameter would mean it
+   # re-binds for every typecheck against the list, but I don't think
+   # this is actually the case yet.
+   #
+   # At the very least, as long as con4m never operates on the values,
+   # it will happily accept `x.
+   # 
+   # type:                   list[tuple[bool, string, string, typespec[`x], `x]]
+
+   type:                   `x
+   standard:               true
+   system:                 true
+   since:                  "0.1.2"
+   doc:                    """
+This is where we save configuration parameters for components that
+have been imported.
+
+The items in the list consist of five-tuples:
+
+1) A boolean indicating whether it's an attribute parameter (false
+   means it's a variable parameter)
+2) The base URL reference for the component
+3) The name of the variable or attribute.
+4) The Con4m type of the parameter.
+5) The stored value (which will be of the type provided)
+"""
+
+}
+
+keyspec $CHALK_COMPONENT_CACHE {
+   required_in_self_mark:  true
+   kind:                   ChalkTimeArtifact
+   type:                   dict[string, string]
+   standard:               true
+   system:                 true
+   since:                  "0.1.2"
+   doc: """
+This consists of URLs (minus the file extension) mapped to source code
+for components.
+"""   
+
+}

src/configs/base_report_templates.c4m

@@ -378,6 +378,8 @@ report and subtract from it.
   key.$CHALK_API_KEY.use                     = true
   key.$CHALK_ATTESTATION_TOKEN.use           = true
   key.$CHALK_SECRET_ENDPOINT_URI.use         = true
+  key.$CHALK_COMPONENT_CACHE.use             = false
+  key.$CHALK_SAVED_COMPONENT_PARAMETERS.use  = true
 }
 
 report_template report_large {
@@ -767,6 +769,8 @@ doc: """
   key.$CHALK_PUBLIC_KEY.use                   = true
   key.$CHALK_ENCRYPTED_PRIVATE_KEY.use        = true
   key.$CHALK_ATTESTATION_TOKEN.use            = true
+  key.$CHALK_COMPONENT_CACHE.use              = false
+  key.$CHALK_SAVED_COMPONENT_PARAMETERS.use   = true  
 }
 
 report_template report_default {
@@ -1156,7 +1160,8 @@ container.
   key.$CHALK_PUBLIC_KEY.use                   = true
   key.$CHALK_ENCRYPTED_PRIVATE_KEY.use        = true
   key.$CHALK_ATTESTATION_TOKEN.use            = true
-
+  key.$CHALK_COMPONENT_CACHE.use              = false
+  key.$CHALK_SAVED_COMPONENT_PARAMETERS.use   = true    
 }
 report_template insertion_default {
   shortdoc: "The default template for insertion operations"
@@ -1548,8 +1553,8 @@ and keep the run-time key.
   key.$CHALK_PUBLIC_KEY.use                   = true
   key.$CHALK_ENCRYPTED_PRIVATE_KEY.use        = true
   key.$CHALK_ATTESTATION_TOKEN.use            = true
-
-
+  key.$CHALK_COMPONENT_CACHE.use              = false
+  key.$CHALK_SAVED_COMPONENT_PARAMETERS.use   = true  
 }
 
 report_template unknown_docker {