* New directive "MDStaplingKeepResponse" for controlling how long OC…

…SP responses are

   kept in the store and older ones get removed at start up.

ChangeLog

@@ -1,3 +1,7 @@
+ * New directive "MDStaplingKeepResponse" for controlling how long OCSP responses are
+   kept in the store and older ones get removed at start up.
+ * Errors reports by an ACME CA may include "subproblems", where several causes may be reported. 
+   These are now part of the md-status reporting and also logged. Test cases added.
  * OCSP status and validity now part of md-status resource.
  * The number of log entries for a single job is now limited to 128. New entries will
    cause the oldest ones to be removed.

README.md

@@ -1213,8 +1213,12 @@ checks by mod_md in v1.1.x which are now eliminated. If you have many domains, t
 * [MDRenewWindow](#mdrenewwindow--when-to-renew)
 * [MDWarnWindow](#mdwarnwindow)
 * [MDServerStatus](#mdserverstatus)
+* [MDStapling](#mdstapling)
+* [MDStapleOthers](#mdstapleothers)
+* [MDStaplingKeepResponse](#mdstaplingkeepresponse)
 * [MDStoreDir](#mdstoredir)
 
+
 ## MDomain
 
 ***Define list of domain names that belong to one group***<BR/>
@@ -1525,6 +1529,46 @@ Default: `on`
 
 Controls if Managed Domains respond to public requests for `/.httpd/certificate-status` or not.
 
+## MDStapling
+
+***Enable stapling for all or a particular MDomain.***<BR/>
+`MDStapling on|off`<BR/>
+Default: `off`
+
+`mod_md` has its own implementation for providing OCSP stapling information. This is an 
+alternative to the one provided by `mod_ssl`. For backward compatiblity reasons, this is
+disabled by default.
+
+The new stapling can be switched on for all certificates on the server or for an individual MDomain. This
+will replace any stapling configurtion in `mod_ssl` for these hosts. When disabled, the `mod_ssl`
+stapling (if configured) will do the work. This allows for a gradual shift over from one 
+implementation to the other.
+
+The stapling of `mod_md` will also work for domains where the certificates are not managed
+by this module (see MDStapleOthers for how to control this). This allows use of the new stapling
+without using any ACME certificate management.
+
+## MDStapleOthers
+
+***Enable stapling for certificates not managed by mod_md.***<BR/>
+`MDStapling on|off`<BR/>
+Default: `on`
+
+This setting only takes effect when `MDStapling` is enabled. It controls if `mod_md` should
+also provide stapling information for certificates that are not directly controlled by it, e.g.
+renewed via an ACME CA.
+
+## MDStaplingKeepResponse
+
+***Controls when responses are considered old and will be removed.***<BR/>
+`MDStaplingKeepResponse duration`<BR/>
+Default: 7d
+
+This time window specifies when OCSP response data used in stapling shall be removed
+from the store again on start up. Response information older than 7 days (default) is
+deleted. This keeps the store from growing when certificates are renewed/reconfigured 
+frequently.
+
 
 ## ServerAdmin / Contact Information
 

mod_md.xcodeproj/project.pbxproj

@@ -23,6 +23,7 @@
 		B2123B8D1F2DD7A200267CAF /* test_md_json.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = test_md_json.c; sourceTree = "<group>"; };
 		B2123B8E1F2DD7A200267CAF /* test_common.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = test_common.h; sourceTree = "<group>"; };
 		B2307ED520E2562100487AD7 /* test.ini */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = test.ini; sourceTree = "<group>"; };
+		B23C47E722F1A45900B8F494 /* test_0740_acme_errors.py */ = {isa = PBXFileReference; lastKnownFileType = text.script.python; path = test_0740_acme_errors.py; sourceTree = "<group>"; };
 		B24052151EF9145000E36701 /* md_core.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = md_core.c; sourceTree = "<group>"; };
 		B24052161EF9145000E36701 /* md_crypt.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = md_crypt.c; sourceTree = "<group>"; };
 		B24052171EF9145000E36701 /* md_crypt.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = md_crypt.h; sourceTree = "<group>"; };
@@ -315,6 +316,7 @@
 				B2C1BE0621F5CB38002EF7B4 /* test_0710_migration.py */,
 				B2C1BE082201AC02002EF7B4 /* test_0720_wildcard.py */,
 				B2830C2422B27C3F005DBFBF /* test_0730_static.py */,
+				B23C47E722F1A45900B8F494 /* test_0740_acme_errors.py */,
 				B2FEEDDF1FFB910D0029E19C /* test_0800_must_staple.py */,
 				B2A3E34D22CA34C2009D35AE /* test_0801_stapling.py */,
 				B2FFC9E920C5776A004A8F58 /* test_0900_notify.py */,

src/md.h

@@ -41,6 +41,7 @@ struct md_pkey_spec_t;
 #define MD_TIME_LIFE_NORM           (apr_time_from_sec(100 * MD_SECS_PER_DAY))
 #define MD_TIME_RENEW_WINDOW_DEF    (apr_time_from_sec(33 * MD_SECS_PER_DAY))
 #define MD_TIME_WARN_WINDOW_DEF     (apr_time_from_sec(10 * MD_SECS_PER_DAY))
+#define MD_TIME_OCSP_KEEP_NORM      (apr_time_from_sec(7 * MD_SECS_PER_DAY))
 
 #define MD_OTHER                "other"
 
@@ -193,6 +194,7 @@ struct md_t {
 #define MD_KEY_STATE            "state"
 #define MD_KEY_STATUS           "status"
 #define MD_KEY_STORE            "store"
+#define MD_KEY_SUBPROBLEMS      "subproblems"
 #define MD_KEY_TEMPORARY        "temporary"
 #define MD_KEY_TOKEN            "token"
 #define MD_KEY_TOTAL            "total"

src/md_acme.c

@@ -175,7 +175,10 @@ static apr_status_t inspect_problem(md_acme_req_t *req, const md_http_response_t
             ptype = md_json_gets(problem, MD_KEY_TYPE, NULL); 
             pdetail = md_json_gets(problem, MD_KEY_DETAIL, NULL);
             req->rv = problem_status_get(ptype);
-            md_result_problem_set(req->result, req->rv, ptype, pdetail);
+            md_result_problem_set(req->result, req->rv, ptype, pdetail,
+                                  md_json_getj(problem, MD_KEY_SUBPROBLEMS, NULL));
+
+
 
             if (APR_STATUS_IS_EAGAIN(req->rv)) {
                 md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, req->rv, req->p,
@@ -494,7 +497,8 @@ void md_acme_report_result(md_acme_t *acme, apr_status_t rv, struct md_result_t
         md_result_set(result, rv, NULL);
     }
     else {
-        md_result_problem_set(result, acme->last->status, acme->last->problem, acme->last->detail);
+        md_result_problem_set(result, acme->last->status, acme->last->problem, 
+                              acme->last->detail, acme->last->subproblems);
     }
 }
 

src/md_acme_authz.c

@@ -157,6 +157,7 @@ static int copy_challenge_error(void *baton, size_t index, md_json_t *json)
     if (md_json_has_key(json, MD_KEY_ERROR, NULL)) {
         ctx->authz->error_type = md_json_dups(ctx->p, json, MD_KEY_ERROR, MD_KEY_TYPE, NULL);
         ctx->authz->error_detail = md_json_dups(ctx->p, json, MD_KEY_ERROR, MD_KEY_DETAIL, NULL);
+        ctx->authz->error_subproblems = md_json_dupj(ctx->p, json, MD_KEY_ERROR, MD_KEY_SUBPROBLEMS, NULL);
     }
     return 1;
 }
@@ -177,6 +178,7 @@ apr_status_t md_acme_authz_update(md_acme_authz_t *authz, md_acme_t *acme, apr_p
     authz->state = MD_ACME_AUTHZ_S_UNKNOWN;
     json = NULL;
     authz->error_type = authz->error_detail = NULL;
+    authz->error_subproblems = NULL;
     err = "unable to parse response";
     log_level = MD_LOG_ERR;
 

src/md_acme_authz.h

@@ -51,6 +51,7 @@ struct md_acme_authz_t {
     apr_time_t expires;
     const char *error_type;
     const char *error_detail;
+    const struct md_json_t *error_subproblems;
     struct md_json_t *resource;
 };
 

src/md_acme_order.c

@@ -468,7 +468,7 @@ apr_status_t md_acme_order_start_challenges(md_acme_order_t *order, md_acme_t *a
             case MD_ACME_AUTHZ_S_INVALID:
                 rv = APR_EINVAL;
                 if (authz->error_type) {
-                    md_result_problem_set(result, rv, authz->error_type, authz->error_detail);
+                    md_result_problem_set(result, rv, authz->error_type, authz->error_detail, NULL);
                     goto leave;
                 }
                 /* fall through */

src/md_json.c

@@ -107,12 +107,12 @@ void md_json_destroy(md_json_t *json)
     }
 }
 
-md_json_t *md_json_copy(apr_pool_t *pool, md_json_t *json)
+md_json_t *md_json_copy(apr_pool_t *pool, const md_json_t *json)
 {
     return json_create(pool, json_copy(json->j));
 }
 
-md_json_t *md_json_clone(apr_pool_t *pool, md_json_t *json)
+md_json_t *md_json_clone(apr_pool_t *pool, const md_json_t *json)
 {
     return json_create(pool, json_deep_copy(json->j));
 }
@@ -314,7 +314,7 @@ int md_json_is(const md_json_type_t jtype, md_json_t *json, ...)
     return 0;
 }
 
-static const char *md_json_type_name(md_json_t *json)
+static const char *md_json_type_name(const md_json_t *json)
 {
     json_t *j = json->j;
     if (json_is_object(j)) return "object";
@@ -463,6 +463,22 @@ md_json_t *md_json_getj(md_json_t *json, ...)
     return NULL;
 }
 
+md_json_t *md_json_dupj(apr_pool_t *p, const md_json_t *json, ...)
+{
+    json_t *j;
+    va_list ap;
+
+    va_start(ap, json);
+    j = jselect(json, ap);
+    va_end(ap);
+
+    if (j) {
+        json_incref(j);
+        return json_create(p, j);
+    }
+    return NULL;
+}
+
 const md_json_t *md_json_getcj(const md_json_t *json, ...)
 {
     json_t *j;
@@ -482,7 +498,7 @@ const md_json_t *md_json_getcj(const md_json_t *json, ...)
     return NULL;
 }
 
-apr_status_t md_json_setj(md_json_t *value, md_json_t *json, ...)
+apr_status_t md_json_setj(const md_json_t *value, md_json_t *json, ...)
 {
     va_list ap;
     apr_status_t rv;
@@ -510,7 +526,7 @@ apr_status_t md_json_setj(md_json_t *value, md_json_t *json, ...)
     return rv;
 }
 
-apr_status_t md_json_addj(md_json_t *value, md_json_t *json, ...)
+apr_status_t md_json_addj(const md_json_t *value, md_json_t *json, ...)
 {
     va_list ap;
     apr_status_t rv;
@@ -675,7 +691,7 @@ apr_status_t md_json_clone_to(void *value, md_json_t *json, apr_pool_t *p, void
     return md_json_setj(md_json_clone(p, value), json, NULL);
 }
 
-apr_status_t md_json_clone_from(void **pvalue, md_json_t *json, apr_pool_t *p, void *baton)
+apr_status_t md_json_clone_from(void **pvalue, const md_json_t *json, apr_pool_t *p, void *baton)
 {
     (void)baton;
     *pvalue = md_json_clone(p, json);
@@ -876,7 +892,7 @@ apr_status_t md_json_setsa(apr_array_header_t *a, md_json_t *json, ...)
 /* formatting, parsing */
 
 typedef struct {
-    md_json_t *json;
+    const md_json_t *json;
     md_json_fmt_t fmt;
     const char *fname;
     apr_file_t *f;
@@ -901,7 +917,7 @@ static int dump_cb(const char *buffer, size_t len, void *baton)
     return (rv == APR_SUCCESS)? 0 : -1;
 }
 
-apr_status_t md_json_writeb(md_json_t *json, md_json_fmt_t fmt, apr_bucket_brigade *bb)
+apr_status_t md_json_writeb(const md_json_t *json, md_json_fmt_t fmt, apr_bucket_brigade *bb)
 {
     int rv = json_dump_callback(json->j, dump_cb, bb, fmt_to_flags(fmt));
     return rv? APR_EGENERAL : APR_SUCCESS;
@@ -921,7 +937,7 @@ static int chunk_cb(const char *buffer, size_t len, void *baton)
     return 0;
 }
 
-const char *md_json_writep(md_json_t *json, apr_pool_t *p, md_json_fmt_t fmt)
+const char *md_json_writep(const md_json_t *json, apr_pool_t *p, md_json_fmt_t fmt)
 {
     apr_array_header_t *chunks;
     int rv;
@@ -944,7 +960,7 @@ const char *md_json_writep(md_json_t *json, apr_pool_t *p, md_json_fmt_t fmt)
     }
 }
 
-apr_status_t md_json_writef(md_json_t *json, apr_pool_t *p, md_json_fmt_t fmt, apr_file_t *f)
+apr_status_t md_json_writef(const md_json_t *json, apr_pool_t *p, md_json_fmt_t fmt, apr_file_t *f)
 {
     apr_status_t rv;
     const char *s;
@@ -963,7 +979,7 @@ apr_status_t md_json_writef(md_json_t *json, apr_pool_t *p, md_json_fmt_t fmt, a
     return rv;
 }
 
-apr_status_t md_json_fcreatex(md_json_t *json, apr_pool_t *p, md_json_fmt_t fmt, 
+apr_status_t md_json_fcreatex(const md_json_t *json, apr_pool_t *p, md_json_fmt_t fmt, 
                               const char *fpath, apr_fileperms_t perms)
 {
     apr_status_t rv;
@@ -987,7 +1003,7 @@ static apr_status_t write_json(void *baton, apr_file_t *f, apr_pool_t *p)
     return rv;
 }
 
-apr_status_t md_json_freplace(md_json_t *json, apr_pool_t *p, md_json_fmt_t fmt, 
+apr_status_t md_json_freplace(const md_json_t *json, apr_pool_t *p, md_json_fmt_t fmt, 
                               const char *fpath, apr_fileperms_t perms)
 {
     j_write_ctx ctx;
@@ -1171,7 +1187,7 @@ apr_status_t md_json_copy_to(md_json_t *dest, const md_json_t *src, ...)
     return rv;
 }
 
-const char *md_json_dump_state(md_json_t *json, apr_pool_t *p)
+const char *md_json_dump_state(const md_json_t *json, apr_pool_t *p)
 {
     if (!json) return "NULL";
     return apr_psprintf(p, "%s, refc=%ld", md_json_type_name(json), (long)json->j->refcount);

src/md_json.h

@@ -47,8 +47,8 @@ typedef enum {
 md_json_t *md_json_create(apr_pool_t *pool);
 void md_json_destroy(md_json_t *json);
 
-md_json_t *md_json_copy(apr_pool_t *pool, md_json_t *json);
-md_json_t *md_json_clone(apr_pool_t *pool, md_json_t *json);
+md_json_t *md_json_copy(apr_pool_t *pool, const md_json_t *json);
+md_json_t *md_json_clone(apr_pool_t *pool, const md_json_t *json);
 
 
 int md_json_has_key(const md_json_t *json, ...);
@@ -74,9 +74,10 @@ apr_status_t md_json_sets(const char *s, md_json_t *json, ...);
 
 /* json manipulation */
 md_json_t *md_json_getj(md_json_t *json, ...);
+md_json_t *md_json_dupj(apr_pool_t *p, const md_json_t *json, ...);
 const md_json_t *md_json_getcj(const md_json_t *json, ...);
-apr_status_t md_json_setj(md_json_t *value, md_json_t *json, ...);
-apr_status_t md_json_addj(md_json_t *value, md_json_t *json, ...);
+apr_status_t md_json_setj(const md_json_t *value, md_json_t *json, ...);
+apr_status_t md_json_addj(const md_json_t *value, md_json_t *json, ...);
 apr_status_t md_json_insertj(md_json_t *value, size_t index, md_json_t *json, ...);
 
 /* Array/Object manipulation */
@@ -96,7 +97,7 @@ apr_status_t md_json_pass_from(void **pvalue, md_json_t *json, apr_pool_t *p, vo
 
 /* conversions from json to json in specified pool */
 apr_status_t md_json_clone_to(void *value, md_json_t *json, apr_pool_t *p, void *baton);
-apr_status_t md_json_clone_from(void **pvalue, md_json_t *json, apr_pool_t *p, void *baton);
+apr_status_t md_json_clone_from(void **pvalue, const md_json_t *json, apr_pool_t *p, void *baton);
 
 /* Manipulating/Iteration on generic Arrays */
 apr_status_t md_json_geta(apr_array_header_t *a, md_json_from_cb *cb, 
@@ -118,13 +119,13 @@ apr_status_t md_json_dupsa(apr_array_header_t *a, apr_pool_t *p, md_json_t *json
 apr_status_t md_json_setsa(apr_array_header_t *a, md_json_t *json, ...);
 
 /* serialization & parsing */
-apr_status_t md_json_writeb(md_json_t *json, md_json_fmt_t fmt, struct apr_bucket_brigade *bb);
-const char *md_json_writep(md_json_t *json, apr_pool_t *p, md_json_fmt_t fmt);
-apr_status_t md_json_writef(md_json_t *json, apr_pool_t *p, 
+apr_status_t md_json_writeb(const md_json_t *json, md_json_fmt_t fmt, struct apr_bucket_brigade *bb);
+const char *md_json_writep(const md_json_t *json, apr_pool_t *p, md_json_fmt_t fmt);
+apr_status_t md_json_writef(const md_json_t *json, apr_pool_t *p, 
                             md_json_fmt_t fmt, struct apr_file_t *f);
-apr_status_t md_json_fcreatex(md_json_t *json, apr_pool_t *p, md_json_fmt_t fmt, 
+apr_status_t md_json_fcreatex(const md_json_t *json, apr_pool_t *p, md_json_fmt_t fmt, 
                               const char *fpath, apr_fileperms_t perms);
-apr_status_t md_json_freplace(md_json_t *json, apr_pool_t *p, md_json_fmt_t fmt, 
+apr_status_t md_json_freplace(const md_json_t *json, apr_pool_t *p, md_json_fmt_t fmt, 
                               const char *fpath, apr_fileperms_t perms);
 
 apr_status_t md_json_readb(md_json_t **pjson, apr_pool_t *pool, struct apr_bucket_brigade *bb);
@@ -140,7 +141,7 @@ apr_status_t md_json_read_http(md_json_t **pjson, apr_pool_t *pool,
 
 apr_status_t md_json_copy_to(md_json_t *dest, const md_json_t *src, ...);
 
-const char *md_json_dump_state(md_json_t *json, apr_pool_t *p);
+const char *md_json_dump_state(const md_json_t *json, apr_pool_t *p);
 
 apr_status_t md_json_timeperiod_set(struct md_timeperiod_t *tp, md_json_t *json, ...);
 

src/md_ocsp.c

@@ -242,7 +242,7 @@ static apr_status_t ocsp_status_refresh(md_ocsp_status_t *ostat, apr_pool_t *pte
     md_data_t resp_der;
     md_timeperiod_t resp_valid;
     md_ocsp_cert_stat_t resp_stat;
-
+    /* Check if the store holds a newer response than the one we have */
     mtime = md_store_get_modified(store, MD_SG_OCSP, ostat->md_name, ostat->file_name, ptemp);
     if (mtime <= ostat->resp_mtime) goto leave;
     rv = md_store_load_json(store, MD_SG_OCSP, ostat->md_name, ostat->file_name, &jprops, ptemp);
@@ -402,8 +402,7 @@ apr_status_t md_ocsp_get_status(unsigned char **pder, int *pderlen,
     *pder = NULL;
     *pderlen = 0;
     if (ostat->resp_der.len <= 0) {
-        /* No resonse known, check the store if out watchdog retrieved one 
-         * in the meantime. */
+        /* No response known, check store for new response. */
         ocsp_status_refresh(ostat, p);
         if (ostat->resp_der.len <= 0) {
             md_log_perror(MD_LOG_MARK, MD_LOG_DEBUG, 0, reg->p,