nvme: Add support for automatic APST payload generation

APST data can be generated automatically based on the apst_max_latency
tunable.  This method offers the same simple configuration as on
Linux but allows for setting an upper latency limit only.

Signed-off-by: Alexey Sukhoguzov <[email protected]>

share/man/man4/nvme.4

@@ -156,6 +156,15 @@ to NOPS 4.
 For more details, please refer to the Autonomous Power State
 Transition section in the NVM Express Base Specification.
 .Pp
+In addition to manually overriding APST table, it is possible to
+specify only the maximum total latency (i.e., enter + exit) in
+milliseconds to generate transition data automatically.
+In this case, the controller will be instructed to sequentially
+switch through all NOPS, excluding those with higher latency:
+.Bd -literal -offset indent
+hw.nvme.apst_max_latency
+.Ed
+.Pp
 The
 .Xr nvd 4
 driver is used to provide a disk driver to the system by default.

sys/dev/nvme/nvme_ctrlr.c

@@ -887,14 +887,42 @@ nvme_ctrlr_configure_int_coalescing(struct nvme_controller *ctrlr)
 	    ctrlr->int_coal_threshold, NULL, NULL);
 }
 
+static void
+nvme_ctrlr_apst_gen_data(uint64_t *data, size_t data_size,
+    int max_latency, struct nvme_power_state *ps, uint8_t npss)
+{
+	uint64_t latency;
+	uint32_t itpt;
+	int i;
+
+	KASSERT(npss < data_size / sizeof(*data),
+	    ("%s: npss=%d, but data_size=%ld\n", __func__, npss, data_size));
+
+	for (i = npss; i > 0; --i) {
+		/* The power state to transition to shall be a NOPS. */
+		if (!NVMEV(NVME_PWR_ST_NOPS, ps[i].mps_nops)) {
+			data[i - 1] = data[i];
+			continue;
+		}
+
+		latency = ps[i].enlat + ps[i].exlat;
+		if (latency > max_latency)
+			continue;
+
+		/* ITPT (24-bit) is 50x the latency in milliseconds. */
+		itpt = MIN(latency * 50 / 1000, (1 << 24) - 1);
+		data[i - 1] = htole64(itpt << 8 | i << 3);
+	}
+}
+
 static void
 nvme_ctrlr_configure_apst(struct nvme_controller *ctrlr)
 {
 	struct nvme_completion_poll_status status;
 	uint64_t *data;
 	size_t data_size;
-	int bytes_read, i;
-	char *env_data;
+	int bytes_read, i, max_latency;
+	char *env_data, *env_mlat;
 	bool enable;
 
 	if (TUNABLE_BOOL_FETCH("hw.nvme.apst_enable", &enable) == 0 ||
@@ -906,13 +934,19 @@ nvme_ctrlr_configure_apst(struct nvme_controller *ctrlr)
 		goto fail;
 
 	env_data = "hw.nvme.apst_data";
+	env_mlat = "hw.nvme.apst_max_latency";
 
 	if (testenv(env_data) != 0) {
 		if (getenv_array(env_data, data, data_size, &bytes_read,
 		    sizeof(*data), GETENV_UNSIGNED) == 0)
 			goto fail;
 		for (i = 0; i < bytes_read / sizeof(*data); ++i)
 			data[i] = htole64(data[i]);
+	} else if (testenv(env_mlat) != 0) {
+		if (TUNABLE_INT_FETCH(env_mlat, &max_latency) == 0)
+			goto fail;
+		nvme_ctrlr_apst_gen_data(data, data_size, max_latency,
+		    ctrlr->cdata.power_state, ctrlr->cdata.npss);
 	} else if (enable) {
 		status.done = 0;
 		nvme_ctrlr_cmd_get_feature(ctrlr,