From f8fa5013d44d0f59d40500d595ff2d6e27d69489 Mon Sep 17 00:00:00 2001 From: stranger80 Date: Mon, 8 May 2023 22:25:58 +0200 Subject: [PATCH 1/2] Updating GCRS with latest state of GAP-4 --- standards/2-service/srv/comp/manifest.md | 128 ++++++++++++++++++ standards/2-service/srv/comp/manifest/net.md | 45 ++++++ .../2-service/srv/comp/manifest/script.md | 60 ++++++++ standards/cheat_sheet.md | 44 +++++- 4 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 standards/2-service/srv/comp/manifest.md create mode 100644 standards/2-service/srv/comp/manifest/net.md create mode 100644 standards/2-service/srv/comp/manifest/script.md diff --git a/standards/2-service/srv/comp/manifest.md b/standards/2-service/srv/comp/manifest.md new file mode 100644 index 00000000..3a46a86a --- /dev/null +++ b/standards/2-service/srv/comp/manifest.md @@ -0,0 +1,128 @@ +# Computation Manifest +This namespace defines properties used to specify the Golem Computation Manifest (as originally designed in [GAP-4](https://github.com/golemfactory/golem-architecture/blob/master/gaps/gap-4_comp_manifest/gap-4_comp_manifest.md)). + +## Computation Manifest Example + +```json +{ + "golem.srv.comp.manifest.script.match": "regex", + "golem.srv.comp.manifest.script.commands": [ + "run /bin/cat /etc/motd", + "{\"run\": { \"args\": \"/bin/date -R\", \"env\": { \"MYVAR\": \"42\", \"match\": \"strict\" }}}" + ], + "golem.srv.comp.manifest.net.inet.out.protocols": [ + "http", + "https" + ], + "golem.srv.comp.manifest.net.inet.out.urls": [ + "http://golemfactory.s3.amazonaws.com/file1", + "http://golemfactory.s3.amazonaws.com/file2" + ] +} +``` + +### Object representation + +1. JSON + +```json +{ + "script": { + "match": "regex", + "commands": [ + "run /bin/cat /etc/motd", + { + "run": { + "args": "/bin/date -R", + "env": { + "MYVAR": "42" + }, + "match": "strict" + } + } + ] + }, + "net": { + "inet": { + "out": { + "protocols": [ + "http", + "https" + ], + "urls": [ + "http://golemfactory.s3.amazonaws.com/file1", + "http://golemfactory.s3.amazonaws.com/file2" + ] + } + } + } +} +``` + +2. Imploded JSON (optional) + +```json +{ + "script.match": "regex", + "script.commands": [ + "run /bin/cat /etc/motd", + { + "run": { + "args": "/bin/date -R", + "env": { + "MYVAR": "42" + }, + "match": "strict" + } + } + ], + "net.inet.out.protocols": [ + "http", + "https" + ], + "net.inet.out.urls": [ + "http://golemfactory.s3.amazonaws.com/file1", + "http://golemfactory.s3.amazonaws.com/file2" + ] +} +``` + +3. YAML + +```yaml +--- +script: + match: regex + commands: + - run /bin/cat /etc/motd + - run: + args: "/bin/date -R" + env: + MYVAR: '42' + match: strict +net: + inet: + out: + protocols: + - http + - https + urls: + - http://golemfactory.s3.amazonaws.com/file1 + - http://golemfactory.s3.amazonaws.com/file2 +``` + +## Common Properties + +N/A + +## Specific Properties + +## `golem.srv.comp.manifest.version : String` + +### Describes: Demand + +Specifies a version (Semantic Versioning 2.0 specification) of the manifest, **defaults** to "0.1.0" + +### **Examples** +* `golem.srv.comp.manifest.version="0.1.0"` - The manifest version is 0.1.0. + diff --git a/standards/2-service/srv/comp/manifest/net.md b/standards/2-service/srv/comp/manifest/net.md new file mode 100644 index 00000000..dfd66825 --- /dev/null +++ b/standards/2-service/srv/comp/manifest/net.md @@ -0,0 +1,45 @@ +# Computation Manifest Net namespace +This namespace defines properties used to specify details the Golem Computation Manifest network aspects. Applies constraints to networking. Currently, outgoing requests to the public Internet network are covered. + +## Common Properties + +N/A + +## Specific Properties + +## `golem.srv.comp.manifest.net.inet.out.protocols : List[String]` + +### Describes: Demand + +List of allowed outbound protocols. Currently **fixed at** `["http", "https"]`. + +### **Examples** +* `golem.srv.comp.manifest.net.inet.out.protocols=["http","https"]` - HTTP/HTTPS allowed. + + +## `golem.srv.comp.manifest.net.inet.out.urls : List[String]` + +### Describes: Demand + +List of allowed external URLs that outbound requests can be sent to. + +If unrestricted outbound access is requested this property must not be set. + + +### **Examples** +* `golem.srv.comp.manifest.net.inet.out.urls=["http://golemfactory.s3.amazonaws.com/file1", "http://golemfactory.s3.amazonaws.com/file2"]` + + +## `golem.srv.comp.manifest.net.inet.out.unrestricted.urls : Boolean` + +### Describes: Demand + +This property means that the payload requires unrestricted outbound access. When present the value is always `true`. Either this property or the URL list in `golem.srv.comp.manifest.net.inet.out.urls` must be present. + +The manifest must be considered invalid and outbound access should not be permitted in the following scenarios: +- neither `golem.srv.comp.manifest.net.inet.out.unrestricted.urls` nor `golem.srv.comp.manifest.net.inet.out.urls` is present +- both `golem.srv.comp.manifest.net.inet.out.unrestricted.urls` and `golem.srv.comp.manifest.net.inet.out.urls` are present +- property `golem.srv.comp.manifest.net.inet.out.unrestricted.urls` is present and it contains a value different from `true` + +### **Examples** +* `golem.srv.comp.manifest.net.inet.out.unrestricted.urls=true` diff --git a/standards/2-service/srv/comp/manifest/script.md b/standards/2-service/srv/comp/manifest/script.md new file mode 100644 index 00000000..d0108a9d --- /dev/null +++ b/standards/2-service/srv/comp/manifest/script.md @@ -0,0 +1,60 @@ +# Computation Manifest Script namespace +This namespace defines properties used to specify details the Golem Computation Manifest ExeScript allowance. Defines a set of allowed ExeScript commands and applies constraints to their arguments. + +## Common Properties + +## Specific Properties + +## `golem.srv.comp.manifest.script.commands : List[String]` + +### Describes: Demand + +Specifies a curated list of commands in form of: + +- UTF-8 encoded strings + + No command context or matching mode need to be specified. + + E.g. `["run /bin/cat /etc/motd", "run /bin/date -R"]` + +- UTF-8 encoded JSON strings + + Command context (e.g. `env`) or argument matching mode need to be + specified for a command. + + E.g. `["{\"run\": { \"args\": \"/bin/date -R\", \"env\": { \"MYVAR\": \"42\", \"match\": \"strict\" }}}"]` + +- mix of both + +`"deploy"`, `"start"` and `"terminate'` commands are always allowed. +These values become the **default** if no `manifest.script.command` property +has been set in the Demand, but the `manifest` namespace is present. + +### **Examples** +* `golem.srv.comp.manifest.script.commands=[ + "run /bin/cat /etc/motd", + "{\"run\": { \"args\": \"/bin/date -R\", \"env\": { \"MYVAR\": \"42\", \"match\": \"strict\" }}}" + ]` + + +## `golem.srv.comp.manifest.script.match : String` + +### Describes: Demand + +Selects a default way of comparing command arguments stated in the manifest +and the ones received in the ExeScript, unless stated otherwise in a +command JSON object. + +### Value enum +|Value| Description | +|---|---| +|`strict`| byte-to-byte argument equality (**default**) | +|`regex`| treat arguments as regular expressions | + +`regex` syntax: Perl-compatible regular expressions (UTF-8 Unicode mode), +w/o the support for look around and backreferences (among others); +for more information read the documentation of the Rust +[regex](https://docs.rs/regex/latest/regex/) crate. + +### **Examples** +* `golem.srv.comp.manifest.script.match="regex"` - The manifest version is 0.1.0. diff --git a/standards/cheat_sheet.md b/standards/cheat_sheet.md index c398e98f..1f830517 100644 --- a/standards/cheat_sheet.md +++ b/standards/cheat_sheet.md @@ -23,6 +23,9 @@ This page contains an aggregated summary of all namespaces and properties specif * [eth](2-service/srv/app/eth.md) * [caps](2-service/srv/caps.md) * [comp](2-service/srv/comp.md) + * [manifest](2-service/srv/comp/manifest.md) + * [net](2-service/srv/comp/manifest/net.md) + * [script](2-service/srv/comp/manifest/script.md) * [wasm](2-service/srv/comp/wasm.md) ### 3-commercial * com @@ -54,6 +57,9 @@ Specifications of CPU computing power assigned to a service. | Property | Type | Applies to | Category | Description | |---|---|---|---|---| |**`golem.inf.cpu.architecture`**|`String`|Offer||CPU architecture. | +|**`golem.inf.cpu.vendor`**|`String`|Offer||CPU vendor. | +|**`golem.inf.cpu.brand`**|`String`|Offer||CPU brand, human-readable. | +|**`golem.inf.cpu.model`**|`String`|Offer||CPU stepping, family and model. | |**`golem.inf.cpu.cores`**|`Number (int32)`|Offer||Total number of CPU cores assigned to service. It is a sum of CPU cores possibly from multiple CPUs. | |**`golem.inf.cpu.threads`**|`Number (int32)`|Offer||Total number of CPU threads assigned to service. It is a sum of CPU threads possibly from multiple CPUs and cores. | |**`golem.inf.cpu.capabilities`**|`List[String]`|Offer||CPU capability flags. For x86 architectures this property is populated with CPU features as returned by CPUID instruction. For full list, see here: https://github.com/golemfactory/ya-runtime-vm/blob/master/runtime/src/cpu.rs#L59 | @@ -155,10 +161,46 @@ Generic properties describing the Computation Platform aspects. | Property | Type | Applies to | Category | Description | |---|---|---|---|---| -|**`golem.srv.comp.expiration`**|`Number (int32)`|Demand||Indicates the expiration time of the Agreement which is being negotiated. This is expressed as Javascript timestamp (number of milliseconds since 1970-01-01 00:00:00 UTC, as returned by `Date.now()`) | +|**`golem.srv.comp.expiration`**|`Number (int32)`|Demand||Indicates the expiration time of the Agreement which is being negotiated. This is expressed as Javascript timestamp (number of milliseconds since 1970-01-01 00:00:00 UTC, as returned by `Date.now()`). After this time both sides are allowed to terminate the Agreement; and Provider actually does that. | |**`golem.srv.comp.task_package`**|`String`|Demand||Indicates the URI of a package/binary which is to be executed by the Provider. This is a generic property, which, however, may be interpreted differently per each Computation Platform. Therefore, in a Computation Platform-specific namespace it is expected to specify the semantics of `golem.srv.comp.task_package` property for that Platform. | --- +## [`srv.comp.manifest`](2-service/srv/comp/manifest.md) + +This namespace defines properties used to specify the Golem Computation Manifest (as originally designed in [GAP-4](https://github.com/golemfactory/golem-architecture/blob/master/gaps/gap-4_comp_manifest/gap-4_comp_manifest.md)). + +### Properties + +| Property | Type | Applies to | Category | Description | +|---|---|---|---|---| +|**`golem.srv.comp.manifest.version`**|`String`|Demand||Specifies a version (Semantic Versioning 2.0 specification) of the manifest, **defaults** to "0.1.0" | +--- + +## [`srv.comp.manifest.net`](2-service/srv/comp/manifest/net.md) + +This namespace defines properties used to specify details the Golem Computation Manifest network aspects. Applies constraints to networking. Currently, outgoing requests to the public Internet network are covered. + +### Properties + +| Property | Type | Applies to | Category | Description | +|---|---|---|---|---| +|**`golem.srv.comp.manifest.net.inet.out.protocols`**|`List[String]`|Demand||List of allowed outbound protocols. Currently **fixed at** `["http", "https"]`. | +|**`golem.srv.comp.manifest.net.inet.out.urls`**|`List[String]`|Demand||List of allowed external URLs that outbound requests can be sent to. | +|**`golem.srv.comp.manifest.net.inet.out.unrestricted.urls`**|`Boolean`|Demand||This property means that the payload requires unrestricted outbound access. When present the value is always `true`. Either this property or the URL list in `golem.srv.comp.manifest.net.inet.out.urls` must be present. | +--- + +## [`srv.comp.manifest.script`](2-service/srv/comp/manifest/script.md) + +This namespace defines properties used to specify details the Golem Computation Manifest ExeScript allowance. Defines a set of allowed ExeScript commands and applies constraints to their arguments. + +### Properties + +| Property | Type | Applies to | Category | Description | +|---|---|---|---|---| +|**`golem.srv.comp.manifest.script.commands`**|`List[String]`|Demand||Specifies a curated list of commands in form of: | +|**`golem.srv.comp.manifest.script.match`**|`String`|Demand||Selects a default way of comparing command arguments stated in the manifest and the ones received in the ExeScript, unless stated otherwise in a command JSON object. | +--- + ## [`srv.comp.wasm`](2-service/srv/comp/wasm.md) This namespace defines properties used to indicate ability to host and execute a WebAssembly program. From 928bb0a1868e87925cd880174e0dd60be3c59977 Mon Sep 17 00:00:00 2001 From: stranger80 Date: Mon, 8 May 2023 22:36:13 +0200 Subject: [PATCH 2/2] GAP-5 properties added --- standards/2-service/srv/caps.md | 9 +++ standards/2-service/srv/comp/payload.md | 90 +++++++++++++++++++++++++ standards/cheat_sheet.md | 16 +++++ 3 files changed, 115 insertions(+) create mode 100644 standards/2-service/srv/comp/payload.md diff --git a/standards/2-service/srv/caps.md b/standards/2-service/srv/caps.md index 655a2c59..83646f4a 100644 --- a/standards/2-service/srv/caps.md +++ b/standards/2-service/srv/caps.md @@ -12,3 +12,12 @@ Indicates the that the Provider supports the multi-activity Agreements. ### **Examples** * `golem.srv.caps.multi-activity=true` - The Provider supports multi-activity Agreements. +## `golem.srv.caps.payload-manifest : Boolean` + +### Describes: Offer + +Providers need to declare that they support Payload Manifests by setting this property to `true`. + +### **Examples** +* `golem.srv.caps.payload-manifest=true` - The Provider supports Payload Manifests. + diff --git a/standards/2-service/srv/comp/payload.md b/standards/2-service/srv/comp/payload.md new file mode 100644 index 00000000..05eaedc3 --- /dev/null +++ b/standards/2-service/srv/comp/payload.md @@ -0,0 +1,90 @@ +# Payload Manifest +This namespace defines properties used to specify the Golem Payload Manifest (as originally designed in [GAP-5](https://github.com/golemfactory/golem-architecture/blob/master/gaps/gap-5_payload_manifest/gap-5_payload_manifest.md)). + +Computation Payload Manifest signatures are verified by either the Provider Agent, the ExeUnit Supervisor or both. +Payload and Computation manifests are not expected to have constraints put on them. + +### Payload Manifest example + +```json + { + "version": "0.1.0", + "createdAt": "2020-12-12T12:12:12.1200012", + "expiresAt": "2022-12-12T12:12:12.1200012", + + "metadata": { + "name": "Service1", + "description": "Description of Service1", + "version": "0.1.1", + "authors": [ + "mf ", + "ng " + ], + "homepage": "https://github.com/golemfactory/s1" + }, + + "payload": [ + { + "platform": { + "arch": "amd64", + "os": "win32", + "osVersion": "6.1.7601" + }, + "urls": [ + "https://golemfactory-payloads.s3.amazonaws.com/payloads/s1-amd64-win32", + "ipfs://Qa.........." + ], + "hash": "sha3-224:deadbeef01" + }, + { + "platform": { + "arch": "ARMv7E-M", + "os": "linux" + }, + "urls": [ + "https://golemfactory-payloads.s3.amazonaws.com/payloads/s1-armv7e-m", + "ipfs://Qb.........." + ], + "hash": "sha3-224:deadbeef02" + } + ], + + "compManifest": {} + } +``` + +`version` and `metadata.version` follow SemVer 2.0 specification. + +## Common Properties + +N/A + +## Specific Properties + +## `golem.srv.comp.payload: String` + +### Describes: Demand + +Base64-encoded JSON manifest. + +## `golem.srv.comp.payload.sig: String` + +### Describes: Demand + +Base64-encoded signature of the base64-encoded manifest. + +## `golem.srv.comp.payload.sig.algorithm: String` + +### Describes: Demand + +Digest algorithm used to generate manifest signature. + +## `golem.srv.comp.payload.cert: String` + +### Describes: Demand + +Base64-encoded certificate in DER format. + + + + diff --git a/standards/cheat_sheet.md b/standards/cheat_sheet.md index 1f830517..40ec67e7 100644 --- a/standards/cheat_sheet.md +++ b/standards/cheat_sheet.md @@ -26,6 +26,7 @@ This page contains an aggregated summary of all namespaces and properties specif * [manifest](2-service/srv/comp/manifest.md) * [net](2-service/srv/comp/manifest/net.md) * [script](2-service/srv/comp/manifest/script.md) + * [payload](2-service/srv/comp/payload.md) * [wasm](2-service/srv/comp/wasm.md) ### 3-commercial * com @@ -151,6 +152,7 @@ Namespace that describes capabilities of a Golem service. | Property | Type | Applies to | Category | Description | |---|---|---|---|---| |**`golem.srv.caps.multi-activity`**|`Boolean`|Demand/Offer||Indicates the that the Provider supports the multi-activity Agreements. | +|**`golem.srv.caps.payload-manifest`**|`Boolean`|Offer||Providers need to declare that they support Payload Manifests by setting this property to `true`. | --- ## [`srv.comp`](2-service/srv/comp.md) @@ -201,6 +203,20 @@ This namespace defines properties used to specify details the Golem Computation |**`golem.srv.comp.manifest.script.match`**|`String`|Demand||Selects a default way of comparing command arguments stated in the manifest and the ones received in the ExeScript, unless stated otherwise in a command JSON object. | --- +## [`srv.comp.payload`](2-service/srv/comp/payload.md) + +This namespace defines properties used to specify the Golem Payload Manifest (as originally designed in [GAP-5](https://github.com/golemfactory/golem-architecture/blob/master/gaps/gap-5_payload_manifest/gap-5_payload_manifest.md)). + +### Properties + +| Property | Type | Applies to | Category | Description | +|---|---|---|---|---| +|**`golem.srv.comp.payload`**|`String`|Demand||Base64-encoded JSON manifest. | +|**`golem.srv.comp.payload.sig`**|`String`|Demand||Base64-encoded signature of the base64-encoded manifest. | +|**`golem.srv.comp.payload.sig.algorithm`**|`String`|Demand||Digest algorithm used to generate manifest signature. | +|**`golem.srv.comp.payload.cert`**|`String`|Demand||Base64-encoded certificate in DER format. | +--- + ## [`srv.comp.wasm`](2-service/srv/comp/wasm.md) This namespace defines properties used to indicate ability to host and execute a WebAssembly program.