refactor: change approach to generating schema.json files

Closes #41

CONTRIBUTING.md

@@ -65,3 +65,20 @@ Just as for types, properties of types can be linked to the other contexts using
       '@id': schema:address
       type: string
 ```
+
+#### The `role` keyword
+
+It is RECOMMENDED to add a `role` property to the schema:
+
+- `base`: base types, not usually instantiated but required for other types e.g `Thing`
+- `primary`: types that are usually the root of a trre generated from a file e.g. `Article`, `Datatable`, `Collection`
+- `secondary`: types usually only referred to by primary types e.g. `Organization` is used for the `publisher` property on a `Article`
+- `tertiary`: types usually only referred to by secondary types e.g. `ContactPoint` is used for the `contactPoints` property on an `Organisation`
+
+#### The `status` keyword
+
+It is RECOMMENDED to add a `status` property to indicate the development status for a type schema e.g.
+
+- `experimental`: new types (i.e. not defined on schema.org or elsewhere) that are still under development
+- `unstable`: types that are defined elsewhere (e.g. https://schema.org/SoftwareApplication) but for which the schema is still being developed
+- `stable`: types for which the schema definition can be considered stable

Gulpfile.js

@@ -11,26 +11,101 @@ const rename = require('gulp-rename')
 const through2 = require('through2')
 const yaml = require('js-yaml')
 
+/**
+ * Generate a `dist/*.schema.json` files from a `schema/*.schema.{yaml,json}` file
+ *
+ * This function processes hand written YAML or JSON schema definitions with the aim
+ * of making schema authoring less tedious and error prone. Including:
+ *
+ * - `$schema`: set to `http://json-schema.org/draft-07/schema#` if not specified
+ * - `$id`: set to `https://stencila.github.com/schema/${schema.title}.schema.json` if not specified
+ * - `properties.*.from`: set to `schema.title`
+ * - `properties.type.enum[0]`: set to `schema.title`
+ * - `additionalProperties`: set to `false` if not specified
+ * - `category`: the category (subdirectory) the schema belongs to
+ *
+ * If the schema defines the `$extends` keyword then in addition:
+ *
+ * - `extends`: the title of the base
+ * - `properties`: merged from bases and current schema
+ * - `required`: merged from bases and current schema
+ */
+function processSchema(filePath) {
+  const schema = yaml.safeLoad(fs.readFileSync(filePath))
+
+  // Do not touch schemas that do not have a title
+  if (!schema.title) return schema
+
+  if (!schema.$schema)
+    schema.$schema = `http://json-schema.org/draft-07/schema#`
+  if (!schema.$id)
+    schema.$id = `https://stencila.github.com/schema/${schema.title}`
+  schema.category = path.relative(
+    path.join(__dirname, 'schema'),
+    path.dirname(filePath)
+  )
+
+  if (schema.properties) {
+    for (let index in schema.properties) {
+      schema.properties[index].from = schema.title
+    }
+  }
+
+  const $extends = schema.$extends
+  if ($extends) {
+    const basePath = path.join(path.dirname(filePath), $extends)
+    if (!fs.existsSync(basePath)) {
+      throw new Error(
+        `When processing "$extends" file not found: "${basePath}"`
+      )
+    }
+    const base = processSchema(basePath)
+    schema.properties = { ...base.properties, ...(schema.properties || {}) }
+    schema.required = [...base.required, ...(schema.required || [])]
+    schema.$extends = base.title
+  }
+
+  if (schema.properties) {
+    schema.type = 'object'
+    if (schema.properties.type && Array.isArray(schema.properties.type.enum)) {
+      schema.properties.type.enum[0] = schema.title
+    }
+    if (schema.additionalProperties === undefined) {
+      schema.additionalProperties = false
+    }
+  }
+
+  return schema
+}
+
 /**
  * Generate `dist/*.schema.json` files from `schema/*.schema.{yaml,json}` files
  */
 function jsonschema() {
   return src('schema/**/*.schema.{yaml,json}')
     .pipe(
       through2.obj((file, enc, cb) => {
-        // Load YAML...
-        const schema = yaml.safeLoad(file.contents)
-        // Replace any `$ref`s to YAML to the JSOn (which will be
-        // in turn generated by this)
+        let schema
+        try {
+          schema = processSchema(file.path)
+        } catch (error) {
+          throw new Error(
+            `Error when processing "${file.path}": "${error.stack}"`
+          )
+        }
+
+        // Replace any `$ref`s to YAML with a ref to the JSON generated
+        // in this function
         function walk(node) {
           if (typeof node !== 'object') return
           for (let [key, child] of Object.entries(node)) {
             if (key === '$ref' && typeof child === 'string')
-              node[key] = child.replace('.yaml', '.json')
+              node[key] = path.basename(child).replace('.yaml', '.json')
             walk(child)
           }
         }
-        walk(schema)
+        if (file.extname === '.yaml') walk(schema)
+
         // Convert to JSON
         file.contents = Buffer.from(JSON.stringify(schema, null, '  '))
 

examples/organization/simple.organization.yaml

@@ -15,8 +15,7 @@ contactPoint:
   telephone:
     - '+649775673'
 departments:
-  - type: organization
-      - department.organization.yaml
+  - type: Organization
 email: [email protected]
 funders:
   - type: Person

examples/software/simple.softwareApplication.yaml

@@ -2,7 +2,7 @@ type: SoftwareApplication
 softwareRequirements:
   - type: SoftwareApplication
     name: SomePackage
-    version: 1.2.3
+    softwareVersion: 1.2.3
   - type: SoftwareApplication
     name: SomeOtherPackage
-    version: 3.2.1
+    softwareVersion: 3.2.1

examples/software/simple.softwareSession.yaml

@@ -1,15 +1,16 @@
 type: SoftwareSession
 environment:
-  - name: StencilaMega
+  type: Environment
+  name: StencilaMega
 volumeMounts:
-  - type: VolumeMount
+  - type: Mount
     mountSource: /foo/bar
     mountDestination: /baz/bax
 cpuResource:
   type: ResourceParameters
-  requested: 500
-  limit: 1000
+  resourceRequested: 500
+  resourceLimit: 1000
 memoryResource:
   type: ResourceParameters
-  requested: 1024
-  limit: 2048
+  resourceRequested: 1024
+  resourceLimit: 2048

schema/Article.schema.yaml

@@ -1,12 +1,14 @@
-$schema: http://json-schema.org/draft-07/schema#
-$id: https://stencila.github.com/schema/Article.schema.json
 title: Article
 '@id': schema:Article
-allOf:
-  - $ref: CreativeWork.schema.yaml
-  - properties:
-      content:
-        '@id': stencila:content
-        type: array
-        items:
-          $ref: Node.schema.yaml
+$extends: CreativeWork.schema.yaml
+properties:
+  content:
+    '@id': stencila:content
+    description: The main content of the article. Equivalent to https://schema.org/articleBody.
+    type: array
+    items:
+      $ref: Node.schema.yaml
+required:
+  # See Google's list of required properties at
+  # https://developers.google.com/search/docs/data-types/article#article_types
+  - authors

schema/Collection.schema.yaml

@@ -1,18 +1,11 @@
-$schema: http://json-schema.org/draft-07/schema#
-$id: https://stencila.github.com/schema/Collection.schema.json
 title: Collection
 '@id': schema:Collection
-allOf:
-  - $ref: CreativeWork.schema.yaml
-  - properties:
-      type:
-        type: string
-        enum: [Collection]
-      parts:
-        '@id': schema:hasParts
-        type: array
-        items:
-          $ref: CreativeWork.schema.yaml
+$extends: CreativeWork.schema.yaml
+properties:
+  parts:
+    '@id': schema:hasParts
+    type: array
+    items:
+      $ref: CreativeWork.schema.yaml
 required:
-  - type
   - parts

schema/CreativeWork.schema.yaml

@@ -1,32 +1,31 @@
-$schema: http://json-schema.org/draft-07/schema#
-$id: https://stencila.github.com/schema/CreativeWork.schema.json
 title: CreativeWork
-description: The most generic kind of creative work, including books, movies, photographs, software programs, etc. https://schema.org/CreativeWork
 '@id': schema:CreativeWork
-allOf:
-  - $ref: Thing.schema.yaml
-  - properties:
-      authors:
-        '@id': schema:author
-        description: The authors of this content.
-        type: array
-        items:
-          anyOf:
-            - $ref: Person.schema.yaml
-            - $ref: Organization.schema.yaml
-      citations:
-        '@id': schema:citation
-        description: Citations or references to another creative work, such as another publication, web page, scholarly article, etc.
-        type: array
-        items:
-          anyOf:
-            - type: string
-            - $ref: CreativeWork.schema.yaml
-      licenses:
-        '@id': schema:license
-        description: License documents that applies to this content, typically indicated by URL.
-        type: array
-        items:
-          anyOf:
-            - $ref: URL.schema.yaml
-            - $ref: CreativeWork.schema.yaml
+$extends: Thing.schema.yaml
+role: base
+status: unstable
+description: The most generic kind of creative work, including books, movies, photographs, software programs, etc. https://schema.org/CreativeWork
+properties:
+  authors:
+    '@id': schema:author
+    description: The authors of this content.
+    type: array
+    items:
+      anyOf:
+        - $ref: Person.schema.yaml
+        - $ref: Organization.schema.yaml
+  citations:
+    '@id': schema:citation
+    description: Citations or references to another creative work, such as another publication, web page, scholarly article, etc.
+    type: array
+    items:
+      anyOf:
+        - type: string
+        - $ref: CreativeWork.schema.yaml
+  licenses:
+    '@id': schema:license
+    description: License documents that applies to this content, typically indicated by URL.
+    type: array
+    items:
+      anyOf:
+        - $ref: URL.schema.yaml
+        - $ref: CreativeWork.schema.yaml

schema/Document.schema.yaml

@@ -1,19 +1,17 @@
-$schema: http://json-schema.org/draft-07/schema#
-$id: https://stencila.github.com/schema/Document.schema.json
 title: Document
 '@id': stencila:Document
+$extends: CreativeWork.schema.yaml
+status: experimental
 description: |
   An executable document.
-allOf:
-  - $ref: CreativeWork.schema.yaml
-  - properties:
-      environment:
-        '@id': stencila:environment
-        description: |
-          The computational environment in which the document should be executed.
-        allOf:
-          - $ref: Environment.schema.yaml
-      body:
-        '@id': stencila:body
-        description: The body of the document.
-        type: array
+properties:
+  environment:
+    '@id': stencila:environment
+    description: |
+      The computational environment in which the document should be executed.
+    allOf:
+      - $ref: Environment.schema.yaml
+  body:
+    '@id': stencila:body
+    description: The body of the document.
+    type: array

schema/ImageObject.schema.yaml

@@ -1,14 +1,13 @@
-$schema: http://json-schema.org/draft-07/schema#
-$id: https://stencila.github.com/schema/ImageObject.schema.json
 title: ImageObject
 '@id': schema:ImageObject
+$extends: MediaObject.schema.yaml
+status: unstable
+role: supporting
 description: |
   An image file. https://schema.org/ImageObject
-allOf:
-  - $ref: MediaObject.schema.yaml
-  - properties:
-      caption:
-        '@id': schema:caption
-        description: |
-          The caption for this object.
-        type: string
+properties:
+  caption:
+    '@id': schema:caption
+    description: |
+      The caption for this object.
+    type: string

schema/Include.schema.yaml

@@ -1,10 +1,9 @@
-$schema: http://json-schema.org/draft-07/schema#
-$id: https://stencila.github.com/schema/Include.schema.json
 title: Include
 '@id': stencila:Include
 description: |
   A directive to include content from an external source (e.g. file, URL) or content.
 type: object
+status: experimental
 properties:
   type:
     description: The type of node.
@@ -28,9 +27,6 @@ properties:
     oneOf:
       - $ref: Datatable.schema.yaml
       - $ref: Table.schema.yaml
-additionalProperties: false
-required:
-  - type
 # Either needs to have `source` specified (from which `content` and `mimetype` are derived)
 # or have `content` and `mimetype` specified.
 anyOf:

schema/MediaObject.schema.yaml

@@ -1,15 +1,14 @@
-$schema: http://json-schema.org/draft-07/schema#
-$id: https://stencila.github.com/schema/MediaObject.schema.json
 title: MediaObject
 '@id': schema:MediaObject
+$extends: CreativeWork.schema.yaml
+role: base
+status: unstable
 description: |
   A media object, such as an image, video, or audio object embedded in a web page or a
   downloadable dataset. https://schema.org/MediaObject
-allOf:
-  - $ref: CreativeWork.schema.yaml
-  - properties:
-      contentUrl:
-        '@id': schema:contentUrl
-        description: Actual bytes of the media object, for example the image file or video file.
-        anyOf:
-          - $ref: URL.schema.yaml
+properties:
+  contentUrl:
+    '@id': schema:contentUrl
+    description: Actual bytes of the media object, for example the image file or video file.
+    anyOf:
+      - $ref: URL.schema.yaml