Skip to content

Commit

Permalink
create ast to python and convert all examples into python
Browse files Browse the repository at this point in the history
  • Loading branch information
Xinyue-Yang committed Nov 20, 2024
1 parent bc10d61 commit d6ad7a5
Showing 95 changed files with 9,640 additions and 99 deletions.
165 changes: 92 additions & 73 deletions bin/prepare-examples.js
Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
#!/usr/bin/env node
import { basename, extname, join, resolve } from 'node:path';
import { copyFile, readdir, readFile, writeFile } from 'node:fs/promises';
import { parseSpec, astToESM } from '@uwdata/mosaic-spec';
import { parse } from 'yaml';
import { basename, extname, join, resolve } from "node:path";
import {
copyFile,
readdir,
readFile,
writeFile,
mkdir,
} from "node:fs/promises";
import { parseSpec, astToESM, astToPython } from "@uwdata/mosaic-spec";
import { parse } from "yaml";

// This script prepares all Mosaic example specifications
// ...AND WILL OVERWRITE EXISTING TEST CASE DATA AND DOCS!
@@ -11,86 +17,99 @@ import { parse } from 'yaml';
// - YAML, non-parsed JSON, and ESM code written to /docs/public/specs
// - Example Markdown pages written to /docs/examples

const specDir = join('specs', 'yaml');
const esmTestDir = join('specs', 'esm');
const jsonTestDir = join('specs', 'json');
const tsTestDir = join('specs', 'ts');
const specDir = join("specs", "yaml");
const esmTestDir = join("specs", "esm");
const jsonTestDir = join("specs", "json");
const tsTestDir = join("specs", "ts");
const pythonTestDir = join("specs", "python");

const docsDir = 'docs';
const yamlDocsDir = join(docsDir, 'public', 'specs', 'yaml');
const jsonDocsDir = join(docsDir, 'public', 'specs', 'json');
const esmDocsDir = join(docsDir, 'public', 'specs', 'esm');
const exampleDir = join(docsDir, 'examples');
const docsDir = "docs";
const yamlDocsDir = join(docsDir, "public", "specs", "yaml");
const jsonDocsDir = join(docsDir, "public", "specs", "json");
const esmDocsDir = join(docsDir, "public", "specs", "esm");
const exampleDir = join(docsDir, "examples");
const pythonDocsDir = join(docsDir, "public", "specs", "python");

const specToTS = spec => {
const specToTS = (spec) => {
return `import { Spec } from '@uwdata/mosaic-spec';
export const spec : Spec = ${JSON.stringify(spec, 0, 2)};
`;
}
};

// Create directories if they don't exist
await Promise.all([
mkdir(pythonTestDir, { recursive: true }),
mkdir(pythonDocsDir, { recursive: true }),
]).catch(console.error);

const files = await Promise.allSettled(
(
await readdir(specDir)
)
.filter((name) => extname(name) === ".yaml")
.map(async (name) => {
const base = basename(name, ".yaml");
const file = resolve(specDir, name);
const text = await readFile(file, "utf8");

const files = await Promise.allSettled((await readdir(specDir))
.filter(name => extname(name) === '.yaml')
.map(async name => {
const base = basename(name, '.yaml');
const file = resolve(specDir, name);
const text = await readFile(file, 'utf8');

// parse spec and perform code generation
// do this first to catch any errors
const spec = parse(text);
const ast = parseSpec(spec);
const code = astToESM(ast);

try {
await Promise.all([
// write ESM DSL spec to tests
writeFile(resolve(esmTestDir, `${base}.js`), code),
// write JSON spec to tests
writeFile(
resolve(jsonTestDir, `${base}.json`),
JSON.stringify(ast.toJSON(), 0, 2)
),
// write TS JSON spec to tests
writeFile(
resolve(tsTestDir, `${base}.ts`),
specToTS(spec)
),
// copy YAML file to docs
copyFile(file, resolve(yamlDocsDir, `${base}.yaml`)),
// write JSON spec to docs
writeFile(
resolve(jsonDocsDir, `${base}.json`),
JSON.stringify(spec, 0, 2)
),
// write ESM DSL spec to docs
writeFile(resolve(esmDocsDir, `${base}.js`), code),
// write examples page to docs
writeFile(
resolve(exampleDir, `${base}.md`),
examplePage(base, spec.meta)
)
]);
} catch (err) {
console.error(err);
}

return base;
})
// parse spec and perform code generation
// do this first to catch any errors
const spec = parse(text);
const ast = parseSpec(spec);
const code = astToESM(ast);

try {
await Promise.all([
// write ESM DSL spec to tests
writeFile(resolve(esmTestDir, `${base}.js`), code),
// write JSON spec to tests
writeFile(
resolve(jsonTestDir, `${base}.json`),
JSON.stringify(ast.toJSON(), 0, 2)
),
// write TS JSON spec to tests
writeFile(resolve(tsTestDir, `${base}.ts`), specToTS(spec)),
// copy YAML file to docs
copyFile(file, resolve(yamlDocsDir, `${base}.yaml`)),
// write JSON spec to docs
writeFile(
resolve(jsonDocsDir, `${base}.json`),
JSON.stringify(spec, 0, 2)
),
// write ESM DSL spec to docs
writeFile(resolve(esmDocsDir, `${base}.js`), code),
// write examples page to docs
writeFile(
resolve(exampleDir, `${base}.md`),
examplePage(base, spec.meta)
),
// write Python spec to tests
writeFile(resolve(pythonTestDir, `${base}.py`), astToPython(ast)),
// write Python spec to docs
writeFile(resolve(pythonDocsDir, `${base}.py`), astToPython(ast)),
]);
} catch (err) {
console.error(err);
}

return base;
})
);

// output successfully written examples
console.log(JSON.stringify(
files
.filter(x => x.status === 'fulfilled')
.map(x => x.value),
0, 2
));
console.log(
JSON.stringify(
files.filter((x) => x.status === "fulfilled").map((x) => x.value),
0,
2
)
);

// output unsuccessful example errors
files
.filter(x => x.status === 'rejected')
.forEach(x => console.error(x.reason));
.filter((x) => x.status === "rejected")
.forEach((x) => console.error(x.reason));

function examplePage(spec, { title = spec, description, credit } = {}) {
return `<script setup>
@@ -99,9 +118,9 @@ function examplePage(spec, { title = spec, description, credit } = {}) {
</script>
# ${title}
${description ? `\n${description.trim()}\n` : ''}
${description ? `\n${description.trim()}\n` : ""}
<Example spec="/specs/yaml/${spec}.yaml" />
${credit ? `\n**Credit**: ${credit}\n` : ''}
${credit ? `\n**Credit**: ${credit}\n` : ""}
## Specification
::: code-group
69 changes: 69 additions & 0 deletions docs/public/specs/python/aeromagnetic-survey.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
from mosaic_spec import *
from typing import Dict, Any, Union

spec = {
"meta": {
"title": "Aeromagnetic Survey",
"description": "A raster visualization of the 1955 [Great Britain aeromagnetic survey](https://www.bgs.ac.uk/datasets/gb-aeromagnetic-survey/), which measured the Earth’s magnetic field by plane. Each sample recorded the longitude and latitude alongside the strength of the [IGRF](https://www.ncei.noaa.gov/products/international-geomagnetic-reference-field) in [nanoteslas](https://en.wikipedia.org/wiki/Tesla_(unit)). This example demonstrates both raster interpolation and smoothing (blur) options.\n",
"credit": "Adapted from an [Observable Plot example](https://observablehq.com/@observablehq/plot-igfr90-raster)."
},
"data": {
"ca55": {
"type": "parquet",
"file": "data/ca55-south.parquet"
}
},
"params": {
"interp": "random-walk",
"blur": 0
},
"vconcat": [
{
"hconcat": [
{
"input": "menu",
"label": "Interpolation Method",
"options": [
"none",
"nearest",
"barycentric",
"random-walk"
],
"as": "$interp"
},
{
"hspace": "1em"
},
{
"input": "slider",
"label": "Blur",
"min": 0,
"max": 100,
"as": "$blur"
}
]
},
{
"vspace": "1em"
},
{
"plot": [
{
"mark": "raster",
"data": {
"from": "ca55"
},
"x": "LONGITUDE",
"y": "LATITUDE",
"fill": {
"max": "MAG_IGRF90"
},
"interpolate": "$interp",
"bandwidth": "$blur"
}
],
"colorScale": "diverging",
"colorDomain": "Fixed"
}
]
}
75 changes: 75 additions & 0 deletions docs/public/specs/python/airline-travelers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
from mosaic_spec import *
from typing import Dict, Any, Union

spec = {
"meta": {
"title": "Airline Travelers",
"description": "A labeled line chart comparing airport travelers in 2019 and 2020.",
"credit": "Adapted from an [Observable Plot example](https://observablehq.com/@observablehq/plot-labeled-line-chart)."
},
"data": {
"travelers": {
"type": "parquet",
"file": "data/travelers.parquet"
},
"endpoint": {
"type": "table",
"query": "SELECT * FROM travelers ORDER BY date DESC LIMIT 1"
}
},
"plot": [
{
"mark": "ruleY",
"data": [
0
]
},
{
"mark": "lineY",
"data": {
"from": "travelers"
},
"x": "date",
"y": "previous",
"strokeOpacity": 0.35
},
{
"mark": "lineY",
"data": {
"from": "travelers"
},
"x": "date",
"y": "current"
},
{
"mark": "text",
"data": {
"from": "endpoint"
},
"x": "date",
"y": "previous",
"text": [
"2019"
],
"fillOpacity": 0.5,
"lineAnchor": "bottom",
"dy": -6
},
{
"mark": "text",
"data": {
"from": "endpoint"
},
"x": "date",
"y": "current",
"text": [
"2020"
],
"lineAnchor": "top",
"dy": 6
}
],
"yGrid": True,
"yLabel": "↑ Travelers per day",
"yTickFormat": "s"
}
Loading

0 comments on commit d6ad7a5

Please sign in to comment.