Skip to content

Commit

Permalink
Merge pull request #1635 from concord-consortium/188544725-v3-remote-…
Browse files Browse the repository at this point in the history
…boundaries

Implement loading of remote boundaries and integration with the formula evaluation/dependency system.
  • Loading branch information
kswenson authored Nov 25, 2024
2 parents 60ecfa1 + 8d80b3b commit 2355700
Show file tree
Hide file tree
Showing 31 changed files with 432 additions and 104 deletions.
1 change: 0 additions & 1 deletion v3/src/boundaries/US_State_Boundaries.json

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ import { clsx } from "clsx"
import { format } from "d3-format"
import React from "react"
import { measureText } from "../../hooks/use-measure-text"
import { boundaryObjectFromBoundaryValue } from "../../models/boundaries/boundary-types"
import { IAttribute } from "../../models/data/attribute"
import { kDefaultNumPrecision } from "../../models/data/attribute-types"
import { boundaryObjectFromBoundaryValue } from "../../utilities/boundary-utils"
import { parseColor } from "../../utilities/color-utils"
import { isStdISODateString } from "../../utilities/date-iso-utils"
import { parseDate } from "../../utilities/date-parser"
Expand Down
14 changes: 12 additions & 2 deletions v3/src/components/common/formula-editor.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import CodeMirror, {
import React, { useCallback, useRef } from "react"
import { useMemo } from "use-memo-one"
import { useDataSetContext } from "../../hooks/use-data-set-context"
import { boundaryManager } from "../../models/boundaries/boundary-manager"
import { IDataSet } from "../../models/data/data-set"
import { typedFnRegistry } from "../../models/formula/functions/math"
import { formulaLanguageWithHighlighting } from "../../models/formula/lezer/formula-language"
Expand All @@ -20,14 +21,15 @@ import { FormulaEditorApi, useFormulaEditorContext } from "./formula-editor-cont

interface ICompletionOptions {
attributes: boolean
boundaries: boolean
constants: boolean
functions: boolean
globals: boolean
specials: boolean
}

const kAllOptions: ICompletionOptions = {
attributes: true, constants: true, functions: true, globals: true, specials: true
attributes: true, boundaries: true, constants: true, functions: true, globals: true, specials: true
}

interface IProps {
Expand Down Expand Up @@ -114,6 +116,9 @@ function cmCodapCompletions(context: CompletionContext): CompletionResult | null
{ label: "π" }
] : []
const specials = options?.specials ? [{ label: "caseIndex" }] : []
const boundaries = options?.boundaries
? Array.from(boundaryManager.boundaryKeys.map(key => ({ label: key })))
: []
const globalManager = dataSet && options?.globals
? getGlobalValueManager(getSharedModelManager(dataSet))
: undefined
Expand All @@ -139,7 +144,9 @@ function cmCodapCompletions(context: CompletionContext): CompletionResult | null
}
}
})) : []
const completions: Completion[] = [...attributes, ...constants, ...specials, ...globals, ...functions]
const completions: Completion[] = [
...attributes, ...constants, ...specials, ...boundaries, ...globals, ...functions
]

if (!before || before.to === before.from) return null

Expand All @@ -164,6 +171,9 @@ const highlightClasses: Record<string, HighlightFn> = {
if (options.attributes && data?.getAttributeByName(nodeText)) return "codap-attribute"
if (options.constants && ["e", "pi", "π"].includes(nodeText)) return "codap-constant"
if (options.specials && ["caseIndex"].includes(nodeText)) return "codap-special"

if (boundaryManager.isBoundarySet(nodeText)) return "codap-boundary"

const globalManager = options.globals ? getGlobalValueManager(getSharedModelManager(data)) : undefined
if (globalManager?.getValueByName(nodeText)) return "codap-global"
}
Expand Down
10 changes: 3 additions & 7 deletions v3/src/components/common/formula-insert-values-menu.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ import { TriangleDownIcon, TriangleUpIcon } from "@chakra-ui/icons"
import {Divider, Flex, List, ListItem,} from "@chakra-ui/react"
import { IAnyStateTreeNode } from "mobx-state-tree"
import React, { useEffect, useRef, useState } from "react"
import { useMemo } from "use-memo-one"
import { useDataSetContext } from "../../hooks/use-data-set-context"
import { boundaryManager } from "../../models/boundaries/boundary-manager"
import { getGlobalValueManager, getSharedModelManager } from "../../models/tiles/tile-environment"
import { useFormulaEditorContext } from "./formula-editor-context"

Expand Down Expand Up @@ -31,10 +31,6 @@ export const InsertValuesMenu = ({setShowValuesMenu}: IProps) => {
.filter(name => name !== undefined)
)
const attributeNames = dataSet?.attributes.map(attr => attr.name)
// TODO_Boundaries
const remoteBoundaryData = useMemo(() => [ "CR_Cantones", "CR_Provincias", "DE_state_boundaries",
"IT_region_boundaries", "JP_Prefectures", "US_congressional_boundaries", "US_county_boundaries",
"US_puma_boundaries", "US_state_boundaries", "country_boundaries" ], [])
const constants = ["e", "false", "true", "π"]
const scrollableContainerRef = useRef<HTMLUListElement>(null)
const [, setScrollPosition] = useState(0)
Expand All @@ -58,7 +54,7 @@ export const InsertValuesMenu = ({setShowValuesMenu}: IProps) => {
maxItemLength.current = attrName.length
}
})
remoteBoundaryData.forEach((boundary) => {
boundaryManager.boundaryKeys.forEach((boundary) => {
if (boundary.length > maxItemLength.current) {
maxItemLength.current = boundary.length
}
Expand Down Expand Up @@ -153,7 +149,7 @@ export const InsertValuesMenu = ({setShowValuesMenu}: IProps) => {
</List>
<Divider className="list-divider"/>
<List className="formula-operand-subset">
{ remoteBoundaryData.map((boundary) => {
{ boundaryManager.boundaryKeys.map((boundary) => {
return (
<ListItem key={boundary} className="formula-operand-list-item"
onClick={() => insertValueToFormula(boundary)}>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ const getTestEnv = () => {
}
const api = {
getDatasets: jest.fn(() => dataSets),
getBoundaryManager: jest.fn(),
getGlobalValueManager: jest.fn(),
getFormulaExtraMetadata: jest.fn(() => extraMetadata),
getFormulaContext: jest.fn(() => context),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ const getTestEnv = () => {
}
const api = {
getDatasets: jest.fn(() => dataSets),
getBoundaryManager: jest.fn(),
getGlobalValueManager: jest.fn(),
getFormulaExtraMetadata: jest.fn(() => extraMetadata),
getFormulaContext: jest.fn(() => context),
Expand Down
2 changes: 1 addition & 1 deletion v3/src/components/map/utilities/map-utils.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import {LatLngBounds, latLngBounds} from 'leaflet'
import {singular} from "pluralize"
import {kPolygonNames} from "../../../models/boundaries/boundary-types"
import {IDataSet} from "../../../models/data/data-set"
import { kPolygonNames } from "../../../utilities/boundary-utils"
import {isFiniteNumber} from "../../../utilities/math-utils"
import {translate} from "../../../utilities/translation/translate"
import {IDataConfigurationModel} from "../../data-display/models/data-configuration-model"
Expand Down
4 changes: 3 additions & 1 deletion v3/src/components/web-view/web-view-utils.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
export const kRootPluginUrl = "https://codap-resources.s3.amazonaws.com/plugins"
import { codapResourcesUrl } from "../../constants"

export const kRootPluginUrl = codapResourcesUrl("plugins")
export const kRelativePluginRoot = "../../../../extn/plugins"

export function processPluginUrl(url: string) {
Expand Down
3 changes: 3 additions & 0 deletions v3/src/constants.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
export const kCodapResourcesUrl = "https://codap-resources.concord.org"

export const codapResourcesUrl = (relUrl: string) => `${kCodapResourcesUrl}/${relUrl}`
3 changes: 2 additions & 1 deletion v3/src/hooks/use-remote-plugins-config.test.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
import { renderHook, waitFor } from "@testing-library/react"
import { codapResourcesUrl } from "../constants"
import { useRemotePluginsConfig } from "./use-remote-plugins-config"

// mock urlParams to have a morePlugins parameter
// url doesn't matter since response is mocked below
jest.mock("../utilities/url-params", () => ({
urlParams: {
morePlugins: "https://codap-resources.s3.amazonaws.com/plugins/published-plugins.json"
morePlugins: codapResourcesUrl("plugins/published-plugins.json")
}
}))

Expand Down
7 changes: 4 additions & 3 deletions v3/src/lib/use-cloud-file-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,15 @@ import { CFMAppOptions, CloudFileManager, CloudFileManagerClientEvent } from "@c
import { useEffect, useRef } from "react"
import { Root, createRoot } from "react-dom/client"
import { useMemo } from "use-memo-one"
import { clientConnect, createCloudFileManager, renderRoot } from "./cfm-utils"
import { handleCFMEvent } from "./handle-cfm-event"
import { codapResourcesUrl } from "../constants"
import { appState } from "../models/app-state"
import { isCodapDocument } from "../models/codap/create-codap-document"
import { gLocale } from "../utilities/translation/locale"
import { t } from "../utilities/translation/translate"
import { removeDevUrlParams, urlParams } from "../utilities/url-params"
import { clientConnect, createCloudFileManager, renderRoot } from "./cfm-utils"
import { DEBUG_CFM_LOCAL_STORAGE } from "./debug"
import { handleCFMEvent } from "./handle-cfm-event"

const locales = [
{
Expand Down Expand Up @@ -199,7 +200,7 @@ export function useCloudFileManager(optionsArg: CFMAppOptions) {
"name": "readOnly",
"displayName": t("DG.fileMenu.provider.examples.displayName"),
"urlDisplayName": "examples",
"src": "https://codap-resources.s3.amazonaws.com/example-documents/index.json",
"src": codapResourcesUrl("example-documents/index.json"),
alphabetize: false
},
{
Expand Down
124 changes: 124 additions & 0 deletions v3/src/models/boundaries/boundary-manager.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
import { waitFor } from "@testing-library/react"
import { BoundaryManager, boundaryManager } from "./boundary-manager"

const kStateBoundaries = "US_state_boundaries"

describe("BoundaryManager", () => {
it("works when initialized/empty", () => {
expect(boundaryManager.boundaryKeys).toEqual([])
expect(boundaryManager.isBoundarySet()).toBe(false)
expect(boundaryManager.isBoundarySet("foo")).toBe(false)
expect(boundaryManager.hasBoundaryData()).toBe(false)
expect(boundaryManager.hasBoundaryData("foo")).toBe(false)
expect(boundaryManager.isBoundaryDataPending()).toBe(false)
expect(boundaryManager.isBoundaryDataPending("foo")).toBe(false)
expect(boundaryManager.getBoundaryData("foo", "bar")).toBeUndefined()
})

it("fetches boundary data successfully", async () => {
// mock the initial boundary specs request
fetchMock.mockResponseOnce(JSON.stringify([
{
"name": kStateBoundaries,
"format": "codap",
"url": "US_State_Boundaries.codap"
}
]), {
headers: {
"content-type": "application/json"
}
})

const _boundaryManager = new BoundaryManager()
// request has been submitted but response has not been received
expect(_boundaryManager.boundaryKeys.length).toBe(0)
await waitFor(() => {
expect(_boundaryManager.boundaryKeys.length).toBe(1)
})
expect(_boundaryManager.isBoundarySet(kStateBoundaries)).toBe(true)

// mock the boundary data request
const kAlaskaBoundaryGuid = 1234
const kAlaskaBoundaryData = { geometry: {}, coordinates: {}, features: {} }
fetchMock.mockResponseOnce(JSON.stringify({
contexts: [{
collections: [
// [0] boundary collection
{
cases: [{
guid: kAlaskaBoundaryGuid,
values: {
boundary: kAlaskaBoundaryData
}
}]
},
// [1] key collection
{
cases: [{
parent: kAlaskaBoundaryGuid,
values: {
key: "alaska"
}
}]
}
]
}]
}))
// request the boundary data
_boundaryManager.getBoundaryData(kStateBoundaries, "Alaska")
// boundary data is pending
expect(_boundaryManager.hasBoundaryData(kStateBoundaries)).toBe(false)
expect(_boundaryManager.isBoundaryDataPending(kStateBoundaries)).toBe(true)
await waitFor(() => {
expect(_boundaryManager.hasBoundaryData(kStateBoundaries)).toBe(true)
})
expect(_boundaryManager.isBoundaryDataPending(kStateBoundaries)).toBe(false)
expect(_boundaryManager.getBoundaryData(kStateBoundaries, "Alaska")).toEqual(kAlaskaBoundaryData)
expect(_boundaryManager.hasBoundaryDataError(kStateBoundaries)).toBe(false)
})

it("logs error when failing to fetch boundary specs", () => {
fetchMock.mockRejectOnce(new Error("not found"))

jestSpyConsole("error", async spy => {
const _boundaryManager = new BoundaryManager()
await waitFor(() => {
expect(spy).toHaveBeenCalledTimes(1)
})
expect(_boundaryManager.boundaryKeys.length).toBe(0)
})
})

it("logs error when failing to fetch boundary data", async () => {
// mock the initial boundary specs request
fetchMock.mockResponseOnce(JSON.stringify([
{
"name": kStateBoundaries,
"format": "codap",
"url": "US_State_Boundaries.codap"
}
]), {
headers: {
"content-type": "application/json"
}
})

const _boundaryManager = new BoundaryManager()
await waitFor(() => {
expect(_boundaryManager.boundaryKeys.length).toBe(1)
})

fetchMock.mockRejectOnce(new Error("not found"))

jestSpyConsole("error", async spy => {
const boundaryData = _boundaryManager.getBoundaryData(kStateBoundaries, "Alaska")
expect(boundaryData).toBeUndefined()
expect(_boundaryManager.isBoundaryDataPending(kStateBoundaries)).toBe(true)
await waitFor(() => {
expect(spy).toHaveBeenCalledTimes(1)
})
expect(_boundaryManager.isBoundaryDataPending(kStateBoundaries)).toBe(false)
expect(_boundaryManager.hasBoundaryDataError(kStateBoundaries)).toBe(true)
})
})
})
Loading

0 comments on commit 2355700

Please sign in to comment.