feat: univariate stats functions

concord-consortium · Oct 29, 2024 · af00d85 · af00d85
1 parent 53874f2
commit af00d85
Show file tree

Hide file tree

Showing 20 changed files with 586 additions and 306 deletions.
diff --git a/v3/src/assets/json/function_strings.json → v3/src/assets/json/function-strings.json5 b/v3/src/assets/json/function_strings.json → v3/src/assets/json/function-strings.json5
@@ -721,7 +721,7 @@
           }
         ],
         "examples": [
-          "greatCircleDistance(40.66, 74, 37.8, 122.4) returns 4128, the distance in kilometers between New York and San Francisco"
+          "greatCircleDistance(40.66, -74, 37.8, -122.4) returns 4128, the distance in kilometers between New York and San Francisco"
         ]
       },
       { "displayName": "if",
@@ -855,13 +855,19 @@
   { "category": "statistical",
     "displayName": "Statistical",
     "functions": [
-      { "displayName": "combine",
+      { "displayName": "combine", // appears in statistical (because it's aggregate) and strings sections
         "description": "Returns the aggregated concatenation of its evaluated expression values.",
         "args": [
           {
             "name": "expression",
             "type": "any",
             "description": "The values to be concatenated."
+          },
+          {
+            "name": "filter",
+            "type": "filter",
+            "description": "A true-false expression that limits the cases considered.",
+            "optional": true
           }
         ],
         "examples": [
@@ -891,33 +897,19 @@
           "count(Height, Age<18) returns the number of cases for the attribute Height with a value for the Age attribute that is less than 18"
         ]
       },
-      { "displayName": "uniqueValues",
-        "description": "Returns the number of unique values produced by the argument.",
-        "args": [
-          {
-            "name": "expression",
-            "type": "any",
-            "description": "Evaluated for each case. The number of these that are unique will be the final value."
-          },
-          {
-            "name": "filter",
-            "type": "boolean",
-            "description": "Serves as a filter for the cases for which the expression will be computed.",
-            "optional": true
-          }
-        ],
-        "examples": [
-          "uniqueValues(State) returns the number unique values of the attribute State. If all state names are present, the result would likely be 50",
-          "uniqueValues(Age, Sex=\"male\") returns the number of different ages for the cases whose sex is male"
-        ]
-      },
       { "displayName": "mad",
         "description": "Computes the mean absolute deviation the given expression.",
         "args": [
           {
             "name": "expression",
             "type": "expression",
             "description": "The expression for which you want to find the MAD."
+          },
+          {
+            "name": "filter",
+            "type": "filter",
+            "description": "A true-false expression that limits the cases considered.",
+            "optional": true
           }
         ],
         "examples": [
@@ -966,31 +958,6 @@
           "mean(Height, Sex = \"F\") computes the mean Height of females"
         ]
       },
-      { "displayName": "rollingMean",
-        "description": "Returns the rolling arithmetic mean of its expression evaluated for every case. The rolling mean averages the n nearest values with the given value where n is the value of \"width\".",
-        "args": [
-          {
-            "name": "expression",
-            "type": "expression",
-            "description": "The expression whose mean will be returned."
-          },
-          {
-            "name": "width",
-            "type": "expression",
-            "description": "The number of values included in each mean. This value is rounded to the nearest whole number."
-          },
-          {
-            "name": "filter",
-            "type": "boolean",
-            "description": "(optional) A true-false expression that selects the cases considered.",
-            "optional": true
-          }
-        ],
-        "examples": [
-          "rollingMean(Height, 5) computes the rolling mean of the attribute Height, including the the current value and the two preceding and following values",
-          "rollingMean(Height, 5, Habitat!=\"water\") computes the rolling mean of Height, excluding water dwelling Mammals"
-        ]
-      },
       { "displayName": "median",
         "description": "Returns the median. Half the values of the attribute will be above this and half will be below.",
         "args": [
@@ -1044,6 +1011,12 @@
             "name": "constant",
             "type": "value",
             "description": "The percentile (in the range 0 to 1, inclusive)."
+          },
+          {
+            "name": "filter",
+            "type": "expression",
+            "description": "Additional filter, numbers, or ranges for which you want the first case.",
+            "optional": true
           }
         ],
         "examples": [
@@ -1052,13 +1025,44 @@
           "percentile(Mass/Volume, 0.1) will return the density corresponding to the 10th percentile"
         ]
       },
+      { "displayName": "rollingMean",
+        "description": "Returns the rolling arithmetic mean of its expression evaluated for every case. The rolling mean averages the n nearest values with the given value where n is the value of \"width\".",
+        "args": [
+          {
+            "name": "expression",
+            "type": "expression",
+            "description": "The expression whose mean will be returned."
+          },
+          {
+            "name": "width",
+            "type": "expression",
+            "description": "The number of values included in each mean. This value is rounded to the nearest whole number."
+          },
+          {
+            "name": "filter",
+            "type": "boolean",
+            "description": "(optional) A true-false expression that selects the cases considered.",
+            "optional": true
+          }
+        ],
+        "examples": [
+          "rollingMean(Height, 5) computes the rolling mean of the attribute Height, including the the current value and the two preceding and following values",
+          "rollingMean(Height, 5, Habitat!=\"water\") computes the rolling mean of Height, excluding water dwelling Mammals"
+        ]
+      },
       { "displayName": "stdDev",
         "description": "Computes the sample standard deviation the given expression.",
         "args": [
           {
             "name": "expression",
             "type": "expression",
             "description": "The expression for which you want to find the standard deviation."
+          },
+          {
+            "name": "filter",
+            "type": "expression",
+            "description": "Additional filter, numbers, or ranges for which you want the first case.",
+            "optional": true
           }
         ],
         "examples": [
@@ -1073,6 +1077,12 @@
             "name": "expression",
             "type": "expression",
             "description": "The expression for which you want to find the standard error."
+          },
+          {
+            "name": "filter",
+            "type": "expression",
+            "description": "Additional filter, numbers, or ranges for which you want the first case.",
+            "optional": true
           }
         ],
         "examples": [
@@ -1100,13 +1110,39 @@
           "sum(Time)/count(Time) returns the sum of the Time attribute divided by the count of the Time attribute"
         ]
       },
+      { "displayName": "uniqueValues",
+        "description": "Returns the number of unique values produced by the argument.",
+        "args": [
+          {
+            "name": "expression",
+            "type": "any",
+            "description": "Evaluated for each case. The number of these that are unique will be the final value."
+          },
+          {
+            "name": "filter",
+            "type": "boolean",
+            "description": "Serves as a filter for the cases for which the expression will be computed.",
+            "optional": true
+          }
+        ],
+        "examples": [
+          "uniqueValues(State) returns the number unique values of the attribute State. If all state names are present, the result would likely be 50",
+          "uniqueValues(Age, Sex=\"male\") returns the number of different ages for the cases whose sex is male"
+        ]
+      },
       { "displayName": "variance",
         "description": "Computes the variance of an attribute.",
         "args": [
           {
             "name": "expression",
             "type": "attribute",
             "description": "The attribute for which you want to find the variance."
+          },
+          {
+            "name": "filter",
+            "type": "expression",
+            "description": "Additional filter, numbers, or ranges for which you want the first case.",
+            "optional": true
           }
         ],
         "examples": [
@@ -1338,6 +1374,26 @@
           "charAt(123, random(4, 1)) returns a random number between 1 and 3"
         ]
       },
+      { "displayName": "combine", // appears in statistical (because it's aggregate) and strings sections
+        "description": "Returns the aggregated concatenation of its evaluated expression values.",
+        "args": [
+          {
+            "name": "expression",
+            "type": "any",
+            "description": "The values to be concatenated."
+          },
+          {
+            "name": "filter",
+            "type": "filter",
+            "description": "A true-false expression that limits the cases considered.",
+            "optional": true
+          }
+        ],
+        "examples": [
+          "combine(flip) might return \"HHT\" when there are three cases each with value \"H\" or \"T\"",
+          "combine(1) returns 1, repeated for the number of cases in the case table"
+        ]
+      },
       { "displayName": "concat",
         "description": "Combines multiple expressions into a single string.",
         "args": [

diff --git a/v3/src/components/case-card/case-attr-view.tsx b/v3/src/components/case-card/case-attr-view.tsx
@@ -2,7 +2,7 @@ import React, { useState } from "react"
 import { observer } from "mobx-react-lite"
 import { Editable, EditablePreview, EditableInput } from "@chakra-ui/react"
 import { clsx } from "clsx"
-import { IValueType } from "../../models/data/attribute"
+import { IValueType } from "../../models/data/attribute-types"
 import { ICollectionModel } from "../../models/data/collection"
 import { ICase } from "../../models/data/data-set-types"
 import { isFiniteNumber } from "../../utilities/math-utils"

diff --git a/v3/src/components/case-card/case-attrs-view.tsx b/v3/src/components/case-card/case-attrs-view.tsx
@@ -1,15 +1,15 @@
 import React, { useCallback, useRef, useState } from "react"
 import { observer } from "mobx-react-lite"
 import { clsx } from "clsx"
+import { IValueType } from "../../models/data/attribute-types"
 import { IGroupedCase } from "../../models/data/data-set-types"
-import { CaseAttrView } from "./case-attr-view"
-import { IValueType } from "../../models/data/attribute"
 import { ICollectionModel } from "../../models/data/collection"
-import { useCaseCardModel } from "./use-case-card-model"
 import { getSharedCaseMetadataFromDataset } from "../../models/shared/shared-data-utils"
 import { AttributeHeader } from "../case-tile-common/attribute-header"
 import { AttributeHeaderDivider } from "../case-tile-common/attribute-header-divider"
 import { kIndexColumnKey } from "../case-tile-common/case-tile-types"
+import { CaseAttrView } from "./case-attr-view"
+import { useCaseCardModel } from "./use-case-card-model"
 
 import "./case-attrs-view.scss"
 

diff --git a/v3/src/components/case-card/case-card-model.ts b/v3/src/components/case-card/case-card-model.ts
@@ -1,10 +1,10 @@
 import { Instance, SnapshotIn, types } from "mobx-state-tree"
+import { IValueType } from "../../models/data/attribute-types"
+import { ICollectionModel } from "../../models/data/collection"
+import { ICaseCreation, IGroupedCase } from "../../models/data/data-set-types"
 import { getTileCaseMetadata, getTileDataSet } from "../../models/shared/shared-data-utils"
 import { ITileContentModel, TileContentModel } from "../../models/tiles/tile-content"
 import { kCaseCardTileType } from "./case-card-defs"
-import { ICollectionModel } from "../../models/data/collection"
-import { ICaseCreation, IGroupedCase } from "../../models/data/data-set-types"
-import { IValueType } from "../../models/data/attribute"
 
 export const CaseCardModel = TileContentModel
   .named("CaseCardModel")

diff --git a/v3/src/components/case-table/use-rows.ts b/v3/src/components/case-table/use-rows.ts
@@ -8,7 +8,7 @@ import { useDataSetContext } from "../../hooks/use-data-set-context"
 import { useLoggingContext } from "../../hooks/use-log-context"
 import { logMessageWithReplacement } from "../../lib/log-message"
 import { appState } from "../../models/app-state"
-import { kDefaultFormatStr } from "../../models/data/attribute"
+import { kDefaultFormatStr } from "../../models/data/attribute-types"
 import { isAddCasesAction, isRemoveCasesAction, isSetCaseValuesAction } from "../../models/data/data-set-actions"
 import { createCasesNotification } from "../../models/data/data-set-notifications"
 import {

diff --git a/v3/src/components/case-tile-common/render-attribute-value.tsx b/v3/src/components/case-tile-common/render-attribute-value.tsx
@@ -1,6 +1,7 @@
 import { format } from "d3-format"
 import React from "react"
-import { IAttribute, kDefaultFormatStr } from "../../models/data/attribute"
+import { IAttribute } from "../../models/data/attribute"
+import { kDefaultFormatStr } from "../../models/data/attribute-types"
 import { parseColor } from "../../utilities/color-utils"
 import { isStdISODateString } from "../../utilities/date-iso-utils"
 import { parseDate } from "../../utilities/date-parser"

diff --git a/v3/src/components/common/formula-insert-function-menu.tsx b/v3/src/components/common/formula-insert-function-menu.tsx
@@ -1,13 +1,31 @@
 import {Divider, Flex, List, ListItem,} from "@chakra-ui/react"
 import React, { useEffect, useState } from "react"
 
-import functionCategoryInfoArray from "../../assets/json/function_strings.json"
+import _functionCategoryInfoArray from "../../assets/json/function-strings.json5"
 import { useFormulaEditorContext } from "./formula-editor-context"
 
 import "./formula-insert-menus.scss"
 
-type FunctionCategoryInfo = typeof functionCategoryInfoArray[number]
-type FunctionInfo = FunctionCategoryInfo["functions"][number]
+interface FunctionArgInfo {
+  name: string
+  type: string
+  description: string
+  optional?: boolean    // defaults to false (i.e. required)
+}
+interface FunctionInfo {
+  displayName: string
+  description: string
+  args: FunctionArgInfo[]
+  examples: string[]
+}
+
+interface FunctionCategoryInfo {
+  category: string
+  displayName: string
+  functions: FunctionInfo[]
+}
+
+const functionCategoryInfoArray: FunctionCategoryInfo[] = _functionCategoryInfoArray
 
 const kMenuGap = 3
 

diff --git a/v3/src/data-interactive/handlers/all-cases-handler.ts b/v3/src/data-interactive/handlers/all-cases-handler.ts
@@ -1,7 +1,7 @@
+import { IValueType } from "../../models/data/attribute-types"
+import { maybeToV2Id, toV2Id } from "../../utilities/codap-utils"
 import { registerDIHandler } from "../data-interactive-handler"
 import { DIHandler, DIResources } from "../data-interactive-types"
-import { IValueType } from "../../models/data/attribute"
-import { maybeToV2Id, toV2Id } from "../../utilities/codap-utils"
 import { collectionNotFoundResult, dataContextNotFoundResult } from "./di-results"
 
 export const diAllCasesHandler: DIHandler = {

diff --git a/v3/src/models/data/attribute-types.ts b/v3/src/models/data/attribute-types.ts
@@ -0,0 +1,6 @@
+export const kDefaultFormatStr = ".3~f"
+
+export const isDevelopment = () => process.env.NODE_ENV !== "production"
+export const isProduction = () => process.env.NODE_ENV === "production"
+
+export type IValueType = string | number | boolean | Date | undefined
diff --git a/v3/src/models/data/attribute.test.ts b/v3/src/models/data/attribute.test.ts
@@ -2,9 +2,9 @@ import { cloneDeep } from "lodash"
 import { reaction } from "mobx"
 import { getSnapshot } from "mobx-state-tree"
 import {
-  Attribute, IAttributeSnapshot, importValueToString, isAttributeType, isFormulaAttr,
-  isValidFormulaAttr, kDefaultFormatStr
+  Attribute, IAttributeSnapshot, importValueToString, isAttributeType, isFormulaAttr, isValidFormulaAttr
 } from "./attribute"
+import { kDefaultFormatStr } from "./attribute-types"
 
 describe("Attribute", () => {
 

diff --git a/v3/src/models/data/attribute.ts b/v3/src/models/data/attribute.ts
@@ -34,15 +34,9 @@ import { cachedFnFactory } from "../../utilities/mst-utils"
 import { Formula, IFormula } from "../formula/formula"
 import { applyModelChange } from "../history/apply-model-change"
 import { withoutUndo } from "../history/without-undo"
+import { isDevelopment, isProduction, IValueType, kDefaultFormatStr } from "./attribute-types"
 import { V2Model } from "./v2-model"
 
-export const kDefaultFormatStr = ".3~f"
-
-const isDevelopment = () => process.env.NODE_ENV !== "production"
-const isProduction = () => process.env.NODE_ENV === "production"
-
-export type IValueType = string | number | boolean | Date | undefined
-
 export interface ISetValueOptions {
   noInvalidate?: boolean
 }

diff --git a/v3/src/models/data/data-set-types.ts b/v3/src/models/data/data-set-types.ts
@@ -1,5 +1,5 @@
 import { RequireAtLeastOne } from "type-fest"
-import { IValueType } from "./attribute"
+import { IValueType } from "./attribute-types"
 
 
 export interface IItemID {